fix imcomplete fetch for some rfa news.

wedojava · wedojava · commit f7a80fc71f70 · 2020-07-10T20:09:22.000+08:00
diff --git a/internal/fetcher/post_test.go b/internal/fetcher/post_test.go
@@ -10,7 +10,7 @@ import (
 
 func TestSetAndSavePost(t *testing.T) {
 	// p := PostFactory("https://www.dwnews.com/经济/60203253")
-	p := PostFactory("https://www.dwnews.com/经济/60203034") // The wrong one
+	p := PostFactory("https://www.rfa.org/mandarin/Xinwen/6-07082020110802.html") // The wrong one
 	raw, doc, err := htmldoc.GetRawAndDoc(p.URL, 1*time.Minute)
 	if err != nil {
 		t.Errorf("GetRawAndDoC error: %v", err)
@@ -30,6 +30,7 @@ func TestTreatPost(t *testing.T) {
 		"https://www.dwnews.com/全球/60203234",
 		"https://www.voachinese.com/a/S-Korea-Says-US-Sees-Importance-Of-N-Korea-Talks-Despite-Tension-20200709/5496028.html",
 		"https://www.rfa.org/mandarin/yataibaodao/shaoshuminzu/gf1-07092020074142.html",
+		"https://www.rfa.org/mandarin/Xinwen/6-07082020110802.html",
 	}
 	for _, tc := range tcs {
 		p := PostFactory(tc)
diff --git a/internal/fetcher/sites/rfa/rfa.go b/internal/fetcher/sites/rfa/rfa.go
@@ -91,7 +91,8 @@ func Rfa(p *Post) (string, error) {
 	for _, v := range plist {
 		if v.FirstChild == nil {
 			continue
-		} else if v.FirstChild.Data == "b" {
+		}
+		if v.FirstChild.Data == "b" {
 			body += "** "
 			blist := htmldoc.ElementsByTagName(v, "b")
 			for _, b := range blist {
@@ -102,7 +103,13 @@ func Rfa(p *Post) (string, error) {
 			}
 			body += " **  \n"
 		} else {
-			body += v.FirstChild.Data + "  \n"
+			// body += v.FirstChild.Data + "  \n"
+			innerNodes := htmldoc.ElementsNext(v)
+			for _, in := range innerNodes {
+				if in.Type == html.TextNode {
+					body += in.Data + "  \n"
+				}
+			}
 		}
 	}
 	body = strings.ReplaceAll(body, "**   **  \n", "")
diff --git a/internal/fetcher/sites/rfa/rfa_test.go b/internal/fetcher/sites/rfa/rfa_test.go
@@ -0,0 +1,47 @@
+package rfa
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/wedojava/fetcher/internal/htmldoc"
+)
+
+func PostFactory(rawurl string) *Post {
+	url, err := url.Parse(rawurl)
+	if err != nil {
+		log.Printf("url parse err: %s", err)
+	}
+	return &Post{
+		Domain: url.Hostname(),
+		URL:    url,
+	}
+}
+
+func TestSetPost(t *testing.T) {
+	p := PostFactory("https://www.rfa.org/mandarin/Xinwen/6-07082020110802.html")
+	raw, doc, err := htmldoc.GetRawAndDoc(p.URL, 1*time.Minute)
+	if err != nil {
+		t.Errorf("GetRawAndDoc err: %v", err)
+	}
+	p.Raw, p.DOC = raw, doc
+	if err := SetPost(p); err != nil {
+		t.Errorf("test SetPost err: %v", doc)
+	}
+	fmt.Println(p.Title)
+	fmt.Println(p.Body)
+}
+
+func TestRfa(t *testing.T) {
+	p := PostFactory("https://www.rfa.org/mandarin/Xinwen/6-07082020110802.html")
+	raw, doc, err := htmldoc.GetRawAndDoc(p.URL, 1*time.Minute)
+	if err != nil {
+		t.Errorf("GetRawAndDoc err: %v", err)
+	}
+	p.Raw, p.DOC = raw, doc
+	tc, err := Rfa(p)
+	fmt.Println(tc)
+}
diff --git a/internal/htmldoc/htmldoc.go b/internal/htmldoc/htmldoc.go
@@ -79,6 +79,17 @@ func ExtractLinks(weburl string) ([]string, error) {
 	return links, nil
 }
 
+func ElementsNext(doc *html.Node) []*html.Node {
+	nodes := []*html.Node{}
+	visitNode := func(n *html.Node) {
+		if n.NextSibling != nil {
+			nodes = append(nodes, n)
+		}
+	}
+	ForEachNode(doc, visitNode, nil)
+	return nodes
+}
+
 func ElementsByTagName(doc *html.Node, name ...string) []*html.Node {
 	var nodes []*html.Node
 	if len(name) == 0 {
