From c06dd6d193a8289636af282a1e9adcb9cfa3e5f1 Mon Sep 17 00:00:00 2001 From: Duke Date: Sat, 18 Jan 2014 12:30:56 -0200 Subject: [PATCH 1/5] starting a way to get extentions #28 --- feed_test.go | 26 +++++++++++++++++++++++ item.go | 2 ++ rss.go | 37 ++++++++++++++++++++++++++++++++ testdata/extension.rss | 48 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+) create mode 100644 testdata/extension.rss diff --git a/feed_test.go b/feed_test.go index dca55b3..f90d5ee 100644 --- a/feed_test.go +++ b/feed_test.go @@ -76,6 +76,32 @@ func Test_RssAuthor(t *testing.T) { } } +func Test_Extensions(t *testing.T) { + content, _ := ioutil.ReadFile("testdata/extension.rss") + feed := New(1, true, chanHandler, itemHandler) + feed.FetchBytes("http://example.com", content, nil) + + extension := feed.Channels[0].Items[0].Extensions["http://www.sec.gov/Archives/edgar"] + + companyExpected := "Cellular Biomedicine Group, Inc." + companyName := *extension.Childrens[0] + if companyName.Value != companyExpected { + t.Errorf("Expected company to be %s but found %s", companyExpected, companyName.Value) + } + + files := *extension.Childrens[11] + fileSizeExpected := 10 + if len(files.Childrens) != 10 { + t.Errorf("Expected files size to be %s but found %s", fileSizeExpected, len(files.Childrens)) + } + + file := *files.Childrens[0] + fileExpected := "cbmg_10qa.htm" + if file.Attrs["file"] != fileExpected { + t.Errorf("Expected file to be %s but found %s", fileExpected, len(file.Attrs["file"])) + } +} + func Test_CData(t *testing.T) { content, _ := ioutil.ReadFile("testdata/iosBoardGameGeek.rss") feed := New(1, true, chanHandler, itemHandler) diff --git a/item.go b/item.go index ea13f85..1a301e8 100644 --- a/item.go +++ b/item.go @@ -23,6 +23,8 @@ type Item struct { Generator *Generator Contributors []string Content *Content + + Extensions map[string]*Extension } func (i *Item) Key() string { diff --git a/rss.go b/rss.go index b6c6d73..6855b3f 100644 --- a/rss.go +++ b/rss.go @@ -6,6 +6,13 @@ import ( xmlx "github.com/jteeuwen/go-pkg-xmlx" ) +type Extension struct { + Name string + Value string + Attrs map[string]string + Childrens []*Extension +} + var days = map[string]int{ "Monday": 1, "Tuesday": 2, @@ -174,9 +181,39 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err error) { } } + tl = item.SelectNodes("*", "*") + i.Extensions = make(map[string]*Extension) + for _, lv := range tl { + e, ok := getExtension(lv) + if ok { + i.Extensions[lv.Name.Space] = e + } + } + ch.Items = append(ch.Items, i) } } this.Channels = foundChannels return } + +func getExtension(node *xmlx.Node) (*Extension, bool) { + if node.Name.Space != "" { + var extension Extension + extension = Extension{Name: node.Name.Local, Value: node.GetValue()} + extension.Attrs = make(map[string]string) + for _, x := range node.Attributes { + extension.Attrs[x.Name.Local] = x.Value + } + + for _, y := range node.Children { + if c, ok := getExtension(y); ok { + extension.Childrens = append(extension.Childrens, c) + } + } + + return &extension, true + } else { + return nil, false + } +} diff --git a/testdata/extension.rss b/testdata/extension.rss new file mode 100644 index 0000000..8619389 --- /dev/null +++ b/testdata/extension.rss @@ -0,0 +1,48 @@ + + + + Extensions Test + http://test.extensions.net + Just a Test + en-US + Wed, 27 Mar 2013 12:30:18 PDT + + http://test.extensions.net + http://test.extensions.net/test.jpg + Extensions Test + + + Cellular Biomedicine Group, Inc. (0001378624) (Filer) + http://www.sec.gov/Archives/edgar/data/1378624/000135448813006749/0001354488-13-006749-index.htm + http://www.sec.gov/Archives/edgar/data/1378624/000135448813006749/0001354488-13-006749-xbrl.zip + + 10-Q/A + Fri, 06 Dec 2013 17:29:22 EST + + Cellular Biomedicine Group, Inc. + 10-Q/A + 12/06/2013 + 0001378624 + 0001354488-13-006749 + 000-52282 + 20131206172922 + 20130630 + 7 + 6199 + 1231 + + + + + + + + + + + + + + + + From 7daa266b189da6ff758b5a2b408e1f6ae382b57e Mon Sep 17 00:00:00 2001 From: Duke Date: Sat, 18 Jan 2014 14:07:37 -0200 Subject: [PATCH 2/5] fixed same extentions space on root level --- feed_test.go | 2 +- item.go | 2 +- rss.go | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/feed_test.go b/feed_test.go index f90d5ee..dc84b76 100644 --- a/feed_test.go +++ b/feed_test.go @@ -81,7 +81,7 @@ func Test_Extensions(t *testing.T) { feed := New(1, true, chanHandler, itemHandler) feed.FetchBytes("http://example.com", content, nil) - extension := feed.Channels[0].Items[0].Extensions["http://www.sec.gov/Archives/edgar"] + extension := feed.Channels[0].Items[0].Extensions["http://www.sec.gov/Archives/edgar"][0] companyExpected := "Cellular Biomedicine Group, Inc." companyName := *extension.Childrens[0] diff --git a/item.go b/item.go index 1a301e8..a9b274f 100644 --- a/item.go +++ b/item.go @@ -24,7 +24,7 @@ type Item struct { Contributors []string Content *Content - Extensions map[string]*Extension + Extensions map[string][]*Extension } func (i *Item) Key() string { diff --git a/rss.go b/rss.go index 6855b3f..34e1eee 100644 --- a/rss.go +++ b/rss.go @@ -182,11 +182,11 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err error) { } tl = item.SelectNodes("*", "*") - i.Extensions = make(map[string]*Extension) + i.Extensions = make(map[string][]*Extension) for _, lv := range tl { e, ok := getExtension(lv) if ok { - i.Extensions[lv.Name.Space] = e + i.Extensions[lv.Name.Space] = append(i.Extensions[lv.Name.Space], e) } } @@ -202,6 +202,7 @@ func getExtension(node *xmlx.Node) (*Extension, bool) { var extension Extension extension = Extension{Name: node.Name.Local, Value: node.GetValue()} extension.Attrs = make(map[string]string) + for _, x := range node.Attributes { extension.Attrs[x.Name.Local] = x.Value } From e895d2d708d585ab12e612a1a814fdadc0d71433 Mon Sep 17 00:00:00 2001 From: Duke Date: Thu, 23 Jan 2014 14:57:26 -0200 Subject: [PATCH 3/5] changed Extensions api --- feed_test.go | 14 +++++++------- item.go | 2 +- rss.go | 41 ++++++++++++++++++++++++++++------------- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/feed_test.go b/feed_test.go index dc84b76..a1345e8 100644 --- a/feed_test.go +++ b/feed_test.go @@ -76,26 +76,26 @@ func Test_RssAuthor(t *testing.T) { } } -func Test_Extensions(t *testing.T) { +func Test_ItemExtensions(t *testing.T) { content, _ := ioutil.ReadFile("testdata/extension.rss") feed := New(1, true, chanHandler, itemHandler) feed.FetchBytes("http://example.com", content, nil) - extension := feed.Channels[0].Items[0].Extensions["http://www.sec.gov/Archives/edgar"][0] + edgarExtension := feed.Channels[0].Items[0].Extensions["http://www.sec.gov/Archives/edgar"] companyExpected := "Cellular Biomedicine Group, Inc." - companyName := *extension.Childrens[0] + companyName := edgarExtension["companyName"][0] if companyName.Value != companyExpected { t.Errorf("Expected company to be %s but found %s", companyExpected, companyName.Value) } - files := *extension.Childrens[11] + files := edgarExtension["xbrlFiles"][0].Childrens["xbrlFile"] fileSizeExpected := 10 - if len(files.Childrens) != 10 { - t.Errorf("Expected files size to be %s but found %s", fileSizeExpected, len(files.Childrens)) + if len(files) != 10 { + t.Errorf("Expected files size to be %s but found %s", fileSizeExpected, len(files)) } - file := *files.Childrens[0] + file := files[0] fileExpected := "cbmg_10qa.htm" if file.Attrs["file"] != fileExpected { t.Errorf("Expected file to be %s but found %s", fileExpected, len(file.Attrs["file"])) diff --git a/item.go b/item.go index a9b274f..74045e4 100644 --- a/item.go +++ b/item.go @@ -24,7 +24,7 @@ type Item struct { Contributors []string Content *Content - Extensions map[string][]*Extension + Extensions map[string]map[string][]Extension } func (i *Item) Key() string { diff --git a/rss.go b/rss.go index 34e1eee..a4797de 100644 --- a/rss.go +++ b/rss.go @@ -10,7 +10,7 @@ type Extension struct { Name string Value string Attrs map[string]string - Childrens []*Extension + Childrens map[string][]Extension } var days = map[string]int{ @@ -182,11 +182,11 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err error) { } tl = item.SelectNodes("*", "*") - i.Extensions = make(map[string][]*Extension) + i.Extensions = make(map[string]map[string][]Extension) for _, lv := range tl { - e, ok := getExtension(lv) + e, ok := getExtentions(lv) if ok { - i.Extensions[lv.Name.Space] = append(i.Extensions[lv.Name.Space], e) + i.Extensions[lv.Name.Space] = e } } @@ -197,24 +197,39 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err error) { return } -func getExtension(node *xmlx.Node) (*Extension, bool) { +func getExtentions(node *xmlx.Node) (extentions map[string][]Extension, ok bool) { + extentions = make(map[string][]Extension, 0) + if node.Name.Space != "" { + for _, y := range node.Children { + extension, ok := getExtention(y) + if ok { + extentions[y.Name.Local] = append(extentions[y.Name.Local], extension) + } + } + ok = true + } else { + ok = false + } + return +} + +func getExtention(node *xmlx.Node) (Extension, bool) { + var extension Extension if node.Name.Space != "" { - var extension Extension extension = Extension{Name: node.Name.Local, Value: node.GetValue()} extension.Attrs = make(map[string]string) - + extension.Childrens = make(map[string][]Extension, 0) for _, x := range node.Attributes { extension.Attrs[x.Name.Local] = x.Value } - for _, y := range node.Children { - if c, ok := getExtension(y); ok { - extension.Childrens = append(extension.Childrens, c) + children, ok := getExtention(y) + if ok { + extension.Childrens[y.Name.Local] = append(extension.Childrens[y.Name.Local], children) } } - - return &extension, true + return extension, true } else { - return nil, false + return extension, false } } From bd84124886c7bb209d0f823f465d58a207bb0a3f Mon Sep 17 00:00:00 2001 From: Duke Date: Thu, 23 Jan 2014 17:37:18 -0200 Subject: [PATCH 4/5] updated getExtentions to don't overwrite root node --- feed_test.go | 7 +++---- rss.go | 32 +++++++++++++++----------------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/feed_test.go b/feed_test.go index a1345e8..cdd39cf 100644 --- a/feed_test.go +++ b/feed_test.go @@ -81,15 +81,14 @@ func Test_ItemExtensions(t *testing.T) { feed := New(1, true, chanHandler, itemHandler) feed.FetchBytes("http://example.com", content, nil) - edgarExtension := feed.Channels[0].Items[0].Extensions["http://www.sec.gov/Archives/edgar"] - + edgarExtensionxbrlFiling := feed.Channels[0].Items[0].Extensions["http://www.sec.gov/Archives/edgar"]["xbrlFiling"][0].Childrens companyExpected := "Cellular Biomedicine Group, Inc." - companyName := edgarExtension["companyName"][0] + companyName := edgarExtensionxbrlFiling["companyName"][0] if companyName.Value != companyExpected { t.Errorf("Expected company to be %s but found %s", companyExpected, companyName.Value) } - files := edgarExtension["xbrlFiles"][0].Childrens["xbrlFile"] + files := edgarExtensionxbrlFiling["xbrlFiles"][0].Childrens["xbrlFile"] fileSizeExpected := 10 if len(files) != 10 { t.Errorf("Expected files size to be %s but found %s", fileSizeExpected, len(files)) diff --git a/rss.go b/rss.go index a4797de..484a8c7 100644 --- a/rss.go +++ b/rss.go @@ -181,13 +181,10 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err error) { } } - tl = item.SelectNodes("*", "*") + tl = item.SelectNodes(ns, ns) i.Extensions = make(map[string]map[string][]Extension) for _, lv := range tl { - e, ok := getExtentions(lv) - if ok { - i.Extensions[lv.Name.Space] = e - } + getExtensions(&i.Extensions, lv) } ch.Items = append(ch.Items, i) @@ -197,23 +194,24 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err error) { return } -func getExtentions(node *xmlx.Node) (extentions map[string][]Extension, ok bool) { - extentions = make(map[string][]Extension, 0) +func getExtensions(extensionsX *map[string]map[string][]Extension, node *xmlx.Node) { + extentions := *extensionsX + if node.Name.Space != "" { - for _, y := range node.Children { - extension, ok := getExtention(y) - if ok { - extentions[y.Name.Local] = append(extentions[y.Name.Local], extension) + extensione, noErrors := getExtension(node) + if noErrors { + if len(extentions[node.Name.Space]) == 0 { + extentions[node.Name.Space] = make(map[string][]Extension, 0) } + if len(extentions[node.Name.Space][node.Name.Local]) == 0 { + extentions[node.Name.Space][node.Name.Local] = make([]Extension, 0) + } + extentions[node.Name.Space][node.Name.Local] = append(extentions[node.Name.Space][node.Name.Local], extensione) } - ok = true - } else { - ok = false } - return } -func getExtention(node *xmlx.Node) (Extension, bool) { +func getExtension(node *xmlx.Node) (Extension, bool) { var extension Extension if node.Name.Space != "" { extension = Extension{Name: node.Name.Local, Value: node.GetValue()} @@ -223,7 +221,7 @@ func getExtention(node *xmlx.Node) (Extension, bool) { extension.Attrs[x.Name.Local] = x.Value } for _, y := range node.Children { - children, ok := getExtention(y) + children, ok := getExtension(y) if ok { extension.Childrens[y.Name.Local] = append(extension.Childrens[y.Name.Local], children) } From 937de70c03b3af6f78338755794385c6f74f7df1 Mon Sep 17 00:00:00 2001 From: Duke Date: Thu, 23 Jan 2014 17:37:50 -0200 Subject: [PATCH 5/5] added Extension to channel --- channel.go | 1 + feed_test.go | 30 ++++++++++++++++++++++++++++++ rss.go | 9 +++++++++ testdata/extension.rss | 9 ++++++++- 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/channel.go b/channel.go index fe3318b..dc3104f 100644 --- a/channel.go +++ b/channel.go @@ -21,6 +21,7 @@ type Channel struct { Items []*Item Cloud Cloud TextInput Input + Extensions map[string]map[string][]Extension // Atom fields Id string diff --git a/feed_test.go b/feed_test.go index cdd39cf..8566533 100644 --- a/feed_test.go +++ b/feed_test.go @@ -101,6 +101,36 @@ func Test_ItemExtensions(t *testing.T) { } } +func Test_ChannelExtensions(t *testing.T) { + content, _ := ioutil.ReadFile("testdata/extension.rss") + feed := New(1, true, chanHandler, itemHandler) + feed.FetchBytes("http://example.com", content, nil) + + channel := feed.Channels[0] + itunesExtentions := channel.Extensions["http://www.itunes.com/dtds/podcast-1.0.dtd"] + + authorExptected := "The Author" + ownerEmailExpected := "test@rss.com" + categoryExpected := "Politics" + imageExptected := "http://golang.org/doc/gopher/project.png" + + if itunesExtentions["author"][0].Value != authorExptected { + t.Errorf("Expected author to be %s but found %s", authorExptected, itunesExtentions["author"][0].Value) + } + + if itunesExtentions["owner"][0].Childrens["email"][0].Value != ownerEmailExpected { + t.Errorf("Expected owner email to be %s but found %s", ownerEmailExpected, itunesExtentions["owner"][0].Childrens["email"][0].Value) + } + + if itunesExtentions["category"][0].Attrs["text"] != categoryExpected { + t.Errorf("Expected category text to be %s but found %s", categoryExpected, itunesExtentions["category"][0].Attrs["text"]) + } + + if itunesExtentions["image"][0].Attrs["href"] != imageExptected { + t.Errorf("Expected image href to be %s but found %s", imageExptected, itunesExtentions["image"][0].Attrs["href"]) + } +} + func Test_CData(t *testing.T) { content, _ := ioutil.ReadFile("testdata/iosBoardGameGeek.rss") feed := New(1, true, chanHandler, itemHandler) diff --git a/rss.go b/rss.go index 484a8c7..f990e59 100644 --- a/rss.go +++ b/rss.go @@ -189,6 +189,15 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err error) { ch.Items = append(ch.Items, i) } + + x := node.SelectNodes(ns, ns) + ch.Extensions = make(map[string]map[string][]Extension) + for _, v := range x { + if v.Name.Space != "" { + getExtensions(&ch.Extensions, v) + } + } + } this.Channels = foundChannels return diff --git a/testdata/extension.rss b/testdata/extension.rss index 8619389..3d7c255 100644 --- a/testdata/extension.rss +++ b/testdata/extension.rss @@ -1,5 +1,5 @@ - + Extensions Test http://test.extensions.net @@ -11,6 +11,13 @@ http://test.extensions.net/test.jpg Extensions Test + The Author + + + test@rss.com + + + Cellular Biomedicine Group, Inc. (0001378624) (Filer) http://www.sec.gov/Archives/edgar/data/1378624/000135448813006749/0001354488-13-006749-index.htm