Did some significant reworking of the Rss and Atom internals. The Feed object now supports passing in 2 funciton handlers which notify your app of any new Channels or Items. These functions are called whenever the feed is updated and new channels or items are found which previously did not exist. See src/feed_test.go for examples. This change also made it prudent to change some of the struct definitions. In particular the fields referencing other structs. They have been changed from Value to Pointer definitions. So beware that this may break some of your existing code. At the very least, the feeder.New() function now takes 2 extra parameters.

2010-12-18 00:25:16 +01:00 · 2010-12-18 00:25:16 +01:00 · a8057b0c92
parent eb15b6a3ef
commit a8057b0c92
7 changed files with 223 additions and 107 deletions
--- a/43
+++ b/43
@ -1,26 +1,33 @@
- Author: jim teeuwen <jimteeuwen@gmail.com>
- Dependencies: go-pkg-xmlx ( http://github.com/jteeuwen/go-pkg-xmlx )
+Author: jim teeuwen <jimteeuwen@gmail.com>
+Dependencies: go-pkg-xmlx ( http://github.com/jteeuwen/go-pkg-xmlx )

- This package allows us to fetch Rss and Atom feeds from the internet.
- They are parsed into an object tree which is a hybrid of both the RSS and Atom
- standards.
+This package allows us to fetch Rss and Atom feeds from the internet.
+They are parsed into an object tree which is a hybrid of both the RSS and Atom
+standards.

- Supported feeds are:
- 	- Rss v0.91, 0.92 and 2.0
- 	- Atom 1.0
+Supported feeds are:
+- Rss v0.91, 0.92 and 2.0
+- Atom 1.0

- The package allows us to maintain cache timeout management. This prevents us
- from querying the servers for feed updates too often and risk ip bans. Apart 
- from setting a cache timeout manually, the package also optionally adheres to
- the TTL, SkipDays and SkipHours values specied in the feeds themselves.
+The package allows us to maintain cache timeout management. This prevents us
+from querying the servers for feed updates too often and risk ip bans. Apart 
+from setting a cache timeout manually, the package also optionally adheres to
+the TTL, SkipDays and SkipHours values specied in the feeds themselves.

- Note that the TTL, SkipDays and SkipHour fields are only part of the RSS spec.
- For Atom feeds, we use the CacheTimeout in the Feed struct.
+Note that the TTL, SkipDays and SkipHour fields are only part of the RSS spec.
+For Atom feeds, we use the CacheTimeout in the Feed struct.

- Because the object structure is a hybrid between both RSS and Atom specs, not
- all fields will be filled when requesting either an RSS or Atom feed. I have
- tried to create as many shared fields as possible but some of them simply do
- not occur in either the RSS or Atom spec.
+Because the object structure is a hybrid between both RSS and Atom specs, not
+all fields will be filled when requesting either an RSS or Atom feed. I have
+tried to create as many shared fields as possible but some of them simply do
+not occur in either the RSS or Atom spec.
+
+The Feed object supports notifications of new channels and items.
+This is achieved by passing 2 function handlers to the feeder.New() function.
+They will be called whenever a feed is updated from the remote source and
+either a new channel or a new item is found that previously did not exist.
+This allows you to easily monitor a feed for changes. See src/feed_test.go for
+an example of how this works.

 ================================================================================
 LICENSE
--- a/src/atom.go
+++ b/src/atom.go
@ -6,14 +6,42 @@ import "xmlx"
 func (this *Feed) readAtom(doc *xmlx.Document) (err os.Error) {
 	ns := "http://www.w3.org/2005/Atom"
 	channels := doc.SelectNodes(ns, "feed")
+
+	getChan := func(id string) *Channel {
+		for _, c := range this.Channels {
+			if c.Id == id {
+				return c
+			}
+		}
+		return nil
+	}
+
+	haveItem := func(ch *Channel, id string) bool {
+		for _, item := range ch.Items {
+			if item.Id == id {
+				return true
+			}
+		}
+		return false
+	}
+
+	var ch *Channel
+	var i *Item
+	var tn *xmlx.Node
+	var list []*xmlx.Node
+
 	for _, node := range channels {
-		ch := Channel{}
+		if ch = getChan(node.GetValue("", "pubDate")); ch == nil {
+			ch = new(Channel)
+			this.Channels = append(this.Channels, ch)
+		}
+
 		ch.Title = node.GetValue(ns, "title")
 		ch.LastBuildDate = node.GetValue(ns, "updated")
 		ch.Id = node.GetValue(ns, "id")
 		ch.Rights = node.GetValue(ns, "rights")

-		list := node.SelectNodes(ns, "link")
+		list = node.SelectNodes(ns, "link")
 		ch.Links = make([]Link, len(list))
 		for i, v := range list {
 			ch.Links[i].Href = v.GetAttr("", "href")
@ -22,8 +50,7 @@ func (this *Feed) readAtom(doc *xmlx.Document) (err os.Error) {
 			ch.Links[i].HrefLang = v.GetAttr("", "hreflang")
 		}

-		tn := node.SelectNode(ns, "subtitle")
-		if tn != nil {
+		if tn = node.SelectNode(ns, "subtitle"); tn != nil {
 			ch.SubTitle = SubTitle{}
 			ch.SubTitle.Type = tn.GetAttr("", "type")
 			ch.SubTitle.Text = tn.Value
@ -43,50 +70,56 @@ func (this *Feed) readAtom(doc *xmlx.Document) (err os.Error) {
 			ch.Author.Email = tn.GetValue("", "email")
 		}

+		itemcount := len(ch.Items)
 		list = node.SelectNodes(ns, "entry")
-		ch.Items = make([]Item, len(list))
-		for _, v := range list {
-			item := Item{}
-			item.Title = v.GetValue(ns, "title")
-			item.Id = v.GetValue(ns, "id")
-			item.PubDate = v.GetValue(ns, "updated")
-			item.Description = v.GetValue(ns, "summary")

-			list = v.SelectNodes(ns, "link")
-			item.Links = make([]Link, 0)
-			for _, lv := range list {
+		for _, item := range list {
+			if haveItem(ch, item.GetValue("", "id")) {
+				continue
+			}
+
+			i = new(Item)
+			i.Title = item.GetValue(ns, "title")
+			i.Id = item.GetValue(ns, "id")
+			i.PubDate = item.GetValue(ns, "updated")
+			i.Description = item.GetValue(ns, "summary")
+
+			links := item.SelectNodes(ns, "link")
+			for _, lv := range links {
 				if tn.GetAttr(ns, "rel") == "enclosure" {
-					enc := Enclosure{}
+					enc := new(Enclosure)
 					enc.Url = lv.GetAttr("", "href")
 					enc.Type = lv.GetAttr("", "type")
-					item.Enclosures = append(item.Enclosures, enc)
+					i.Enclosures = append(i.Enclosures, enc)
 				} else {
-					lnk := Link{}
+					lnk := new(Link)
 					lnk.Href = lv.GetAttr("", "href")
 					lnk.Rel = lv.GetAttr("", "rel")
 					lnk.Type = lv.GetAttr("", "type")
 					lnk.HrefLang = lv.GetAttr("", "hreflang")
-					item.Links = append(item.Links, lnk)
+					i.Links = append(i.Links, lnk)
 				}
 			}

-			list = v.SelectNodes(ns, "contributor")
-			item.Contributors = make([]string, len(list))
-			for ci, cv := range list {
-				item.Contributors[ci] = cv.GetValue("", "name")
+			list = item.SelectNodes(ns, "contributor")
+			for _, cv := range list {
+				i.Contributors = append(i.Contributors, cv.GetValue("", "name"))
 			}

-			if tn = v.SelectNode(ns, "content"); tn != nil {
-				item.Content = Content{}
-				item.Content.Type = tn.GetAttr("", "type")
-				item.Content.Lang = tn.GetValue("xml", "lang")
-				item.Content.Base = tn.GetValue("xml", "base")
-				item.Content.Text = tn.Value
+			if tn = item.SelectNode(ns, "content"); tn != nil {
+				i.Content = new(Content)
+				i.Content.Type = tn.GetAttr("", "type")
+				i.Content.Lang = tn.GetValue("xml", "lang")
+				i.Content.Base = tn.GetValue("xml", "base")
+				i.Content.Text = tn.Value
 			}
-			ch.Items = append(ch.Items, item)
+
+			ch.Items = append(ch.Items, i)
 		}

-		this.Channels = append(this.Channels, ch)
+		if itemcount != len(ch.Items) && this.itemhandler != nil {
+			this.itemhandler(this, ch, ch.Items[itemcount:])
+		}
 	}
 	return
 }
--- a/src/channel.go
+++ b/src/channel.go
@ -11,14 +11,14 @@ type Channel struct {
 	PubDate        string
 	LastBuildDate  string
 	Docs           string
-	Categories     []Category
+	Categories     []*Category
 	Generator      Generator
 	TTL            int
 	Rating         string
 	SkipHours      []int
 	SkipDays       []int
 	Image          Image
-	Items          []Item
+	Items          []*Item
 	Cloud          Cloud
 	TextInput      Input

--- a/src/feed.go
+++ b/src/feed.go
@ -32,6 +32,9 @@ import "fmt"
 import "strconv"
 import "strings"

+type ChannelHandler func(f *Feed, newchannels []*Channel)
+type ItemHandler func(f *Feed, ch *Channel, newitems []*Item)
+
 type Feed struct {
 	// Custom cache timeout in minutes.
 	CacheTimeout int
@ -47,21 +50,31 @@ type Feed struct {
 	Version [2]int

 	// Channels with content.
-	Channels []Channel
+	Channels []*Channel

 	// Url from which this feed was created.
 	Url string

+	// A notification function, used to notify the host when a new channel
+	// has been found.
+	chanhandler ChannelHandler
+
+	// A notification function, used to notify the host when a new item
+	// has been found for a given channel.
+	itemhandler ItemHandler
+
 	// Last time content was fetched. Used in conjunction with CacheTimeout
 	// to ensure we don't get content too often.
 	lastupdate int64
 }

-func New(cachetimeout int, enforcecachelimit bool) *Feed {
+func New(cachetimeout int, enforcecachelimit bool, ch ChannelHandler, ih ItemHandler) *Feed {
 	v := new(Feed)
 	v.CacheTimeout = cachetimeout
 	v.EnforceCacheLimit = enforcecachelimit
 	v.Type = "none"
+	v.chanhandler = ch
+	v.itemhandler = ih
 	return v
 }

@ -71,7 +84,6 @@ func (this *Feed) Fetch(uri string) (err os.Error) {
 	}

 	this.Url = uri
-	this.Channels = nil

 	// Extract type and version of the feed so we can have the appropriate
 	// function parse it (rss 0.91, rss 0.92, rss 2, atom etc).
@ -86,14 +98,21 @@ func (this *Feed) Fetch(uri string) (err os.Error) {
 		return
 	}

+	chancount := len(this.Channels)
 	if err = this.buildFeed(doc); err != nil || len(this.Channels) == 0 {
 		return
 	}

+	// Notify host of new channels
+	if chancount != len(this.Channels) && this.chanhandler != nil {
+		this.chanhandler(this, this.Channels[chancount:])
+	}
+
 	// reset cache timeout values according to feed specified values (TTL)
 	if this.EnforceCacheLimit && this.CacheTimeout < this.Channels[0].TTL {
 		this.CacheTimeout = this.Channels[0].TTL
 	}
+
 	return
 }

--- a/src/feed_test.go
+++ b/src/feed_test.go
@ -15,10 +15,31 @@ func TestFeed(t *testing.T) {
 	var err os.Error

 	for _, uri := range urilist {
-		feed = New(5, true)
+		feed = New(5, true, chanHandler, itemHandler)

 		if err = feed.Fetch(uri); err != nil {
 			t.Errorf("%s >>> %s", uri, err)
 		}
 	}
+	
+	/*
+Output of handlers:
+
+6 new item(s) in WriteTheWeb of http://cyber.law.harvard.edu/rss/examples/sampleRss091.xml
+1 new channel(s) in http://cyber.law.harvard.edu/rss/examples/sampleRss091.xml
+21 new item(s) in Dave Winer: Grateful Dead of http://cyber.law.harvard.edu/rss/examples/sampleRss092.xml
+1 new channel(s) in http://cyber.law.harvard.edu/rss/examples/sampleRss092.xml
+4 new item(s) in Liftoff News of http://cyber.law.harvard.edu/rss/examples/rss2sample.xml
+1 new channel(s) in http://cyber.law.harvard.edu/rss/examples/rss2sample.xml
+15 new item(s) in Blog@Case of http://blog.case.edu/news/feed.atom
+1 new channel(s) in http://blog.case.edu/news/feed.atom
+	*/
+}
+
+func chanHandler(feed *Feed, newchannels []*Channel) {
+	//println(len(newchannels), "new channel(s) in", feed.Url)
+}
+
+func itemHandler(feed *Feed, ch *Channel, newitems []*Item) {
+	//println(len(newitems), "new item(s) in", ch.Title, "of", feed.Url)
 }
--- a/src/item.go
+++ b/src/item.go
@ -3,19 +3,19 @@ package feeder
 type Item struct {
 	// RSS and Shared fields
 	Title       string
-	Links       []Link
+	Links       []*Link
 	Description string
 	Author      Author
-	Categories  []Category
+	Categories  []*Category
 	Comments    string
-	Enclosures  []Enclosure
+	Enclosures  []*Enclosure
 	Guid        string
 	PubDate     string
-	Source      Source
+	Source      *Source

 	// Atom specific fields
 	Id           string
-	Generator    Generator
+	Generator    *Generator
 	Contributors []string
-	Content      Content
+	Content      *Content
 }
--- a/src/rss.go
+++ b/src/rss.go
@ -4,13 +4,60 @@ import "os"
 import "xmlx"

 func (this *Feed) readRss2(doc *xmlx.Document) (err os.Error) {
+	days := make(map[string]int)
+	days["Monday"] = 1
+	days["Tuesday"] = 2
+	days["Wednesday"] = 3
+	days["Thursday"] = 4
+	days["Friday"] = 5
+	days["Saturday"] = 6
+	days["Sunday"] = 7
+
+	getChan := func(pubdate string) *Channel {
+		for _, c := range this.Channels {
+			if c.PubDate == pubdate {
+				return c
+			}
+		}
+		return nil
+	}
+
+	haveItem := func(ch *Channel, id, title, desc string) bool {
+		for _, item := range ch.Items {
+			switch {
+			case len(id) > 0:
+				if item.Id == id {
+					return true
+				}
+			case len(title) > 0:
+				if item.Title == title {
+					return true
+				}
+			default:
+				if item.Description == desc {
+					return true
+				}
+			}
+		}
+		return false
+	}
+
+	var ch *Channel
+	var i *Item
+	var n *xmlx.Node
+	var list, tl []*xmlx.Node
+
 	channels := doc.SelectNodes("", "channel")
 	for _, node := range channels {
-		ch := Channel{}
-		ch.Title = node.GetValue("", "title")
+		if ch = getChan(node.GetValue("", "pubDate")); ch == nil {
+			ch = new(Channel)
+			this.Channels = append(this.Channels, ch)
+		}

-		list := node.SelectNodes("", "link")
+		ch.Title = node.GetValue("", "title")
+		list = node.SelectNodes("", "link")
 		ch.Links = make([]Link, len(list))
+
 		for i, v := range list {
 			ch.Links[i].Href = v.Value
 		}
@ -25,14 +72,14 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err os.Error) {
 		ch.Docs = node.GetValue("", "docs")

 		list = node.SelectNodes("", "category")
-		ch.Categories = make([]Category, len(list))
+		ch.Categories = make([]*Category, len(list))
 		for i, v := range list {
+			ch.Categories[i] = new(Category)
 			ch.Categories[i].Domain = v.GetAttr("", "domain")
 			ch.Categories[i].Text = v.Value
 		}

-		n := node.SelectNode("", "generator")
-		if n != nil {
+		if n = node.SelectNode("", "generator"); n != nil {
 			ch.Generator = Generator{}
 			ch.Generator.Text = n.Value
 		}
@ -49,7 +96,7 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err os.Error) {
 		list = node.SelectNodes("", "days")
 		ch.SkipDays = make([]int, len(list))
 		for i, v := range list {
-			ch.SkipDays[i] = mapDay(v.Value)
+			ch.SkipDays[i] = days[v.Value]
 		}

 		if n = node.SelectNode("", "image"); n != nil {
@ -78,16 +125,22 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err os.Error) {
 			ch.TextInput.Link = n.GetValue("", "link")
 		}

+		itemcount := len(ch.Items)
 		list = node.SelectNodes("", "item")
+
 		for _, item := range list {
-			i := Item{}
+			if haveItem(ch, item.GetValue("", "pubDate"),
+				item.GetValue("", "title"), item.GetValue("", "description")) {
+				continue
+			}
+
+			i = new(Item)
 			i.Title = item.GetValue("", "title")
 			i.Description = item.GetValue("", "description")

-			list = node.SelectNodes("", "link")
-			i.Links = make([]Link, 0)
-			for _, v := range list {
-				lnk := Link{}
+			tl = node.SelectNodes("", "link")
+			for _, v := range tl {
+				lnk := new(Link)
 				lnk.Href = v.Value
 				i.Links = append(i.Links, lnk)
 			}
@ -101,24 +154,25 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err os.Error) {
 			i.Guid = item.GetValue("", "guid")
 			i.PubDate = item.GetValue("", "pubDate")

-			list := item.SelectNodes("", "category")
-			i.Categories = make([]Category, len(list))
-			for li, lv := range list {
-				i.Categories[li].Domain = lv.GetAttr("", "domain")
-				i.Categories[li].Text = lv.Value
+			tl = item.SelectNodes("", "category")
+			for _, lv := range tl {
+				cat := new(Category)
+				cat.Domain = lv.GetAttr("", "domain")
+				cat.Text = lv.Value
+				i.Categories = append(i.Categories, cat)
 			}

-			list = item.SelectNodes("", "enclosure")
-			i.Enclosures = make([]Enclosure, len(list))
-			for li, lv := range list {
-				i.Enclosures[li].Url = lv.GetAttr("", "url")
-				i.Enclosures[li].Length = lv.GetAttri64("", "length")
-				i.Enclosures[li].Type = lv.GetAttr("", "type")
+			tl = item.SelectNodes("", "enclosure")
+			for _, lv := range tl {
+				enc := new(Enclosure)
+				enc.Url = lv.GetAttr("", "url")
+				enc.Length = lv.GetAttri64("", "length")
+				enc.Type = lv.GetAttr("", "type")
+				i.Enclosures = append(i.Enclosures, enc)
 			}

-			src := item.SelectNode("", "source")
-			if src != nil {
-				i.Source = Source{}
+			if src := item.SelectNode("", "source"); src != nil {
+				i.Source = new(Source)
 				i.Source.Url = src.GetAttr("", "url")
 				i.Source.Text = src.Value
 			}
@ -126,27 +180,9 @@ func (this *Feed) readRss2(doc *xmlx.Document) (err os.Error) {
 			ch.Items = append(ch.Items, i)
 		}

-		this.Channels = append(this.Channels, ch)
+		if itemcount != len(ch.Items) && this.itemhandler != nil {
+			this.itemhandler(this, ch, ch.Items[itemcount:])
+		}
 	}
 	return
 }
-
-func mapDay(day string) int {
-	switch day {
-	case "Monday":
-		return 1
-	case "Tuesday":
-		return 2
-	case "Wednesday":
-		return 3
-	case "Thursday":
-		return 4
-	case "Friday":
-		return 5
-	case "Saturday":
-		return 6
-	case "Sunday":
-		return 7
-	}
-	return 1
-}