From 9c39e3cd415b68dbd7be399a20ec53d23d8a9426 Mon Sep 17 00:00:00 2001 From: skkiesel Date: Wed, 10 May 2017 15:15:24 -0400 Subject: [PATCH 1/2] barebones rss headline skimmer --- main.go | 2 + plugins/rss/rss.go | 92 +++++++++++++++++++++++++++++++++++++++++ plugins/rss/rss_test.go | 44 ++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 plugins/rss/rss.go create mode 100644 plugins/rss/rss_test.go diff --git a/main.go b/main.go index 6d8f86d..f542e30 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( "github.com/velour/catbase/plugins/fact" "github.com/velour/catbase/plugins/leftpad" "github.com/velour/catbase/plugins/reminder" + "github.com/velour/catbase/plugins/rss" "github.com/velour/catbase/plugins/talker" "github.com/velour/catbase/plugins/your" "github.com/velour/catbase/plugins/zork" @@ -57,6 +58,7 @@ func main() { b.AddHandler("reminder", reminder.New(b)) b.AddHandler("babbler", babbler.New(b)) b.AddHandler("zork", zork.New(b)) + b.AddHandler("rss", rss.New(b)) // catches anything left, will always return true b.AddHandler("factoid", fact.New(b)) diff --git a/plugins/rss/rss.go b/plugins/rss/rss.go new file mode 100644 index 0000000..c12a935 --- /dev/null +++ b/plugins/rss/rss.go @@ -0,0 +1,92 @@ +package rss + +import ( + "fmt" + "log" + "strings" + "time" + + "github.com/velour/catbase/bot" + "github.com/velour/catbase/bot/msg" + + "github.com/mmcdole/gofeed" +) + +type RSSPlugin struct { + Bot bot.Bot + cache map[string]*cacheItem + shelfLife time.Duration +} + +type cacheItem struct { + key string + data string + expiration time.Time +} + +func New(bot bot.Bot) *RSSPlugin { + return &RSSPlugin{ + Bot: bot, + cache: map[string]*cacheItem{}, + shelfLife: time.Minute * 20, + } +} + +func (p *RSSPlugin) Message(message msg.Message) bool { + tokens := strings.Fields(message.Body) + numTokens := len(tokens) + + if numTokens == 2 && strings.ToLower(tokens[0]) == "rss" { + if data, ok := p.cache[strings.ToLower(tokens[1])]; ok && time.Now().Before(data.expiration) { + log.Printf("rss cache hit") + p.Bot.SendMessage(message.Channel, data.data) + return true + } else { + fp := gofeed.NewParser() + feed, err := fp.ParseURL(tokens[1]) + if err != nil { + p.Bot.SendMessage(message.Channel, fmt.Sprintf("RSS error: %s", err.Error())) + return true + } + response := feed.Title + for _, item := range feed.Items { + response += fmt.Sprintf("\n%s", item.Title) + } + + p.cache[strings.ToLower(tokens[1])] = &cacheItem{ + key: strings.ToLower(tokens[1]), + data: response, + expiration: time.Now().Add(p.shelfLife), + } + + p.Bot.SendMessage(message.Channel, response) + return true + } + } + + return false +} + +func (p *RSSPlugin) LoadData() { + // This bot has no data to load +} + +// Help responds to help requests. Every plugin must implement a help function. +func (p *RSSPlugin) Help(channel string, parts []string) { + p.Bot.SendMessage(channel, "try '!rss http://rss.cnn.com/rss/edition.rss'") +} + +// Empty event handler because this plugin does not do anything on event recv +func (p *RSSPlugin) Event(kind string, message msg.Message) bool { + return false +} + +// Handler for bot's own messages +func (p *RSSPlugin) BotMessage(message msg.Message) bool { + return false +} + +// Register any web URLs desired +func (p *RSSPlugin) RegisterWeb() *string { + return nil +} diff --git a/plugins/rss/rss_test.go b/plugins/rss/rss_test.go new file mode 100644 index 0000000..3a00bec --- /dev/null +++ b/plugins/rss/rss_test.go @@ -0,0 +1,44 @@ +package rss + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/velour/catbase/bot" + "github.com/velour/catbase/bot/msg" + "github.com/velour/catbase/bot/user" +) + +func makeMessage(payload string) msg.Message { + isCmd := strings.HasPrefix(payload, "!") + if isCmd { + payload = payload[1:] + } + return msg.Message{ + User: &user.User{Name: "tester"}, + Channel: "test", + Body: payload, + Command: isCmd, + } +} + +func TestRSS(t *testing.T) { + mb := bot.NewMockBot() + c := New(mb) + assert.NotNil(t, c) + res := c.Message(makeMessage("!rss http://rss.cnn.com/rss/edition.rss")) + assert.Len(t, mb.Messages, 1) + assert.True(t, res) +} + +func TestRSSCache(t *testing.T) { + mb := bot.NewMockBot() + c := New(mb) + assert.NotNil(t, c) + res := c.Message(makeMessage("!rss http://rss.cnn.com/rss/edition.rss")) + assert.True(t, res) + res = c.Message(makeMessage("!rss http://rss.cnn.com/rss/edition.rss")) + assert.Len(t, mb.Messages, 2) + assert.True(t, res) +} From 889372dcb4fb831a3100d901d01c953c2b5411e9 Mon Sep 17 00:00:00 2001 From: skkiesel Date: Wed, 10 May 2017 15:56:03 -0400 Subject: [PATCH 2/2] add paging to rss feeds --- plugins/rss/rss.go | 57 ++++++++++++++++++++++++++++++----------- plugins/rss/rss_test.go | 26 ++++++++++++++----- 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/plugins/rss/rss.go b/plugins/rss/rss.go index c12a935..bc7bee5 100644 --- a/plugins/rss/rss.go +++ b/plugins/rss/rss.go @@ -2,7 +2,6 @@ package rss import ( "fmt" - "log" "strings" "time" @@ -16,12 +15,38 @@ type RSSPlugin struct { Bot bot.Bot cache map[string]*cacheItem shelfLife time.Duration + maxLines int } type cacheItem struct { - key string - data string - expiration time.Time + key string + data []string + currentLine int + expiration time.Time +} + +func (c *cacheItem) getCurrentPage(maxLines int) string { + if len(c.data) <= maxLines { + return strings.Join(c.data, "\n") + } + + start := c.currentLine + end := start + maxLines + if end > len(c.data) { + end = len(c.data) + } + + page := strings.Join(c.data[start:end], "\n") + + if end - start == maxLines { + c.currentLine = end + } else { + c.currentLine = maxLines-(end-start) + page += "\n" + page += strings.Join(c.data[0:c.currentLine], "\n") + } + + return page } func New(bot bot.Bot) *RSSPlugin { @@ -29,6 +54,7 @@ func New(bot bot.Bot) *RSSPlugin { Bot: bot, cache: map[string]*cacheItem{}, shelfLife: time.Minute * 20, + maxLines: 5, } } @@ -37,9 +63,8 @@ func (p *RSSPlugin) Message(message msg.Message) bool { numTokens := len(tokens) if numTokens == 2 && strings.ToLower(tokens[0]) == "rss" { - if data, ok := p.cache[strings.ToLower(tokens[1])]; ok && time.Now().Before(data.expiration) { - log.Printf("rss cache hit") - p.Bot.SendMessage(message.Channel, data.data) + if item, ok := p.cache[strings.ToLower(tokens[1])]; ok && time.Now().Before(item.expiration) { + p.Bot.SendMessage(message.Channel, item.getCurrentPage(p.maxLines)) return true } else { fp := gofeed.NewParser() @@ -48,18 +73,20 @@ func (p *RSSPlugin) Message(message msg.Message) bool { p.Bot.SendMessage(message.Channel, fmt.Sprintf("RSS error: %s", err.Error())) return true } - response := feed.Title - for _, item := range feed.Items { - response += fmt.Sprintf("\n%s", item.Title) + item := &cacheItem{ + key: strings.ToLower(tokens[1]), + data: []string{feed.Title}, + expiration: time.Now().Add(p.shelfLife), + currentLine: 0, } - p.cache[strings.ToLower(tokens[1])] = &cacheItem{ - key: strings.ToLower(tokens[1]), - data: response, - expiration: time.Now().Add(p.shelfLife), + for _, feedItem := range feed.Items { + item.data = append(item.data, feedItem.Title) } - p.Bot.SendMessage(message.Channel, response) + p.cache[strings.ToLower(tokens[1])] = item + + p.Bot.SendMessage(message.Channel, item.getCurrentPage(p.maxLines)) return true } } diff --git a/plugins/rss/rss_test.go b/plugins/rss/rss_test.go index 3a00bec..7c20fd1 100644 --- a/plugins/rss/rss_test.go +++ b/plugins/rss/rss_test.go @@ -1,6 +1,7 @@ package rss import ( + "fmt" "strings" "testing" @@ -32,13 +33,26 @@ func TestRSS(t *testing.T) { assert.True(t, res) } -func TestRSSCache(t *testing.T) { +func TestRSSPaging(t *testing.T) { mb := bot.NewMockBot() c := New(mb) assert.NotNil(t, c) - res := c.Message(makeMessage("!rss http://rss.cnn.com/rss/edition.rss")) - assert.True(t, res) - res = c.Message(makeMessage("!rss http://rss.cnn.com/rss/edition.rss")) - assert.Len(t, mb.Messages, 2) - assert.True(t, res) + for i := 0; i < 20; i++ { + res := c.Message(makeMessage("!rss http://rss.cnn.com/rss/edition.rss")) + assert.True(t, res) + } + + assert.Len(t, mb.Messages, 20) + + for i := 0; i < len(mb.Messages); i++ { + if i > 0 && strings.Contains(mb.Messages[i], "CNN.com - RSS Channel - Intl Homepage - News") { + fmt.Println("----------------") + fmt.Println(mb.Messages[i]) + fmt.Println("----------------") + break + } + fmt.Println("----------------") + fmt.Println(mb.Messages[i]) + fmt.Println("----------------") + } }