newsbid: make the !bids command look nicer

plugins/newsbid/newsbid.go

@@ -48,10 +48,16 @@ func (p *NewsBid) message(conn bot.Connector, k bot.Kind, message msg.Message, a
 			p.bot.Send(conn, bot.Message, ch, "No bids to report.")
 			return true
 		}
-		sort.Slice(bids, func(i, j int) bool { return bids[i].User < bids[j].User })
+		sort.Slice(bids, func(i, j int) bool {
+			if bids[i].User == bids[j].User {
+				return bids[i].Bid > bids[j].Bid
+			}
+			return bids[i].User < bids[j].User
+		})
 		out := "Bids:\n"
 		for _, b := range bids {
-			out += fmt.Sprintf("%s bid %d on <%s|%s> \n", b.User, b.Bid, b.URL, b.Title)
+			hnURL := fmt.Sprintf("https://news.ycombinator.com/item?id=%d", b.HNID)
+			out += fmt.Sprintf("• %s bid %s <%s|%s> (<%s|Comments>)\n", b.User, b.BidStr, b.URL, b.Title, hnURL)
 		}
 		p.bot.Send(conn, bot.Message, ch, out)
 		return true
@@ -82,7 +88,7 @@ func (p *NewsBid) message(conn bot.Connector, k bot.Kind, message msg.Message, a
 		}
 		amount, _ := strconv.Atoi(parts[1])
 		url := parts[2]
-		if bid, err := p.ws.Bid(message.User.Name, amount, url); err != nil {
+		if bid, err := p.ws.Bid(message.User.Name, amount, parts[1], url); err != nil {
 			p.bot.Send(conn, bot.Message, ch, fmt.Sprintf("Error placing bid: %s", err))
 		} else {
 			p.bot.Send(conn, bot.Message, ch, fmt.Sprintf("Your bid has been placed on %s", bid.Title))
@@ -97,11 +103,13 @@ func (p *NewsBid) message(conn bot.Connector, k bot.Kind, message msg.Message, a
 }
 
 func (p *NewsBid) check(conn bot.Connector, ch string) {
-	wr, err := p.ws.Check()
+	last := p.bot.Config().GetInt64("newsbid.lastprocessed", 0)
+	wr, pubTime, err := p.ws.Check(last)
 	if err != nil {
 		p.bot.Send(conn, bot.Message, ch, fmt.Sprintf("Error checking ngate: %s", err))
 		return
 	}
+	p.bot.Config().Set("newsbid.lastprocessed", strconv.FormatInt(pubTime, 10))
 
 	topWon := 0
 	topSpread := 0
@@ -126,10 +134,10 @@ func (p *NewsBid) check(conn bot.Connector, ch string) {
 		msg := fmt.Sprintf("%s%s won %d for a score of %d",
 			icon, res.User, res.Won, res.Score)
 		if len(res.WinningArticles) > 0 {
-			msg += "\nWinning articles: " + res.WinningArticles.Titles()
+			msg += "\nWinning articles: \n" + res.WinningArticles.Titles()
 		}
 		if len(res.LosingArticles) > 0 {
-			msg += "\nLosing articles: " + res.LosingArticles.Titles()
+			msg += "\nLosing articles: \n" + res.LosingArticles.Titles()
 		}
 		p.bot.Send(conn, bot.Message, ch, msg)
 	}

plugins/newsbid/webshit/hn/api.go

@@ -0,0 +1,41 @@
+package hn
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+const BASE = `https://hacker-news.firebaseio.com/v0`
+
+func get(url string) (*http.Response, error) {
+	c := &http.Client{}
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("User-Agent", "catbase/1.0")
+	return c.Do(req)
+}
+
+func GetItem(id int) (Item, error) {
+	u := fmt.Sprintf("%s/item/%d.json", BASE, id)
+	resp, err := get(u)
+	if err != nil {
+		return Item{}, err
+	}
+	dec := json.NewDecoder(resp.Body)
+	i := Item{}
+	if err := dec.Decode(&i); err != nil {
+		return Item{}, err
+	}
+	return i, nil
+}
+
+type Items []Item
+
+func (is Items) Titles() string {
+	out := ""
+	for _, v := range is {
+		hnURL := fmt.Sprintf("https://news.ycombinator.com/item?id=%d", v.ID)
+		out += fmt.Sprintf("• %s <%s|%s> (<%s|Comments>)\n", v.Bid, v.URL, v.Title, hnURL)
+	}
+	return out
+}

plugins/newsbid/webshit/hn/structs.go

@@ -0,0 +1,24 @@
+package hn
+
+type Item struct {
+	ID          int    `json:"id"`
+	Deleted     bool   `json:"deleted"`
+	Type        string `json:"type"`
+	By          string `json:"by"`
+	Time        int    `json:"time"`
+	Text        string `json:"text"`
+	Dead        bool   `json:"dead"`
+	Parent      int    `json:"parent"`
+	Poll        []int  `json:"poll"` // check this
+	Kids        []int  `json:"kids"`
+	URL         string `json:"url"`
+	Score       int    `json:"score"`
+	Title       string `json:"title"`
+	Parts       []int  `json:"parts"`
+	Descendants int    `json:"descendants"`
+
+	// This is not in the API but it's
+	// easier to hack it in here than
+	// fix my code.
+	Bid string `json:"bid"`
+}

plugins/newsbid/webshit/webshit.go

@@ -4,15 +4,12 @@ import (
 	"bytes"
 	"fmt"
 	"math"
-	"net/http"
 	"net/url"
 	"strconv"
-	"strings"
 	"time"
 
-	"github.com/gocolly/colly"
+	"github.com/velour/catbase/plugins/newsbid/webshit/hn"
 
-	hacknews "github.com/PaulRosset/go-hacknews"
 	"github.com/PuerkitoBio/goquery"
 	"github.com/jmoiron/sqlx"
 	"github.com/mmcdole/gofeed"
@@ -36,30 +33,16 @@ type Webshit struct {
 	config Config
 }
 
-type Story struct {
-	Title string
-	URL   string
-}
-
-type Stories []Story
-
-func (s Stories) Titles() string {
-	out := ""
-	for i, v := range s {
-		if i > 0 {
-			out += ", "
-		}
-		out += fmt.Sprintf("<%s|%s>", v.URL, v.Title)
-	}
-	return out
-}
-
 type Bid struct {
 	ID             int
 	User           string
 	Title          string
 	URL            string
+	HNID           int `db:"hnid"`
 	Bid            int
+	BidStr         string
+	PlacedScore    int `db:"placed_score"`
+	ProcessedScore int `db:"processed_score"`
 	Placed         int64
 	Processed      int64
 }
@@ -77,8 +60,8 @@ type Balance struct {
 type WeeklyResult struct {
 	User            string
 	Won             int
-	WinningArticles Stories
+	WinningArticles hn.Items
-	LosingArticles  Stories
+	LosingArticles  hn.Items
 	Score           int
 }
 
@@ -99,8 +82,12 @@ func (w *Webshit) setup() {
 		user string,
 		title string,
 		url string,
+		hnid string,
 		bid integer,
-		placed integer
+		bidstr string,
+		placed_score integer,
+		processed_score integer,
+		placed integer,
 		processed integer
 	)`)
 	w.db.MustExec(`create table if not exists webshit_balances (
@@ -110,57 +97,61 @@ func (w *Webshit) setup() {
 	)`)
 }
 
-func (w *Webshit) Check() ([]WeeklyResult, error) {
+func (w *Webshit) Check(last int64) ([]WeeklyResult, int64, error) {
 	stories, published, err := w.GetWeekly()
 	if err != nil {
-		return nil, err
+		return nil, 0, err
+	}
+
+	if published.Unix() <= last {
+		log.Debug().Msgf("No new ngate: %v vs %v", published.Unix(), last)
+		return nil, 0, fmt.Errorf("no new ngate")
 	}
 
 	var bids []Bid
-	if err = w.db.Select(&bids, `select user,title,url,bid from webshit_bids where placed < ? and processed=0`,
+	if err = w.db.Select(&bids, `select user,title,url,hnid,bid,bidstr from webshit_bids where processed=0`); err != nil {
-		published.Unix()); err != nil {
+		return nil, 0, err
-		return nil, err
 	}
 
+	log.Debug().
+		Interface("bids", bids).
+		Interface("ngate", stories).
+		Interface("published", published).
+		Msg("checking ngate")
+
 	// Assuming no bids earlier than the weekly means there hasn't been a new weekly
 	if len(bids) == 0 {
-		return nil, fmt.Errorf("there are no bids against the current ngate post")
+		return nil, 0, fmt.Errorf("there are no bids against the current ngate post")
 	}
 
-	storyMap := map[string]Story{}
+	storyMap := map[string]hn.Item{}
 	for _, s := range stories {
-		u, err := url.Parse(s.URL)
+		storyMap[s.URL] = s
-		if err != nil {
-			log.Error().Err(err).Msg("couldn't parse URL")
-			continue
-		}
-		id := u.Query().Get("id")
-		storyMap[id] = s
 	}
 
 	wr := w.checkBids(bids, storyMap)
 
 	// Update all balance scores in a tx
 	if err := w.updateScores(wr); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	// Delete all those bids
-	if _, err = w.db.Exec(`update webshit_bids set processed=? where placed < ?`,
+	if _, err = w.db.Exec(`update webshit_bids set processed=? where processed=0`,
-		time.Now().Unix(), published.Unix()); err != nil {
+		time.Now().Unix()); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	// Set all balances to 100
 	if _, err = w.db.Exec(`update webshit_balances set balance=?`,
 		w.config.BalanceReferesh); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
-	return wr, nil
+	return wr, published.Unix(), nil
 }
 
-func (w *Webshit) checkBids(bids []Bid, storyMap map[string]Story) []WeeklyResult {
+func (w *Webshit) checkBids(bids []Bid, storyMap map[string]hn.Item) []WeeklyResult {
 
 	var wins []Bid
 	total, totalWinning := 0.0, 0.0
@@ -176,26 +167,30 @@ func (w *Webshit) checkBids(bids []Bid, storyMap map[string]Story) []WeeklyResul
 		}
 		rec := wr[b.User]
 
-		u, err := url.Parse(b.URL)
+		if s, ok := storyMap[b.URL]; ok {
-		if err != nil {
-			log.Error().Err(err).Msg("couldn't parse URL")
-			continue
-		}
-		id := u.Query().Get("id")
-
-		if s, ok := storyMap[id]; ok {
 			wins = append(wins, b)
+			s.Bid = b.BidStr
 			rec.WinningArticles = append(rec.WinningArticles, s)
 			totalWinning += float64(b.Bid)
 		} else {
-			rec.LosingArticles = append(rec.LosingArticles, Story{Title: b.Title, URL: b.URL})
+			bid := hn.Item{
+				ID:    b.HNID,
+				URL:   b.URL,
+				Title: b.Title,
+				Bid:   b.BidStr,
+			}
+			rec.LosingArticles = append(rec.LosingArticles, bid)
 		}
 		total += float64(b.Bid)
 		wr[b.User] = rec
 	}
 
 	for _, b := range wins {
-		score, comments, err := scrapeScoreAndComments(b.URL)
+		u, _ := url.Parse(b.URL)
+		id, _ := strconv.Atoi(u.Query().Get("id"))
+		item, err := hn.GetItem(id)
+		score := item.Score
+		comments := item.Descendants
 		ratio := 1.0
 		if err != nil {
 			ratio = float64(score) / math.Max(float64(comments), 1.0)
@@ -210,64 +205,8 @@ func (w *Webshit) checkBids(bids []Bid, storyMap map[string]Story) []WeeklyResul
 	return wrMapToSlice(wr)
 }
 
-func scrapeScoreAndComments(url string) (int, int, error) {
-	c := colly.NewCollector()
-
-	// why do I need this to break out of these stupid callbacks?
-	c.Async = true
-
-	finished := make(chan bool)
-
-	score := 0
-	comments := 0
-	var err error = nil
-
-	c.OnHTML("td.subtext > span.score", func(r *colly.HTMLElement) {
-		score, _ = strconv.Atoi(strings.Fields(r.Text)[0])
-	})
-
-	c.OnHTML("td.subtext > a[href*='item?id=']:last-of-type", func(r *colly.HTMLElement) {
-		comments, _ = strconv.Atoi(strings.Fields(r.Text)[0])
-	})
-
-	c.OnScraped(func(r *colly.Response) {
-		finished <- true
-	})
-
-	c.OnError(func(r *colly.Response, e error) {
-		log.Error().Err(err).Msgf("could not scrape %s", r.Request.URL)
-		err = e
-		finished <- true
-	})
-
-	c.Visit(url)
-	<-finished
-	return score, comments, err
-}
-
-// GetHeadlines will return the current possible news headlines for bidding
-func (w *Webshit) GetHeadlines() ([]Story, error) {
-	news := hacknews.Initializer{Story: w.config.HNFeed, NbPosts: w.config.HNLimit}
-	ids, err := news.GetCodesStory()
-	if err != nil {
-		return nil, err
-	}
-	posts, err := news.GetPostStory(ids)
-	if err != nil {
-		return nil, err
-	}
-	var stories []Story
-	for _, p := range posts {
-		stories = append(stories, Story{
-			Title: p.Title,
-			URL:   p.Url,
-		})
-	}
-	return stories, nil
-}
-
 // GetWeekly will return the headlines in the last webshit weekly report
-func (w *Webshit) GetWeekly() ([]Story, *time.Time, error) {
+func (w *Webshit) GetWeekly() (hn.Items, *time.Time, error) {
 	fp := gofeed.NewParser()
 	feed, err := fp.ParseURL("http://n-gate.com/hackernews/index.rss")
 	if err != nil {
@@ -285,17 +224,20 @@ func (w *Webshit) GetWeekly() ([]Story, *time.Time, error) {
 		return nil, nil, err
 	}
 
-	var items []Story
+	var items hn.Items
 	doc.Find(".storylink").Each(func(i int, s *goquery.Selection) {
-		story := Story{
+		url, err := url.Parse(s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", ""))
-			Title: s.Find("a").Text(),
+		if err != nil {
-			URL:   s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", ""),
+			log.Error().Err(err).Msg("Could not parse URL from ngate")
+			return
 		}
-		items = append(items, story)
+		id, _ := strconv.Atoi(url.Query().Get("id"))
-		log.Debug().
+		item, err := hn.GetItem(id)
-			Str("URL", story.URL).
+		if err != nil {
-			Str("Title", story.Title).
+			log.Error().Err(err).Msg("Could not get story from ngate")
-			Msg("Parsed webshit story")
+			return
+		}
+		items = append(items, item)
 	})
 
 	return items, published, nil
@@ -342,7 +284,7 @@ func (w *Webshit) GetAllBalances() ([]Balance, error) {
 }
 
 // Bid allows a user to place a bid on a particular story
-func (w *Webshit) Bid(user string, amount int, URL string) (Bid, error) {
+func (w *Webshit) Bid(user string, amount int, bidStr, URL string) (Bid, error) {
 	bal := w.GetBalance(user)
 	if amount < 0 {
 		return Bid{}, fmt.Errorf("cannot bid less than 0")
@@ -358,10 +300,12 @@ func (w *Webshit) Bid(user string, amount int, URL string) (Bid, error) {
 	ts := time.Now().Unix()
 
 	tx := w.db.MustBegin()
-	_, err = tx.Exec(`insert into webshit_bids (user,title,url,bid,placed,processed) values (?,?,?,?,?,0)`,
+	_, err = tx.Exec(`insert into webshit_bids (user,title,url,hnid,bid,bidstr,placed,processed,placed_score,processed_score) values (?,?,?,?,?,?,?,0,?,0)`,
-		user, story.Title, story.URL, amount, ts)
+		user, story.Title, story.URL, story.ID, amount, bidStr, ts, story.Score)
 	if err != nil {
-		tx.Rollback()
+		if err := tx.Rollback(); err != nil {
+			return Bid{}, err
+		}
 		return Bid{}, err
 	}
 	q := `insert into webshit_balances (user,balance,score) values (?,?,0)
@@ -371,7 +315,7 @@ func (w *Webshit) Bid(user string, amount int, URL string) (Bid, error) {
 		tx.Rollback()
 		return Bid{}, err
 	}
-	tx.Commit()
+	err = tx.Commit()
 
 	return Bid{
 		User:   user,
@@ -382,36 +326,19 @@ func (w *Webshit) Bid(user string, amount int, URL string) (Bid, error) {
 }
 
 // getStoryByURL scrapes the URL for a title
-func (w *Webshit) getStoryByURL(URL string) (Story, error) {
+func (w *Webshit) getStoryByURL(URL string) (hn.Item, error) {
 	u, err := url.Parse(URL)
 	if err != nil {
-		return Story{}, err
+		return hn.Item{}, err
 	}
 	if u.Host != "news.ycombinator.com" {
-		return Story{}, fmt.Errorf("expected HN link")
+		return hn.Item{}, fmt.Errorf("expected HN link")
 	}
-	res, err := http.Get(URL)
+	id, err := strconv.Atoi(u.Query().Get("id"))
-	if err != nil {
+	if id == 0 || err != nil {
-		return Story{}, err
+		return hn.Item{}, fmt.Errorf("invalid item ID")
 	}
-	defer res.Body.Close()
+	return hn.GetItem(id)
-	if res.StatusCode != 200 {
-		return Story{}, fmt.Errorf("bad response code: %d", res.StatusCode)
-	}
-
-	// Load the HTML document
-	doc, err := goquery.NewDocumentFromReader(res.Body)
-	if err != nil {
-		return Story{}, err
-	}
-
-	// Find the review items
-	title := doc.Find("title").Text()
-	title = strings.ReplaceAll(title, " | Hacker News", "")
-	return Story{
-		Title: title,
-		URL:   URL,
-	}, nil
 }
 
 func (w *Webshit) updateScores(results []WeeklyResult) error {
@@ -419,7 +346,9 @@ func (w *Webshit) updateScores(results []WeeklyResult) error {
 	for _, res := range results {
 		if _, err := tx.Exec(`update webshit_balances set score=? where user=?`,
 			res.Score, res.User); err != nil {
-			tx.Rollback()
+			if err := tx.Rollback(); err != nil {
+				return err
+			}
 			return err
 		}
 	}

plugins/newsbid/webshit/webshit_test.go

@@ -0,0 +1,7 @@
+package webshit
+
+//func TestWebshit_Check(t *testing.T) {
+//	mb := bot.NewMockBot()
+//	ws := New(mb.DB())
+//	ws.checkBids()
+//}