wip

plugins/newsbid/newsbid.go

@@ -97,11 +97,13 @@ func (p *NewsBid) message(conn bot.Connector, k bot.Kind, message msg.Message, a
 }
 
 func (p *NewsBid) check(conn bot.Connector, ch string) {
-	wr, err := p.ws.Check()
+	last := p.bot.Config().GetInt64("newsbid.lastprocessed", 0)
+	wr, pubTime, err := p.ws.Check(last)
 	if err != nil {
 		p.bot.Send(conn, bot.Message, ch, fmt.Sprintf("Error checking ngate: %s", err))
 		return
 	}
+	p.bot.Config().Set("newsbid.lastprocessed", strconv.FormatInt(pubTime, 10))
 
 	topWon := 0
 	topSpread := 0

plugins/newsbid/webshit/hn/api.go

@@ -4,10 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"path"
 )
 
-const BASE = "https://hacker-news.firebaseio.com/v0/"
+const BASE = `https://hacker-news.firebaseio.com/v0`
 
 func get(url string) (*http.Response, error) {
 	c := &http.Client{}
@@ -17,7 +16,7 @@ func get(url string) (*http.Response, error) {
 }
 
 func GetItem(id int) (Item, error) {
-	u := path.Join(BASE, "item", fmt.Sprintf("%d.json", id))
+	u := fmt.Sprintf("%s/item/%d.json", BASE, id)
 	resp, err := get(u)
 	if err != nil {
 		return Item{}, err

plugins/newsbid/webshit/webshit.go

@@ -39,8 +39,8 @@ type Bid struct {
 	Title          string
 	URL            string
 	Bid            int
-	PlacedScore    int
+	PlacedScore    int `db:"placed_score"`
-	ProcessedScore int
+	ProcessedScore int `db:"processed_score"`
 	Placed         int64
 	Processed      int64
 }
@@ -93,31 +93,29 @@ func (w *Webshit) setup() {
 	)`)
 }
 
-func (w *Webshit) Check() ([]WeeklyResult, error) {
+func (w *Webshit) Check(last int64) ([]WeeklyResult, int64, error) {
 	stories, published, err := w.GetWeekly()
 	if err != nil {
-		return nil, err
+		return nil, 0, err
+	}
+
+	if published.Unix() <= last {
+		return nil, 0, fmt.Errorf("no new ngate")
 	}
 
 	var bids []Bid
-	if err = w.db.Select(&bids, `select user,title,url,bid from webshit_bids where placed < ? and processed=0`,
+	if err = w.db.Select(&bids, `select user,title,url,bid from webshit_bids where processed=0`); err != nil {
-		published.Unix()); err != nil {
+		return nil, 0, err
-		return nil, err
 	}
 
 	// Assuming no bids earlier than the weekly means there hasn't been a new weekly
 	if len(bids) == 0 {
-		return nil, fmt.Errorf("there are no bids against the current ngate post")
+		return nil, 0, fmt.Errorf("there are no bids against the current ngate post")
 	}
 
 	storyMap := map[string]hn.Item{}
 	for _, s := range stories {
-		u, err := url.Parse(s.URL)
+		id := strconv.Itoa(s.ID)
-		if err != nil {
-			log.Error().Err(err).Msg("couldn't parse URL")
-			continue
-		}
-		id := u.Query().Get("id")
 		storyMap[id] = s
 	}
 
@@ -125,22 +123,22 @@ func (w *Webshit) Check() ([]WeeklyResult, error) {
 
 	// Update all balance scores in a tx
 	if err := w.updateScores(wr); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	// Delete all those bids
 	if _, err = w.db.Exec(`update webshit_bids set processed=? where placed < ?`,
 		time.Now().Unix(), published.Unix()); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	// Set all balances to 100
 	if _, err = w.db.Exec(`update webshit_balances set balance=?`,
 		w.config.BalanceReferesh); err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
-	return wr, nil
+	return wr, published.Unix(), nil
 }
 
 func (w *Webshit) checkBids(bids []Bid, storyMap map[string]hn.Item) []WeeklyResult {
@@ -218,15 +216,18 @@ func (w *Webshit) GetWeekly() (hn.Items, *time.Time, error) {
 
 	var items hn.Items
 	doc.Find(".storylink").Each(func(i int, s *goquery.Selection) {
-		story := hn.Item{
+		url, err := url.Parse(s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", ""))
-			Title: s.Find("a").Text(),
+		if err != nil {
-			URL:   s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", ""),
+			log.Error().Err(err).Msg("Could not parse URL from ngate")
+			return
 		}
-		items = append(items, story)
+		id, _ := strconv.Atoi(url.Query().Get("id"))
-		log.Debug().
+		item, err := hn.GetItem(id)
-			Str("URL", story.URL).
+		if err != nil {
-			Str("Title", story.Title).
+			log.Error().Err(err).Msg("Could not get story from ngate")
-			Msg("Parsed webshit story")
+			return
+		}
+		items = append(items, item)
 	})
 
 	return items, published, nil
@@ -289,8 +290,8 @@ func (w *Webshit) Bid(user string, amount int, URL string) (Bid, error) {
 	ts := time.Now().Unix()
 
 	tx := w.db.MustBegin()
-	_, err = tx.Exec(`insert into webshit_bids (user,title,url,bid,placed,processed) values (?,?,?,?,?,0)`,
+	_, err = tx.Exec(`insert into webshit_bids (user,title,url,bid,placed,processed,placed_score,processed_score) values (?,?,?,?,?,0,?,0)`,
-		user, story.Title, story.URL, amount, ts)
+		user, story.Title, story.URL, amount, ts, story.Score)
 	if err != nil {
 		if err := tx.Rollback(); err != nil {
 			return Bid{}, err