From 7b8f37d67d8bd0563a05a883b79dd02e6f1bc3c2 Mon Sep 17 00:00:00 2001 From: Chris Sexton Date: Fri, 22 Nov 2019 11:51:48 -0500 Subject: [PATCH] wip --- plugins/newsbid/newsbid.go | 4 ++- plugins/newsbid/webshit/hn/api.go | 5 ++- plugins/newsbid/webshit/webshit.go | 57 +++++++++++++++--------------- 3 files changed, 34 insertions(+), 32 deletions(-) diff --git a/plugins/newsbid/newsbid.go b/plugins/newsbid/newsbid.go index accc59c..090b1f1 100644 --- a/plugins/newsbid/newsbid.go +++ b/plugins/newsbid/newsbid.go @@ -97,11 +97,13 @@ func (p *NewsBid) message(conn bot.Connector, k bot.Kind, message msg.Message, a } func (p *NewsBid) check(conn bot.Connector, ch string) { - wr, err := p.ws.Check() + last := p.bot.Config().GetInt64("newsbid.lastprocessed", 0) + wr, pubTime, err := p.ws.Check(last) if err != nil { p.bot.Send(conn, bot.Message, ch, fmt.Sprintf("Error checking ngate: %s", err)) return } + p.bot.Config().Set("newsbid.lastprocessed", strconv.FormatInt(pubTime, 10)) topWon := 0 topSpread := 0 diff --git a/plugins/newsbid/webshit/hn/api.go b/plugins/newsbid/webshit/hn/api.go index 8fb4e90..1061f67 100644 --- a/plugins/newsbid/webshit/hn/api.go +++ b/plugins/newsbid/webshit/hn/api.go @@ -4,10 +4,9 @@ import ( "encoding/json" "fmt" "net/http" - "path" ) -const BASE = "https://hacker-news.firebaseio.com/v0/" +const BASE = `https://hacker-news.firebaseio.com/v0` func get(url string) (*http.Response, error) { c := &http.Client{} @@ -17,7 +16,7 @@ func get(url string) (*http.Response, error) { } func GetItem(id int) (Item, error) { - u := path.Join(BASE, "item", fmt.Sprintf("%d.json", id)) + u := fmt.Sprintf("%s/item/%d.json", BASE, id) resp, err := get(u) if err != nil { return Item{}, err diff --git a/plugins/newsbid/webshit/webshit.go b/plugins/newsbid/webshit/webshit.go index c83d9c4..c16862a 100644 --- a/plugins/newsbid/webshit/webshit.go +++ b/plugins/newsbid/webshit/webshit.go @@ -39,8 +39,8 @@ type Bid struct { Title string URL string Bid int - PlacedScore int - ProcessedScore int + PlacedScore int `db:"placed_score"` + ProcessedScore int `db:"processed_score"` Placed int64 Processed int64 } @@ -93,31 +93,29 @@ func (w *Webshit) setup() { )`) } -func (w *Webshit) Check() ([]WeeklyResult, error) { +func (w *Webshit) Check(last int64) ([]WeeklyResult, int64, error) { stories, published, err := w.GetWeekly() if err != nil { - return nil, err + return nil, 0, err + } + + if published.Unix() <= last { + return nil, 0, fmt.Errorf("no new ngate") } var bids []Bid - if err = w.db.Select(&bids, `select user,title,url,bid from webshit_bids where placed < ? and processed=0`, - published.Unix()); err != nil { - return nil, err + if err = w.db.Select(&bids, `select user,title,url,bid from webshit_bids where processed=0`); err != nil { + return nil, 0, err } // Assuming no bids earlier than the weekly means there hasn't been a new weekly if len(bids) == 0 { - return nil, fmt.Errorf("there are no bids against the current ngate post") + return nil, 0, fmt.Errorf("there are no bids against the current ngate post") } storyMap := map[string]hn.Item{} for _, s := range stories { - u, err := url.Parse(s.URL) - if err != nil { - log.Error().Err(err).Msg("couldn't parse URL") - continue - } - id := u.Query().Get("id") + id := strconv.Itoa(s.ID) storyMap[id] = s } @@ -125,22 +123,22 @@ func (w *Webshit) Check() ([]WeeklyResult, error) { // Update all balance scores in a tx if err := w.updateScores(wr); err != nil { - return nil, err + return nil, 0, err } // Delete all those bids if _, err = w.db.Exec(`update webshit_bids set processed=? where placed < ?`, time.Now().Unix(), published.Unix()); err != nil { - return nil, err + return nil, 0, err } // Set all balances to 100 if _, err = w.db.Exec(`update webshit_balances set balance=?`, w.config.BalanceReferesh); err != nil { - return nil, err + return nil, 0, err } - return wr, nil + return wr, published.Unix(), nil } func (w *Webshit) checkBids(bids []Bid, storyMap map[string]hn.Item) []WeeklyResult { @@ -218,15 +216,18 @@ func (w *Webshit) GetWeekly() (hn.Items, *time.Time, error) { var items hn.Items doc.Find(".storylink").Each(func(i int, s *goquery.Selection) { - story := hn.Item{ - Title: s.Find("a").Text(), - URL: s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", ""), + url, err := url.Parse(s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", "")) + if err != nil { + log.Error().Err(err).Msg("Could not parse URL from ngate") + return } - items = append(items, story) - log.Debug(). - Str("URL", story.URL). - Str("Title", story.Title). - Msg("Parsed webshit story") + id, _ := strconv.Atoi(url.Query().Get("id")) + item, err := hn.GetItem(id) + if err != nil { + log.Error().Err(err).Msg("Could not get story from ngate") + return + } + items = append(items, item) }) return items, published, nil @@ -289,8 +290,8 @@ func (w *Webshit) Bid(user string, amount int, URL string) (Bid, error) { ts := time.Now().Unix() tx := w.db.MustBegin() - _, err = tx.Exec(`insert into webshit_bids (user,title,url,bid,placed,processed) values (?,?,?,?,?,0)`, - user, story.Title, story.URL, amount, ts) + _, err = tx.Exec(`insert into webshit_bids (user,title,url,bid,placed,processed,placed_score,processed_score) values (?,?,?,?,?,0,?,0)`, + user, story.Title, story.URL, amount, ts, story.Score) if err != nil { if err := tx.Rollback(); err != nil { return Bid{}, err