package webshit import ( "bytes" "fmt" bh "github.com/timshannon/bolthold" "net/url" "strconv" "time" "github.com/velour/catbase/plugins/newsbid/webshit/hn" "github.com/PuerkitoBio/goquery" "github.com/mmcdole/gofeed" "github.com/rs/zerolog/log" ) type Config struct { HNFeed string HNLimit int BalanceReferesh int HouseName string } type Webshit struct { store *bh.Store config Config } type Bid struct { ID int64 `boltholdKey:"ID"` User string Title string URL string HNID int `db:"hnid"` Bid int BidStr string PlacedScore int `db:"placed_score"` ProcessedScore int `db:"processed_score"` Placed time.Time Processed bool } type Balance struct { User string `boltholdKey:"User"` Balance int Score int } type Balances []Balance func (b Balances) Len() int { return len(b) } func (b Balances) Swap(i, j int) { b[i], b[j] = b[j], b[i] } func (b Balances) Less(i, j int) bool { return b[i].Score > b[j].Score } type WeeklyResult struct { User string Won int WinningArticles hn.Items LosingArticles hn.Items Score int } func NewConfig(store *bh.Store, cfg Config) *Webshit { w := &Webshit{store: store, config: cfg} return w } func (w *Webshit) Check(last int64) ([]WeeklyResult, int64, error) { stories, published, err := w.GetWeekly() if err != nil { return nil, 0, err } if published.Unix() <= last { log.Debug().Msgf("No new ngate: %v vs %v", published.Unix(), last) return nil, 0, fmt.Errorf("no new ngate") } var bids []Bid if err = w.store.Find(&bids, bh.Where("Processed").Eq(false)); err != nil { return nil, 0, err } log.Debug(). Interface("bids", bids). Interface("ngate", stories). Interface("published", published). Msg("checking ngate") // Assuming no bids earlier than the weekly means there hasn't been a new weekly if len(bids) == 0 { return nil, 0, fmt.Errorf("there are no bids against the current ngate post") } storyMap := map[string]hn.Item{} for _, s := range stories { storyMap[s.URL] = s } wr := w.checkBids(bids, storyMap) // Update all balance scores in a tx if err := w.updateScores(wr); err != nil { return nil, 0, err } // Delete all those bids if err = w.store.UpdateMatching(Bid{}, bh.Where("Processed").Eq(false), func(record interface{}) error { r := record.(*Bid) r.Processed = true return w.store.Update(r.ID, r) }); err != nil { return nil, 0, err } // Set all balances to 100 if err = w.store.UpdateMatching(Balance{}, &bh.Query{}, func(record interface{}) error { r := record.(*Balance) r.Balance = w.config.BalanceReferesh return w.store.Update(r.User, r) }); err != nil { return nil, 0, err } return wr, published.Unix(), nil } func (w *Webshit) checkBids(bids []Bid, storyMap map[string]hn.Item) []WeeklyResult { var wins []Bid total, totalWinning := 0.0, 0.0 wr := map[string]WeeklyResult{} houseName := w.config.HouseName houseScore := 0 for _, b := range bids { score := w.GetBalance(b.User).Score if _, ok := wr[b.User]; !ok { wr[b.User] = WeeklyResult{ User: b.User, Score: score, } } rec := wr[b.User] if s, ok := storyMap[b.URL]; ok { wins = append(wins, b) s.Bid = b.BidStr rec.WinningArticles = append(rec.WinningArticles, s) totalWinning += float64(b.Bid) } else { bid := hn.Item{ ID: b.HNID, URL: b.URL, Title: b.Title, Bid: b.BidStr, } rec.LosingArticles = append(rec.LosingArticles, bid) houseScore += b.Bid } total += float64(b.Bid) wr[b.User] = rec } for _, b := range wins { rec := wr[b.User] rec.Won += b.Bid rec.Score += b.Bid wr[b.User] = rec } wr[houseName] = WeeklyResult{ User: houseName, Score: w.GetBalance(houseName).Score + houseScore, Won: houseScore, } return wrMapToSlice(wr) } // GetWeekly will return the headlines in the last webshit weekly report func (w *Webshit) GetWeekly() (hn.Items, *time.Time, error) { fp := gofeed.NewParser() feed, err := fp.ParseURL("http://n-gate.com/hackernews/index.rss") if err != nil { return nil, nil, err } if len(feed.Items) <= 0 { return nil, nil, fmt.Errorf("no webshit weekly found") } published := feed.Items[0].PublishedParsed buf := bytes.NewBufferString(feed.Items[0].Description) doc, err := goquery.NewDocumentFromReader(buf) if err != nil { return nil, nil, err } var items hn.Items doc.Find(".storylink").Each(func(i int, s *goquery.Selection) { url, err := url.Parse(s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", "")) if err != nil { log.Error().Err(err).Msg("Could not parse URL from ngate") return } id, _ := strconv.Atoi(url.Query().Get("id")) item, err := hn.GetItem(id) if err != nil { log.Error().Err(err).Msg("Could not get story from ngate") return } items = append(items, item) }) return items, published, nil } // GetBalances returns the current balance for all known users // Any unknown user has a default balance on their first bid func (w *Webshit) GetBalance(user string) Balance { var balance Balance err := w.store.Get(user, &balance) if err != nil { return Balance{ User: user, Balance: 100, Score: 0, } } return balance } func (w *Webshit) GetAllBids() ([]Bid, error) { var bids []Bid err := w.store.Find(&bids, bh.Where("Processed").Eq(false)) if err != nil { return nil, err } return bids, nil } func (w *Webshit) GetAllBalances() (Balances, error) { var balances Balances err := w.store.Find(&balances, &bh.Query{}) if err != nil { return nil, err } return balances, nil } // Bid allows a user to place a bid on a particular story func (w *Webshit) Bid(user string, amount int, bidStr, URL string) (Bid, error) { bal := w.GetBalance(user).Balance if amount < 0 { return Bid{}, fmt.Errorf("cannot bid less than 0") } if bal < amount { return Bid{}, fmt.Errorf("cannot bid more than balance, %d", bal) } story, err := w.getStoryByURL(URL) if err != nil { return Bid{}, err } bid := Bid{ User: user, Title: story.Title, URL: story.URL, Placed: time.Now(), } err = w.store.Insert(Bid{}, bid) if err != nil { return Bid{}, err } err = w.store.Upsert(user, bal) if err != nil { return Bid{}, err } return bid, nil } // getStoryByURL scrapes the URL for a title func (w *Webshit) getStoryByURL(URL string) (hn.Item, error) { u, err := url.Parse(URL) if err != nil { return hn.Item{}, err } if u.Host != "news.ycombinator.com" { return hn.Item{}, fmt.Errorf("expected HN link") } id, err := strconv.Atoi(u.Query().Get("id")) if id == 0 || err != nil { return hn.Item{}, fmt.Errorf("invalid item ID") } return hn.GetItem(id) } func (w *Webshit) updateScores(results []WeeklyResult) error { for _, res := range results { bal := w.GetBalance(res.User) bal.Score = res.Score if err := w.store.Insert(res.User, bal); err != nil { return err } } return nil } func wrMapToSlice(wr map[string]WeeklyResult) []WeeklyResult { var out = []WeeklyResult{} for _, r := range wr { out = append(out, r) } return out }