package webshit import ( "bytes" "fmt" "net/url" "strconv" "time" "github.com/velour/catbase/plugins/newsbid/webshit/hn" "github.com/PuerkitoBio/goquery" "github.com/jmoiron/sqlx" "github.com/mmcdole/gofeed" "github.com/rs/zerolog/log" ) type Config struct { HNFeed string HNLimit int BalanceReferesh int HouseName string } type Webshit struct { db *sqlx.DB config Config } type Bid struct { ID int User string Title string URL string HNID int `db:"hnid"` Bid int BidStr string PlacedScore int `db:"placed_score"` ProcessedScore int `db:"processed_score"` Placed int64 Processed int64 } func (b Bid) PlacedParsed() time.Time { return time.Unix(b.Placed, 0) } type Balance struct { User string Balance int Score int } type Balances []Balance func (b Balances) Len() int { return len(b) } func (b Balances) Swap(i, j int) { b[i], b[j] = b[j], b[i] } func (b Balances) Less(i, j int) bool { return b[i].Score > b[j].Score } type WeeklyResult struct { User string Won int WinningArticles hn.Items LosingArticles hn.Items Score int } func NewConfig(db *sqlx.DB, cfg Config) *Webshit { w := &Webshit{db: db, config: cfg} w.setup() return w } // setup will create any necessary SQL tables and populate them with minimal data func (w *Webshit) setup() { w.db.MustExec(`create table if not exists webshit_bids ( id integer primary key autoincrement, user string, title string, url string, hnid string, bid integer, bidstr string, placed_score integer, processed_score integer, placed integer, processed integer )`) w.db.MustExec(`create table if not exists webshit_balances ( user string primary key, balance int, score int )`) } func (w *Webshit) Check(last int64) ([]WeeklyResult, int64, error) { stories, published, err := w.GetWeekly() if err != nil { return nil, 0, err } if published.Unix() <= last { log.Debug().Msgf("No new ngate: %v vs %v", published.Unix(), last) return nil, 0, fmt.Errorf("no new ngate") } var bids []Bid if err = w.db.Select(&bids, `select user,title,url,hnid,bid,bidstr from webshit_bids where processed=0`); err != nil { return nil, 0, err } log.Debug(). Interface("bids", bids). Interface("ngate", stories). Interface("published", published). Msg("checking ngate") // Assuming no bids earlier than the weekly means there hasn't been a new weekly if len(bids) == 0 { return nil, 0, fmt.Errorf("there are no bids against the current ngate post") } storyMap := map[string]hn.Item{} for _, s := range stories { storyMap[s.URL] = s } wr := w.checkBids(bids, storyMap) // Update all balance scores in a tx if err := w.updateScores(wr); err != nil { return nil, 0, err } // Delete all those bids if _, err = w.db.Exec(`update webshit_bids set processed=? where processed=0`, time.Now().Unix()); err != nil { return nil, 0, err } // Set all balances to 100 if _, err = w.db.Exec(`update webshit_balances set balance=?`, w.config.BalanceReferesh); err != nil { return nil, 0, err } return wr, published.Unix(), nil } func (w *Webshit) checkBids(bids []Bid, storyMap map[string]hn.Item) []WeeklyResult { var wins []Bid total, totalWinning := 0.0, 0.0 wr := map[string]WeeklyResult{} houseName := w.config.HouseName houseScore := 0 for _, b := range bids { score := w.GetScore(b.User) if _, ok := wr[b.User]; !ok { wr[b.User] = WeeklyResult{ User: b.User, Score: score, } } rec := wr[b.User] if s, ok := storyMap[b.URL]; ok { wins = append(wins, b) s.Bid = b.BidStr rec.WinningArticles = append(rec.WinningArticles, s) totalWinning += float64(b.Bid) } else { bid := hn.Item{ ID: b.HNID, URL: b.URL, Title: b.Title, Bid: b.BidStr, } rec.LosingArticles = append(rec.LosingArticles, bid) houseScore += b.Bid } total += float64(b.Bid) wr[b.User] = rec } for _, b := range wins { rec := wr[b.User] rec.Won += b.Bid rec.Score += b.Bid wr[b.User] = rec } wr[houseName] = WeeklyResult{ User: houseName, Score: w.GetScore(houseName) + houseScore, Won: houseScore, } return wrMapToSlice(wr) } // GetWeekly will return the headlines in the last webshit weekly report func (w *Webshit) GetWeekly() (hn.Items, *time.Time, error) { fp := gofeed.NewParser() feed, err := fp.ParseURL("http://n-gate.com/hackernews/index.rss") if err != nil { return nil, nil, err } if len(feed.Items) <= 0 { return nil, nil, fmt.Errorf("no webshit weekly found") } published := feed.Items[0].PublishedParsed buf := bytes.NewBufferString(feed.Items[0].Description) doc, err := goquery.NewDocumentFromReader(buf) if err != nil { return nil, nil, err } var items hn.Items doc.Find(".storylink").Each(func(i int, s *goquery.Selection) { url, err := url.Parse(s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", "")) if err != nil { log.Error().Err(err).Msg("Could not parse URL from ngate") return } id, _ := strconv.Atoi(url.Query().Get("id")) item, err := hn.GetItem(id) if err != nil { log.Error().Err(err).Msg("Could not get story from ngate") return } items = append(items, item) }) return items, published, nil } // GetBalances returns the current balance for all known users // Any unknown user has a default balance on their first bid func (w *Webshit) GetBalance(user string) int { q := `select balance from webshit_balances where user=?` var balance int err := w.db.Get(&balance, q, user) if err != nil { return 100 } return balance } func (w *Webshit) GetScore(user string) int { q := `select score from webshit_balances where user=?` var score int err := w.db.Get(&score, q, user) if err != nil { return 0 } return score } func (w *Webshit) GetAllBids() ([]Bid, error) { var bids []Bid err := w.db.Select(&bids, `select * from webshit_bids where processed=0`) if err != nil { return nil, err } return bids, nil } func (w *Webshit) GetAllBalances() (Balances, error) { var balances Balances err := w.db.Select(&balances, `select * from webshit_balances`) if err != nil { return nil, err } return balances, nil } // Bid allows a user to place a bid on a particular story func (w *Webshit) Bid(user string, amount int, bidStr, URL string) (Bid, error) { bal := w.GetBalance(user) if amount < 0 { return Bid{}, fmt.Errorf("cannot bid less than 0") } if bal < amount { return Bid{}, fmt.Errorf("cannot bid more than balance, %d", bal) } story, err := w.getStoryByURL(URL) if err != nil { return Bid{}, err } ts := time.Now().Unix() tx := w.db.MustBegin() _, err = tx.Exec(`insert into webshit_bids (user,title,url,hnid,bid,bidstr,placed,processed,placed_score,processed_score) values (?,?,?,?,?,?,?,0,?,0)`, user, story.Title, story.URL, story.ID, amount, bidStr, ts, story.Score) if err != nil { if err := tx.Rollback(); err != nil { return Bid{}, err } return Bid{}, err } q := `insert into webshit_balances (user,balance,score) values (?,?,0) on conflict(user) do update set balance=?` _, err = tx.Exec(q, user, bal-amount, bal-amount) if err != nil { tx.Rollback() return Bid{}, err } err = tx.Commit() return Bid{ User: user, Title: story.Title, URL: story.URL, Placed: ts, }, err } // getStoryByURL scrapes the URL for a title func (w *Webshit) getStoryByURL(URL string) (hn.Item, error) { u, err := url.Parse(URL) if err != nil { return hn.Item{}, err } if u.Host != "news.ycombinator.com" { return hn.Item{}, fmt.Errorf("expected HN link") } id, err := strconv.Atoi(u.Query().Get("id")) if id == 0 || err != nil { return hn.Item{}, fmt.Errorf("invalid item ID") } return hn.GetItem(id) } func (w *Webshit) updateScores(results []WeeklyResult) error { tx := w.db.MustBegin() for _, res := range results { if _, err := tx.Exec(`insert into webshit_balances (user, balance, score) values (?, 0, ?) on conflict(user) do update set score=excluded.score`, res.User, res.Score); err != nil { if err := tx.Rollback(); err != nil { return err } return err } } err := tx.Commit() return err } func wrMapToSlice(wr map[string]WeeklyResult) []WeeklyResult { var out = []WeeklyResult{} for _, r := range wr { out = append(out, r) } return out }