catbase/plugins/newsbid/webshit/webshit.go

package webshit

import (
	"bytes"
	"fmt"
	bh "github.com/timshannon/bolthold"
	"net/url"
	"strconv"
	"time"

	"github.com/velour/catbase/plugins/newsbid/webshit/hn"

	"github.com/PuerkitoBio/goquery"
	"github.com/mmcdole/gofeed"
	"github.com/rs/zerolog/log"
)

type Config struct {
	HNFeed          string
	HNLimit         int
	BalanceReferesh int
	HouseName       string
}

type Webshit struct {
	store  *bh.Store
	config Config
}

type Bid struct {
	ID             int64 `boltholdKey:"ID"`
	User           string
	Title          string
	URL            string
	HNID           int `db:"hnid"`
	Bid            int
	BidStr         string
	PlacedScore    int `db:"placed_score"`
	ProcessedScore int `db:"processed_score"`
	Placed         time.Time
	Processed      time.Time
}

type Balance struct {
	User    string `boltholdKey:"User"`
	Balance int
	Score   int
}

type Balances []Balance

func (b Balances) Len() int           { return len(b) }
func (b Balances) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
func (b Balances) Less(i, j int) bool { return b[i].Score > b[j].Score }

type WeeklyResult struct {
	User            string
	Won             int
	WinningArticles hn.Items
	LosingArticles  hn.Items
	Score           int
}

func NewConfig(store *bh.Store, cfg Config) *Webshit {
	w := &Webshit{store: store, config: cfg}
	return w
}

func (w *Webshit) Check(last int64) ([]WeeklyResult, int64, error) {
	stories, published, err := w.GetWeekly()
	if err != nil {
		return nil, 0, err
	}

	if published.Unix() <= last {
		log.Debug().Msgf("No new ngate: %v vs %v", published.Unix(), last)
		return nil, 0, fmt.Errorf("no new ngate")
	}

	var bids []Bid
	if err = w.store.Find(&bids, bh.Where("Processed").Eq(false)); err != nil {
		return nil, 0, err
	}

	log.Debug().
		Interface("bids", bids).
		Interface("ngate", stories).
		Interface("published", published).
		Msg("checking ngate")

	// Assuming no bids earlier than the weekly means there hasn't been a new weekly
	if len(bids) == 0 {
		return nil, 0, fmt.Errorf("there are no bids against the current ngate post")
	}

	storyMap := map[string]hn.Item{}
	for _, s := range stories {
		storyMap[s.URL] = s
	}

	wr := w.checkBids(bids, storyMap)

	// Update all balance scores in a tx
	if err := w.updateScores(wr); err != nil {
		return nil, 0, err
	}

	// Delete all those bids
	if err = w.store.UpdateMatching(Bid{}, bh.Where("Processed").Eq(false), func(record interface{}) error {
		r := record.(*Bid)
		r.Processed = time.Now()
		return w.store.Update(r.ID, r)
	}); err != nil {
		return nil, 0, err
	}

	// Set all balances to 100
	if err = w.store.UpdateMatching(Balance{}, &bh.Query{}, func(record interface{}) error {
		r := record.(*Balance)
		r.Balance = w.config.BalanceReferesh
		return w.store.Update(r.User, r)
	}); err != nil {
		return nil, 0, err
	}

	return wr, published.Unix(), nil
}

func (w *Webshit) checkBids(bids []Bid, storyMap map[string]hn.Item) []WeeklyResult {
	var wins []Bid
	total, totalWinning := 0.0, 0.0
	wr := map[string]WeeklyResult{}

	houseName := w.config.HouseName
	houseScore := 0

	for _, b := range bids {
		score := w.GetBalance(b.User).Score
		if _, ok := wr[b.User]; !ok {
			wr[b.User] = WeeklyResult{
				User:  b.User,
				Score: score,
			}
		}
		rec := wr[b.User]

		if s, ok := storyMap[b.URL]; ok {
			wins = append(wins, b)
			s.Bid = b.BidStr
			rec.WinningArticles = append(rec.WinningArticles, s)
			totalWinning += float64(b.Bid)
		} else {
			bid := hn.Item{
				ID:    b.HNID,
				URL:   b.URL,
				Title: b.Title,
				Bid:   b.BidStr,
			}
			rec.LosingArticles = append(rec.LosingArticles, bid)
			houseScore += b.Bid
		}
		total += float64(b.Bid)
		wr[b.User] = rec
	}

	for _, b := range wins {
		rec := wr[b.User]
		rec.Won += b.Bid
		rec.Score += b.Bid
		wr[b.User] = rec
	}

	wr[houseName] = WeeklyResult{
		User:  houseName,
		Score: w.GetBalance(houseName).Score + houseScore,
		Won:   houseScore,
	}

	return wrMapToSlice(wr)
}

// GetWeekly will return the headlines in the last webshit weekly report
func (w *Webshit) GetWeekly() (hn.Items, *time.Time, error) {
	fp := gofeed.NewParser()
	feed, err := fp.ParseURL("http://n-gate.com/hackernews/index.rss")
	if err != nil {
		return nil, nil, err
	}
	if len(feed.Items) <= 0 {
		return nil, nil, fmt.Errorf("no webshit weekly found")
	}

	published := feed.Items[0].PublishedParsed

	buf := bytes.NewBufferString(feed.Items[0].Description)
	doc, err := goquery.NewDocumentFromReader(buf)
	if err != nil {
		return nil, nil, err
	}

	var items hn.Items
	doc.Find(".storylink").Each(func(i int, s *goquery.Selection) {
		url, err := url.Parse(s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", ""))
		if err != nil {
			log.Error().Err(err).Msg("Could not parse URL from ngate")
			return
		}
		id, _ := strconv.Atoi(url.Query().Get("id"))
		item, err := hn.GetItem(id)
		if err != nil {
			log.Error().Err(err).Msg("Could not get story from ngate")
			return
		}
		items = append(items, item)
	})

	return items, published, nil
}

// GetBalances returns the current balance for all known users
// Any unknown user has a default balance on their first bid
func (w *Webshit) GetBalance(user string) Balance {
	var balance Balance
	err := w.store.Get(user, &balance)
	if err != nil {
		return Balance{
			User:    user,
			Balance: 100,
			Score:   0,
		}
	}
	return balance
}

func (w *Webshit) GetAllBids() ([]Bid, error) {
	var bids []Bid
	err := w.store.Find(&bids, bh.Where("Processed").Eq(false))
	if err != nil {
		return nil, err
	}
	return bids, nil
}

func (w *Webshit) GetAllBalances() (Balances, error) {
	var balances Balances
	err := w.store.Find(&balances, &bh.Query{})
	if err != nil {
		return nil, err
	}
	return balances, nil
}

// Bid allows a user to place a bid on a particular story
func (w *Webshit) Bid(user string, amount int, bidStr, URL string) (Bid, error) {
	bal := w.GetBalance(user).Balance
	if amount < 0 {
		return Bid{}, fmt.Errorf("cannot bid less than 0")
	}
	if bal < amount {
		return Bid{}, fmt.Errorf("cannot bid more than balance, %d", bal)
	}
	story, err := w.getStoryByURL(URL)
	if err != nil {
		return Bid{}, err
	}

	bid := Bid{
		User:   user,
		Title:  story.Title,
		URL:    story.URL,
		Placed: time.Now(),
	}

	err = w.store.Insert(Bid{}, bid)
	if err != nil {
		return Bid{}, err
	}

	err = w.store.Upsert(user, bal)
	if err != nil {
		return Bid{}, err
	}

	return bid, nil
}

// getStoryByURL scrapes the URL for a title
func (w *Webshit) getStoryByURL(URL string) (hn.Item, error) {
	u, err := url.Parse(URL)
	if err != nil {
		return hn.Item{}, err
	}
	if u.Host != "news.ycombinator.com" {
		return hn.Item{}, fmt.Errorf("expected HN link")
	}
	id, err := strconv.Atoi(u.Query().Get("id"))
	if id == 0 || err != nil {
		return hn.Item{}, fmt.Errorf("invalid item ID")
	}
	return hn.GetItem(id)
}

func (w *Webshit) updateScores(results []WeeklyResult) error {
	for _, res := range results {
		bal := w.GetBalance(res.User)
		bal.Score = res.Score
		if err := w.store.Insert(res.User, bal); err != nil {
			return err
		}
	}
	return nil
}

func wrMapToSlice(wr map[string]WeeklyResult) []WeeklyResult {
	var out = []WeeklyResult{}
	for _, r := range wr {
		out = append(out, r)
	}
	return out
}