mirror of https://github.com/velour/catbase.git
Compare commits
No commits in common. "47c3def7228b0c762dd8e61fae979bc38a4ad07c" and "958a45427115de65a3a399c210f1acc46507ab48" have entirely different histories.
47c3def722
...
958a454271
|
|
@ -48,16 +48,10 @@ func (p *NewsBid) message(conn bot.Connector, k bot.Kind, message msg.Message, a
|
|||
p.bot.Send(conn, bot.Message, ch, "No bids to report.")
|
||||
return true
|
||||
}
|
||||
sort.Slice(bids, func(i, j int) bool {
|
||||
if bids[i].User == bids[j].User {
|
||||
return bids[i].Bid > bids[j].Bid
|
||||
}
|
||||
return bids[i].User < bids[j].User
|
||||
})
|
||||
sort.Slice(bids, func(i, j int) bool { return bids[i].User < bids[j].User })
|
||||
out := "Bids:\n"
|
||||
for _, b := range bids {
|
||||
hnURL := fmt.Sprintf("https://news.ycombinator.com/item?id=%d", b.HNID)
|
||||
out += fmt.Sprintf("• %s bid %s <%s|%s> (<%s|Comments>)\n", b.User, b.BidStr, b.URL, b.Title, hnURL)
|
||||
out += fmt.Sprintf("%s bid %d on <%s|%s> \n", b.User, b.Bid, b.URL, b.Title)
|
||||
}
|
||||
p.bot.Send(conn, bot.Message, ch, out)
|
||||
return true
|
||||
|
|
@ -88,7 +82,7 @@ func (p *NewsBid) message(conn bot.Connector, k bot.Kind, message msg.Message, a
|
|||
}
|
||||
amount, _ := strconv.Atoi(parts[1])
|
||||
url := parts[2]
|
||||
if bid, err := p.ws.Bid(message.User.Name, amount, parts[1], url); err != nil {
|
||||
if bid, err := p.ws.Bid(message.User.Name, amount, url); err != nil {
|
||||
p.bot.Send(conn, bot.Message, ch, fmt.Sprintf("Error placing bid: %s", err))
|
||||
} else {
|
||||
p.bot.Send(conn, bot.Message, ch, fmt.Sprintf("Your bid has been placed on %s", bid.Title))
|
||||
|
|
@ -103,13 +97,11 @@ func (p *NewsBid) message(conn bot.Connector, k bot.Kind, message msg.Message, a
|
|||
}
|
||||
|
||||
func (p *NewsBid) check(conn bot.Connector, ch string) {
|
||||
last := p.bot.Config().GetInt64("newsbid.lastprocessed", 0)
|
||||
wr, pubTime, err := p.ws.Check(last)
|
||||
wr, err := p.ws.Check()
|
||||
if err != nil {
|
||||
p.bot.Send(conn, bot.Message, ch, fmt.Sprintf("Error checking ngate: %s", err))
|
||||
return
|
||||
}
|
||||
p.bot.Config().Set("newsbid.lastprocessed", strconv.FormatInt(pubTime, 10))
|
||||
|
||||
topWon := 0
|
||||
topSpread := 0
|
||||
|
|
@ -134,10 +126,10 @@ func (p *NewsBid) check(conn bot.Connector, ch string) {
|
|||
msg := fmt.Sprintf("%s%s won %d for a score of %d",
|
||||
icon, res.User, res.Won, res.Score)
|
||||
if len(res.WinningArticles) > 0 {
|
||||
msg += "\nWinning articles: \n" + res.WinningArticles.Titles()
|
||||
msg += "\nWinning articles: " + res.WinningArticles.Titles()
|
||||
}
|
||||
if len(res.LosingArticles) > 0 {
|
||||
msg += "\nLosing articles: \n" + res.LosingArticles.Titles()
|
||||
msg += "\nLosing articles: " + res.LosingArticles.Titles()
|
||||
}
|
||||
p.bot.Send(conn, bot.Message, ch, msg)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
package hn
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
const BASE = `https://hacker-news.firebaseio.com/v0`
|
||||
|
||||
func get(url string) (*http.Response, error) {
|
||||
c := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
req.Header.Set("User-Agent", "catbase/1.0")
|
||||
return c.Do(req)
|
||||
}
|
||||
|
||||
func GetItem(id int) (Item, error) {
|
||||
u := fmt.Sprintf("%s/item/%d.json", BASE, id)
|
||||
resp, err := get(u)
|
||||
if err != nil {
|
||||
return Item{}, err
|
||||
}
|
||||
dec := json.NewDecoder(resp.Body)
|
||||
i := Item{}
|
||||
if err := dec.Decode(&i); err != nil {
|
||||
return Item{}, err
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
type Items []Item
|
||||
|
||||
func (is Items) Titles() string {
|
||||
out := ""
|
||||
for _, v := range is {
|
||||
hnURL := fmt.Sprintf("https://news.ycombinator.com/item?id=%d", v.ID)
|
||||
out += fmt.Sprintf("• %s <%s|%s> (<%s|Comments>)\n", v.Bid, v.URL, v.Title, hnURL)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
package hn
|
||||
|
||||
type Item struct {
|
||||
ID int `json:"id"`
|
||||
Deleted bool `json:"deleted"`
|
||||
Type string `json:"type"`
|
||||
By string `json:"by"`
|
||||
Time int `json:"time"`
|
||||
Text string `json:"text"`
|
||||
Dead bool `json:"dead"`
|
||||
Parent int `json:"parent"`
|
||||
Poll []int `json:"poll"` // check this
|
||||
Kids []int `json:"kids"`
|
||||
URL string `json:"url"`
|
||||
Score int `json:"score"`
|
||||
Title string `json:"title"`
|
||||
Parts []int `json:"parts"`
|
||||
Descendants int `json:"descendants"`
|
||||
|
||||
// This is not in the API but it's
|
||||
// easier to hack it in here than
|
||||
// fix my code.
|
||||
Bid string `json:"bid"`
|
||||
}
|
||||
|
|
@ -4,12 +4,15 @@ import (
|
|||
"bytes"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/velour/catbase/plugins/newsbid/webshit/hn"
|
||||
"github.com/gocolly/colly"
|
||||
|
||||
hacknews "github.com/PaulRosset/go-hacknews"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/mmcdole/gofeed"
|
||||
|
|
@ -33,18 +36,32 @@ type Webshit struct {
|
|||
config Config
|
||||
}
|
||||
|
||||
type Story struct {
|
||||
Title string
|
||||
URL string
|
||||
}
|
||||
|
||||
type Stories []Story
|
||||
|
||||
func (s Stories) Titles() string {
|
||||
out := ""
|
||||
for i, v := range s {
|
||||
if i > 0 {
|
||||
out += ", "
|
||||
}
|
||||
out += fmt.Sprintf("<%s|%s>", v.URL, v.Title)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type Bid struct {
|
||||
ID int
|
||||
User string
|
||||
Title string
|
||||
URL string
|
||||
HNID int `db:"hnid"`
|
||||
Bid int
|
||||
BidStr string
|
||||
PlacedScore int `db:"placed_score"`
|
||||
ProcessedScore int `db:"processed_score"`
|
||||
Placed int64
|
||||
Processed int64
|
||||
ID int
|
||||
User string
|
||||
Title string
|
||||
URL string
|
||||
Bid int
|
||||
Placed int64
|
||||
Processed int64
|
||||
}
|
||||
|
||||
func (b Bid) PlacedParsed() time.Time {
|
||||
|
|
@ -60,8 +77,8 @@ type Balance struct {
|
|||
type WeeklyResult struct {
|
||||
User string
|
||||
Won int
|
||||
WinningArticles hn.Items
|
||||
LosingArticles hn.Items
|
||||
WinningArticles Stories
|
||||
LosingArticles Stories
|
||||
Score int
|
||||
}
|
||||
|
||||
|
|
@ -82,12 +99,8 @@ func (w *Webshit) setup() {
|
|||
user string,
|
||||
title string,
|
||||
url string,
|
||||
hnid string,
|
||||
bid integer,
|
||||
bidstr string,
|
||||
placed_score integer,
|
||||
processed_score integer,
|
||||
placed integer,
|
||||
placed integer
|
||||
processed integer
|
||||
)`)
|
||||
w.db.MustExec(`create table if not exists webshit_balances (
|
||||
|
|
@ -97,61 +110,57 @@ func (w *Webshit) setup() {
|
|||
)`)
|
||||
}
|
||||
|
||||
func (w *Webshit) Check(last int64) ([]WeeklyResult, int64, error) {
|
||||
func (w *Webshit) Check() ([]WeeklyResult, error) {
|
||||
stories, published, err := w.GetWeekly()
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if published.Unix() <= last {
|
||||
log.Debug().Msgf("No new ngate: %v vs %v", published.Unix(), last)
|
||||
return nil, 0, fmt.Errorf("no new ngate")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var bids []Bid
|
||||
if err = w.db.Select(&bids, `select user,title,url,hnid,bid,bidstr from webshit_bids where processed=0`); err != nil {
|
||||
return nil, 0, err
|
||||
if err = w.db.Select(&bids, `select user,title,url,bid from webshit_bids where placed < ? and processed=0`,
|
||||
published.Unix()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Interface("bids", bids).
|
||||
Interface("ngate", stories).
|
||||
Interface("published", published).
|
||||
Msg("checking ngate")
|
||||
|
||||
// Assuming no bids earlier than the weekly means there hasn't been a new weekly
|
||||
if len(bids) == 0 {
|
||||
return nil, 0, fmt.Errorf("there are no bids against the current ngate post")
|
||||
return nil, fmt.Errorf("there are no bids against the current ngate post")
|
||||
}
|
||||
|
||||
storyMap := map[string]hn.Item{}
|
||||
storyMap := map[string]Story{}
|
||||
for _, s := range stories {
|
||||
storyMap[s.URL] = s
|
||||
u, err := url.Parse(s.URL)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("couldn't parse URL")
|
||||
continue
|
||||
}
|
||||
id := u.Query().Get("id")
|
||||
storyMap[id] = s
|
||||
}
|
||||
|
||||
wr := w.checkBids(bids, storyMap)
|
||||
|
||||
// Update all balance scores in a tx
|
||||
if err := w.updateScores(wr); err != nil {
|
||||
return nil, 0, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Delete all those bids
|
||||
if _, err = w.db.Exec(`update webshit_bids set processed=? where processed=0`,
|
||||
time.Now().Unix()); err != nil {
|
||||
return nil, 0, err
|
||||
if _, err = w.db.Exec(`update webshit_bids set processed=? where placed < ?`,
|
||||
time.Now().Unix(), published.Unix()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set all balances to 100
|
||||
if _, err = w.db.Exec(`update webshit_balances set balance=?`,
|
||||
w.config.BalanceReferesh); err != nil {
|
||||
return nil, 0, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return wr, published.Unix(), nil
|
||||
return wr, nil
|
||||
}
|
||||
|
||||
func (w *Webshit) checkBids(bids []Bid, storyMap map[string]hn.Item) []WeeklyResult {
|
||||
func (w *Webshit) checkBids(bids []Bid, storyMap map[string]Story) []WeeklyResult {
|
||||
|
||||
var wins []Bid
|
||||
total, totalWinning := 0.0, 0.0
|
||||
|
|
@ -167,30 +176,26 @@ func (w *Webshit) checkBids(bids []Bid, storyMap map[string]hn.Item) []WeeklyRes
|
|||
}
|
||||
rec := wr[b.User]
|
||||
|
||||
if s, ok := storyMap[b.URL]; ok {
|
||||
u, err := url.Parse(b.URL)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("couldn't parse URL")
|
||||
continue
|
||||
}
|
||||
id := u.Query().Get("id")
|
||||
|
||||
if s, ok := storyMap[id]; ok {
|
||||
wins = append(wins, b)
|
||||
s.Bid = b.BidStr
|
||||
rec.WinningArticles = append(rec.WinningArticles, s)
|
||||
totalWinning += float64(b.Bid)
|
||||
} else {
|
||||
bid := hn.Item{
|
||||
ID: b.HNID,
|
||||
URL: b.URL,
|
||||
Title: b.Title,
|
||||
Bid: b.BidStr,
|
||||
}
|
||||
rec.LosingArticles = append(rec.LosingArticles, bid)
|
||||
rec.LosingArticles = append(rec.LosingArticles, Story{Title: b.Title, URL: b.URL})
|
||||
}
|
||||
total += float64(b.Bid)
|
||||
wr[b.User] = rec
|
||||
}
|
||||
|
||||
for _, b := range wins {
|
||||
u, _ := url.Parse(b.URL)
|
||||
id, _ := strconv.Atoi(u.Query().Get("id"))
|
||||
item, err := hn.GetItem(id)
|
||||
score := item.Score
|
||||
comments := item.Descendants
|
||||
score, comments, err := scrapeScoreAndComments(b.URL)
|
||||
ratio := 1.0
|
||||
if err != nil {
|
||||
ratio = float64(score) / math.Max(float64(comments), 1.0)
|
||||
|
|
@ -205,8 +210,64 @@ func (w *Webshit) checkBids(bids []Bid, storyMap map[string]hn.Item) []WeeklyRes
|
|||
return wrMapToSlice(wr)
|
||||
}
|
||||
|
||||
func scrapeScoreAndComments(url string) (int, int, error) {
|
||||
c := colly.NewCollector()
|
||||
|
||||
// why do I need this to break out of these stupid callbacks?
|
||||
c.Async = true
|
||||
|
||||
finished := make(chan bool)
|
||||
|
||||
score := 0
|
||||
comments := 0
|
||||
var err error = nil
|
||||
|
||||
c.OnHTML("td.subtext > span.score", func(r *colly.HTMLElement) {
|
||||
score, _ = strconv.Atoi(strings.Fields(r.Text)[0])
|
||||
})
|
||||
|
||||
c.OnHTML("td.subtext > a[href*='item?id=']:last-of-type", func(r *colly.HTMLElement) {
|
||||
comments, _ = strconv.Atoi(strings.Fields(r.Text)[0])
|
||||
})
|
||||
|
||||
c.OnScraped(func(r *colly.Response) {
|
||||
finished <- true
|
||||
})
|
||||
|
||||
c.OnError(func(r *colly.Response, e error) {
|
||||
log.Error().Err(err).Msgf("could not scrape %s", r.Request.URL)
|
||||
err = e
|
||||
finished <- true
|
||||
})
|
||||
|
||||
c.Visit(url)
|
||||
<-finished
|
||||
return score, comments, err
|
||||
}
|
||||
|
||||
// GetHeadlines will return the current possible news headlines for bidding
|
||||
func (w *Webshit) GetHeadlines() ([]Story, error) {
|
||||
news := hacknews.Initializer{Story: w.config.HNFeed, NbPosts: w.config.HNLimit}
|
||||
ids, err := news.GetCodesStory()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
posts, err := news.GetPostStory(ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var stories []Story
|
||||
for _, p := range posts {
|
||||
stories = append(stories, Story{
|
||||
Title: p.Title,
|
||||
URL: p.Url,
|
||||
})
|
||||
}
|
||||
return stories, nil
|
||||
}
|
||||
|
||||
// GetWeekly will return the headlines in the last webshit weekly report
|
||||
func (w *Webshit) GetWeekly() (hn.Items, *time.Time, error) {
|
||||
func (w *Webshit) GetWeekly() ([]Story, *time.Time, error) {
|
||||
fp := gofeed.NewParser()
|
||||
feed, err := fp.ParseURL("http://n-gate.com/hackernews/index.rss")
|
||||
if err != nil {
|
||||
|
|
@ -224,20 +285,17 @@ func (w *Webshit) GetWeekly() (hn.Items, *time.Time, error) {
|
|||
return nil, nil, err
|
||||
}
|
||||
|
||||
var items hn.Items
|
||||
var items []Story
|
||||
doc.Find(".storylink").Each(func(i int, s *goquery.Selection) {
|
||||
url, err := url.Parse(s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", ""))
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Could not parse URL from ngate")
|
||||
return
|
||||
story := Story{
|
||||
Title: s.Find("a").Text(),
|
||||
URL: s.SiblingsFiltered(".small").First().Find("a").AttrOr("href", ""),
|
||||
}
|
||||
id, _ := strconv.Atoi(url.Query().Get("id"))
|
||||
item, err := hn.GetItem(id)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Could not get story from ngate")
|
||||
return
|
||||
}
|
||||
items = append(items, item)
|
||||
items = append(items, story)
|
||||
log.Debug().
|
||||
Str("URL", story.URL).
|
||||
Str("Title", story.Title).
|
||||
Msg("Parsed webshit story")
|
||||
})
|
||||
|
||||
return items, published, nil
|
||||
|
|
@ -284,7 +342,7 @@ func (w *Webshit) GetAllBalances() ([]Balance, error) {
|
|||
}
|
||||
|
||||
// Bid allows a user to place a bid on a particular story
|
||||
func (w *Webshit) Bid(user string, amount int, bidStr, URL string) (Bid, error) {
|
||||
func (w *Webshit) Bid(user string, amount int, URL string) (Bid, error) {
|
||||
bal := w.GetBalance(user)
|
||||
if amount < 0 {
|
||||
return Bid{}, fmt.Errorf("cannot bid less than 0")
|
||||
|
|
@ -300,12 +358,10 @@ func (w *Webshit) Bid(user string, amount int, bidStr, URL string) (Bid, error)
|
|||
ts := time.Now().Unix()
|
||||
|
||||
tx := w.db.MustBegin()
|
||||
_, err = tx.Exec(`insert into webshit_bids (user,title,url,hnid,bid,bidstr,placed,processed,placed_score,processed_score) values (?,?,?,?,?,?,?,0,?,0)`,
|
||||
user, story.Title, story.URL, story.ID, amount, bidStr, ts, story.Score)
|
||||
_, err = tx.Exec(`insert into webshit_bids (user,title,url,bid,placed,processed) values (?,?,?,?,?,0)`,
|
||||
user, story.Title, story.URL, amount, ts)
|
||||
if err != nil {
|
||||
if err := tx.Rollback(); err != nil {
|
||||
return Bid{}, err
|
||||
}
|
||||
tx.Rollback()
|
||||
return Bid{}, err
|
||||
}
|
||||
q := `insert into webshit_balances (user,balance,score) values (?,?,0)
|
||||
|
|
@ -315,7 +371,7 @@ func (w *Webshit) Bid(user string, amount int, bidStr, URL string) (Bid, error)
|
|||
tx.Rollback()
|
||||
return Bid{}, err
|
||||
}
|
||||
err = tx.Commit()
|
||||
tx.Commit()
|
||||
|
||||
return Bid{
|
||||
User: user,
|
||||
|
|
@ -326,19 +382,36 @@ func (w *Webshit) Bid(user string, amount int, bidStr, URL string) (Bid, error)
|
|||
}
|
||||
|
||||
// getStoryByURL scrapes the URL for a title
|
||||
func (w *Webshit) getStoryByURL(URL string) (hn.Item, error) {
|
||||
func (w *Webshit) getStoryByURL(URL string) (Story, error) {
|
||||
u, err := url.Parse(URL)
|
||||
if err != nil {
|
||||
return hn.Item{}, err
|
||||
return Story{}, err
|
||||
}
|
||||
if u.Host != "news.ycombinator.com" {
|
||||
return hn.Item{}, fmt.Errorf("expected HN link")
|
||||
return Story{}, fmt.Errorf("expected HN link")
|
||||
}
|
||||
id, err := strconv.Atoi(u.Query().Get("id"))
|
||||
if id == 0 || err != nil {
|
||||
return hn.Item{}, fmt.Errorf("invalid item ID")
|
||||
res, err := http.Get(URL)
|
||||
if err != nil {
|
||||
return Story{}, err
|
||||
}
|
||||
return hn.GetItem(id)
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != 200 {
|
||||
return Story{}, fmt.Errorf("bad response code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
// Load the HTML document
|
||||
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||
if err != nil {
|
||||
return Story{}, err
|
||||
}
|
||||
|
||||
// Find the review items
|
||||
title := doc.Find("title").Text()
|
||||
title = strings.ReplaceAll(title, " | Hacker News", "")
|
||||
return Story{
|
||||
Title: title,
|
||||
URL: URL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (w *Webshit) updateScores(results []WeeklyResult) error {
|
||||
|
|
@ -346,9 +419,7 @@ func (w *Webshit) updateScores(results []WeeklyResult) error {
|
|||
for _, res := range results {
|
||||
if _, err := tx.Exec(`update webshit_balances set score=? where user=?`,
|
||||
res.Score, res.User); err != nil {
|
||||
if err := tx.Rollback(); err != nil {
|
||||
return err
|
||||
}
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
package webshit
|
||||
|
||||
//func TestWebshit_Check(t *testing.T) {
|
||||
// mb := bot.NewMockBot()
|
||||
// ws := New(mb.DB())
|
||||
// ws.checkBids()
|
||||
//}
|
||||
Loading…
Reference in New Issue