diff --git a/Makefile b/Makefile index 17c2ad7..91f94e7 100644 --- a/Makefile +++ b/Makefile @@ -4,11 +4,12 @@ TW_URL := 'https://twitter.com/TwitterDev/status/1443269993676763138' .PHONY: all test build clean install all: test build -build: ; go build -x -v ./... +build: ; go build -race -x -v ./... clean: ; go clean -x ./... install: ; go install ./... test: go vet ./... go test ./... -# go run cmd/hackerview/main.go -t 20s $(HN_URL) - go run cmd/twitterview/main.go -t 20s $(TW_URL) +testrun: + go run cmd/hackerview/main.go -t 20s $(HN_URL) +# go run cmd/twitterview/main.go -t 20s $(TW_URL) diff --git a/cmd/hackerview/main.go b/cmd/hackerview/main.go index d60e169..4edb06b 100644 --- a/cmd/hackerview/main.go +++ b/cmd/hackerview/main.go @@ -44,11 +44,12 @@ func run(args []string, stdout io.Writer) error { return errors.New("missing URL argument") } url = flag.Args()[0] - doc, err := hackernews.Fetch(url, opt.userAgent, opt.timeout) + op, err := hackernews.Fetch(url, opt.userAgent, opt.timeout) if err != nil { return errors.New("could not fetch url") } - hackernews.PrintDoc(doc) + println(op.String()) + // hackernews.PrintDoc(doc) return nil } diff --git a/go.mod b/go.mod index cefb227..4eb6203 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( require ( github.com/andybalholm/cascadia v1.2.0 // indirect + github.com/caser/gophernews v0.0.0-20150107061403-f17255d5b792 github.com/containerd/console v1.0.2 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.13 // indirect diff --git a/go.sum b/go.sum index 55f5993..16d6005 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBb github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/atotto/clipboard v0.1.2/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/caser/gophernews v0.0.0-20150107061403-f17255d5b792 h1:yWTZbMC92aSFyZm+f7iJTrLdV2Q3eEIreRoYmJE86lk= +github.com/caser/gophernews v0.0.0-20150107061403-f17255d5b792/go.mod h1:XAJyumv6flVGfp36Ii0+4mjkLLHZYlu+osSOpgsLHLs= github.com/charmbracelet/bubbles v0.9.0 h1:lqJ8FXwoLceQF2J0A+dWo1Cuu1dNyjbW4Opgdi2vkhw= github.com/charmbracelet/bubbles v0.9.0/go.mod h1:NWT/c+0rYEnYChz5qCyX4Lj6fDw9gGToh9EFJPajghU= github.com/charmbracelet/bubbletea v0.14.1/go.mod h1:b5lOf5mLjMg1tRn1HVla54guZB+jvsyV0yYAQja95zE= diff --git a/internal/hackernews/fetch.go b/internal/hackernews/fetch.go index f7fcbd4..d6cf07e 100644 --- a/internal/hackernews/fetch.go +++ b/internal/hackernews/fetch.go @@ -1,25 +1,102 @@ package hackernews import ( - "strings" + "errors" + "github.com/caser/gophernews" "time" - - "github.com/PuerkitoBio/goquery" - "github.com/azimut/cli-view/internal/fetch" ) -func Fetch(url, ua string, timeout time.Duration) (doc *goquery.Document, err error) { - url, err = effectiveUrl(url) +const NUM_OF_WORKERS = 3 + +func unix2time(t int) time.Time { + return time.Unix(int64(t), 0) +} + +func Fetch(rawUrl, ua string, timeout time.Duration) (doc *Op, err error) { + url, storyId, err := effectiveUrl(rawUrl) if err != nil { return nil, err } - res, err := fetch.Fetch(url, ua, timeout) + client := gophernews.NewClient() + story, err := fetchStory(client, storyId) if err != nil { return nil, err } - doc, err = goquery.NewDocumentFromReader(strings.NewReader(res)) + if story.Type != "story" { + return nil, errors.New("is not a story") + } + // TODO: ncomments + op := Op{ + url: url, + title: story.Title, + score: story.Score, + user: story.By, + date: unix2time(story.Time), + } + // comments := fetchComments(story.Kids) // TODO: error + return &op, nil +} + +func fetchStory(client *gophernews.Client, id int) (*gophernews.Story, error) { + story, err := client.GetStory(id) if err != nil { return nil, err } - return + return &story, nil +} + +func fetchComments(comments []int) (state map[int]Comment) { + storiesCh := make(chan int, 5) + commentsCh := make(chan result, 5) + go sendWork(comments, storiesCh) + spawnWorkers(storiesCh, commentsCh) + collector(storiesCh, commentsCh, state) + return state +} + +func sendWork(ids []int, input chan<- int) { + for _, id := range ids { + input <- id + } +} + +func spawnWorkers(in <-chan int, out chan<- result) { + for i := 0; i < NUM_OF_WORKERS; i++ { + go worker(in, out) + } +} + +type result struct { + comment gophernews.Comment + err error +} + +func worker(stories <-chan int, output chan<- result) { + client := gophernews.NewClient() + for storyId := range stories { + comment, err := client.GetComment(storyId) + if err != nil { + output <- result{err: err} + } + output <- result{comment: comment} + } +} + +func collector(commentsCh chan<- int, responseCh <-chan result, state map[int]Comment) { + for response := range responseCh { + if response.err != nil { + continue + } + c := response.comment + state[c.ID] = Comment{ + id: c.ID, + msg: c.Text, + user: c.By, + kids: c.Kids, + } + for _, id := range c.Kids { + commentsCh <- id + } + } + close(commentsCh) } diff --git a/internal/hackernews/type.go b/internal/hackernews/type.go index 8fa9fe1..d2d4d8b 100644 --- a/internal/hackernews/type.go +++ b/internal/hackernews/type.go @@ -1,6 +1,10 @@ package hackernews -import "time" +import ( + "time" + + "github.com/caser/gophernews" +) type Op struct { url string @@ -15,7 +19,30 @@ type Comment struct { id int msg string Childs []*Comment + kids []int indent int user string date time.Time } + +func comment2comment(c gophernews.Comment, indent int) Comment { + return Comment{ + id: c.ID, + msg: c.Text, + user: c.By, + kids: c.Kids, + date: time.Unix(int64(c.Time), 0), + indent: indent, + } +} + +func story2op(s gophernews.Story) Op { + return Op{ + url: s.URL, + title: s.Title, + score: s.Score, + ncomments: len(s.Kids), + user: s.By, + date: time.Unix(int64(s.Time), 0), + } +} diff --git a/internal/hackernews/validate.go b/internal/hackernews/validate.go index c77f6c1..1d1321f 100644 --- a/internal/hackernews/validate.go +++ b/internal/hackernews/validate.go @@ -1,21 +1,31 @@ package hackernews import ( - "errors" + "fmt" "net/url" + "strconv" + "strings" ) -func effectiveUrl(rawUrl string) (string, error) { +func effectiveUrl(rawUrl string) (string, int, error) { uri, err := url.Parse(rawUrl) if err != nil { - return "", err + return "", 0, err } if uri.Host != "news.ycombinator.com" { - return "", errors.New("invalid hostname") + return "", 0, fmt.Errorf("invalid Host: %s", uri.Host) } if uri.Path != "/item" { - return "", errors.New("invalid path") + return "", 0, fmt.Errorf("invalid Path: %s", uri.Path) + } + params := strings.Split(uri.RawQuery, "=") + if params[0] != "id" || len(params) != 2 { + return "", 0, fmt.Errorf("invalid RawQuery: %s", params) + } + id, err := strconv.Atoi(params[1]) + if err != nil { + return "", 0, err } uri.Scheme = "https" - return uri.String(), nil + return uri.String(), id, nil }