-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
gophernews, no subcomments support...
Squashed commit of the following: commit 908dabb Author: azimut <azimut.github@protonmail.com> Date: Tue Jul 26 23:23:26 2022 -0300 take width from opts commit b045b41 Author: azimut <azimut.github@protonmail.com> Date: Tue Jul 26 22:39:45 2022 -0300 wg wait before sending the work possible fixing some race condition that caused comments not being displayed commit 4e96c1d Author: azimut <azimut.github@protonmail.com> Date: Tue Jul 26 21:23:46 2022 -0300 cleanup commit 8008c5d Author: azimut <azimut.github@protonmail.com> Date: Tue Jul 26 21:20:21 2022 -0300 skip deleted(? messages, add date of comments commit 394bf4d Author: azimut <azimut.github@protonmail.com> Date: Tue Jul 26 21:03:31 2022 -0300 comments fetch working, only for the first level of comments commit 8045e55 Author: azimut <azimut.github@protonmail.com> Date: Tue Jul 26 07:46:33 2022 -0300 minor cleanup commit 46f8ac5 Author: azimut <azimut.github@protonmail.com> Date: Tue Jul 26 07:26:23 2022 -0300 add selfurl to op commit a7e5ba4 Author: azimut <azimut.github@protonmail.com> Date: Tue Jul 26 07:19:02 2022 -0300 just return story
- Loading branch information
Showing
10 changed files
with
164 additions
and
142 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,25 +1,130 @@ | ||
| package hackernews | ||
|
|
||
| import ( | ||
| "strings" | ||
| "errors" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/PuerkitoBio/goquery" | ||
| "github.com/azimut/cli-view/internal/fetch" | ||
| "github.com/caser/gophernews" | ||
| ) | ||
|
|
||
| func Fetch(url, ua string, timeout time.Duration) (doc *goquery.Document, err error) { | ||
| url, err = effectiveUrl(url) | ||
| const NUM_OF_WORKERS = 3 | ||
|
|
||
| func unix2time(t int) time.Time { | ||
| return time.Unix(int64(t), 0) | ||
| } | ||
|
|
||
| func newOp(story *gophernews.Story, selfUrl string) Op { | ||
| return Op{ | ||
| url: story.URL, | ||
| title: story.Title, | ||
| score: story.Score, | ||
| user: story.By, | ||
| date: unix2time(story.Time), | ||
| ncomments: len(story.Kids), // ? | ||
| selfUrl: selfUrl, | ||
| } | ||
| } | ||
|
|
||
| func Fetch(rawUrl, ua string, timeout time.Duration) (doc *Op, c *[]Comment, err error) { | ||
| url, storyId, err := effectiveUrl(rawUrl) | ||
| if err != nil { | ||
| return nil, err | ||
| return nil, nil, err | ||
| } | ||
| res, err := fetch.Fetch(url, ua, timeout) | ||
| client := gophernews.NewClient() | ||
| story, err := fetchStory(client, storyId) | ||
| if err != nil { | ||
| return nil, err | ||
| return nil, nil, err | ||
| } | ||
| doc, err = goquery.NewDocumentFromReader(strings.NewReader(res)) | ||
| op := newOp(story, url) | ||
| comments := fetchComments(story.Kids) // TODO: error | ||
| return &op, &comments, nil | ||
| } | ||
|
|
||
| func fetchStory(client *gophernews.Client, id int) (*gophernews.Story, error) { | ||
| story, err := client.GetStory(id) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if story.Type != "story" { | ||
| return nil, errors.New("is not a story") | ||
| } | ||
| return &story, nil | ||
| } | ||
|
|
||
| func fetchComments(comments []int) []Comment { | ||
| idsChan := make(chan int, 5) | ||
| commentsCh := make(chan result, 5) | ||
| var wg sync.WaitGroup | ||
| wg.Add(len(comments)) | ||
| go sendWork(comments, idsChan) | ||
| go closeAfterWait(&wg, commentsCh) | ||
| spawnWorkers(&wg, idsChan, commentsCh) | ||
| return collector(&wg, idsChan, commentsCh) | ||
| } | ||
|
|
||
| func closeAfterWait(wg *sync.WaitGroup, commentsCh chan<- result) { | ||
| wg.Wait() | ||
| close(commentsCh) | ||
| } | ||
|
|
||
| func sendWork(ids []int, input chan<- int) { | ||
| for _, id := range ids { | ||
| input <- id | ||
| } | ||
| close(input) | ||
| } | ||
|
|
||
| func spawnWorkers(wg *sync.WaitGroup, in <-chan int, out chan<- result) { | ||
| for i := 0; i < NUM_OF_WORKERS; i++ { | ||
| go worker(wg, in, out) | ||
| } | ||
| } | ||
|
|
||
| type result struct { | ||
| comment gophernews.Comment | ||
| err error | ||
| } | ||
|
|
||
| func worker(wg *sync.WaitGroup, commentsChan <-chan int, output chan<- result) { | ||
| client := gophernews.NewClient() | ||
| for commentId := range commentsChan { | ||
| comment, err := client.GetComment(commentId) | ||
| if err != nil { | ||
| output <- result{err: err} | ||
| } | ||
| output <- result{comment: comment} | ||
| wg.Done() | ||
| } | ||
| } | ||
|
|
||
| func collector( | ||
| wg *sync.WaitGroup, | ||
| commentsCh chan<- int, | ||
| responseCh <-chan result, | ||
|
|
||
| ) (state []Comment) { | ||
| for response := range responseCh { | ||
| if response.err != nil { | ||
| continue | ||
| } | ||
|
|
||
| c := response.comment | ||
|
|
||
| if c.Text == "" && len(c.Kids) == 0 { | ||
| continue | ||
| } | ||
|
|
||
| state = append(state, Comment{ | ||
| id: c.ID, | ||
| msg: c.Text, | ||
| user: c.By, | ||
| kids: c.Kids, | ||
| date: unix2time(c.Time), | ||
| }) | ||
| // for _, id := range c.Kids { | ||
| // commentsCh <- id | ||
| // } | ||
| } | ||
| return | ||
| } |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.