diff --git a/app.go b/app.go index 262f7d2..cd7425a 100644 --- a/app.go +++ b/app.go @@ -8,6 +8,8 @@ import ( "path/filepath" "scraper" "strconv" + + "github.com/wk8/go-ordered-map/v2" ) func checkIndex(index int) int { @@ -30,7 +32,7 @@ func app(s scraper.Scraper) { data = scraper.ScrapeData(s.AlbumList) for true { cli.CallClear() - cli.PrintMap(data.Metadata, s.StyleColor()) + cli.PrintMap(s.StyleColor(), data.Metadata) index = checkIndex(cli.PrintTable(data.Rows, data.Columns.Title, data.Columns.Width)) s.SetLink(data.Links[index]) albumData := scraper.ScrapeData(s.Album) @@ -38,7 +40,7 @@ func app(s scraper.Scraper) { if albumData.Image != nil { cli.PrintImage(albumData.Image) } - cli.PrintMap(albumData.Metadata, s.StyleColor()) + cli.PrintMap(s.StyleColor(), albumData.Metadata) cli.PrintLink(data.Links[index]) _ = checkIndex(cli.PrintTable(albumData.Rows, albumData.Columns.Title, albumData.Columns.Width)) listIndex := checkIndex(cli.PrintList(s.ListChoices())) @@ -47,7 +49,7 @@ func app(s scraper.Scraper) { } gotCredits := false gotReviews := false - var creditsData map[string]string + var creditsData *orderedmap.OrderedMap[string, string] var reviewData scraper.ScrapedData for true { switch listIndex { @@ -56,7 +58,7 @@ func app(s scraper.Scraper) { creditsData = s.Credits() gotCredits = true } - cli.PrintMap(creditsData, s.StyleColor()) + cli.PrintMap(s.StyleColor(), creditsData) case 2: if !gotReviews { @@ -81,7 +83,8 @@ func app(s scraper.Scraper) { } fmt.Println("Wrong rating value") } - s.AdditionalFunctions()[3].(func(string, string))(rating, albumData.Metadata["ID"]) + id, _ := albumData.Metadata.Get("ID") + s.AdditionalFunctions()[3].(func(string, string))(rating, id) } listIndex = checkIndex(cli.PrintList(s.ListChoices())) if listIndex == 0 { diff --git a/cli/cli.go b/cli/cli.go index 16e80d7..aa8252d 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -12,6 +12,7 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/qeesung/image2ascii/convert" + "github.com/wk8/go-ordered-map/v2" "golang.org/x/term" ) @@ -48,13 +49,14 @@ func PrintLink(link string) { } // use utf8 for non-English alphabets -func PrintMap(metadata map[string]string, color string) { +func PrintMap(color string, metadata *orderedmap.OrderedMap[string, string]) { w, _, e := term.GetSize(0) if e != nil { panic(e) } maxKeyLength := 0 - for k := range metadata { + for pair := metadata.Oldest(); pair != nil; pair = pair.Next() { + k := pair.Key if utf8.RuneCountInString(k) > maxKeyLength { maxKeyLength = utf8.RuneCountInString(k) } @@ -62,7 +64,9 @@ func PrintMap(metadata map[string]string, color string) { maxKeyLength += 4 style := lipgloss.NewStyle().Foreground(lipgloss.Color(color)) valueSpace := w - maxKeyLength - for key, value := range metadata { + for pair := metadata.Oldest(); pair != nil; pair = pair.Next() { + key := pair.Key + value := pair.Value if len(value) > valueSpace { fmt.Printf("%s%s", style.Render(key), strings.Repeat(" ", maxKeyLength-utf8.RuneCountInString(key))) words := strings.Fields(value) diff --git a/cli/go.mod b/cli/go.mod index 86d12ed..356148a 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -7,6 +7,7 @@ require ( github.com/charmbracelet/bubbletea v0.23.2 github.com/charmbracelet/lipgloss v0.6.0 github.com/qeesung/image2ascii v1.0.1 + github.com/wk8/go-ordered-map/v2 v2.1.8 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 ) @@ -14,8 +15,11 @@ require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect github.com/aymanbagabas/go-osc52 v1.2.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/containerd/console v1.0.3 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect @@ -27,9 +31,9 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect github.com/stretchr/objx v0.5.0 // indirect - github.com/stretchr/testify v1.8.0 // indirect github.com/wayneashleyberry/terminal-dimensions v1.1.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect golang.org/x/text v0.3.7 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cli/go.sum b/cli/go.sum index deefcf2..1c22566 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -5,6 +5,10 @@ github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E= github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/charmbracelet/bubbles v0.15.0 h1:c5vZ3woHV5W2b8YZI1q7v4ZNQaPetfHuoHzx+56Z6TI= github.com/charmbracelet/bubbles v0.15.0/go.mod h1:Y7gSFbBzlMpUDR/XM9MhZI374Q+1p1kluf1uLl8iK74= github.com/charmbracelet/bubbletea v0.23.1/go.mod h1:JAfGK/3/pPKHTnAS8JIE2u9f61BjWTQY57RbT25aMXU= @@ -18,10 +22,13 @@ github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkX github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= @@ -60,10 +67,12 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/wayneashleyberry/terminal-dimensions v1.1.0 h1:EB7cIzBdsOzAgmhTUtTTQXBByuPheP/Zv1zL2BRPY6g= github.com/wayneashleyberry/terminal-dimensions v1.1.0/go.mod h1:2lc/0eWCObmhRczn2SdGSQtgBooLUzIotkkEGXqghyg= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -77,6 +86,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go.mod b/go.mod index 692ee16..c9103e0 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( cli v0.0.0-00010101000000-000000000000 + github.com/wk8/go-ordered-map/v2 v2.1.8 scraper v0.0.0-00010101000000-000000000000 ) @@ -16,6 +17,8 @@ require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect github.com/aymanbagabas/go-osc52 v1.2.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/charmbracelet/bubbles v0.15.0 // indirect github.com/charmbracelet/bubbletea v0.23.2 // indirect github.com/charmbracelet/lipgloss v0.6.0 // indirect @@ -26,6 +29,7 @@ require ( github.com/golang/protobuf v1.3.1 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect @@ -46,7 +50,7 @@ require ( golang.org/x/term v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect google.golang.org/appengine v1.6.7 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace cli => ./cli diff --git a/go.sum b/go.sum index b0a582e..616d116 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,10 @@ github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E= github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/charmbracelet/bubbles v0.15.0 h1:c5vZ3woHV5W2b8YZI1q7v4ZNQaPetfHuoHzx+56Z6TI= github.com/charmbracelet/bubbles v0.15.0/go.mod h1:Y7gSFbBzlMpUDR/XM9MhZI374Q+1p1kluf1uLl8iK74= github.com/charmbracelet/bubbletea v0.23.1/go.mod h1:JAfGK/3/pPKHTnAS8JIE2u9f61BjWTQY57RbT25aMXU= @@ -25,8 +29,8 @@ github.com/charmbracelet/lipgloss v0.6.0 h1:1StyZB9vBSOyuZxQUcUwGr17JmojPNm87ini github.com/charmbracelet/lipgloss v0.6.0/go.mod h1:tHh2wr34xcHjC2HCXIlGSG1jaDF0S0atAUvBMP6Ppuk= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= @@ -35,12 +39,15 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= @@ -79,11 +86,13 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/wayneashleyberry/terminal-dimensions v1.1.0 h1:EB7cIzBdsOzAgmhTUtTTQXBByuPheP/Zv1zL2BRPY6g= github.com/wayneashleyberry/terminal-dimensions v1.1.0/go.mod h1:2lc/0eWCObmhRczn2SdGSQtgBooLUzIotkkEGXqghyg= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -136,6 +145,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scraper/go.mod b/scraper/go.mod index d66b158..3464237 100644 --- a/scraper/go.mod +++ b/scraper/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/PuerkitoBio/goquery v1.8.1 github.com/gocolly/colly v1.2.0 + github.com/wk8/go-ordered-map/v2 v2.1.8 golang.org/x/term v0.5.0 ) @@ -13,14 +14,18 @@ require ( github.com/antchfx/htmlquery v1.3.0 // indirect github.com/antchfx/xmlquery v1.3.15 // indirect github.com/antchfx/xpath v1.2.3 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.3.1 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/temoto/robotstxt v1.1.2 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect google.golang.org/appengine v1.6.7 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/scraper/go.sum b/scraper/go.sum index affd9e2..9de7171 100644 --- a/scraper/go.sum +++ b/scraper/go.sum @@ -8,8 +8,12 @@ github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8 github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes= github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= @@ -18,17 +22,22 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -71,3 +80,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scraper/metallum.go b/scraper/metallum.go index 6faff82..145bf4e 100644 --- a/scraper/metallum.go +++ b/scraper/metallum.go @@ -12,14 +12,11 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/gocolly/colly" + "github.com/wk8/go-ordered-map/v2" ) type SearchResponse struct { - Error string `json:"error"` - ITotalRecords int `json:"iTotalRecords"` - ITotalDisplayRecords int `json:"iTotalDisplayRecords"` - SEcho int `json:"sEcho"` - AaData [][]string `json:"aaData"` + AaData [][]string `json:"aaData"` } const METALLUMSTYLECOLOR string = "#b57614" @@ -39,7 +36,7 @@ type Metallum struct { Link string } -func getMetadata(h *colly.HTMLElement, metadata map[string]string) { +func getMetadata(h *colly.HTMLElement, metadata *orderedmap.OrderedMap[string, string]) { keys, values := []string{}, []string{} h.ForEach("dt", func(_ int, h *colly.HTMLElement) { keys = append(keys, h.Text) @@ -48,7 +45,7 @@ func getMetadata(h *colly.HTMLElement, metadata map[string]string) { values = append(values, strings.Replace(h.Text, "\n", "", -1)) }) for i, k := range keys { - metadata[k] = values[i] + metadata.Set(k, values[i]) } } @@ -90,7 +87,7 @@ func (m *Metallum) SearchBand(data *ScrapedData) ([]int, []string) { func (m *Metallum) AlbumList(data *ScrapedData) ([]int, []string) { c := colly.NewCollector() data.Links = make([]string, 0) - data.Metadata = make(map[string]string) + data.Metadata = orderedmap.New[string, string]() c.OnHTML("#band_disco a[href*='all']", func(e *colly.HTMLElement) { e.Request.Visit(e.Attr("href")) @@ -117,7 +114,7 @@ func (m *Metallum) AlbumList(data *ScrapedData) ([]int, []string) { func (m *Metallum) Album(data *ScrapedData) ([]int, []string) { c := colly.NewCollector() data.Links = make([]string, 0) - data.Metadata = make(map[string]string) + data.Metadata = orderedmap.New[string, string]() c.OnHTML("div#album_tabs_tracklist tr.even, div#album_tabs_tracklist tr.odd", func(h *colly.HTMLElement) { var row [4]string @@ -189,14 +186,14 @@ func (m *Metallum) ReviewsList(data *ScrapedData) ([]int, []string) { return mReviewColumnWidths[:], mReviewColumnTitles[:] } -func (m *Metallum) Credits() map[string]string { +func (m *Metallum) Credits() *orderedmap.OrderedMap[string, string] { c := colly.NewCollector() - credits := make(map[string]string) + credits := orderedmap.New[string, string]() c.OnHTML("div#album_members_lineup table.lineupTable > tbody > tr.lineupRow", func(h *colly.HTMLElement) { artist := h.ChildText("td:has(a)") credit := h.ChildText("td:not(:has(a))") - credits[artist] = credit + credits.Set(artist, credit) }) c.Visit(m.Link) diff --git a/scraper/rym.go b/scraper/rym.go index 518c805..c68ad98 100644 --- a/scraper/rym.go +++ b/scraper/rym.go @@ -11,6 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/gocolly/colly" + "github.com/wk8/go-ordered-map/v2" ) const ( @@ -19,6 +20,7 @@ const ( LOGIN string = "https://rateyourmusic.com/httprequest/Login" RATING string = "https://rateyourmusic.com/httprequest/CatalogSetRating" USERDATA string = "https://rateyourmusic.com/user_albums_export?album_list_id=" + USERAGENT string = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0" ) var ( @@ -45,11 +47,7 @@ type AlbumTable struct { } func createCrawler(delay int, cookies map[string]string) *colly.Collector { - c := colly.NewCollector( - colly.Async(true), - colly.UserAgent( - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0", - )) + c := colly.NewCollector(colly.Async(true), colly.UserAgent(USERAGENT)) if delay > 0 { c.Limit(&colly.LimitRule{ DomainGlob: "*", @@ -81,13 +79,13 @@ func getAlbumListDiscography( hasBio bool, ) { c.OnHTML("div#column_container_right div.section_artist_image > a > div", func(h *colly.HTMLElement) { - data.Metadata["Top Album"] = h.Text + data.Metadata.Set("Top Album", h.Text) }) if hasBio { c.OnHTML( "div#column_container_right div.section_artist_biography > span.rendered_text", func(h *colly.HTMLElement) { - data.Metadata["Biography"] = h.Text + data.Metadata.Set("Biography", h.Text) }) } @@ -104,10 +102,8 @@ func getAlbumListDiscography( if h.ChildAttr("div.disco_info b.disco_mainline_recommended", "title") == "Recommended" { recommended = "" } - data.Rows = append( - data.Rows, - []string{recommended, title, year, reviews, ratings, average, albumTable.Section, rating}, - ) + row := []string{recommended, title, year, reviews, ratings, average, albumTable.Section, rating} + data.Rows = append(data.Rows, row) data.Links = append(data.Links, DOMAIN+h.ChildAttr("div.disco_info > a", "href")) }) } @@ -141,7 +137,7 @@ func (r *RateYourMusic) AlbumList(data *ScrapedData) ([]int, []string) { var hasBio bool var visitLink string data.Links = make([]string, 0) - data.Metadata = make(map[string]string) + data.Metadata = orderedmap.New[string, string]() if r.GetCredits { albumTables = []AlbumTable{{Query: "div.disco_search_results > div.disco_release", Section: "Credits"}} @@ -169,7 +165,7 @@ func (r *RateYourMusic) AlbumList(data *ScrapedData) ([]int, []string) { func (r *RateYourMusic) Album(data *ScrapedData) ([]int, []string) { c := createCrawler(r.Delay, r.Cookies) - data.Metadata = make(map[string]string) + data.Metadata = orderedmap.New[string, string]() c.OnHTML("div#column_container_left div.page_release_art_frame", func(h *colly.HTMLElement) { image_url := h.ChildAttr("img", "src") @@ -190,19 +186,19 @@ func (r *RateYourMusic) Album(data *ScrapedData) ([]int, []string) { key := h.ChildText("th") value := strings.Join(strings.Fields(strings.Replace(h.ChildText("td"), "\n", "", -1)), " ") if key != "Share" { - data.Metadata[key] = value + data.Metadata.Set(key, value) } }) c.OnHTML("div.album_title > input.album_shortcut", func(h *colly.HTMLElement) { albumId := h.Attr("value") - data.Metadata["ID"] = albumId[6 : len(albumId)-1] + data.Metadata.Set("ID", albumId[6:len(albumId)-1]) }) c.OnHTML("div#column_container_left div.section_tracklisting ul#tracks", func(h *colly.HTMLElement) { h.ForEach("li.track", func(_ int, h *colly.HTMLElement) { if len(h.ChildText("span.tracklist_total")) > 0 { value := strings.Fields(h.ChildText("span.tracklist_total")) - data.Metadata["Total Length"] = value[len(value)-1] + data.Metadata.Set("Total Length", value[len(value)-1]) } else { number := h.ChildText("span.tracklist_num") title := h.ChildText("span[itemprop=name] span.rendered_text") @@ -250,9 +246,9 @@ func (r *RateYourMusic) ReviewsList(data *ScrapedData) ([]int, []string) { return rReviewColumnWidths[:], rReviewColumnTitles[:] } -func (r *RateYourMusic) Credits() map[string]string { +func (r *RateYourMusic) Credits() *orderedmap.OrderedMap[string, string] { c := createCrawler(r.Delay, r.Cookies) - credits := make(map[string]string) + credits := orderedmap.New[string, string]() c.OnHTML("div.section_credits > ul.credits", func(h *colly.HTMLElement) { h.ForEach("li[class!='expand_button']:not([style='display:none;'])", func(_ int, h *colly.HTMLElement) { @@ -266,7 +262,7 @@ func (r *RateYourMusic) Credits() map[string]string { credit = append(credit, strings.ToUpper(s.Text()[:1])+s.Text()[1:]) }) }) - credits[artist] = strings.Join(credit, ", ") + credits.Set(artist, strings.Join(credit, ", ")) }) }) @@ -275,19 +271,21 @@ func (r *RateYourMusic) Credits() map[string]string { return credits } +var loginForm = map[string][]byte{ + "remember": []byte("false"), + "maintain_session": []byte("true"), + "rym_ajax_req": []byte("1"), + "action": []byte("Login"), +} + func (r *RateYourMusic) Login() { user, password, err := credentials() if err != nil { panic(err) } - formRequest := map[string][]byte{ - "user": []byte(user), - "password": []byte(password), - "remember": []byte("false"), - "maintain_session": []byte("true"), - "rym_ajax_req": []byte("1"), - "action": []byte("Login"), - } + loginForm["user"] = []byte(user) + loginForm["password"] = []byte(password) + r.Cookies = make(map[string]string) c := createCrawler(r.Delay, r.Cookies) @@ -303,20 +301,21 @@ func (r *RateYourMusic) Login() { } }) - c.PostMultipart(LOGIN, formRequest) + c.PostMultipart(LOGIN, loginForm) c.Wait() } +var ratingForm = map[string][]byte{ + "rym_ajax_req": []byte("1"), + "action": []byte("CatalogSetRating"), + "type": []byte("l"), +} + func (r *RateYourMusic) SendRating(rating string, id string) { c := createCrawler(r.Delay, r.Cookies) - formRequest := map[string][]byte{ - "type": []byte("l"), - "assoc_id": []byte(id), - "rating": []byte(rating), - "action": []byte("CatalogSetRating"), - "rym_ajax_req": []byte("1"), - "request_token": []byte(strings.ReplaceAll(r.Cookies["ulv"], "%2e", ".")), - } + ratingForm["assoc_id"] = []byte(id) + ratingForm["rating"] = []byte(rating) + ratingForm["request_token"] = []byte(strings.ReplaceAll(r.Cookies["ulv"], "%2e", ".")) c.OnResponse(func(r *colly.Response) { fmt.Println(r.StatusCode, "Vote has been uploaded.") @@ -326,7 +325,7 @@ func (r *RateYourMusic) SendRating(rating string, id string) { fmt.Println("Something went wrong:", err) }) - c.PostMultipart("https://rateyourmusic.com/httprequest/CatalogSetRating", formRequest) + c.PostMultipart("https://rateyourmusic.com/httprequest/CatalogSetRating", ratingForm) c.Wait() } diff --git a/scraper/scraper.go b/scraper/scraper.go index e27e554..340fd17 100644 --- a/scraper/scraper.go +++ b/scraper/scraper.go @@ -2,6 +2,8 @@ package scraper import ( "image" + + "github.com/wk8/go-ordered-map/v2" ) type ColumnData struct { @@ -14,7 +16,7 @@ type ScrapedData struct { Columns ColumnData // optionals Links []string - Metadata map[string]string + Metadata *orderedmap.OrderedMap[string, string] Image image.Image } @@ -23,7 +25,7 @@ type Scraper interface { AlbumList(*ScrapedData) ([]int, []string) Album(*ScrapedData) ([]int, []string) ReviewsList(*ScrapedData) ([]int, []string) - Credits() map[string]string + Credits() *orderedmap.OrderedMap[string, string] StyleColor() string SetLink(string) ListChoices() []string