From adba766908bc34dfb8e236ea978b6e87ccdfbc4b Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Fri, 24 Feb 2023 11:48:23 -0500 Subject: [PATCH 1/3] Place results in subdirs Subdirs are simply named by taking the prefix of the input UUID. This is a common technique to prevent the chance of overflowing filesystem limits on the number of files allowed in a single directory. e.g. Before - /srv/gitea/custom/public/90fa9a89-5e50-4bd4-834d-d42655e1ee8e.html - http://127.0.0.1:3000/assets/90fa9a89-5e50-4bd4-834d-d42655e1ee8e.html After: - /srv/gitea/custom/public/bids-validator/84/b1/84b10e1c-b188-41e2-a92d-fb6ded9c6889.html - http://127.0.0.1:3000/assets/bids-validator/84/b1/84b10e1c-b188-41e2-a92d-fb6ded9c6889.html --- bids-hook.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bids-hook.go b/bids-hook.go index a792ec6..f32815b 100644 --- a/bids-hook.go +++ b/bids-hook.go @@ -279,13 +279,13 @@ type job struct { // web link to the results page for this job // see also j.resultPath() func (j job) resultUrl() string { - return giteaRootUrl.JoinPath("assets", fmt.Sprintf("%s.html", j.uuid)).String() + return giteaRootUrl.JoinPath("assets", "bids-validator", j.uuid[:2], j.uuid[2:4], fmt.Sprintf("%s.html", j.uuid)).String() } // file path to the results page for this job // see also j.resultUrl() func (j job) resultPath() string { - return filepath.Join(giteaCustom, "public", fmt.Sprintf("%s.html", j.uuid)) + return filepath.Join(giteaCustom, "public", "bids-validator", j.uuid[:2], j.uuid[2:4], fmt.Sprintf("%s.html", j.uuid)) } // file path to the log file for this job @@ -360,7 +360,12 @@ func (j job) run() (state string, _ error) { ) // redirect stdout to the result file - stdout, err := os.OpenFile(j.resultPath(), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + resultPath := j.resultPath() + err := os.MkdirAll(filepath.Dir(resultPath), 0750) + if err != nil { + return stateError, err + } + stdout, err := os.OpenFile(resultPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0640) if err != nil { return stateError, err } @@ -464,7 +469,7 @@ func readConfig() { if err != nil { log.Fatalf("invalid GITEA_CUSTOM: %v", err) } - err = os.MkdirAll(filepath.Join(giteaCustom, "public"), 0750) + err = os.MkdirAll(filepath.Join(giteaCustom, "public", "bids-validator"), 0750) if err != nil { log.Fatalf("error creating output folder: %v", err) } From 0b41065083bc1239dae396d8304b4a5d591ce6d3 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sat, 25 Feb 2023 01:12:17 -0500 Subject: [PATCH 2/3] Create webhook automatically --- README.md | 7 +++ bids-hook.go | 141 +++++++++++++++++++++++++++++++++++++++++++++++---- start | 1 - 3 files changed, 139 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 257ea7a..f92bd13 100644 --- a/README.md +++ b/README.md @@ -12,3 +12,10 @@ It assumes the URL `%(ROOT_URL)s/static/assets/` loads from Gitea's `%(GITEA_CUSTOM)/public/`; it is **not** compatible with configuring Gitea's `%(STATIC_URL_PREFIX)` so that static files are hosted on a different server or CDN. + +`%(GITEA_TOKEN)` must be from an admin account with the "all" scope for two reasons: + +1. To install the webhook that notifies on pushes +2. To be able to post status icons on any repo without being a member of all repos + +Perhaps in the future Gitea will offer even more finely-grained scopes, but today is not that day. diff --git a/bids-hook.go b/bids-hook.go index f32815b..9435694 100644 --- a/bids-hook.go +++ b/bids-hook.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/hmac" + "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" @@ -11,7 +12,9 @@ import ( "fmt" "io" "io/fs" + "io/ioutil" "log" + "net" "net/http" "net/url" "os" @@ -38,8 +41,7 @@ var ( bidsHookUrl *url.URL // secret used to authenticate api calls from Gitea to bids-hook - // read from environment variable BIDS_HOOK_SECRET - // this should be entered as-in in Gitea to configure the webhook + // generated fresh on each startup bidsHookSecret []byte // the base URL to reach Gitea @@ -111,8 +113,20 @@ func main() { WriteTimeout: 3 * time.Second, MaxHeaderBytes: 4 << 10, } + addr := server.Addr + if addr == "" { + addr = ":http" + } + sock, err := net.Listen("tcp", addr) + if err != nil { + log.Fatal(err) + } log.Printf("main: listening on %q", bidsHookUrl) - log.Fatal(server.ListenAndServe()) + err = installWebhook() + if err != nil { + log.Fatalf("error installing webhook: %v", err) + } + log.Fatal(server.Serve(sock)) } // router checks the host, method and target of the request, @@ -156,6 +170,121 @@ func router(w http.ResponseWriter, r *http.Request) { return } +type Hook struct { + ID int64 `json:"id"` + Type string `json:"type"` + URL string `json:"-"` + Config map[string]string `json:"config"` + Events []string `json:"events"` + AuthorizationHeader string `json:"authorization_header"` + Active bool `json:"active"` + IsSystemWebhook bool `json:"is_system_webhook"` + Updated time.Time `json:"updated_at"` + Created time.Time `json:"created_at"` +} + +// set up the webhook +func installWebhook() error { + url := giteaRootUrl.JoinPath("api", "v1", "admin", "hooks") + + _bidsHookSecret := make([]byte, 32) + _, err := rand.Read(_bidsHookSecret) + if err != nil { + return err + } + // hex-encode, to avoid any trouble with unusual characters + bidsHookSecret = make([]byte, hex.EncodedLen(len(_bidsHookSecret))) + hex.Encode(bidsHookSecret, _bidsHookSecret) + + // This is the hook we want to exist + // Note: Gitea internally uses a CreateHookOption or a EditHookOption for these + // which are two different subsets of the Hook type. So we're not being 100% correct here. + hook := Hook{ + Type: "gitea", + Config: map[string]string{ + "content_type": "json", + "url": bidsHookUrl.String(), + // notice: we *regenerate* the secret *each time* + // it is entirely ephemeral, and thus we must always create/patch the hook + "secret": string(bidsHookSecret), + }, + Events: []string{"push"}, + Active: true, + IsSystemWebhook: true, + } + + hookJSON, err := json.Marshal(hook) + if err != nil { + return err + } + + // Check if the hook already exists + req, err := http.NewRequest(http.MethodGet, url.String(), nil) + if err != nil { + return err + } + req.Header.Add("Authorization", fmt.Sprintf("token %s", giteaToken)) + req.Header.Add("Accept", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return errors.New(fmt.Sprintf("got http status code %d", resp.StatusCode)) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + var hooks []Hook + err = json.Unmarshal(body, &hooks) + if err != nil { + return err + } + // search the result for pre-existing webhook + found := false + var id int64 + for _, _hook := range hooks { + if _hook.URL == hook.URL { + found = true + id = _hook.ID + break + } + } + + // depending on whether we found, either use POST or PATCH + method := http.MethodPost + if found { + method = http.MethodPatch + url = url.JoinPath(fmt.Sprintf("%d", id)) + } + + req, err = http.NewRequest(method, url.String(), bytes.NewReader(hookJSON)) + if err != nil { + return err + } + req.Header.Add("Authorization", fmt.Sprintf("token %s", giteaToken)) + req.Header.Add("Accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + resp, err = http.DefaultClient.Do(req) + if err != nil { + return err + } + if resp.StatusCode != 200 && resp.StatusCode != 201 { + return errors.New(fmt.Sprintf("got http status code %d", resp.StatusCode)) + } + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + defer resp.Body.Close() + + return nil +} + // postHandler deals with requests that have successfully passed // through the router based on their host, method and target. func postHandler(w http.ResponseWriter, r *http.Request) { @@ -440,12 +569,6 @@ func readConfig() { log.Fatalf("error parsing BIDS_HOOK_URL: %v", err) } - val, ok = os.LookupEnv("BIDS_HOOK_SECRET") - if !ok { - log.Fatal("missing environment variable BIDS_HOOK_SECRET") - } - bidsHookSecret = []byte(val) - val, ok = os.LookupEnv("GITEA_ROOT_URL") if !ok { log.Fatal("missing environment variable GITEA_ROOT_URL") diff --git a/start b/start index 89be690..ea08dc3 100755 --- a/start +++ b/start @@ -17,7 +17,6 @@ export GITEA_CUSTOM # 127.0.0.1 is localhost, and 2845 is 0xB1D export BIDS_HOOK_URL='http://127.0.0.1:2845/bids-hook' -export BIDS_HOOK_SECRET='blabla' export GITEA_ROOT_URL='http://127.0.0.1:3000' export GITEA_TOKEN='69e45fa9cfa75a7497633c6be8dd2347226e2f62' From df9b6f1f56333a71fe40df8d7227ce24c63dab1a Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sat, 25 Feb 2023 01:13:33 -0500 Subject: [PATCH 3/3] Debug --- bids-hook.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bids-hook.go b/bids-hook.go index 9435694..703e32d 100644 --- a/bids-hook.go +++ b/bids-hook.go @@ -199,6 +199,7 @@ func installWebhook() error { // This is the hook we want to exist // Note: Gitea internally uses a CreateHookOption or a EditHookOption for these // which are two different subsets of the Hook type. So we're not being 100% correct here. + log.Printf("SECERT: %s", string(bidsHookSecret)) hook := Hook{ Type: "gitea", Config: map[string]string{ @@ -225,6 +226,7 @@ func installWebhook() error { } req.Header.Add("Authorization", fmt.Sprintf("token %s", giteaToken)) req.Header.Add("Accept", "application/json") + log.Printf("Calling %s %s\n", req.Method, url.String()) // DEBUG resp, err := http.DefaultClient.Do(req) if err != nil { @@ -243,11 +245,15 @@ func installWebhook() error { if err != nil { return err } + fmt.Printf("Read this: |%s|\n", body) // search the result for pre-existing webhook found := false var id int64 + fmt.Printf("hook: %#v\n", hook) // DEBUG for _, _hook := range hooks { + fmt.Printf("hook: %#v\n", _hook) // DEBUG if _hook.URL == hook.URL { + fmt.Printf("found at id: %#v\n", _hook.ID) // DEBUG found = true id = _hook.ID break @@ -260,6 +266,7 @@ func installWebhook() error { method = http.MethodPatch url = url.JoinPath(fmt.Sprintf("%d", id)) } + log.Printf("Calling %s %s\n", method, url.String()) // DEBUG req, err = http.NewRequest(method, url.String(), bytes.NewReader(hookJSON)) if err != nil { @@ -274,12 +281,14 @@ func installWebhook() error { return err } if resp.StatusCode != 200 && resp.StatusCode != 201 { + log.Printf("Bailing\n") return errors.New(fmt.Sprintf("got http status code %d", resp.StatusCode)) } body, err = ioutil.ReadAll(resp.Body) if err != nil { return err } + fmt.Printf("Read: |%s|\n", body) defer resp.Body.Close() return nil