Tidy up tools/rumble to only run needed queries

.github/workflows/rumble-vulnerability-data.yaml

@@ -44,13 +44,13 @@ jobs:
         with:
           go-version: '^1.20.0'
 
-      - name: set up bigqueryrc
+      - name: Fetch latest Grype vulnerability database
         shell: bash
         run: |
-          # gcloud config set auth/impersonate_service_account "${{ env.SERVICE_ACCOUNT }}"
-          # the following is just used to quiet the bigqueryrc init message, the query result is unused
-          # bq query --use_legacy_sql=false --format=csv --max_rows=1 'SELECT COUNT(*) FROM base-image-rumble.rumble.scheduled;' 2>&1 > /dev/null
-          curl -s $(curl -s https://toolbox-data.anchore.io/grype/databases/listing.json |jq -r '.available."5" | .[0] .url') -o-|tar xvz
+          curl -s \
+            $(curl -s https://toolbox-data.anchore.io/grype/databases/listing.json \
+            |jq -r '.available."5" | .[0] .url') -o- \
+            |tar xvz
      
       - name: Generate Rumble JSON files
         run: go run .

tools/rumble/bigquery.go

@@ -53,53 +53,11 @@ func NewBqClient() (bqClient, error) {
 	return b, nil
 }
 
-const csvQuery = `
-SELECT
-ROW_NUMBER() OVER (ORDER BY time),
-image,
-scanner,
-scanner_version,
-scanner_db_version,
-FORMAT_DATETIME("%Y-%m-%d %H:%M:%S", DATE(time)) as time,
-low_cve_count as low_cve_cnt,
-med_cve_count as med_cve_cnt,
-high_cve_count as high_cve_cnt,
-crit_cve_count as crit_cve_cnt,
-unknown_cve_count as unknown_cve_cnt,
-low_cve_count + med_cve_count + high_cve_count + crit_cve_count + unknown_cve_count AS tot_cve_cnt,
-digest FROM base-image-rumble.rumble.scheduled
-WHERE DATE(time) BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) AND CURRENT_DATE()
-	AND scanner = "grype"
-	LIMIT 10
-`
-
 const allVulnsQuery = `
 SELECT DISTINCT vulnerability
 FROM base-image-rumble.rumble.scheduled_vulns
 `
 
-const cveQuery = `
-WITH ruuuumble AS (
-	SELECT s1.image,
-			s1.time as t,
-			s1.raw_grype_json,
-			s2.vulnerability,
-			s2.installed as version,
-			s2.type,
-			s2.severity
-	FROM base-image-rumble.rumble.scheduled_vulns
-	AS s2
-	INNER JOIN base-image-rumble.rumble.scheduled
-	AS s1
-	ON s1.id = s2.scan_id
-	WHERE s1.image = "golang:latest"
-	OR s1.image = "cgr.dev/chainguard/golang:latest"
-  )
-  SELECT image, t, vulnerability, version, type, severity FROM ruuuumble
-  WHERE DATE(t) BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND CURRENT_DATE()
-  GROUP BY vulnerability, t, image, version, type, severity
-`
-
 const affectedImagesQuery = `
 SELECT s1.image, s1.time as time,
 FROM base-image-rumble.rumble.scheduled_vulns
@@ -115,9 +73,10 @@ ORDER BY s1.image, s1.time
 func (b *bqClient) queryAffectedImages(qr string, vulns []vuln) ([]vuln, error) {
 	eg := new(errgroup.Group)
 	eg.SetLimit(50)
-	for i, v := range vulns {
+	for idx, v := range vulns {
+		vulnerability := v
+		i := idx
 		eg.Go(func() error {
-			vulnerability := v
 			fmt.Printf("querying %v\n", vulnerability.Id)
 			q := b.Client.Query(qr)
 			q.Parameters = []bigquery.QueryParameter{

tools/rumble/go.mod

@@ -6,6 +6,7 @@ require (
 	cloud.google.com/go/bigquery v1.52.0
 	cloud.google.com/go/storage v1.30.1
 	github.com/mattn/go-sqlite3 v1.14.17
+	golang.org/x/sync v0.2.0
 	google.golang.org/api v0.126.0
 )
 
@@ -39,7 +40,6 @@ require (
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
@@ -50,4 +50,4 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
 	google.golang.org/grpc v1.55.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
-)
+)

tools/rumble/storage.go

@@ -10,12 +10,10 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"fmt"
-	"log"
 	"os"
 
 	"cloud.google.com/go/storage"
 	"golang.org/x/sync/errgroup"
-	"google.golang.org/api/iterator"
 )
 
 type gcsClient struct {
@@ -64,10 +62,6 @@ func (g *gcsClient) saveJSON(vulns []vuln) error {
 				return err
 			}
 
-			// err = os.WriteFile("/tmp/cves/"+v.Id+".json", js, os.ModePerm)
-			// if err != nil {
-			// 	return err
-			// }
 			fmt.Printf("Wrote %s\n", fName)
 			return nil
 		})
@@ -79,33 +73,7 @@ func (g *gcsClient) saveJSON(vulns []vuln) error {
 	return nil
 }
 
-func (c *gcsClient) query() {
-	bkt := c.Client.Bucket("chainguard-academy")
-	query := &storage.Query{Prefix: "cve-data"}
-
-	var names [][]string
-	it := bkt.Objects(c.Ctx, query)
-	for {
-		attrs, err := it.Next()
-		if err == iterator.Done {
-			break
-		}
-		if err != nil {
-			log.Fatal(err)
-		}
-		names = append(names, []string{attrs.Name})
-	}
-
-	w := csv.NewWriter(os.Stdout)
-
-	for _, v := range names {
-		if err := w.Write(v); err != nil {
-			log.Fatalln("error writing record to csv:", err)
-		}
-
-	}
-}
-
+// unused, but can print a csv to stdout if needed
 func printRecords(records []interface{}, queryType string) error {
 	w := csv.NewWriter(os.Stdout)