feat: create cron nft-ttr to measure nft time to retrievability

.github/workflows/cron-nft-ttr.yml

@@ -0,0 +1,40 @@
+name: Measure NFT Time to Retrievability (nft-ttr)
+
+on:
+  schedule:
+    - cron: '30 * * * *'
+  workflow_dispatch:
+
+jobs:
+  measure:
+    name: measure nft time to retrievability
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        env: ['staging', 'production']
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+      - uses: bahmutov/npm-install@v1
+      - name: Run job
+        env:
+          DEBUG: '*'
+          ENV: ${{ matrix.env }}
+          NFT_STORAGE_API_KEY: ${{ secrets.NFT_STORAGE_API_KEY }}
+          PUSHGATEWAY_JOBNAME: nftstorage_ci/instance/github_action
+          PUSHGATEWAY_URL: https://pushgateway.k8s.locotorp.info/metrics/job/nftstorage_ci/instance/github_action
+          PUSHGATEWAY_BASIC_AUTH: ${{ secrets.PUSHGATEWAY_BASIC_AUTH }}
+        run: |
+          yarn workspace cron run start:nft-ttr measure --logConfigAndExit
+          yarn workspace cron run start:nft-ttr measure \
+            --minImageSizeBytes=10000000 \
+            --gateways https://nftstorage.link https://dweb.link \
+            --metricsPushGateway $PUSHGATEWAY_URL \
+            --metricsPushGatewayJobName $PUSHGATEWAY_JOBNAME \
+
+      - name: Heartbeat
+        if: ${{ success() }}
+        run: ./packages/tools/cli.js heartbeat --token ${{ secrets.OPSGENIE_KEY }} --name cron-nft-ttr

.github/workflows/cron.yml

@@ -6,7 +6,54 @@ on:
     paths:
       - 'packages/cron/**'
       - '.github/workflows/cron.yml'
+      - 'yarn.lock'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'packages/cron/**'
+      - '.github/workflows/cron.yml'
+      - 'yarn.lock'
+
 jobs:
+  lint:
+    name: lint cron
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+      - uses: bahmutov/npm-install@v1
+      - run: yarn workspace nft.storage prepare
+      - run: yarn workspace cron lint
+
+  check:
+    name: Typecheck cron
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+      - uses: bahmutov/npm-install@v1
+      - run: yarn workspace nft.storage prepare
+      - run: yarn workspace cron typecheck
+
+  test:
+    name: Test cron
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+      - uses: bahmutov/npm-install@v1
+      - name: Test (ES)
+        run: yarn --cwd packages/cron test
+        env:
+          NFT_STORAGE_API_KEY: ${{ secrets.NFT_STORAGE_API_KEY }}
+
   release:
     name: Release
     runs-on: ubuntu-latest

package.json

@@ -1,8 +1,17 @@
 {
   "private": true,
-  "workspaces": [
-    "packages/*"
-  ],
+  "workspaces": {
+    "packages": [
+      "packages/*"
+    ],
+    "nohoist": [
+      "@cloudflare/workers-types",
+      "@types/jest",
+      "@types/mocha",
+      "@types/node",
+      "@types/webpack-env"
+    ]
+  },
   "scripts": {
     "dev:api": "cd packages/api && yarn dev",
     "dev:website": "cd packages/website && yarn dev",

packages/cron/package.json

@@ -11,29 +11,170 @@
     "nftstorage-pinata": "src/bin/pinata.js"
   },
   "scripts": {
+    "lint": "eslint 'src/**/*.js'",
+    "lint:fix": "npm run lint -- --fix",
     "start": "run-s start:*",
+    "start:nft-ttr": "node src/bin/nft-ttr.js",
     "start:pins": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/pins.js",
     "start:pins-failed": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/pins-failed.js",
     "start:pinata": "node src/bin/pinata.js",
     "start:dagcargo:sizes": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/dagcargo-sizes.js",
-    "start:metrics": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/metrics.js"
+    "start:metrics": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/metrics.js",
+    "test": "c8 ava",
+    "typecheck": "npx tsc --build tsconfig.json"
   },
   "author": "Alan Shaw",
   "license": "(Apache-2.0 OR MIT)",
   "dependencies": {
     "@nftstorage/ipfs-cluster": "^5.0.1",
     "@web-std/fetch": "^4.0.0",
-    "debug": "^4.3.2",
+    "ava": "^4.3.0",
     "dotenv": "^10.0.0",
     "form-data": "^4.0.0",
+    "it-to-buffer": "^2.0.2",
     "limiter": "2.0.1",
     "nft.storage": "^6.0.0",
     "p-retry": "^4.6.1",
     "p-settle": "^5.0.0",
     "pg": "^8.7.1",
+    "prom-client": "^14.0.1",
+    "sade": "^1.8.1",
+    "sharp": "^0.30.6",
+    "stream-mime-type": "^1.0.2",
     "streaming-iterables": "^6.0.0"
   },
   "devDependencies": {
+    "@types/mime-types": "^2.1.1",
+    "@types/node": "^17.0.35",
+    "@types/sharp": "^0.30.2",
+    "@typescript-eslint/eslint-plugin": "^5.27.0",
+    "c8": "^7.11.3",
+    "eslint-plugin-prettier": "^4.0.0",
     "npm-run-all": "^4.1.5"
-  }
+  },
+  "eslintConfig": {
+    "plugins": [
+      "@typescript-eslint"
+    ],
+    "extends": [
+      "eslint:recommended",
+      "plugin:@typescript-eslint/recommended",
+      "plugin:@typescript-eslint/recommended-requiring-type-checking",
+      "standard",
+      "plugin:prettier/recommended"
+    ],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+      "ecmaVersion": 13,
+      "project": [
+        "./tsconfig.json"
+      ]
+    },
+    "env": {
+      "es2022": true,
+      "browser": true,
+      "node": true
+    },
+    "rules": {},
+    "overrides": [
+      {
+        "files": [
+          "./src/bin/pins-failed.js",
+          "./src/jobs/dagcargo.js"
+        ],
+        "rules": {
+          "@typescript-eslint/no-unsafe-assignment": "off"
+        }
+      },
+      {
+        "files": [
+          "./src/jobs/dagcargo.js"
+        ],
+        "rules": {
+          "camelcase": "off"
+        }
+      },
+      {
+        "files": [
+          "./src/jobs/dagcargo.js",
+          "./src/jobs/metrics.js",
+          "./src/jobs/pins.js",
+          "./src/lib/ipfs.js"
+        ],
+        "rules": {
+          "@typescript-eslint/no-unsafe-member-access": "off"
+        }
+      },
+      {
+        "files": [
+          "./src/lib/ipfs.js"
+        ],
+        "rules": {
+          "@typescript-eslint/no-unsafe-argument": "off",
+          "@typescript-eslint/no-unsafe-assignment": "off"
+        }
+      },
+      {
+        "files": [
+          "./src/lib/fetch.js"
+        ],
+        "rules": {
+          "@typescript-eslint/no-unsafe-assignment": "off"
+        }
+      },
+      {
+        "files": [
+          "./src/jobs/dagcargo.js",
+          "./src/jobs/pinata.js",
+          "./src/jobs/pins.js",
+          "./src/lib/ipfs.js"
+        ],
+        "rules": {
+          "@typescript-eslint/restrict-template-expressions": "off"
+        }
+      },
+      {
+        "files": [
+          "./src/jobs/dagcargo.js",
+          "./src/jobs/pins.js",
+          "./src/lib/fetch.js",
+          "./src/lib/pinata.js"
+        ],
+        "rules": {
+          "@typescript-eslint/no-unsafe-return": "off"
+        }
+      },
+      {
+        "files": [
+          "./src/bin/dagcargo-sizes.js",
+          "./src/bin/metrics.js",
+          "./src/bin/pinata.js",
+          "./src/bin/pins-failed.js",
+          "./src/bin/pins.js"
+        ],
+        "rules": {
+          "@typescript-eslint/no-floating-promises": "off"
+        }
+      },
+      {
+        "files": [
+          "./src/bin/dagcargo-sizes.js",
+          "./src/bin/metrics.js",
+          "./src/bin/pinata.js",
+          "./src/bin/pins-failed.js",
+          "./src/bin/pins.js",
+          "./src/jobs/dagcargo.js",
+          "./src/jobs/pinata.js",
+          "./src/jobs/pins.js",
+          "./src/lib/pinata.js"
+        ],
+        "rules": {
+          "@typescript-eslint/naming-convention": "off"
+        }
+      }
+    ]
+  },
+  "eslintIgnore": [
+    "node_modules"
+  ]
 }