Feat: Included API handling for fetching from daily_node_metrics (#71)

* Included API handling for fetching from daily_node_metrics * Matching switch in spark-evaluate from metric_date to day * Further separated out platform metric functionality and tests * Health check in platform routes, Naming changes, small test changes --------- Co-authored-by: Julian Gruber <julian@juliangruber.com>
filecoin-station · Apr 30, 2024 · 5ea9490 · 5ea9490
1 parent 3257ee6
commit 5ea9490
Show file tree

Hide file tree

Showing 8 changed files with 233 additions and 101 deletions.
diff --git a/.gitignore b/.gitignore
@@ -129,3 +129,6 @@ dist
 .yarn/install-state.gz
 .pnp.*
 secrets
+
+# MacOS file for folder view display
+.DS_Store
diff --git a/lib/handler.js b/lib/handler.js
@@ -1,7 +1,6 @@
-import assert from 'http-assert'
-import { json } from 'http-responders'
 import Sentry from '@sentry/node'
-import { URLSearchParams } from 'node:url'
+
+import { getStatsWithFilterAndCaching } from './request-helpers.js'
 
 import {
   fetchDailyParticipants,
@@ -11,6 +10,8 @@ import {
   fetchRetrievalSuccessRate
 } from './stats-fetchers.js'
 
+import { handlePlatformRoutes } from './platform-routes.js'
+
 /**
  * @param {object} args
  * @param {import('pg').Pool} args.pgPool
@@ -76,9 +77,8 @@ const handler = async (req, res, pgPool) => {
       res,
       pgPool,
       fetchMinersRSRSummary)
-  } else if (req.method === 'GET' && segs.length === 0) {
-    // health check - required by Grafana datasources
-    res.end('OK')
+  } else if (await handlePlatformRoutes(req, res, pgPool)) {
+    // no-op, request was handled by handlePlatformRoute
   } else {
     notFound(res)
   }
@@ -106,86 +106,3 @@ const notFound = (res) => {
   res.statusCode = 404
   res.end('Not Found')
 }
-
-const getDayAsISOString = (d) => d.toISOString().split('T')[0]
-
-export const today = () => getDayAsISOString(new Date())
-
-/**
- * @param {string} pathname
- * @param {URLSearchParams} searchParams
- * @param {import('node:http').ServerResponse} res
- * @param {import('pg').Pool} pgPool
- * @param {(import('pg').Pool, import('./typings').Filter) => Promise<object[]>} fetchStatsFn
- */
-const getStatsWithFilterAndCaching = async (pathname, searchParams, res, pgPool, fetchStatsFn) => {
-  let from = searchParams.get('from')
-  let to = searchParams.get('to')
-  let shouldRedirect = false
-
-  // Provide default values for "from" and "to" when not specified
-
-  if (!to) {
-    to = today()
-    shouldRedirect = true
-  }
-  if (!from) {
-    from = to
-    shouldRedirect = true
-  }
-  if (shouldRedirect) {
-    res.setHeader('cache-control', `public, max-age=${600 /* 10min */}`)
-    res.setHeader('location', `${pathname}?${new URLSearchParams({ from, to })}`)
-    res.writeHead(302) // Found
-    res.end()
-    return { from, to }
-  }
-
-  // Trim time from date-time values that are typically provided by Grafana
-
-  const matchFrom = from.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
-  assert(matchFrom, 400, '"from" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ')
-  if (matchFrom[2]) {
-    from = matchFrom[1]
-    shouldRedirect = true
-  }
-
-  const matchTo = to.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
-  assert(matchTo, 400, '"to" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ')
-  if (matchTo[2]) {
-    to = matchTo[1]
-    shouldRedirect = true
-  }
-
-  if (shouldRedirect) {
-    res.setHeader('cache-control', `public, max-age=${24 * 3600 /* one day */}`)
-    res.setHeader('location', `${pathname}?${new URLSearchParams({ from, to })}`)
-    res.writeHead(301) // Found
-    res.end()
-    return { from, to }
-  }
-
-  // We have well-formed from & to dates now, let's fetch the requested stats from the DB
-  const filter = { from, to }
-  const stats = await fetchStatsFn(pgPool, filter)
-  setCacheControlForStatsResponse(res, filter)
-  json(res, stats)
-}
-
-/**
- * @param {import('node:http').ServerResponse} res
- * @param {import('./typings').Filter} filter
- */
-const setCacheControlForStatsResponse = (res, filter) => {
-  // We cannot simply compare filter.to vs today() because there may be a delay in finalizing
-  // stats for the previous day. Let's allow up to one hour for the finalization.
-  const boundary = getDayAsISOString(new Date(Date.now() - 3600_000))
-
-  if (filter.to >= boundary) {
-    // response includes partial data for today, cache it for 10 minutes only
-    res.setHeader('cache-control', 'public, max-age=600')
-  } else {
-    // historical data should never change, cache it for one year
-    res.setHeader('cache-control', `public, max-age=${365 * 24 * 3600}, immutable`)
-  }
-}
diff --git a/lib/platform-routes.js b/lib/platform-routes.js
@@ -0,0 +1,23 @@
+import { getStatsWithFilterAndCaching } from './request-helpers.js'
+import { fetchDailyStationMetrics } from './platform-stats-fetchers.js'
+
+export const handlePlatformRoutes = async (req, res, pgPool) => {
+  // Caveat! `new URL('//foo', 'http://127.0.0.1')` would produce "http://foo/" - not what we want!
+  const { pathname, searchParams } = new URL(`http://127.0.0.1${req.url}`)
+  const segs = pathname.split('/').filter(Boolean)
+  if (req.method === 'GET' && segs[0] === 'stations' && segs[1] === 'raw' && segs.length === 2) {
+    await getStatsWithFilterAndCaching(
+      pathname,
+      searchParams,
+      res,
+      pgPool,
+      fetchDailyStationMetrics)
+    return true
+  } else if (req.method === 'GET' && segs.length === 0) {
+    // health check - required by Grafana datasources
+    res.end('OK')
+    return true
+  }
+
+  return false
+}
diff --git a/lib/platform-stats-fetchers.js b/lib/platform-stats-fetchers.js
@@ -0,0 +1,16 @@
+/**
+ * @param {import('pg').Pool} pgPool
+ * @param {import('./typings').Filter} filter
+ */
+export const fetchDailyStationMetrics = async (pgPool, filter) => {
+  const { rows } = await pgPool.query(`
+    SELECT day::TEXT, station_id
+    FROM daily_stations
+    WHERE day >= $1 AND day <= $2
+    GROUP BY day, station_id
+    `, [
+    filter.from,
+    filter.to
+  ])
+  return rows
+}
diff --git a/lib/request-helpers.js b/lib/request-helpers.js
@@ -0,0 +1,86 @@
+import assert from 'http-assert'
+import { json } from 'http-responders'
+import { URLSearchParams } from 'node:url'
+
+const getDayAsISOString = (d) => d.toISOString().split('T')[0]
+
+export const today = () => getDayAsISOString(new Date())
+
+/**
+ * @param {string} pathname
+ * @param {URLSearchParams} searchParams
+ * @param {import('node:http').ServerResponse} res
+ * @param {import('pg').Pool} pgPool
+ * @param {(import('pg').Pool, import('./typings').Filter) => Promise<object[]>} fetchStatsFn
+ */
+export const getStatsWithFilterAndCaching = async (pathname, searchParams, res, pgPool, fetchStatsFn) => {
+  let from = searchParams.get('from')
+  let to = searchParams.get('to')
+  let shouldRedirect = false
+
+  // Provide default values for "from" and "to" when not specified
+
+  if (!to) {
+    to = today()
+    shouldRedirect = true
+  }
+  if (!from) {
+    from = to
+    shouldRedirect = true
+  }
+  if (shouldRedirect) {
+    res.setHeader('cache-control', `public, max-age=${600 /* 10min */}`)
+    res.setHeader('location', `${pathname}?${new URLSearchParams({ from, to })}`)
+    res.writeHead(302) // Found
+    res.end()
+    return { from, to }
+  }
+
+  // Trim time from date-time values that are typically provided by Grafana
+
+  const matchFrom = from.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
+  assert(matchFrom, 400, '"from" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ')
+  if (matchFrom[2]) {
+    from = matchFrom[1]
+    shouldRedirect = true
+  }
+
+  const matchTo = to.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
+  assert(matchTo, 400, '"to" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ')
+  if (matchTo[2]) {
+    to = matchTo[1]
+    shouldRedirect = true
+  }
+
+  if (shouldRedirect) {
+    res.setHeader('cache-control', `public, max-age=${24 * 3600 /* one day */}`)
+    res.setHeader('location', `${pathname}?${new URLSearchParams({ from, to })}`)
+    res.writeHead(301) // Found
+    res.end()
+    return { from, to }
+  }
+
+  // We have well-formed from & to dates now, let's fetch the requested stats from the DB
+  const filter = { from, to }
+  const stats = await fetchStatsFn(pgPool, filter)
+  setCacheControlForStatsResponse(res, filter)
+  json(res, stats)
+}
+
+/**
+ * @param {import('node:http').ServerResponse} res
+ * @param {import('./typings').Filter} filter
+ */
+const setCacheControlForStatsResponse = (res, filter) => {
+  // We cannot simply compare filter.to vs today() because there may be a delay in finalizing
+  // stats for the previous day. Let's allow up to one hour for the finalization.
+  const boundary = getDayAsISOString(new Date(Date.now() - 3600_000))
+
+  if (filter.to >= boundary) {
+    // response includes partial data for today, cache it for 10 minutes only
+    res.setHeader('cache-control', 'public, max-age=600')
+  } else {
+    // historical data should never change, cache it for one year
+    res.setHeader('cache-control', `public, max-age=${365 * 24 * 3600}, immutable`)
+  }
+}
diff --git a/test/handler.test.js b/test/handler.test.js
@@ -1,11 +1,13 @@
 import http from 'node:http'
 import { once } from 'node:events'
-import assert, { AssertionError } from 'node:assert'
+import assert from 'node:assert'
 import pg from 'pg'
 import createDebug from 'debug'
 import { mapParticipantsToIds } from 'spark-evaluate/lib/public-stats.js'
 
-import { createHandler, today } from '../lib/handler.js'
+import { assertResponseStatus } from './test-helpers.js'
+import { createHandler } from '../lib/handler.js'
+import { today } from '../lib/request-helpers.js'
 import { DATABASE_URL } from '../lib/config.js'
 
 const debug = createDebug('test')
@@ -346,16 +348,6 @@ describe('HTTP request handler', () => {
   })
 })
 
-const assertResponseStatus = async (res, status) => {
-  if (res.status !== status) {
-    throw new AssertionError({
-      actual: res.status,
-      expected: status,
-      message: await res.text()
-    })
-  }
-}
-
 const givenRetrievalStats = async (pgPool, { day, minerId, total, successful }) => {
   await pgPool.query(
     'INSERT INTO retrieval_stats (day, miner_id, total, successful) VALUES ($1, $2, $3, $4)',

diff --git a/test/platform-routes.test.js b/test/platform-routes.test.js
@@ -0,0 +1,84 @@
+import http from 'node:http'
+import { once } from 'node:events'
+import assert from 'node:assert'
+import pg from 'pg'
+import createDebug from 'debug'
+
+import { assertResponseStatus } from './test-helpers.js'
+import { createHandler } from '../lib/handler.js'
+import { DATABASE_URL } from '../lib/config.js'
+
+const debug = createDebug('test')
+
+describe('Platform Routes HTTP request handler', () => {
+  /** @type {pg.Pool} */
+  let pgPool
+  /** @type {http.Server} */
+  let server
+  /** @type {string} */
+  let baseUrl
+
+  before(async () => {
+    pgPool = new pg.Pool({ connectionString: DATABASE_URL })
+
+    const handler = createHandler({
+      pgPool,
+      logger: {
+        info: debug,
+        error: console.error,
+        request: debug
+      }
+    })
+
+    server = http.createServer(handler)
+    server.listen()
+    await once(server, 'listening')
+    baseUrl = `http://127.0.0.1:${server.address().port}`
+  })
+
+  after(async () => {
+    server.closeAllConnections()
+    server.close()
+    await pgPool.end()
+  })
+
+  beforeEach(async () => {
+    await pgPool.query('DELETE FROM daily_stations')
+  })
+
+  describe('GET /stations/raw', () => {
+    it('returns daily station metrics for the given date range', async () => {
+      await givenDailyStationMetrics(pgPool, '2024-01-10', ['station1'])
+      await givenDailyStationMetrics(pgPool, '2024-01-11', ['station2'])
+      await givenDailyStationMetrics(pgPool, '2024-01-12', ['station2', 'station3'])
+      await givenDailyStationMetrics(pgPool, '2024-01-13', ['station1'])
+
+      const res = await fetch(
+        new URL(
+          '/stations/raw?from=2024-01-11&to=2024-01-12',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      const metrics = await res.json()
+      assert.deepStrictEqual(metrics, [
+        { day: '2024-01-11', station_id: 'station2' },
+        { day: '2024-01-12', station_id: 'station2' },
+        { day: '2024-01-12', station_id: 'station3' }
+      ])
+    })
+  })
+})
+
+const givenDailyStationMetrics = async (pgPool, day, stationIds) => {
+  await pgPool.query(`
+    INSERT INTO daily_stations (day, station_id)
+    SELECT $1 AS day, UNNEST($2::text[]) AS station_id
+    ON CONFLICT DO NOTHING
+    `, [
+    day,
+    stationIds
+  ])
+}
diff --git a/test/test-helpers.js b/test/test-helpers.js
@@ -0,0 +1,11 @@
+import { AssertionError } from 'node:assert'
+
+export const assertResponseStatus = async (res, status) => {
+  if (res.status !== status) {
+    throw new AssertionError({
+      actual: res.status,
+      expected: status,
+      message: await res.text()
+    })
+  }
+}