feat: fetch RSR for miners with non-zero RSR only (#136)

Signed-off-by: Miroslav Bajtoš <oss@bajtos.net>
Co-authored-by: Julian Gruber <julian@juliangruber.com>

README.md

@@ -6,6 +6,16 @@ API exposing public statistics about Spark
 
 Base URL: http://stats.filspark.com/
 
+- `GET /retrieval-success-rate?from=2024-01-01&to=2024-01-31`
+
+  http://stats.filspark.com/retrieval-success-rate
+
+- `GET /retrieval-success-rate?from=2024-01-01&to=2024-01-31&nonZero=true`
+
+  _Miners with no successful retrievals are excluded from the RSR calculation._
+
+  http://stats.filspark.com/retrieval-success-rate?nonZero=true
+
 - `GET /miners/retrieval-success-rate/summary?from=<day>&to=<day>`
 
   http://stats.filspark.com/miners/retrieval-success-rate/summary

stats/lib/request-helpers.js

@@ -8,61 +8,60 @@ const getDayAsISOString = (d) => d.toISOString().split('T')[0]
 export const today = () => getDayAsISOString(new Date())
 
 /**
+ * @template {import('./typings.d.ts').DateRangeFilter} FilterType
  * @param {string} pathname
  * @param {URLSearchParams} searchParams
  * @param {import('node:http').ServerResponse} res
- * @param {import('pg').Pool} pgPool
- * @param {(import('pg').Pool, import('./typings.d.ts').DateRangeFilter) => Promise<object[]>} fetchStatsFn
+ * @param {pg.Pool} pgPool
+ * @param {(pg.Pool, FilterType) => Promise<object[]>} fetchStatsFn
  */
 export const getStatsWithFilterAndCaching = async (pathname, searchParams, res, pgPool, fetchStatsFn) => {
-  let from = searchParams.get('from')
-  let to = searchParams.get('to')
+  const filter = Object.fromEntries(searchParams)
   let shouldRedirect = false
 
   // Provide default values for "from" and "to" when not specified
 
-  if (!to) {
-    to = today()
+  if (!filter.to) {
+    filter.to = today()
     shouldRedirect = true
   }
-  if (!from) {
-    from = to
+  if (!filter.from) {
+    filter.from = filter.to
     shouldRedirect = true
   }
   if (shouldRedirect) {
     res.setHeader('cache-control', `public, max-age=${600 /* 10min */}`)
-    res.setHeader('location', `${pathname}?${new URLSearchParams({ from, to })}`)
+    res.setHeader('location', `${pathname}?${new URLSearchParams(Object.entries(filter))}`)
     res.writeHead(302) // Found
     res.end()
-    return { from, to }
+    return
   }
 
   // Trim time from date-time values that are typically provided by Grafana
 
-  const matchFrom = from.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
+  const matchFrom = filter.from.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
   assert(matchFrom, 400, '"from" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ')
   if (matchFrom[2]) {
-    from = matchFrom[1]
+    filter.from = matchFrom[1]
     shouldRedirect = true
   }
 
-  const matchTo = to.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
+  const matchTo = filter.to.match(/^(\d{4}-\d{2}-\d{2})(T\d{2}:\d{2}:\d{2}\.\d{3}Z)?$/)
   assert(matchTo, 400, '"to" must have format YYYY-MM-DD or YYYY-MM-DDThh:mm:ss.sssZ')
   if (matchTo[2]) {
-    to = matchTo[1]
+    filter.to = matchTo[1]
     shouldRedirect = true
   }
 
   if (shouldRedirect) {
     res.setHeader('cache-control', `public, max-age=${24 * 3600 /* one day */}`)
-    res.setHeader('location', `${pathname}?${new URLSearchParams({ from, to })}`)
+    res.setHeader('location', `${pathname}?${new URLSearchParams(Object.entries(filter))}`)
     res.writeHead(301) // Found
     res.end()
-    return { from, to }
+    return
   }
 
   // We have well-formed from & to dates now, let's fetch the requested stats from the DB
-  const filter = { from, to }
   const stats = await fetchStatsFn(pgPool, filter)
   setCacheControlForStatsResponse(res, filter)
   json(res, stats)

stats/lib/stats-fetchers.js

@@ -2,15 +2,15 @@ import { getDailyDistinctCount, getMonthlyDistinctCount } from './request-helper
 
 /**
  * @param {import('pg').Pool} pgPool
- * @param {import('./typings').DateRangeFilter} filter
+ * @param {import('./typings').DateRangeFilter & {nonZero?: 'true'}} filter
  */
 export const fetchRetrievalSuccessRate = async (pgPool, filter) => {
   // Fetch the "day" (DATE) as a string (TEXT) to prevent node-postgres for converting it into
   // a JavaScript Date with a timezone, as that could change the date one day forward or back.
   const { rows } = await pgPool.query(`
     SELECT day::text, SUM(total) as total, SUM(successful) as successful
     FROM retrieval_stats
-    WHERE day >= $1 AND day <= $2
+    WHERE day >= $1 AND day <= $2 ${filter.nonZero === 'true' ? 'AND successful > 0' : ''}
     GROUP BY day
     ORDER BY day
     `, [

stats/test/handler.test.js

@@ -185,6 +185,38 @@ describe('HTTP request handler', () => {
         { day: '2024-01-20', success_rate: 1 / 10, total: '10', successful: '1' }
       ])
     })
+
+    it('filters out miners with zero RSR when asked', async () => {
+      await givenRetrievalStats(pgPool, { day: '2024-01-20', total: 10, successful: 1, minerId: 'f1one' })
+      await givenRetrievalStats(pgPool, { day: '2024-01-20', total: 10, successful: 0, minerId: 'f1two' })
+
+      const res = await fetch(
+        new URL(
+          '/retrieval-success-rate?from=2024-01-01&to=2024-01-31&nonZero=true',
+          baseUrl
+        ), {
+          redirect: 'manual'
+        }
+      )
+      await assertResponseStatus(res, 200)
+      /** @type {{ day: string, success_rate: number }[]} */
+      const stats = await res.json()
+      assert.deepStrictEqual(stats, [
+        { day: '2024-01-20', success_rate: 1 / 10, successful: '1', total: '10' }
+      ])
+    })
+
+    it('preserves additional query string arguments when redirecting', async () => {
+      const day = today()
+      await givenRetrievalStats(pgPool, { day, total: 10, successful: 1, minerId: 'f1one' })
+      await givenRetrievalStats(pgPool, { day, total: 10, successful: 0, minerId: 'f1two' })
+      const res = await fetch(new URL('/retrieval-success-rate?nonZero=true', baseUrl), { redirect: 'follow' })
+      await assertResponseStatus(res, 200)
+      const stats = await res.json()
+      assert.deepStrictEqual(stats, [
+        { day, success_rate: 0.1, successful: '1', total: '10' }
+      ])
+    })
   })
 
   describe('GET /participants/daily', () => {