implement backup scanner, don't try to fetch api on parsed matches

odota · Jun 22, 2024 · d5f530c · d5f530c
1 parent 6b83b23
commit d5f530c
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 22 deletions.
diff --git a/config.ts b/config.ts
@@ -75,6 +75,7 @@ const defaults = {
   API_KEY_GEN_THRESHOLD: '0', // Account ID requirement (delta from max) for generating API keys
   SERVICE_REGISTRY_HOST: '', // Host for external services to register themselves at
   USE_SERVICE_REGISTRY: '', // Use the service registry for determining gc and parser urls
+  SCANNER_OFFSET: '0', // Delay in match seq num value to run secondary scanner (to pick up missing matches)
 };
 if (process.env.NODE_ENV === 'development') {
   // force PORT to null in development so we can run multiple web services without conflict

diff --git a/ecosystem.config.js b/ecosystem.config.js
@@ -51,6 +51,13 @@ let arr = [
     name: 'scanner',
     group: 'backend',
   },
+  // {
+  //   name: 'scanner2',
+  //   group: 'backend',
+  //   env: {
+  //     SCANNER_OFFSET: '50000',
+  //   }
+  // },
   {
     name: 'backupscanner',
     group: 'disabled',

diff --git a/global.d.ts b/global.d.ts
@@ -454,7 +454,8 @@ type MetricName =
   | 'incomplete_archive'
   | 'gen_api_key_invalid'
   | 'parser_job'
-  | 'oldparse';
+  | 'oldparse'
+  | 'secondary_scanner';
 
 // Object to map player_slot to basic info
 type PGroup = {

diff --git a/store/buildStatus.ts b/store/buildStatus.ts
@@ -139,6 +139,7 @@ export async function buildStatus() {
     error_last_day: async () => countDay('500_error'),
     web_crash_last_day: async () => countDay('web_crash'),
     skip_seq_num_last_day: async () => countDay('skip_seq_num'),
+    secondary_scanner_last_day: async () => countDay('secondary_scanner'),
     api_paths: async () => {
       const results = await redis.zrangebyscore(
         'api_paths',

diff --git a/store/queries.ts b/store/queries.ts
@@ -731,31 +731,33 @@ export async function getMatchDataFromBlobWithMetadata(
     has_gcdata: Boolean(gcdata),
     has_parsed: Boolean(parsed),
   };
-
-  if (!api && backfill) {
-    redisCount(redis, 'steam_api_backfill');
-    api = await tryFetchApiData(matchId, true);
-    if (api) {
-      odData.backfill_api = true;
-    }
-  }
-  if (!api) {
-    return [null, null];
-  }
-  if (!gcdata && backfill) {
-    redisCount(redis, 'steam_gc_backfill');
-    // gcdata = await tryFetchGcData(matchId, getPGroup(api));
-    if (gcdata) {
-      odData.backfill_gc = true;
-    }
-  }
   if (backfill) {
     archived = await tryReadArchivedMatch(matchId);
     if (archived) {
       odData.archive = true;
     }
   }
 
+  if (!archived) {
+    if (!api && backfill) {
+      redisCount(redis, 'steam_api_backfill');
+      api = await tryFetchApiData(matchId, true);
+      if (api) {
+        odData.backfill_api = true;
+      }
+    }
+    if (!api) {
+      return [null, null];
+    }
+    if (!gcdata && backfill) {
+      redisCount(redis, 'steam_gc_backfill');
+      // gcdata = await tryFetchGcData(matchId, getPGroup(api));
+      if (gcdata) {
+        odData.backfill_gc = true;
+      }
+    }
+  }
+
   // Merge the results together
   const final: Match | ParsedMatch = {
     ...archived,

diff --git a/svc/scanner.ts b/svc/scanner.ts
@@ -14,7 +14,7 @@ const SCANNER_WAIT = 5000;
 const SCANNER_WAIT_CATCHUP = SCANNER_WAIT / parallelism;
 
 async function scanApi(seqNum: number) {
-  let nextSeqNum = seqNum;
+  let nextSeqNum = seqNum - Number(config.SCANNER_OFFSET);
   while (true) {
     const container = generateJob('api_sequence', {
       start_at_match_seq_num: nextSeqNum,
@@ -64,15 +64,20 @@ async function processMatch(match: ApiMatch) {
     return;
   }
   // check if match was previously processed
-  const result = await redis.get(`scanner_insert:${match.match_id}`);
+  const result = await redis.zscore('scanner_insert', match.match_id);
   // console.log(match.match_id, result);
   // don't insert this match if we already processed it recently
   if (!result) {
+    if (Number(config.SCANNER_OFFSET)) {
+      // secondary scanner picked up a missing match
+      redisCount(redis, 'secondary_scanner');
+    }
     await insertMatch(match, {
       type: 'api',
       origin: 'scanner',
     });
-    await redis.setex(`scanner_insert:${match.match_id}`, 3600 * 4, 1);
+    await redis.zadd('scanner_insert', match.match_id, match.match_id);
+    await redis.zremrangebyrank('scanner_insert', '0', '-100000');
   }
 }