From a243633a99c32ad3d68055d8a3b8e33c1e9ab05f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Tue, 28 May 2024 19:19:30 +0200 Subject: [PATCH] Avoid breaking up log entries containing `}{` (#491) * Avoid breaking up log entries containing `}{` * fixed types --- .changeset/fresh-kiwis-drop.md | 5 + .../src/utils/recordings/getRecordings.ts | 408 +++++++++--------- .../src/utils/recordings/readRecordingLog.ts | 34 ++ .../utils/recordings/readRecordingLogLines.ts | 11 - .../src/utils/recordings/removeFromDisk.ts | 33 +- .../utils/recordings/updateRecordingLog.ts | 7 +- 6 files changed, 257 insertions(+), 241 deletions(-) create mode 100644 .changeset/fresh-kiwis-drop.md create mode 100644 packages/replayio/src/utils/recordings/readRecordingLog.ts delete mode 100644 packages/replayio/src/utils/recordings/readRecordingLogLines.ts diff --git a/.changeset/fresh-kiwis-drop.md b/.changeset/fresh-kiwis-drop.md new file mode 100644 index 00000000..8c5be7b2 --- /dev/null +++ b/.changeset/fresh-kiwis-drop.md @@ -0,0 +1,5 @@ +--- +"replayio": patch +--- + +Improved the way `recordings.log` gets processed. It should fix parsing issues when the log contains strings with `}{` inside them. diff --git a/packages/replayio/src/utils/recordings/getRecordings.ts b/packages/replayio/src/utils/recordings/getRecordings.ts index f352247b..6e6488b7 100644 --- a/packages/replayio/src/utils/recordings/getRecordings.ts +++ b/packages/replayio/src/utils/recordings/getRecordings.ts @@ -3,243 +3,237 @@ import { existsSync } from "fs-extra"; import { basename } from "path"; import { recordingLogPath } from "./config"; import { debug } from "./debug"; -import { readRecordingLogLines } from "./readRecordingLogLines"; -import { LocalRecording, LogEntry, RECORDING_LOG_KIND } from "./types"; +import { readRecordingLog } from "./readRecordingLog"; +import { LocalRecording, RECORDING_LOG_KIND } from "./types"; export function getRecordings(processGroupIdFilter?: string): LocalRecording[] { const recordings: LocalRecording[] = []; const idToRecording: Record = {}; if (existsSync(recordingLogPath)) { - const rawTextLines = readRecordingLogLines(); + const entries = readRecordingLog(); - debug("Reading recording log %s\n%s", recordingLogPath, rawTextLines.join("\n")); + debug("Reading recording log %s\n%s", recordingLogPath, JSON.stringify(entries)); const idToStartTimestamp: Record = {}; - for (let line of rawTextLines) { - try { - const entry = JSON.parse(line) as LogEntry; - switch (entry.kind) { - case RECORDING_LOG_KIND.addMetadata: { - const { id, metadata = {} } = entry; - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - - Object.assign(recording.metadata, metadata); - - const { argv, process, processGroupId, uri } = metadata; - - if (uri) { - let host = uri; - if (host && typeof host === "string") { - try { - recording.metadata.host = new URL(host).host; - } catch (error) { - recording.metadata.host = host; - } - } - } else if (Array.isArray(argv) && typeof argv[0] === "string") { - recording.metadata.host = basename(argv[0]); - } + for (let entry of entries) { + switch (entry.kind) { + case RECORDING_LOG_KIND.addMetadata: { + const { id, metadata = {} } = entry; + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); - if (process) { - recording.metadata.processType = process; - } + Object.assign(recording.metadata, metadata); - if (processGroupId) { - recording.metadata.processGroupId = processGroupId; - } - break; - } - case RECORDING_LOG_KIND.crashData: { - const { data, id } = entry; - - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - if (recording.crashData) { - recording.crashData.push(data); - } else { - recording.crashData = [data]; + const { argv, process, processGroupId, uri } = metadata; + + if (uri) { + let host = uri; + if (host && typeof host === "string") { + try { + recording.metadata.host = new URL(host).host; + } catch (error) { + recording.metadata.host = host; + } } - break; + } else if (Array.isArray(argv) && typeof argv[0] === "string") { + recording.metadata.host = basename(argv[0]); } - case RECORDING_LOG_KIND.crashed: { - const { id } = entry; - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - recording.recordingStatus = "crashed"; - break; + if (process) { + recording.metadata.processType = process; } - case RECORDING_LOG_KIND.crashUploaded: { - // No-op - break; - } - case RECORDING_LOG_KIND.createRecording: { - const recording: LocalRecording = { - buildId: entry.buildId as string, - crashData: undefined, - date: new Date(entry.timestamp), - driverVersion: entry.driverVersion as string, - duration: undefined, - id: entry.id, - metadata: { - host: undefined, - processGroupId: undefined, - processType: undefined, - sourceMaps: [], - uri: undefined, - }, - path: undefined, - processingStatus: undefined, - recordingStatus: "recording", - uploadStatus: undefined, - }; - - idToRecording[entry.id] = recording; - - recordings.push(recording); - break; + + if (processGroupId) { + recording.metadata.processGroupId = processGroupId; } - case RECORDING_LOG_KIND.originalSourceAdded: { - const { recordingId, parentId, path, parentOffset } = entry; - assert(recordingId, '"originalSourceAdded" entry must have a "recordingId"'); - assert(parentId, '"originalSourceAdded" entry must have a "parentId"'); - assert(path, '"originalSourceAdded" entry must have a "path"'); - assert( - typeof parentOffset === "number", - '"originalSourceAdded" entry must have a numeric "parentOffset"' - ); - - const recording = idToRecording[recordingId]; - assert(recording, `Recording with ID "${recordingId}" not found`); - - const sourceMap = recording.metadata.sourceMaps.find( - sourceMap => sourceMap.id === parentId - ); - assert(sourceMap, `Source map with ID "${parentId}" not found`); - - sourceMap.originalSources.push({ - path, - parentOffset, - }); - break; + break; + } + case RECORDING_LOG_KIND.crashData: { + const { data, id } = entry; + + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + if (recording.crashData) { + recording.crashData.push(data); + } else { + recording.crashData = [data]; } - case RECORDING_LOG_KIND.processingFailed: { - const { id } = entry; + break; + } + case RECORDING_LOG_KIND.crashed: { + const { id } = entry; - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - recording.processingStatus = "failed"; - break; - } - case RECORDING_LOG_KIND.processingFinished: { - const { id } = entry; + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + recording.recordingStatus = "crashed"; + break; + } + case RECORDING_LOG_KIND.crashUploaded: { + // No-op + break; + } + case RECORDING_LOG_KIND.createRecording: { + const recording: LocalRecording = { + buildId: entry.buildId as string, + crashData: undefined, + date: new Date(entry.timestamp), + driverVersion: entry.driverVersion as string, + duration: undefined, + id: entry.id, + metadata: { + host: undefined, + processGroupId: undefined, + processType: undefined, + sourceMaps: [], + uri: undefined, + }, + path: undefined, + processingStatus: undefined, + recordingStatus: "recording", + uploadStatus: undefined, + }; + + idToRecording[entry.id] = recording; + + recordings.push(recording); + break; + } + case RECORDING_LOG_KIND.originalSourceAdded: { + const { recordingId, parentId, path, parentOffset } = entry; + assert(recordingId, '"originalSourceAdded" entry must have a "recordingId"'); + assert(parentId, '"originalSourceAdded" entry must have a "parentId"'); + assert(path, '"originalSourceAdded" entry must have a "path"'); + assert( + typeof parentOffset === "number", + '"originalSourceAdded" entry must have a numeric "parentOffset"' + ); + + const recording = idToRecording[recordingId]; + assert(recording, `Recording with ID "${recordingId}" not found`); + + const sourceMap = recording.metadata.sourceMaps.find( + sourceMap => sourceMap.id === parentId + ); + assert(sourceMap, `Source map with ID "${parentId}" not found`); + + sourceMap.originalSources.push({ + path, + parentOffset, + }); + break; + } + case RECORDING_LOG_KIND.processingFailed: { + const { id } = entry; - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - recording.processingStatus = "processed"; - break; - } - case RECORDING_LOG_KIND.processingStarted: { - const { id } = entry; + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + recording.processingStatus = "failed"; + break; + } + case RECORDING_LOG_KIND.processingFinished: { + const { id } = entry; - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - recording.processingStatus = "processing"; - break; - } - case RECORDING_LOG_KIND.recordingUnusable: { - const { id } = entry; - const recording = idToRecording[id]; + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + recording.processingStatus = "processed"; + break; + } + case RECORDING_LOG_KIND.processingStarted: { + const { id } = entry; - assert(recording, `Recording with ID "${id}" not found`); - recording.recordingStatus = "unusable"; + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + recording.processingStatus = "processing"; + break; + } + case RECORDING_LOG_KIND.recordingUnusable: { + const { id } = entry; + const recording = idToRecording[id]; - const index = recordings.indexOf(recording); - recordings.splice(index, 1); - break; - } - case RECORDING_LOG_KIND.sourcemapAdded: { - const { - path, - recordingId, - id, - baseURL, - targetContentHash, - targetURLHash, - targetMapURLHash, - } = entry; - assert(recordingId, '"sourcemapAdded" entry must have a "recordingId"'); - assert(path, '"sourcemapAdded" entry must have a "path"'); - assert(baseURL, '"sourcemapAdded" entry must have a "baseURL"'); - assert(targetMapURLHash, '"sourcemapAdded" entry must have a "targetMapURLHash"'); - - const recording = idToRecording[recordingId]; - assert(recording, `Recording with ID "${recordingId}" not found`); - - recording.metadata.sourceMaps.push({ - id, - path, - baseURL, - targetContentHash, - targetURLHash, - targetMapURLHash, - originalSources: [], - }); - break; - } - case RECORDING_LOG_KIND.uploadFailed: { - const { id } = entry; + assert(recording, `Recording with ID "${id}" not found`); + recording.recordingStatus = "unusable"; - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - recording.uploadStatus = "failed"; - break; - } - case RECORDING_LOG_KIND.uploadFinished: { - const { id } = entry; + const index = recordings.indexOf(recording); + recordings.splice(index, 1); + break; + } + case RECORDING_LOG_KIND.sourcemapAdded: { + const { + path, + recordingId, + id, + baseURL, + targetContentHash, + targetURLHash, + targetMapURLHash, + } = entry; + assert(recordingId, '"sourcemapAdded" entry must have a "recordingId"'); + assert(path, '"sourcemapAdded" entry must have a "path"'); + assert(baseURL, '"sourcemapAdded" entry must have a "baseURL"'); + assert(targetMapURLHash, '"sourcemapAdded" entry must have a "targetMapURLHash"'); + + const recording = idToRecording[recordingId]; + assert(recording, `Recording with ID "${recordingId}" not found`); + + recording.metadata.sourceMaps.push({ + id, + path, + baseURL, + targetContentHash, + targetURLHash, + targetMapURLHash, + originalSources: [], + }); + break; + } + case RECORDING_LOG_KIND.uploadFailed: { + const { id } = entry; - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - recording.uploadStatus = "uploaded"; - break; - } - case RECORDING_LOG_KIND.uploadStarted: { - const { id } = entry; + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + recording.uploadStatus = "failed"; + break; + } + case RECORDING_LOG_KIND.uploadFinished: { + const { id } = entry; - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - recording.uploadStatus = "uploading"; - break; - } - case RECORDING_LOG_KIND.writeFinished: { - const { id, timestamp } = entry; + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + recording.uploadStatus = "uploaded"; + break; + } + case RECORDING_LOG_KIND.uploadStarted: { + const { id } = entry; + + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + recording.uploadStatus = "uploading"; + break; + } + case RECORDING_LOG_KIND.writeFinished: { + const { id, timestamp } = entry; - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - recording.recordingStatus = "finished"; + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + recording.recordingStatus = "finished"; - const startTimestamp = idToStartTimestamp[id]; - if (startTimestamp != undefined) { - recording.duration = timestamp - idToStartTimestamp[id]; - } - break; - } - case RECORDING_LOG_KIND.writeStarted: { - const { id, path, timestamp } = entry; - - const recording = idToRecording[id]; - assert(recording, `Recording with ID "${id}" not found`); - recording.path = path; - idToStartTimestamp[id] = timestamp; - break; + const startTimestamp = idToStartTimestamp[id]; + if (startTimestamp != undefined) { + recording.duration = timestamp - idToStartTimestamp[id]; } + break; + } + case RECORDING_LOG_KIND.writeStarted: { + const { id, path, timestamp } = entry; + + const recording = idToRecording[id]; + assert(recording, `Recording with ID "${id}" not found`); + recording.path = path; + idToStartTimestamp[id] = timestamp; + break; } - } catch (error) { - debug(`Error parsing line:\n${line}`); - continue; } } } diff --git a/packages/replayio/src/utils/recordings/readRecordingLog.ts b/packages/replayio/src/utils/recordings/readRecordingLog.ts new file mode 100644 index 00000000..9582fa93 --- /dev/null +++ b/packages/replayio/src/utils/recordings/readRecordingLog.ts @@ -0,0 +1,34 @@ +import { readFileSync } from "fs-extra"; +import { recordingLogPath } from "./config"; +import { debug } from "./debug"; +import { LogEntry } from "./types"; + +export function readRecordingLog() { + const rawText = readFileSync(recordingLogPath, "utf8"); + return rawText + .split(/[\n\r]+/) + .map(text => text.trim()) + .filter(Boolean) + .flatMap(line => { + try { + return JSON.parse(line) as LogEntry; + } catch (err) { + debug(`Error parsing line:\n${line}`); + + const replaced = line.replace(/\}\{/g, "}\n{"); + + if (replaced.length === line.length) { + return; + } + + return replaced.split(/[\n\r]+/).map(splitted => { + try { + return JSON.parse(splitted) as LogEntry; + } catch (err) { + debug(`Error parsing splitted line:\n${splitted}`); + } + }); + } + }) + .filter((v): v is LogEntry => !!v); +} diff --git a/packages/replayio/src/utils/recordings/readRecordingLogLines.ts b/packages/replayio/src/utils/recordings/readRecordingLogLines.ts deleted file mode 100644 index 3f2c73b5..00000000 --- a/packages/replayio/src/utils/recordings/readRecordingLogLines.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { readFileSync } from "fs-extra"; -import { recordingLogPath } from "./config"; - -export function readRecordingLogLines() { - const rawText = readFileSync(recordingLogPath, "utf8"); - return rawText - .replace(/\}\{/g, "}\n{") - .split(/[\n\r]+/) - .map(text => text.trim()) - .filter(Boolean); -} diff --git a/packages/replayio/src/utils/recordings/removeFromDisk.ts b/packages/replayio/src/utils/recordings/removeFromDisk.ts index 66efa124..a350eb6c 100644 --- a/packages/replayio/src/utils/recordings/removeFromDisk.ts +++ b/packages/replayio/src/utils/recordings/removeFromDisk.ts @@ -3,8 +3,8 @@ import { join } from "path"; import { recordingLogPath, recordingsPath } from "./config"; import { debug } from "./debug"; import { getRecordings } from "./getRecordings"; -import { readRecordingLogLines } from "./readRecordingLogLines"; -import { LocalRecording, LogEntry, RECORDING_LOG_KIND } from "./types"; +import { readRecordingLog } from "./readRecordingLog"; +import { LocalRecording, RECORDING_LOG_KIND } from "./types"; function getAssetsUsageMap(recordings: LocalRecording[]) { const usageMap: Record = {}; @@ -57,26 +57,23 @@ export function removeFromDisk(id?: string) { } // Remove entries from log - const filteredLines = readRecordingLogLines().filter(text => { - if (text) { - try { - const entry = JSON.parse(text) as LogEntry; - switch (entry.kind) { - case RECORDING_LOG_KIND.originalSourceAdded: - case RECORDING_LOG_KIND.sourcemapAdded: { - return entry.recordingId !== id; - } - default: { - return entry.id !== id; - } - } - } catch (error) { - console.error("Error parsing log text:\n%s\n%s", text, error); + const filteredLogs = readRecordingLog().filter(entry => { + switch (entry.kind) { + case RECORDING_LOG_KIND.originalSourceAdded: + case RECORDING_LOG_KIND.sourcemapAdded: { + return entry.recordingId !== id; + } + default: { + return entry.id !== id; } } }); - writeFileSync(recordingLogPath, filteredLines.join("\n"), "utf8"); + writeFileSync( + recordingLogPath, + filteredLogs.map(log => JSON.stringify(log)).join("\n"), + "utf8" + ); } else { console.log("Recording not found"); } diff --git a/packages/replayio/src/utils/recordings/updateRecordingLog.ts b/packages/replayio/src/utils/recordings/updateRecordingLog.ts index 5fe0fa68..e58c8e62 100644 --- a/packages/replayio/src/utils/recordings/updateRecordingLog.ts +++ b/packages/replayio/src/utils/recordings/updateRecordingLog.ts @@ -1,7 +1,6 @@ -import { writeFileSync } from "fs-extra"; +import { appendFileSync } from "fs-extra"; import { recordingLogPath } from "./config"; import { debug } from "./debug"; -import { readRecordingLogLines } from "./readRecordingLogLines"; import { LocalRecording, LogEntry } from "./types"; export function updateRecordingLog( @@ -18,9 +17,7 @@ export function updateRecordingLog( timestamp: Date.now(), }; - const rawTextLines = readRecordingLogLines(); - - writeFileSync(recordingLogPath, `${rawTextLines.join("\n")}\n${JSON.stringify(entry)}\n`, { + appendFileSync(recordingLogPath, `\n${JSON.stringify(entry)}\n`, { encoding: "utf8", }); }