feat(benchmark): store benchmark results in original subdirectories

src/benchmark/summarizer/summarizer.ts

@@ -45,17 +45,18 @@ export class BenchmarkSummarizer extends Summarizer<UltimateSlicerStats, Benchma
 		this.removeIfExists(this.config.intermediateOutputPath)
 		fs.mkdirSync(this.config.intermediateOutputPath)
 
-		const filesToSummarize = fs.readdirSync(this.config.inputPath)
+		// recursively find all files in all the input path subdirectories
+		const filesToSummarize = fs.readdirSync(this.config.inputPath, { encoding: 'utf-8', recursive: true })
+			.map(e => path.join(this.config.inputPath, e)).filter(e => fs.statSync(e).isFile())
 
 		const outputPathsPerRun = new DefaultMap<number, string[]>(() => [])
 		for(let i = 0; i < filesToSummarize.length; i++) {
-			const fileInputPath = path.join(this.config.inputPath, filesToSummarize[i])
-			const outputDir = path.join(this.config.intermediateOutputPath, path.parse(filesToSummarize[i]).name)
-			fs.mkdirSync(outputDir)
+			const outputDir = path.join(this.config.intermediateOutputPath, path.relative(this.config.inputPath, filesToSummarize[i]))
+			fs.mkdirSync(outputDir, { recursive: true })
 			const textOutputPath = path.join(outputDir, 'summary.log')
 
 			// generate measurements for each run
-			await readLineByLine(fileInputPath, (line, lineNumber) => {
+			await readLineByLine(filesToSummarize[i], (line, lineNumber) => {
 				const runOutputPath = path.join(outputDir, `run-${lineNumber}.json`)
 				outputPathsPerRun.get(lineNumber).push(runOutputPath)
 				return processRunMeasurement(line, i, lineNumber, textOutputPath, runOutputPath)

src/cli/benchmark-app.ts

@@ -1,7 +1,7 @@
 import fs from 'fs'
 import path from 'path'
 import { guard } from '../util/assert'
-import { allRFilesFrom } from '../util/files'
+import { allRFiles } from '../util/files'
 import { log } from '../util/log'
 import { LimitedThreadPool } from '../util/parallel'
 import { processCommandLineArgs } from './common/script'
@@ -52,9 +52,11 @@ async function benchmark() {
 	console.log(`Storing output in ${options.output}`)
 	console.log(`Using ${options.parallel} parallel executors`)
 	// we do not use the limit argument to be able to pick the limit randomly
-	const files: RParseRequestFromFile[] = []
-	for await (const file of allRFilesFrom(options.input)) {
-		files.push(file)
+	const files: {request: RParseRequestFromFile, baseDir: string}[] = []
+	for(const input of options.input) {
+		for await (const file of allRFiles(input)) {
+			files.push({ request: file, baseDir: input })
+		}
 	}
 
 	if(options.limit) {
@@ -66,9 +68,9 @@ async function benchmark() {
 
 	const verboseAdd = options.verbose ? ['--verbose'] : []
 	const args = files.map((f,i) => [
-		'--input', f.content,
+		'--input', f.request.content,
 		'--file-id', `${i}`,
-		'--output', path.join(options.output, `${path.parse(f.content).name}.json`),
+		'--output', path.join(options.output, path.relative(f.baseDir, `${f.request.content}.json`)),
 		'--slice', options.slice, ...verboseAdd])
 
 	const runs = options.runs ?? 1

src/cli/benchmark-helper-app.ts

@@ -6,6 +6,7 @@ import { processCommandLineArgs } from './common/script'
 import type { RParseRequestFromFile } from '../r-bridge/retriever'
 import { BenchmarkSlicer } from '../benchmark/slicer'
 import { DefaultAllVariablesFilter } from '../slicing/criterion/filters/all-variables'
+import path from 'path'
 
 
 export interface SingleBenchmarkCliOptions {
@@ -44,6 +45,7 @@ async function benchmark() {
 	// prefix for printing to console, includes file id and run number if present
 	const prefix = `[${options.input }${options['file-id'] !== undefined ? ` (file ${options['file-id']}, run ${options['run-num']})` : ''}]`
 	console.log(`${prefix} Appending output to ${options.output}`)
+	fs.mkdirSync(path.parse(options.output).dir, { recursive: true })
 
 	// ensure the file exists
 	const fileStat = fs.statSync(options.input)