Fix benchmarks again ... (#839)

flowr-analysis · Jun 12, 2024 · cdeed92 · cdeed92 · github-actions · Jun 12, 2024
2 parents 6ba4997 + adf996c
commit cdeed92
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 3 deletions.
diff --git a/src/benchmark/summarizer/summarizer.ts b/src/benchmark/summarizer/summarizer.ts
@@ -14,6 +14,7 @@ import { readLineByLine, readLineByLineSync } from '../../util/files'
 import { jsonReplacer } from '../../util/json'
 import { ultimateStats2String } from '../stats/print'
 import { DefaultMap } from '../../util/defaultmap'
+import { guard } from '../../util/assert'
 
 export interface BenchmarkSummarizerConfiguration extends CommonSummarizerConfiguration {
 	/**
@@ -43,12 +44,15 @@ export class BenchmarkSummarizer extends Summarizer<UltimateSlicerStats, Benchma
 		this.removeIfExists(this.summaryFile())
 
 		this.removeIfExists(this.config.intermediateOutputPath)
-		fs.mkdirSync(this.config.intermediateOutputPath)
+		fs.mkdirSync(this.config.intermediateOutputPath, { recursive: true })
 
 		// recursively find all files in all the input path subdirectories
 		const filesToSummarize = fs.readdirSync(this.config.inputPath, { encoding: 'utf-8', recursive: true })
 			.map(e => path.join(this.config.inputPath, e)).filter(e => fs.statSync(e).isFile())
 
+		guard(filesToSummarize.length > 0, `Found no files to summarize in ${this.config.inputPath}`)
+		this.log(`Found ${filesToSummarize.length} files to summarize in ${this.config.inputPath}`)
+
 		const outputPathsPerRun = new DefaultMap<number, string[]>(() => [])
 		for(let i = 0; i < filesToSummarize.length; i++) {
 			const outputDir = path.join(this.config.intermediateOutputPath, path.relative(this.config.inputPath, filesToSummarize[i]))

diff --git a/src/cli/benchmark-app.ts b/src/cli/benchmark-app.ts
@@ -47,7 +47,7 @@ function removeIfExists(summarizedRaw: string) {
 
 async function benchmark() {
 	removeIfExists(options.output)
-	fs.mkdirSync(options.output)
+	fs.mkdirSync(options.output, { recursive: true })
 
 	console.log(`Storing output in ${options.output}`)
 	console.log(`Using ${options.parallel} parallel executors`)

diff --git a/src/cli/benchmark-helper-app.ts b/src/cli/benchmark-helper-app.ts
@@ -79,7 +79,9 @@ async function benchmark() {
 			'run-num': options['run-num'],
 			stats
 		}
+
 		// append line by line
+		console.log(`Appending benchmark of ${options.input} to ${options.output}`)
 		fs.appendFileSync(options.output, `${JSON.stringify(output, jsonReplacer)}\n`)
 	} catch(e: unknown) {
 		if(e instanceof Error) {

diff --git a/test/performance/run-all-suites.sh b/test/performance/run-all-suites.sh
@@ -10,7 +10,7 @@ ONLY_SUITE="${3-}"
 
 
 SUITE_PREFIX="suite-"
-OUTPUT_DIR="results/"
+OUTPUT_DIR="results"
 
 mkdir -p "${OUTPUT_DIR}"
Benchmark suite	Current: `cdeed92`	Previous: `c919f62`	Ratio
`Retrieve AST from R code`	`239.60870940909092` ms (`105.57312979769767`)	`238.8422665` ms (`100.36542442978394`)	`1.00`
`Normalize R AST`	`31.613356181818183` ms (`62.791939115848734`)	`31.483756772727272` ms (`61.76510267487319`)	`1.00`
`Produce dataflow information`	`55.97008422727273` ms (`137.99595407304932`)	`56.44745127272727` ms (`139.82495087699667`)	`0.99`
`Total per-file`	`1270.2330369545455` ms (`3095.020759074828`)	`1271.1114238181817` ms (`3087.0669050474426`)	`1.00`
`Static slicing`	`1.2143042206451704` ms (`1.1488861877931393`)	`1.2020123265030156` ms (`1.1211635564324134`)	`1.01`
`Reconstruct code`	`0.40831558892666786` ms (`0.22643744595455462`)	`0.4129428068869898` ms (`0.22972322292335046`)	`0.99`
`Total per-slice`	`1.641306228803399` ms (`1.1991530804223784`)	`1.6328556957834646` ms (`1.1753183659101543`)	`1.01`
`failed to reconstruct/re-parse`	`0` #	`0` #	`1`
`times hit threshold`	`0` #	`0` #	`1`
`reduction (characters)`	`0.7974469447714406` #	`0.7974469447714406` #	`1`
`reduction (normalized tokens)`	`0.774073695901592` #	`0.774073695901592` #	`1`
`memory (df-graph)`	`151615.13636363635` Bytes (`369422.56988247525`)	`151615.13636363635` Bytes (`369422.56988247525`)	`1`