Skip to content

Commit

Permalink
Benchmark: compare reductions without comments and empty lines (#797)
Browse files Browse the repository at this point in the history
  • Loading branch information
@EagleoutIce
EagleoutIce committed May 18, 2024
2 parents d370560 + 48698aa commit 1f92d6e
Show file tree
Hide file tree
Showing 10 changed files with 282 additions and 129 deletions.
42 changes: 34 additions & 8 deletions src/benchmark/slicer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,11 @@ import { RShell } from '../r-bridge/shell'
import { DEFAULT_SLICING_PIPELINE } from '../core/steps/pipeline/default-pipelines'
import type { RParseRequestFromFile, RParseRequestFromText } from '../r-bridge/retriever'
import { retrieveNumberOfRTokensOfLastParse } from '../r-bridge/retriever'
import { collectAllIds } from '../r-bridge/lang-4.x/ast/model/collect'
import type { PipelineStepNames, PipelineStepOutputWithName } from '../core/steps/pipeline/pipeline'
import type { SlicingCriteriaFilter } from '../slicing/criterion/collect-all'
import { collectAllSlicingCriteria } from '../slicing/criterion/collect-all'
import { RType } from '../r-bridge/lang-4.x/ast/model/type'
import { visitAst } from '../r-bridge/lang-4.x/ast/model/processing/visitor'

export const benchmarkLogger = log.getSubLogger({ name: 'benchmark' })

Expand Down Expand Up @@ -120,6 +121,7 @@ export class BenchmarkSlicer {
const loadedContent = request.request === 'text' ? request.content : fs.readFileSync(request.content, 'utf-8')
// retrieve number of R tokens - flowr_parsed should still contain the last parsed code
const numberOfRTokens = await retrieveNumberOfRTokensOfLastParse(this.shell)
const numberOfRTokensNoComments = await retrieveNumberOfRTokensOfLastParse(this.shell, true)

guard(this.normalizedAst !== undefined, 'normalizedAst should be defined after initialization')
guard(this.dataflow !== undefined, 'dataflow should be defined after initialization')
Expand All @@ -140,16 +142,40 @@ export class BenchmarkSlicer {
}
}

let nodes = 0
let nodesNoComments = 0
let commentChars = 0
let commentCharsNoWhitespace = 0
visitAst(this.normalizedAst.ast, t => {
nodes++
const comments = t.info.additionalTokens?.filter(t => t.type === RType.Comment)
if(comments && comments.length > 0) {
const content = comments.map(c => c.lexeme ?? '').join('')
commentChars += content.length
commentCharsNoWhitespace += withoutWhitespace(content).length
} else {
nodesNoComments++
}
return false
})

const split = loadedContent.split('\n')
const nonWhitespace = withoutWhitespace(loadedContent).length
this.stats = {
commonMeasurements: new Map<CommonSlicerMeasurements, ElapsedTime>(),
perSliceMeasurements: this.perSliceMeasurements,
request,
input: {
numberOfLines: loadedContent.split('\n').length,
numberOfCharacters: loadedContent.length,
numberOfNonWhitespaceCharacters: withoutWhitespace(loadedContent).length,
numberOfRTokens: numberOfRTokens,
numberOfNormalizedTokens: [...collectAllIds(this.normalizedAst.ast)].length
numberOfLines: split.length,
numberOfNonEmptyLines: split.filter(l => l.trim().length > 0).length,
numberOfCharacters: loadedContent.length,
numberOfCharactersNoComments: loadedContent.length - commentChars,
numberOfNonWhitespaceCharacters: nonWhitespace,
numberOfNonWhitespaceCharactersNoComments: nonWhitespace - commentCharsNoWhitespace,
numberOfRTokens: numberOfRTokens,
numberOfRTokensNoComments: numberOfRTokensNoComments,
numberOfNormalizedTokens: nodes,
numberOfNormalizedTokensNoComments: nodesNoComments
},
dataflow: {
numberOfNodes: [...this.dataflow.graph.vertices(true)].length,
Expand Down Expand Up @@ -180,8 +206,8 @@ export class BenchmarkSlicer {
numberOfDataflowNodesSliced: 0,
timesHitThreshold: 0,
reconstructedCode: {
code: '',
autoSelected: 0
code: '',
linesWithAutoSelected: 0
}
}
this.perSliceMeasurements.set(slicingCriteria, stats)
Expand Down
40 changes: 30 additions & 10 deletions src/benchmark/stats/print.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* @module
*/
import type { ElapsedTime, PerSliceMeasurements } from './stats'
import type { SummarizedPerSliceStats, SummarizedSlicerStats, UltimateSlicerStats } from '../summarizer/data'
import type { Reduction, SummarizedPerSliceStats, SummarizedSlicerStats, UltimateSlicerStats } from '../summarizer/data'
import { guard } from '../../util/assert'
import type { SummarizedMeasurement } from '../../util/summarizer'

Expand Down Expand Up @@ -54,6 +54,7 @@ function asPercentage(num: number): string {
if(isNaN(num)) {
return '??%'
}
guard(num >= 0 && num <= 1, `Percentage ${num} should be between 0 and 1`)
return pad(`${roundTo(num * 100, 3)}%`)
}

Expand Down Expand Up @@ -100,11 +101,14 @@ Slicing summary for ${stats.perSliceMeasurements.numberOfSlices} slice${stats.pe
Used Slice Criteria Sizes: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceCriteriaSizes)}
Result Slice Sizes:
Number of lines: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.lines)}
Number of non-empty lines: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.nonEmptyLines)}
Number of characters: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.characters)}
Number of non whitespace characters: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.nonWhitespaceCharacters)}
Number of auto selected: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.autoSelected)}
Number of auto selected lines: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.linesWithAutoSelected)}
Number of R tokens: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.tokens)}
Number of R tokens (w/o comments): ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.tokensNoComments)}
Normalized R tokens: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.normalizedTokens)}
Normalized R tokens (w/o comments): ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.normalizedTokensNoComments)}
Number of dataflow nodes: ${printCountSummarizedMeasurements(stats.perSliceMeasurements.sliceSize.dataflowNodes)}
`
}
Expand All @@ -115,10 +119,15 @@ Total: ${print(stats.commonMeasurements, 'total')}
Input:
Number of lines: ${pad(stats.input.numberOfLines)}
Number of non empty lines: ${pad(stats.input.numberOfNonEmptyLines)}
Number of characters: ${pad(stats.input.numberOfCharacters)}
Number of characters (w/o comments): ${pad(stats.input.numberOfCharactersNoComments)}
Number of non whitespace characters: ${pad(stats.input.numberOfNonWhitespaceCharacters)}
Number of n. w. c. (w/o comments): ${pad(stats.input.numberOfNonWhitespaceCharactersNoComments)}
Number of tokens: ${pad(stats.input.numberOfRTokens)}
Number of tokens (w/o comments): ${pad(stats.input.numberOfRTokensNoComments)}
Normalized R tokens: ${pad(stats.input.numberOfNormalizedTokens)}
Normalized R tokens (w/o comments): ${pad(stats.input.numberOfNormalizedTokensNoComments)}
Dataflow:
Number of nodes: ${pad(stats.dataflow.numberOfNodes)}
Expand All @@ -142,28 +151,39 @@ Slice summary for:
Reconstruction: ${formatSummarizedTimeMeasure(stats.perSliceMeasurements.get('reconstruct code'))}
Failed to Re-Parse: ${pad(stats.failedToRepParse)}/${stats.totalSlices}
Times hit Threshold: ${pad(stats.timesHitThreshold)}/${stats.totalSlices}
Reductions (reduced by x%):
Number of lines: ${formatSummarizedMeasure(stats.reduction.numberOfLines, asPercentage)}
Number of lines no auto: ${formatSummarizedMeasure(stats.reduction.numberOfLinesNoAutoSelection, asPercentage)}
Number of characters: ${formatSummarizedMeasure(stats.reduction.numberOfCharacters, asPercentage)}
Number of non whitespace characters: ${formatSummarizedMeasure(stats.reduction.numberOfNonWhitespaceCharacters, asPercentage)}
Number of R tokens: ${formatSummarizedMeasure(stats.reduction.numberOfRTokens, asPercentage)}
Normalized R tokens: ${formatSummarizedMeasure(stats.reduction.numberOfNormalizedTokens, asPercentage)}
Number of dataflow nodes: ${formatSummarizedMeasure(stats.reduction.numberOfDataflowNodes, asPercentage)}
${reduction2String('Reductions', stats.reduction)}
${reduction2String('Reductions without comments and empty lines', stats.reductionNoFluff)}
Shell close: ${formatSummarizedTimeMeasure(stats.commonMeasurements.get('close R session'))}
Total: ${formatSummarizedTimeMeasure(stats.commonMeasurements.get('total'))}
Input:
Number of lines: ${formatSummarizedMeasure(stats.input.numberOfLines)}
Number of non empty lines: ${formatSummarizedMeasure(stats.input.numberOfNonEmptyLines)}
Number of characters: ${formatSummarizedMeasure(stats.input.numberOfCharacters)}
Number of characters (w/o comments): ${formatSummarizedMeasure(stats.input.numberOfCharactersNoComments)}
Number of non whitespace characters: ${formatSummarizedMeasure(stats.input.numberOfNonWhitespaceCharacters)}
Number of n. w. c. (w/o comments): ${formatSummarizedMeasure(stats.input.numberOfNonWhitespaceCharactersNoComments)}
Number of tokens: ${formatSummarizedMeasure(stats.input.numberOfRTokens)}
Number of tokens (w/o comments): ${formatSummarizedMeasure(stats.input.numberOfRTokensNoComments)}
Normalized R tokens: ${formatSummarizedMeasure(stats.input.numberOfNormalizedTokens)}
Normalized R tokens (w/o comments): ${formatSummarizedMeasure(stats.input.numberOfNormalizedTokensNoComments)}
Dataflow:
Number of nodes: ${formatSummarizedMeasure(stats.dataflow.numberOfNodes)}
Number of edges: ${formatSummarizedMeasure(stats.dataflow.numberOfEdges)}
Number of calls: ${formatSummarizedMeasure(stats.dataflow.numberOfCalls)}
Number of function defs: ${formatSummarizedMeasure(stats.dataflow.numberOfFunctionDefinitions)}`
}

function reduction2String(title: string, reduction: Reduction<SummarizedMeasurement>) {
return `
${title} (reduced by x%):
Number of lines: ${formatSummarizedMeasure(reduction.numberOfLines, asPercentage)}
Number of lines no auto: ${formatSummarizedMeasure(reduction.numberOfLinesNoAutoSelection, asPercentage)}
Number of characters: ${formatSummarizedMeasure(reduction.numberOfCharacters, asPercentage)}
Number of non whitespace characters: ${formatSummarizedMeasure(reduction.numberOfNonWhitespaceCharacters, asPercentage)}
Number of R tokens: ${formatSummarizedMeasure(reduction.numberOfRTokens, asPercentage)}
Normalized R tokens: ${formatSummarizedMeasure(reduction.numberOfNormalizedTokens, asPercentage)}
Number of dataflow nodes: ${formatSummarizedMeasure(reduction.numberOfDataflowNodes, asPercentage)}`
}
15 changes: 10 additions & 5 deletions src/benchmark/stats/stats.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,16 @@ export interface PerSliceStats {
}

export interface SlicerStatsInput<T = number> {
numberOfLines: T
numberOfCharacters: T
numberOfNonWhitespaceCharacters: T
numberOfRTokens: T
numberOfNormalizedTokens: T
numberOfLines: T
numberOfNonEmptyLines: T
numberOfCharacters: T
numberOfCharactersNoComments: T
numberOfNonWhitespaceCharacters: T
numberOfNonWhitespaceCharactersNoComments: T
numberOfRTokens: T
numberOfRTokensNoComments: T
numberOfNormalizedTokens: T
numberOfNormalizedTokensNoComments: T
}


Expand Down
23 changes: 16 additions & 7 deletions src/benchmark/summarizer/data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,19 @@ import type {


export interface SliceSizeCollection {
lines: number[]
characters: number[]
nonWhitespaceCharacters: number[]
lines: number[]
nonEmptyLines: number[]
characters: number[]
charactersNoComments: number[]
nonWhitespaceCharacters: number[]
nonWhitespaceCharactersNoComments: number[]
/** like library statements during reconstruction */
autoSelected: number[]
dataflowNodes: number[]
tokens: number[]
normalizedTokens: number[]
linesWithAutoSelected: number[]
dataflowNodes: number[]
tokens: number[]
tokensNoComments: number[]
normalizedTokens: number[]
normalizedTokensNoComments: number[]
}

/**
Expand Down Expand Up @@ -44,6 +49,8 @@ export interface SummarizedPerSliceStats {
sliceCriteriaSizes: SummarizedMeasurement
measurements: Map<PerSliceMeasurements, SummarizedMeasurement>
reduction: Reduction<SummarizedMeasurement>
/** reduction, but without taking into account comments and empty lines */
reductionNoFluff: Reduction<SummarizedMeasurement>
failedToRepParse: number
timesHitThreshold: number
sliceSize: {
Expand All @@ -61,6 +68,8 @@ export interface UltimateSlicerStats {
/** sum */
timesHitThreshold: number
reduction: Reduction<SummarizedMeasurement>
/** reduction, but without taking into account comments and empty lines */
reductionNoFluff: Reduction<SummarizedMeasurement>
input: SlicerStatsInput<SummarizedMeasurement>
dataflow: SlicerStatsDataflow<SummarizedMeasurement>
}
Loading

2 comments on commit 1f92d6e

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"artificial" Benchmark Suite

Benchmark suite Current: 1f92d6e Previous: eddce74 Ratio
Retrieve AST from R code 234.33753595454547 ms 282.3815673181818 ms 0.83
Normalize R AST 30.930480181818183 ms 36.45953090909091 ms 0.85
Produce dataflow information 57.28076859090909 ms 66.17186054545455 ms 0.87
Total per-file 1265.8251616818181 ms 1557.712024590909 ms 0.81
Static slicing 1.2414823242585065 ms (1.0322022134680742) 1.4160048835200516 ms (1.1715105189277293) 0.88
Reconstruct code 0.4034536519410049 ms (0.2174657389084703) 0.48975330853571625 ms (0.2799126163528146) 0.82
Total per-slice 1.6634474118556402 ms (1.0913290972899217) 1.9280102618968917 ms (1.2491682419110006) 0.86
failed to reconstruct/re-parse 0 # 0 # 1
times hit threshold 0 # 0 # 1
reduction (characters) 0.797431685913541 # 0.797431685913541 # 1
reduction (normalized tokens) 0.7740577588998524 # 0.7740577588998524 # 1

This comment was automatically generated by workflow using github-action-benchmark.

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"social-science" Benchmark Suite

Benchmark suite Current: 1f92d6e Previous: eddce74 Ratio
Retrieve AST from R code 243.97464256 ms 236.60308994 ms 1.03
Normalize R AST 32.8926248 ms 31.77161632 ms 1.04
Produce dataflow information 83.85733323999999 ms 81.86384715999999 ms 1.02
Total per-file 2729.426583 ms 2674.55292236 ms 1.02
Static slicing 5.512771831024629 ms (10.289585776091592) 5.448939336693711 ms (10.154387662460568) 1.01
Reconstruct code 0.39278268347501233 ms (0.2167305926294885) 0.35616079097800935 ms (0.1827576285958724) 1.10
Total per-slice 5.914859928431791 ms (10.365845573992187) 5.814113487370482 ms (10.211794878365124) 1.02
failed to reconstruct/re-parse 2 # 2 # 1
times hit threshold 0 # 0 # 1
reduction (characters) 0.9241844105867956 # 0.9241844105867956 # 1
reduction (normalized tokens) 0.8924953600737399 # 0.8924953600737399 # 1

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.