Skip to content

Commit

Permalink
feat: add processinfo index, add externalId (#1055)
Browse files Browse the repository at this point in the history
If a NYC_PROCESSINFO_EXTERNAL_ID environment variable is set, then it is
saved in the processinfo as `externalId`.

Furthermore, when this file is generated, some additional helpful
metadata is memoized to the processinfo json files, to minimize the cost
of repeated generation.  (This isn't necessarily a breaking change, but
it is an update to the de facto schema for those files.)

As soon as possible, index generation and process tree display should be
migrated out to a new 'istanbul-lib-processinfo' library.

This opens the door to add features in the v14 release family to improve
support for partial/resumed test runs and file watching.

- When a process is run with --clean=false and a previously seen
  externalId, clear away all the coverage files in the set for that
  externalId.
- When a file is changed, a test runner can use the index to determine
  which tests (by externalId) ought to be re-run.
- Adds a NYC_PROCESS_ID to environment
- Adds `parent` to processInfo object, a uuid referring to parent.
- Rebase onto processinfo-numeric-pids branch
- Avoid re-writing the processinfo/{uuid}.json files
- Update process tree output to rely on process index instead of
  duplicating effort.

BREAKING CHANGE: This adds a file named 'index.json' to the
.nyc_output/processinfo directory, which has a different format from the
other files in this dir.
  • Loading branch information
isaacs authored and coreyfarrell committed Apr 6, 2019
1 parent 32f75b0 commit 8dcf180
Show file tree
Hide file tree
Showing 5 changed files with 214 additions and 34 deletions.
4 changes: 4 additions & 0 deletions bin/nyc.js
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ if ([
), function (done) {
var mainChildExitCode = process.exitCode

if (argv.showProcessTree || argv.buildProcessTree) {
nyc.writeProcessIndex()
}

if (argv.checkCoverage) {
nyc.checkCoverage({
lines: argv.lines,
Expand Down
5 changes: 5 additions & 0 deletions bin/wrap.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,13 @@ config.isChildProcess = true
config._processInfo = {
pid: process.pid,
ppid: process.ppid,
parent: process.env.NYC_PROCESS_ID || null,
root: process.env.NYC_ROOT_ID
}
if (process.env.NYC_PROCESSINFO_EXTERNAL_ID) {
config._processInfo.externalId = process.env.NYC_PROCESSINFO_EXTERNAL_ID
delete process.env.NYC_PROCESSINFO_EXTERNAL_ID
}

;(new NYC(config)).wrap()

Expand Down
102 changes: 92 additions & 10 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,7 @@ NYC.prototype._wrapExit = function () {
}

NYC.prototype.wrap = function (bin) {
process.env.NYC_PROCESS_ID = this.processInfo.uuid
this._addRequireHooks()
this._wrapExit()
this._loadAdditionalModules()
Expand Down Expand Up @@ -341,7 +342,7 @@ NYC.prototype.writeCoverageFile = function () {
coverage = this.sourceMaps.remapCoverage(coverage)
}

var id = this.generateUniqueID()
var id = this.processInfo.uuid
var coverageFilename = path.resolve(this.tempDirectory(), id + '.json')

fs.writeFileSync(
Expand All @@ -355,6 +356,7 @@ NYC.prototype.writeCoverageFile = function () {
}

this.processInfo.coverageFilename = coverageFilename
this.processInfo.files = Object.keys(coverage)

fs.writeFileSync(
path.resolve(this.processInfoDirectory(), id + '.json'),
Expand Down Expand Up @@ -412,6 +414,80 @@ NYC.prototype.report = function () {
}
}

// XXX(@isaacs) Index generation should move to istanbul-lib-processinfo
NYC.prototype.writeProcessIndex = function () {
const dir = this.processInfoDirectory()
const pidToUid = new Map()
const infoByUid = new Map()
const eidToUid = new Map()
const infos = fs.readdirSync(dir).filter(f => f !== 'index.json').map(f => {
try {
const info = JSON.parse(fs.readFileSync(path.resolve(dir, f), 'utf-8'))
info.children = []
pidToUid.set(info.uuid, info.pid)
pidToUid.set(info.pid, info.uuid)
infoByUid.set(info.uuid, info)
if (info.externalId) {
eidToUid.set(info.externalId, info.uuid)
}
return info
} catch (er) {
return null
}
}).filter(Boolean)

// create all the parent-child links and write back the updated info
infos.forEach(info => {
if (info.parent) {
const parentInfo = infoByUid.get(info.parent)
if (parentInfo.children.indexOf(info.uuid) === -1) {
parentInfo.children.push(info.uuid)
}
}
})

// figure out which files were touched by each process.
const files = infos.reduce((files, info) => {
info.files.forEach(f => {
files[f] = files[f] || []
files[f].push(info.uuid)
})
return files
}, {})

// build the actual index!
const index = infos.reduce((index, info) => {
index.processes[info.uuid] = {}
index.processes[info.uuid].parent = info.parent
if (info.externalId) {
if (index.externalIds[info.externalId]) {
throw new Error(`External ID ${info.externalId} used by multiple processes`)
}
index.processes[info.uuid].externalId = info.externalId
index.externalIds[info.externalId] = {
root: info.uuid,
children: info.children
}
}
index.processes[info.uuid].children = Array.from(info.children)
return index
}, { processes: {}, files: files, externalIds: {} })

// flatten the descendant sets of all the externalId procs
Object.keys(index.externalIds).forEach(eid => {
const { children } = index.externalIds[eid]
// push the next generation onto the list so we accumulate them all
for (let i = 0; i < children.length; i++) {
const nextGen = index.processes[children[i]].children
if (nextGen && nextGen.length) {
children.push(...nextGen.filter(uuid => children.indexOf(uuid) === -1))
}
}
})

fs.writeFileSync(path.resolve(dir, 'index.json'), JSON.stringify(index))
}

NYC.prototype.showProcessTree = function () {
var processTree = ProcessInfo.buildProcessTree(this._loadProcessInfos())

Expand Down Expand Up @@ -448,19 +524,25 @@ NYC.prototype._checkCoverage = function (summary, thresholds, file) {
}

NYC.prototype._loadProcessInfos = function () {
var _this = this
var files = fs.readdirSync(this.processInfoDirectory())

return files.map(function (f) {
return fs.readdirSync(this.processInfoDirectory()).map(f => {
let data
try {
return new ProcessInfo(JSON.parse(fs.readFileSync(
path.resolve(_this.processInfoDirectory(), f),
data = JSON.parse(fs.readFileSync(
path.resolve(this.processInfoDirectory(), f),
'utf-8'
)))
))
} catch (e) { // handle corrupt JSON output.
return {}
return null
}
})
if (f !== 'index.json') {
data.nodes = []
data = new ProcessInfo(data)
}
return { file: path.basename(f, '.json'), data: data }
}).filter(Boolean).reduce((infos, info) => {
infos[info.file] = info.data
return infos
}, {})
}

NYC.prototype.eachReport = function (filenames, iterator, baseDirectory) {
Expand Down
42 changes: 18 additions & 24 deletions lib/process.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
const archy = require('archy')
const libCoverage = require('istanbul-lib-coverage')
const uuid = require('uuid/v4')

function ProcessInfo (defaults) {
defaults = defaults || {}

this.uuid = null
this.parent = null
this.pid = String(process.pid)
this.argv = process.argv
this.execArgv = process.execArgv
Expand All @@ -12,13 +15,14 @@ function ProcessInfo (defaults) {
this.ppid = null
this.root = null
this.coverageFilename = null
this.nodes = [] // list of children, filled by buildProcessTree()

this._coverageMap = null

for (var key in defaults) {
this[key] = defaults[key]
}

if (!this.uuid) {
this.uuid = uuid()
}
}

Object.defineProperty(ProcessInfo.prototype, 'label', {
Expand All @@ -36,29 +40,19 @@ Object.defineProperty(ProcessInfo.prototype, 'label', {
})

ProcessInfo.buildProcessTree = function (infos) {
var treeRoot = new ProcessInfo({ _label: 'nyc' })
var nodes = { }

infos = infos.sort(function (a, b) {
return a.time - b.time
})

infos.forEach(function (p) {
nodes[p.root + ':' + p.pid] = p
})

infos.forEach(function (p) {
if (!p.ppid) {
return
const treeRoot = new ProcessInfo({ _label: 'nyc', nodes: [] })
const index = infos.index
for (const id in index.processes) {
const node = infos[id]
if (!node) {
throw new Error(`Invalid entry in processinfo index: ${id}`)
}

var parent = nodes[p.root + ':' + p.ppid]
if (!parent) {
parent = treeRoot
const idx = index.processes[id]
node.nodes = idx.children.map(id => infos[id]).sort((a, b) => a.time - b.time)
if (!node.parent) {
treeRoot.nodes.push(node)
}

parent.nodes.push(p)
})
}

return treeRoot
}
Expand Down
95 changes: 95 additions & 0 deletions test/processinfo.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
const { resolve } = require('path')
const bin = resolve(__dirname, '../self-coverage/bin/nyc')
const { spawn } = require('child_process')
const t = require('tap')
const rimraf = require('rimraf')
const node = process.execPath
const fixturesCLI = resolve(__dirname, './fixtures/cli')
const tmp = 'processinfo-test'
const fs = require('fs')
const resolvedJS = resolve(fixturesCLI, 'selfspawn-fibonacci.js')

rimraf.sync(resolve(fixturesCLI, tmp))
t.teardown(() => rimraf.sync(resolve(fixturesCLI, tmp)))

t.test('build some processinfo', t => {
var args = [
bin, '-t', tmp, '--build-process-tree',
node, 'selfspawn-fibonacci.js', '5'
]
var proc = spawn(process.execPath, args, {
cwd: fixturesCLI,
env: {
PATH: process.env.PATH,
NYC_PROCESSINFO_EXTERNAL_ID: 'blorp'
}
})
// don't actually care about the output for this test, just the data
proc.stderr.resume()
proc.stdout.resume()
proc.on('close', (code, signal) => {
t.equal(code, 0)
t.equal(signal, null)
t.end()
})
})

t.test('validate the created processinfo data', t => {
const covs = fs.readdirSync(resolve(fixturesCLI, tmp))
.filter(f => f !== 'processinfo')
t.plan(covs.length * 2)

covs.forEach(f => {
fs.readFile(resolve(fixturesCLI, tmp, f), 'utf8', (er, covjson) => {
if (er) {
throw er
}
const covdata = JSON.parse(covjson)
t.same(Object.keys(covdata), [resolvedJS])
// should have matching processinfo for each cov json
const procInfoFile = resolve(fixturesCLI, tmp, 'processinfo', f)
fs.readFile(procInfoFile, 'utf8', (er, procInfoJson) => {
if (er) {
throw er
}
const procInfoData = JSON.parse(procInfoJson)
t.match(procInfoData, {
pid: Number,
ppid: Number,
uuid: f.replace(/\.json$/, ''),
argv: [
node,
resolvedJS,
/[1-5]/
],
execArgv: [],
cwd: fixturesCLI,
time: Number,
root: /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/,
coverageFilename: resolve(fixturesCLI, tmp, f),
files: [ resolvedJS ]
})
})
})
})
})

t.test('check out the index', t => {
const indexFile = resolve(fixturesCLI, tmp, 'processinfo', 'index.json')
const indexJson = fs.readFileSync(indexFile, 'utf-8')
const index = JSON.parse(indexJson)
const u = /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/
t.match(index, {
processes: {},
files: {
[resolvedJS]: [ u, u, u, u, u, u, u, u, u ]
},
externalIds: {
blorp: {
root: u,
children: [ u, u, u, u, u, u, u, u ]
}
}
})
t.end()
})

0 comments on commit 8dcf180

Please sign in to comment.