From 42522f3e8cc91fcb0d9d29d5d91a2f95ac3b858a Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 12 Oct 2018 16:03:40 +0200 Subject: [PATCH 01/21] import default.nix, mkcache.js from serokell-closure:yorickvp-review --- default.nix | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++ mkcache.js | 38 +++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 default.nix create mode 100644 mkcache.js diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..0291e38 --- /dev/null +++ b/default.nix @@ -0,0 +1,74 @@ +{ stdenvNoCC, writeShellScriptBin, writeText, stdenv, fetchurl, makeWrapper, nodejs-10_x }: +with stdenv.lib; +let + inherit (builtins) fromJSON toJSON split; + + depsToFetches = deps: concatMap depToFetch (attrValues deps); + depFetchOwn = { resolved, integrity, ... }: let + ssri = split "-" integrity; # standard subresource integrity + hashType = head ssri; + hash = elemAt ssri 2; + in nameValuePair resolved (fetchurl { + url = resolved; + "${hashType}" = hash; + }); + + depToFetch = args @ { resolved ? null, dependencies ? {}, ... }: + (optional (resolved != null) (depFetchOwn args)) ++ (depsToFetches dependencies); + + npmCacheInput = lock: writeText "npm-cache-input.json" (toJSON (listToAttrs (depToFetch lock))); + + patchShebangs = writeShellScriptBin "patchShebangs.sh" '' + set -e + source ${stdenvNoCC}/setup + patchShebangs "$@" + ''; + + shellWrap = writeShellScriptBin "npm-shell-wrap.sh" '' + set -e + if [ ! -e .shebangs_patched ]; then + ${patchShebangs}/bin/patchShebangs.sh . + touch .shebangs_patched + fi + exec bash "$@" + ''; +in +args @ { lockfile, src, buildInputs ? [], npmFlags ? [], ... }: +let + lock = fromJSON (readFile lockfile); +in +stdenv.mkDerivation ({ + inherit (lock) version; + name = "${lock.name}-${lock.version}"; + + XDG_CONFIG_DIRS = "."; + NO_UPDATE_NOTIFIER = true; + preBuildPhases = [ "npmCachePhase" ]; + preInstallPhases = [ "npmPackPhase" ]; + installJavascript = true; + npmCachePhase = '' + node ${./mkcache.js} ${npmCacheInput lock} + ''; + buildPhase = '' + runHook preBuild + npm ci $npmFlags + runHook postBuild + ''; + # make a package .tgz (no way around it) + npmPackPhase = '' + npm prune --production $npmFlags + npm pack --ignore-scripts $npmFlags + ''; + # unpack the .tgz into output directory and add npm wrapper + installPhase = '' + mkdir -p $out/bin + tar xzvf ./${lock.name}-${lock.version}.tgz -C $out --strip-components=1 + if [ "$installJavascript" -eq "1" ]; then + cp -R node_modules $out/ + makeWrapper ${nodejs-10_x}/bin/npm $out/bin/npm --run "cd $out" + fi + ''; +} // args // { + buildInputs = [ nodejs-10_x makeWrapper ] ++ buildInputs; + npmFlags = [ "--cache=./npm-cache" "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ] ++ npmFlags; + }) diff --git a/mkcache.js b/mkcache.js new file mode 100644 index 0000000..36ff6b5 --- /dev/null +++ b/mkcache.js @@ -0,0 +1,38 @@ +const assert = require('assert') +const fs = require('fs') +const path = require('path') +const {promisify} = require('util') + +// find pacote from npm dependencies +module.paths.push(path.join(process.argv[0], "../../lib/node_modules/npm/node_modules")) +const pacote = require('pacote') + +function traverseDeps(pkg, fn) { + Object.values(pkg.dependencies).forEach(dep => { + if (dep.resolved && dep.integrity) fn(dep) + if (dep.dependencies) traverseDeps(dep, fn) + }) +} + +async function main(lockfile, nix, cache) { + var promises = Object.keys(nix).map(async function(url) { + var tar = nix[url] + const manifest = await pacote.manifest(tar, {offline: true, cache}) + return [url, manifest._integrity] + }) + var hashes = new Map(await Promise.all(promises)) + traverseDeps(lockfile, dep => { + if (dep.integrity.startsWith("sha1-")) { + assert(hashes.has(dep.resolved)) + dep.integrity = hashes.get(dep.resolved) + } + else { + assert(dep.integrity == hashes.get(dep.resolved)) + } + }) + await promisify(fs.writeFile)("./package-lock.json", JSON.stringify(lock, null, 4)) +} +const lock = JSON.parse(fs.readFileSync('./package-lock.json', 'utf8')) +const nix_pkgs = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')) +main(lock, nix_pkgs, './npm-cache/_cacache') + From 2205e62633a5bcac55f0b66caa09883a1c6b25cd Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 12 Oct 2018 16:27:26 +0200 Subject: [PATCH 02/21] looks OK * improve formatting, DRY, add TODOs * refactor * + mklock.js * use runCommand to create package-lock.json from yarn.lock * allow adding integreties * better variable names * inherit removeAttrs * fixes --- default.nix | 125 ++++++++++++++++++++++++++++++++-------------------- mkcache.js | 63 +++++++++++++------------- mklock.js | 66 +++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 80 deletions(-) create mode 100644 mklock.js diff --git a/default.nix b/default.nix index 0291e38..3e9f5ff 100644 --- a/default.nix +++ b/default.nix @@ -1,15 +1,17 @@ -{ stdenvNoCC, writeShellScriptBin, writeText, stdenv, fetchurl, makeWrapper, nodejs-10_x }: -with stdenv.lib; -let - inherit (builtins) fromJSON toJSON split; +{ stdenvNoCC, writeShellScriptBin, writeText, runCommand, + stdenv, fetchurl, makeWrapper, nodejs-10_x, yarn2nix }: +with stdenv.lib; let + inherit (builtins) fromJSON toJSON split removeAttrs; depsToFetches = deps: concatMap depToFetch (attrValues deps); - depFetchOwn = { resolved, integrity, ... }: let - ssri = split "-" integrity; # standard subresource integrity - hashType = head ssri; - hash = elemAt ssri 2; - in nameValuePair resolved (fetchurl { - url = resolved; + + depFetchOwn = { resolved, integrity, ... }: + let + ssri = split "-" integrity; # standard subresource integrity + hashType = head ssri; + hash = elemAt ssri 2; + in nameValuePair resolved (fetchurl { + url = resolved; "${hashType}" = hash; }); @@ -33,42 +35,69 @@ let exec bash "$@" ''; in -args @ { lockfile, src, buildInputs ? [], npmFlags ? [], ... }: -let - lock = fromJSON (readFile lockfile); -in -stdenv.mkDerivation ({ - inherit (lock) version; - name = "${lock.name}-${lock.version}"; + args @ { src, useYarn ? false, yarnIntegreties ? {}, + npmBuild ? "npm ci", npmBuildMore ? "", + buildInputs ? [], npmFlags ? [], ... }: + let + packageJson = src + "/package.json"; + packageLockJson = src + "/package-lock.json"; + yarnLock = src + "/yarn.lock"; + yarnIntFile = writeText "integreties.json" (toJSON yarnIntegreties); + lockfile = if useYarn then mkLockFileFromYarn else packageLockJson; + info = fromJSON (readFile packageJson); + lock = fromJSON (readFile lockfile); + name = "${info.name}-${info.version}"; + npm = "${nodejs-10_x}/bin/npm"; + npmAlias = ''npm() { ${npm} "$@" $npmFlags; }''; - XDG_CONFIG_DIRS = "."; - NO_UPDATE_NOTIFIER = true; - preBuildPhases = [ "npmCachePhase" ]; - preInstallPhases = [ "npmPackPhase" ]; - installJavascript = true; - npmCachePhase = '' - node ${./mkcache.js} ${npmCacheInput lock} - ''; - buildPhase = '' - runHook preBuild - npm ci $npmFlags - runHook postBuild - ''; - # make a package .tgz (no way around it) - npmPackPhase = '' - npm prune --production $npmFlags - npm pack --ignore-scripts $npmFlags - ''; - # unpack the .tgz into output directory and add npm wrapper - installPhase = '' - mkdir -p $out/bin - tar xzvf ./${lock.name}-${lock.version}.tgz -C $out --strip-components=1 - if [ "$installJavascript" -eq "1" ]; then - cp -R node_modules $out/ - makeWrapper ${nodejs-10_x}/bin/npm $out/bin/npm --run "cd $out" - fi - ''; -} // args // { - buildInputs = [ nodejs-10_x makeWrapper ] ++ buildInputs; - npmFlags = [ "--cache=./npm-cache" "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ] ++ npmFlags; - }) + mkLockFileFromYarn = runCommand "yarn-package-lock.json" {} '' + addToSearchPath NODE_PATH ${nodejs-10_x}/lib/node_modules/npm/node_modules + addToSearchPath NODE_PATH ${yarn2nix.node_modules} + ${nodejs-10_x}/bin/node ${./mklock.js} $out ${packageJson} ${yarnLock} ${yarnIntFile} + ''; + in stdenv.mkDerivation ({ + inherit name src; + inherit (info) version; + + XDG_CONFIG_DIRS = "."; + NO_UPDATE_NOTIFIER = true; + preBuildPhases = [ "npmCachePhase" ]; + preInstallPhases = [ "npmPackPhase" ]; + installJavascript = true; + + npmCachePhase = '' + if [ "$useYarn" -eq "1" ]; then + cp ${lockfile} ${builtins.baseNameOf packageLockJson} + fi + addToSearchPath NODE_PATH ${nodejs-10_x}/lib/node_modules/npm/node_modules + node ${./mkcache.js} ${npmCacheInput lock} + ''; + + buildPhase = '' + ${npmAlias} + runHook preBuild + ${npmBuild} + ${npmBuildMore} + runHook postBuild + ''; + + # make a package .tgz (no way around it) + npmPackPhase = '' + ${npmAlias} + npm prune --production + npm pack --ignore-scripts + ''; + + # unpack the .tgz into output directory and add npm wrapper + installPhase = '' + mkdir -p $out/bin + tar xzvf ./${name}.tgz -C $out --strip-components=1 + if [ "$installJavascript" -eq "1" ]; then + cp -R node_modules $out/ + makeWrapper ${npm} $out/bin/npm --run "cd $out" + fi + ''; + } // removeAttrs args [ "yarnIntegreties" ] // { + buildInputs = [ nodejs-10_x makeWrapper ] ++ buildInputs; + npmFlags = [ "--cache=./npm-cache" "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ] ++ npmFlags; + }) diff --git a/mkcache.js b/mkcache.js index 36ff6b5..97e8ee8 100644 --- a/mkcache.js +++ b/mkcache.js @@ -1,38 +1,37 @@ -const assert = require('assert') -const fs = require('fs') -const path = require('path') -const {promisify} = require('util') - -// find pacote from npm dependencies -module.paths.push(path.join(process.argv[0], "../../lib/node_modules/npm/node_modules")) -const pacote = require('pacote') +const assert = require('assert') +const fs = require('fs') +const path = require('path') +const pacote = require('pacote') function traverseDeps(pkg, fn) { - Object.values(pkg.dependencies).forEach(dep => { - if (dep.resolved && dep.integrity) fn(dep) - if (dep.dependencies) traverseDeps(dep, fn) - }) + Object.values(pkg.dependencies).forEach(dep => { + if (dep.resolved && dep.integrity) fn(dep) + if (dep.dependencies) traverseDeps(dep, fn) + }) } -async function main(lockfile, nix, cache) { - var promises = Object.keys(nix).map(async function(url) { - var tar = nix[url] - const manifest = await pacote.manifest(tar, {offline: true, cache}) - return [url, manifest._integrity] - }) - var hashes = new Map(await Promise.all(promises)) - traverseDeps(lockfile, dep => { - if (dep.integrity.startsWith("sha1-")) { - assert(hashes.has(dep.resolved)) - dep.integrity = hashes.get(dep.resolved) - } - else { - assert(dep.integrity == hashes.get(dep.resolved)) - } - }) - await promisify(fs.writeFile)("./package-lock.json", JSON.stringify(lock, null, 4)) +function main(lockfile, nix, cache) { + let hashes = new Map(Object.keys(nix).map(url => { + let tar = nix[url] + let manifest = pacote.manifest(tar, {offline: true, cache}) + return [url, manifest._integrity] + })) + traverseDeps(lockfile, dep => { + if (dep.integrity.startsWith("sha1-")) { + assert(hashes.has(dep.resolved)) + dep.integrity = hashes.get(dep.resolved) + } else { + assert(dep.integrity == hashes.get(dep.resolved)) + } + }) + // TODO: why? + // fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 4)) } -const lock = JSON.parse(fs.readFileSync('./package-lock.json', 'utf8')) -const nix_pkgs = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')) -main(lock, nix_pkgs, './npm-cache/_cacache') +let nixPkgsFile = process.argv[2] + +const pkgLockFile = "./package-lock.json" +const lock = JSON.parse(fs.readFileSync(pkgLockFile, 'utf8')) +const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, 'utf8')) + +main(lock, nixPkgs, './npm-cache/_cacache') diff --git a/mklock.js b/mklock.js new file mode 100644 index 0000000..5f60f12 --- /dev/null +++ b/mklock.js @@ -0,0 +1,66 @@ +const assert = require('assert') +const fs = require('fs') +const lockfile = require("@yarnpkg/lockfile") +const semver = require('semver') + +let pkgLockFile = process.argv[2] +let pkgJsonFile = process.argv[3] +let yarnLockFile = process.argv[4] +let intFile = process.argv[5] + +let pkgData = fs.readFileSync(pkgJsonFile, 'utf8') +let yarnData = fs.readFileSync(yarnLockFile, 'utf8') +let pkgJson = JSON.parse(pkgData) +let yarnJson = lockfile.parse(yarnData).object +let integrities = intFile ? JSON.parse(fs.readFileSync(intFile)) : {} + +let pkgDeps = { ...(pkgJson.devDependencies || {}), + ...(pkgJson.dependencies || {}) } + +function splitNameVsn (key) { + if (key[0] == "@") { + let [name, vsn] = key.slice(1).split("@") + return ["@"+name, vsn] + } else { + return key.split("@") + } +} + +function addDeps(obj, pkg, vsn, seen) { + if (seen[pkg + "@" + vsn]) return // loop + seen[pkg + "@" + vsn] = true + let pkgdeps = deps[pkg][vsn]._dependencies || {} + obj.dependencies = {} + Object.keys(pkgdeps).forEach(key => { + let depvsn = pkgdeps[key] + obj.dependencies[key] = {...deps[key][depvsn]} + addDeps(obj.dependencies[key], key, depvsn, {...seen}) + }) +} + +let deps = {} +Object.keys(yarnJson).forEach(key => { + let dep = yarnJson[key] + let [name, vsn] = splitNameVsn(key) + let [url, sha1] = dep.resolved.split("#", 2) + if (!sha1 && integrities[url]) sha1 = integrities[url] + assert(sha1, "missing sha1 for " + JSON.stringify(dep)) // TODO + if (!deps[name]) deps[name] = {} + deps[name][vsn] = { version: dep.version, resolved: url, + integrity: "sha1-" + sha1, + _dependencies: dep.dependencies } +}) + +let depsTree = {} +Object.keys(pkgDeps).forEach(key => { + let vsn = pkgDeps[key] + depsTree[key] = {...deps[key][vsn]} + addDeps(depsTree[key], key, vsn, {}) +}) + +// NB: dependencies flattened by yarn; should work! +let lock = { name: pkgJson.name, version: pkgJson.version, + lockfileVersion: 1, requires: true, + dependencies: depsTree } + +fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 2)) From 8b8c0f9250d224dcd2f5ec89be9eb5c2c3f92702 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Tue, 16 Oct 2018 19:45:32 +0200 Subject: [PATCH 03/21] README, cleanup, review, some fixes --- README.md | 33 +++++++++++++++++++++++++++++++-- default.nix | 36 +++++++++++++++++++++++------------- mkcache.js | 35 +++++++++++++++++++---------------- mklock.js | 31 +++++++++++++++++-------------- 4 files changed, 90 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 9b2b084..e7093dd 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,31 @@ -# nix-npm-buildpackage -Build nix packages that use npm/yarn packages. +## Description + +nix-npm-buildpackage - build nix packages that use npm/yarn packages + +You can use `buildNpmPackage` to: +* convert a `yarn.lock` file to a `packages-lock.json` file if needed +* use a `packages-lock.json` file to: + - download the dependencies to the nix store + - build an offline npm cache that uses those +* build a nix package from the npm package + +## Examples + +```nix +{ pkgs ? import {} }: +let + buildNpmPackage = pkgs.callPackage .../nix-npm-buildpackage {}; + integreties = { # in case the yarn.lock file is missing sha1 hashes + "https://codeload.github.com/foo/bar.js/tar.gz/..." = "sha512-..."; + }; +in buildNpmPackage { + src = ./.; + useYarnLock = true; + yarnIntegreties = integreties; +} +``` + +## TODO: + +* use symlinks to avoid duplication? +* make npm `bin` and `scripts` work? diff --git a/default.nix b/default.nix index 3e9f5ff..0ab0c9c 100644 --- a/default.nix +++ b/default.nix @@ -35,28 +35,33 @@ with stdenv.lib; let exec bash "$@" ''; in - args @ { src, useYarn ? false, yarnIntegreties ? {}, + args @ { src, useYarnLock ? false, yarnIntegreties ? {}, npmBuild ? "npm ci", npmBuildMore ? "", buildInputs ? [], npmFlags ? [], ... }: let - packageJson = src + "/package.json"; - packageLockJson = src + "/package-lock.json"; + pkgJson = src + "/package.json"; + pkgLockJson = src + "/package-lock.json"; yarnLock = src + "/yarn.lock"; + yarnIntFile = writeText "integreties.json" (toJSON yarnIntegreties); - lockfile = if useYarn then mkLockFileFromYarn else packageLockJson; - info = fromJSON (readFile packageJson); + lockfile = if useYarnLock then mkLockFileFromYarn else pkgLockJson; + + info = fromJSON (readFile pkgJson); lock = fromJSON (readFile lockfile); + name = "${info.name}-${info.version}"; npm = "${nodejs-10_x}/bin/npm"; npmAlias = ''npm() { ${npm} "$@" $npmFlags; }''; + npmModules = "${nodejs-10_x}/lib/node_modules/npm/node_modules"; mkLockFileFromYarn = runCommand "yarn-package-lock.json" {} '' - addToSearchPath NODE_PATH ${nodejs-10_x}/lib/node_modules/npm/node_modules + set -e + addToSearchPath NODE_PATH ${npmModules} addToSearchPath NODE_PATH ${yarn2nix.node_modules} - ${nodejs-10_x}/bin/node ${./mklock.js} $out ${packageJson} ${yarnLock} ${yarnIntFile} + ${nodejs-10_x}/bin/node ${./mklock.js} $out ${pkgJson} ${yarnLock} ${yarnIntFile} ''; in stdenv.mkDerivation ({ - inherit name src; + inherit name; inherit (info) version; XDG_CONFIG_DIRS = "."; @@ -66,14 +71,17 @@ in installJavascript = true; npmCachePhase = '' - if [ "$useYarn" -eq "1" ]; then - cp ${lockfile} ${builtins.baseNameOf packageLockJson} + set -e + if [ "$useYarnLock" = "1" ]; then + cp ${lockfile} ${builtins.baseNameOf pkgLockJson} + chmod u+w ${builtins.baseNameOf pkgLockJson} fi - addToSearchPath NODE_PATH ${nodejs-10_x}/lib/node_modules/npm/node_modules + addToSearchPath NODE_PATH ${npmModules} node ${./mkcache.js} ${npmCacheInput lock} ''; buildPhase = '' + set -e ${npmAlias} runHook preBuild ${npmBuild} @@ -83,6 +91,7 @@ in # make a package .tgz (no way around it) npmPackPhase = '' + set -e ${npmAlias} npm prune --production npm pack --ignore-scripts @@ -90,14 +99,15 @@ in # unpack the .tgz into output directory and add npm wrapper installPhase = '' + set -e mkdir -p $out/bin tar xzvf ./${name}.tgz -C $out --strip-components=1 - if [ "$installJavascript" -eq "1" ]; then + if [ "$installJavascript" = "1" ]; then cp -R node_modules $out/ makeWrapper ${npm} $out/bin/npm --run "cd $out" fi ''; } // removeAttrs args [ "yarnIntegreties" ] // { buildInputs = [ nodejs-10_x makeWrapper ] ++ buildInputs; - npmFlags = [ "--cache=./npm-cache" "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ] ++ npmFlags; + npmFlags = [ "--cache=./npm-cache" "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ] ++ npmFlags; }) diff --git a/mkcache.js b/mkcache.js index 97e8ee8..ab32b5d 100644 --- a/mkcache.js +++ b/mkcache.js @@ -1,7 +1,11 @@ -const assert = require('assert') -const fs = require('fs') -const path = require('path') -const pacote = require('pacote') +const assert = require("assert") +const fs = require("fs") +const pacote = require("pacote") +const path = require("path") + +// Usage: node mkcache.js npm-cache-input.json + +let nixPkgsFile = process.argv[2] function traverseDeps(pkg, fn) { Object.values(pkg.dependencies).forEach(dep => { @@ -10,12 +14,13 @@ function traverseDeps(pkg, fn) { }) } -function main(lockfile, nix, cache) { - let hashes = new Map(Object.keys(nix).map(url => { - let tar = nix[url] - let manifest = pacote.manifest(tar, {offline: true, cache}) +async function main(lockfile, nix, cache) { + let promises = Object.keys(nix).map(async function (url) { + let tar = nix[url] + let manifest = await pacote.manifest(tar, { offline: true, cache }) return [url, manifest._integrity] - })) + }) + let hashes = new Map(await Promise.all(promises)) traverseDeps(lockfile, dep => { if (dep.integrity.startsWith("sha1-")) { assert(hashes.has(dep.resolved)) @@ -24,14 +29,12 @@ function main(lockfile, nix, cache) { assert(dep.integrity == hashes.get(dep.resolved)) } }) - // TODO: why? - // fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 4)) + // rewrite lock file to use sha512 hashes from pacote + fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 2)) } -let nixPkgsFile = process.argv[2] - const pkgLockFile = "./package-lock.json" -const lock = JSON.parse(fs.readFileSync(pkgLockFile, 'utf8')) -const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, 'utf8')) +const lock = JSON.parse(fs.readFileSync(pkgLockFile, "utf8")) +const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) -main(lock, nixPkgs, './npm-cache/_cacache') +main(lock, nixPkgs, "./npm-cache/_cacache") diff --git a/mklock.js b/mklock.js index 5f60f12..55a4ed9 100644 --- a/mklock.js +++ b/mklock.js @@ -1,23 +1,26 @@ -const assert = require('assert') -const fs = require('fs') +const assert = require("assert") +const fs = require("fs") const lockfile = require("@yarnpkg/lockfile") -const semver = require('semver') +const semver = require("semver") + +// Usage: node mklock.js package-lock.json package.json yarn.lock integrities.json let pkgLockFile = process.argv[2] -let pkgJsonFile = process.argv[3] +let pkgFile = process.argv[3] let yarnLockFile = process.argv[4] let intFile = process.argv[5] -let pkgData = fs.readFileSync(pkgJsonFile, 'utf8') -let yarnData = fs.readFileSync(yarnLockFile, 'utf8') -let pkgJson = JSON.parse(pkgData) -let yarnJson = lockfile.parse(yarnData).object +let pkgJson = JSON.parse(fs.readFileSync(pkgFile, "utf8")) +let yarnJson = lockfile.parse(fs.readFileSync(yarnLockFile, "utf8")).object let integrities = intFile ? JSON.parse(fs.readFileSync(intFile)) : {} let pkgDeps = { ...(pkgJson.devDependencies || {}), ...(pkgJson.dependencies || {}) } +const hex2base64 = s => Buffer.from(s, "hex").toString("base64") + function splitNameVsn (key) { + // foo@vsn or @foo/bar@vsn if (key[0] == "@") { let [name, vsn] = key.slice(1).split("@") return ["@"+name, vsn] @@ -27,8 +30,9 @@ function splitNameVsn (key) { } function addDeps(obj, pkg, vsn, seen) { - if (seen[pkg + "@" + vsn]) return // loop - seen[pkg + "@" + vsn] = true + let pkgvsn = pkg + "@" + vsn + if (seen[pkgvsn]) return // break cycle + seen[pkgvsn] = true let pkgdeps = deps[pkg][vsn]._dependencies || {} obj.dependencies = {} Object.keys(pkgdeps).forEach(key => { @@ -43,11 +47,11 @@ Object.keys(yarnJson).forEach(key => { let dep = yarnJson[key] let [name, vsn] = splitNameVsn(key) let [url, sha1] = dep.resolved.split("#", 2) - if (!sha1 && integrities[url]) sha1 = integrities[url] - assert(sha1, "missing sha1 for " + JSON.stringify(dep)) // TODO + let integrity = dep.integrity || integrities[url] || (sha1 && "sha1-" + hex2base64(sha1)) + assert(integrity, "missing integrity for " + JSON.stringify(dep)) if (!deps[name]) deps[name] = {} deps[name][vsn] = { version: dep.version, resolved: url, - integrity: "sha1-" + sha1, + integrity: integrity, _dependencies: dep.dependencies } }) @@ -58,7 +62,6 @@ Object.keys(pkgDeps).forEach(key => { addDeps(depsTree[key], key, vsn, {}) }) -// NB: dependencies flattened by yarn; should work! let lock = { name: pkgJson.name, version: pkgJson.version, lockfileVersion: 1, requires: true, dependencies: depsTree } From f26dce3fee93f3fc2b020d229661cb0c7a990c6e Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Tue, 16 Oct 2018 19:47:27 +0200 Subject: [PATCH 04/21] README: small fixes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e7093dd..e443e39 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ You can use `buildNpmPackage` to: { pkgs ? import {} }: let buildNpmPackage = pkgs.callPackage .../nix-npm-buildpackage {}; - integreties = { # in case the yarn.lock file is missing sha1 hashes + integreties = { # in case the yarn.lock file is missing hashes "https://codeload.github.com/foo/bar.js/tar.gz/..." = "sha512-..."; }; in buildNpmPackage { @@ -25,7 +25,7 @@ in buildNpmPackage { } ``` -## TODO: +## TODO * use symlinks to avoid duplication? * make npm `bin` and `scripts` work? From 938136dcdadfb707b1b19645f914785c9e7f33fd Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Tue, 16 Oct 2018 20:36:37 +0200 Subject: [PATCH 05/21] use .tgz suffix for pacote; use ssri; catch unhandledRejection --- default.nix | 3 +++ mkcache.js | 5 +++++ mklock.js | 5 ++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index 0ab0c9c..5fdfc0a 100644 --- a/default.nix +++ b/default.nix @@ -10,9 +10,12 @@ with stdenv.lib; let ssri = split "-" integrity; # standard subresource integrity hashType = head ssri; hash = elemAt ssri 2; + fname = baseNameOf resolved; in nameValuePair resolved (fetchurl { url = resolved; "${hashType}" = hash; + name = if hasSuffix ".tgz" fname || hasSuffix ".tar.gz" fname + then fname else fname + ".tgz"; }); depToFetch = args @ { resolved ? null, dependencies ? {}, ... }: diff --git a/mkcache.js b/mkcache.js index ab32b5d..a7fc1da 100644 --- a/mkcache.js +++ b/mkcache.js @@ -37,4 +37,9 @@ const pkgLockFile = "./package-lock.json" const lock = JSON.parse(fs.readFileSync(pkgLockFile, "utf8")) const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) +process.on("unhandledRejection", error => { + console.log("unhandledRejection", error.message); + process.exit(1) +}); + main(lock, nixPkgs, "./npm-cache/_cacache") diff --git a/mklock.js b/mklock.js index 55a4ed9..360a651 100644 --- a/mklock.js +++ b/mklock.js @@ -2,6 +2,7 @@ const assert = require("assert") const fs = require("fs") const lockfile = require("@yarnpkg/lockfile") const semver = require("semver") +const ssri = require("ssri") // Usage: node mklock.js package-lock.json package.json yarn.lock integrities.json @@ -17,8 +18,6 @@ let integrities = intFile ? JSON.parse(fs.readFileSync(intFile)) : {} let pkgDeps = { ...(pkgJson.devDependencies || {}), ...(pkgJson.dependencies || {}) } -const hex2base64 = s => Buffer.from(s, "hex").toString("base64") - function splitNameVsn (key) { // foo@vsn or @foo/bar@vsn if (key[0] == "@") { @@ -47,7 +46,7 @@ Object.keys(yarnJson).forEach(key => { let dep = yarnJson[key] let [name, vsn] = splitNameVsn(key) let [url, sha1] = dep.resolved.split("#", 2) - let integrity = dep.integrity || integrities[url] || (sha1 && "sha1-" + hex2base64(sha1)) + let integrity = dep.integrity || integrities[url] || ssri.fromHex(sha1, "sha1").toString() assert(integrity, "missing integrity for " + JSON.stringify(dep)) if (!deps[name]) deps[name] = {} deps[name][vsn] = { version: dep.version, resolved: url, From a3891e1734e1ff26550476b226409e3f6d5e8c62 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Tue, 16 Oct 2018 23:39:39 +0200 Subject: [PATCH 06/21] mklock.js: naively merge dependencies --- mklock.js | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/mklock.js b/mklock.js index 360a651..cd42341 100644 --- a/mklock.js +++ b/mklock.js @@ -18,7 +18,13 @@ let integrities = intFile ? JSON.parse(fs.readFileSync(intFile)) : {} let pkgDeps = { ...(pkgJson.devDependencies || {}), ...(pkgJson.dependencies || {}) } -function splitNameVsn (key) { +function objKeys(o) { + let keys = Object.keys(o) + keys.sort() + return keys +} + +function splitNameVsn(key) { // foo@vsn or @foo/bar@vsn if (key[0] == "@") { let [name, vsn] = key.slice(1).split("@") @@ -33,16 +39,26 @@ function addDeps(obj, pkg, vsn, seen) { if (seen[pkgvsn]) return // break cycle seen[pkgvsn] = true let pkgdeps = deps[pkg][vsn]._dependencies || {} + obj.requires = {} obj.dependencies = {} - Object.keys(pkgdeps).forEach(key => { + objKeys(pkgdeps).forEach(key => { let depvsn = pkgdeps[key] - obj.dependencies[key] = {...deps[key][depvsn]} - addDeps(obj.dependencies[key], key, depvsn, {...seen}) + let dep = deps[key][depvsn] + let dep_ = { ...dep, _dependencies: undefined } + obj.requires[key] = depvsn + if (!depsTree[key]) { + depsTree[key] = dep_ + } else if (depsTree[key].version != dep.version) { + obj.dependencies[key] = dep_ + } else { + return + } + addDeps(dep_, key, depvsn, { ...seen }) }) } let deps = {} -Object.keys(yarnJson).forEach(key => { +objKeys(yarnJson).forEach(key => { let dep = yarnJson[key] let [name, vsn] = splitNameVsn(key) let [url, sha1] = dep.resolved.split("#", 2) @@ -55,9 +71,9 @@ Object.keys(yarnJson).forEach(key => { }) let depsTree = {} -Object.keys(pkgDeps).forEach(key => { +objKeys(pkgDeps).forEach(key => { let vsn = pkgDeps[key] - depsTree[key] = {...deps[key][vsn]} + depsTree[key] = { ...deps[key][vsn], _dependencies: undefined } addDeps(depsTree[key], key, vsn, {}) }) From e43fb1d042304c1b6e9524cf126e7b99d45dd15c Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Wed, 17 Oct 2018 02:55:32 +0200 Subject: [PATCH 07/21] mklock.js: small improvements, fix ordering w/ second loop --- mklock.js | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/mklock.js b/mklock.js index cd42341..c5adc79 100644 --- a/mklock.js +++ b/mklock.js @@ -35,34 +35,36 @@ function splitNameVsn(key) { } function addDeps(obj, pkg, vsn, seen) { - let pkgvsn = pkg + "@" + vsn + let pkgvsn = pkg + "@" + vsn if (seen[pkgvsn]) return // break cycle - seen[pkgvsn] = true - let pkgdeps = deps[pkg][vsn]._dependencies || {} - obj.requires = {} - obj.dependencies = {} + seen[pkgvsn] = true + let pkgdeps = deps[pkg][vsn]._dependencies || {} + obj.requires = {} + obj.dependencies = {} objKeys(pkgdeps).forEach(key => { - let depvsn = pkgdeps[key] - let dep = deps[key][depvsn] - let dep_ = { ...dep, _dependencies: undefined } - obj.requires[key] = depvsn - if (!depsTree[key]) { - depsTree[key] = dep_ - } else if (depsTree[key].version != dep.version) { - obj.dependencies[key] = dep_ - } else { - return + let depvsn = pkgdeps[key] + let dep = deps[key][depvsn] + let ass = !depsTree[key] ? depsTree : + depsTree[key].version != dep.version ? + obj.dependencies : null + obj.requires[key] = depvsn + if (ass) { + let dep_ = { ...dep, _dependencies: undefined } + ass[key] = dep_ + addDeps(dep_, key, depvsn, { ...seen }) } - addDeps(dep_, key, depvsn, { ...seen }) }) } -let deps = {} +let deps = {} +let depsTree = {} + objKeys(yarnJson).forEach(key => { let dep = yarnJson[key] let [name, vsn] = splitNameVsn(key) let [url, sha1] = dep.resolved.split("#", 2) - let integrity = dep.integrity || integrities[url] || ssri.fromHex(sha1, "sha1").toString() + let integrity = dep.integrity || integrities[url] || + ssri.fromHex(sha1, "sha1").toString() assert(integrity, "missing integrity for " + JSON.stringify(dep)) if (!deps[name]) deps[name] = {} deps[name][vsn] = { version: dep.version, resolved: url, @@ -70,11 +72,12 @@ objKeys(yarnJson).forEach(key => { _dependencies: dep.dependencies } }) -let depsTree = {} objKeys(pkgDeps).forEach(key => { - let vsn = pkgDeps[key] - depsTree[key] = { ...deps[key][vsn], _dependencies: undefined } - addDeps(depsTree[key], key, vsn, {}) + depsTree[key] = { ...deps[key][pkgDeps[key]], _dependencies: undefined } +}) + +objKeys(pkgDeps).forEach(key => { + addDeps(depsTree[key], key, pkgDeps[key], {}) }) let lock = { name: pkgJson.name, version: pkgJson.version, From 7b322b167dbab8141d45ee25a797178b44382ca5 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 19 Oct 2018 15:36:18 +0200 Subject: [PATCH 08/21] rm redundant set -e; s/let/const/; use argv.slice --- default.nix | 4 --- mkcache.js | 23 ++++++++++------- mklock.js | 74 +++++++++++++++++++++++++++-------------------------- 3 files changed, 52 insertions(+), 49 deletions(-) diff --git a/default.nix b/default.nix index 5fdfc0a..5d0b842 100644 --- a/default.nix +++ b/default.nix @@ -74,7 +74,6 @@ in installJavascript = true; npmCachePhase = '' - set -e if [ "$useYarnLock" = "1" ]; then cp ${lockfile} ${builtins.baseNameOf pkgLockJson} chmod u+w ${builtins.baseNameOf pkgLockJson} @@ -84,7 +83,6 @@ in ''; buildPhase = '' - set -e ${npmAlias} runHook preBuild ${npmBuild} @@ -94,7 +92,6 @@ in # make a package .tgz (no way around it) npmPackPhase = '' - set -e ${npmAlias} npm prune --production npm pack --ignore-scripts @@ -102,7 +99,6 @@ in # unpack the .tgz into output directory and add npm wrapper installPhase = '' - set -e mkdir -p $out/bin tar xzvf ./${name}.tgz -C $out --strip-components=1 if [ "$installJavascript" = "1" ]; then diff --git a/mkcache.js b/mkcache.js index a7fc1da..9b7e0e7 100644 --- a/mkcache.js +++ b/mkcache.js @@ -3,9 +3,14 @@ const fs = require("fs") const pacote = require("pacote") const path = require("path") -// Usage: node mkcache.js npm-cache-input.json +const USAGE = "node mkcache.js npm-cache-input.json" -let nixPkgsFile = process.argv[2] +if (process.argv.length != USAGE.split(/\s+/).length) { + console.error("Usage:", USAGE) + process.exit(1) +} + +const [nixPkgsFile] = process.argv.slice(2) function traverseDeps(pkg, fn) { Object.values(pkg.dependencies).forEach(dep => { @@ -15,12 +20,12 @@ function traverseDeps(pkg, fn) { } async function main(lockfile, nix, cache) { - let promises = Object.keys(nix).map(async function (url) { - let tar = nix[url] - let manifest = await pacote.manifest(tar, { offline: true, cache }) + const promises = Object.keys(nix).map(async function (url) { + const tar = nix[url] + const manifest = await pacote.manifest(tar, { offline: true, cache }) return [url, manifest._integrity] }) - let hashes = new Map(await Promise.all(promises)) + const hashes = new Map(await Promise.all(promises)) traverseDeps(lockfile, dep => { if (dep.integrity.startsWith("sha1-")) { assert(hashes.has(dep.resolved)) @@ -33,9 +38,9 @@ async function main(lockfile, nix, cache) { fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 2)) } -const pkgLockFile = "./package-lock.json" -const lock = JSON.parse(fs.readFileSync(pkgLockFile, "utf8")) -const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) +const pkgLockFile = "./package-lock.json" +const lock = JSON.parse(fs.readFileSync(pkgLockFile, "utf8")) +const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) process.on("unhandledRejection", error => { console.log("unhandledRejection", error.message); diff --git a/mklock.js b/mklock.js index c5adc79..2c7aefd 100644 --- a/mklock.js +++ b/mklock.js @@ -1,25 +1,27 @@ -const assert = require("assert") -const fs = require("fs") -const lockfile = require("@yarnpkg/lockfile") -const semver = require("semver") -const ssri = require("ssri") +const assert = require("assert") +const fs = require("fs") +const lockfile = require("@yarnpkg/lockfile") +const semver = require("semver") +const ssri = require("ssri") -// Usage: node mklock.js package-lock.json package.json yarn.lock integrities.json +const USAGE = "node mklock.js package-lock.json package.json yarn.lock integrities.json" -let pkgLockFile = process.argv[2] -let pkgFile = process.argv[3] -let yarnLockFile = process.argv[4] -let intFile = process.argv[5] +if (process.argv.length != USAGE.split(/\s+/).length) { + console.error("Usage:", USAGE) + process.exit(1) +} + +const [pkgLockFile, pkgFile, yarnLockFile, intFile] = process.argv.slice(2) -let pkgJson = JSON.parse(fs.readFileSync(pkgFile, "utf8")) -let yarnJson = lockfile.parse(fs.readFileSync(yarnLockFile, "utf8")).object -let integrities = intFile ? JSON.parse(fs.readFileSync(intFile)) : {} +const pkgJson = JSON.parse(fs.readFileSync(pkgFile, "utf8")) +const yarnJson = lockfile.parse(fs.readFileSync(yarnLockFile, "utf8")).object +const integrities = intFile ? JSON.parse(fs.readFileSync(intFile)) : {} -let pkgDeps = { ...(pkgJson.devDependencies || {}), - ...(pkgJson.dependencies || {}) } +const pkgDeps = { ...(pkgJson.devDependencies || {}), + ...(pkgJson.dependencies || {}) } function objKeys(o) { - let keys = Object.keys(o) + const keys = Object.keys(o) keys.sort() return keys } @@ -27,7 +29,7 @@ function objKeys(o) { function splitNameVsn(key) { // foo@vsn or @foo/bar@vsn if (key[0] == "@") { - let [name, vsn] = key.slice(1).split("@") + const [name, vsn] = key.slice(1).split("@") return ["@"+name, vsn] } else { return key.split("@") @@ -35,41 +37,41 @@ function splitNameVsn(key) { } function addDeps(obj, pkg, vsn, seen) { - let pkgvsn = pkg + "@" + vsn + const pkgvsn = pkg + "@" + vsn if (seen[pkgvsn]) return // break cycle seen[pkgvsn] = true - let pkgdeps = deps[pkg][vsn]._dependencies || {} + const pkgdeps = deps[pkg][vsn]._dependencies || {} obj.requires = {} obj.dependencies = {} objKeys(pkgdeps).forEach(key => { - let depvsn = pkgdeps[key] - let dep = deps[key][depvsn] - let ass = !depsTree[key] ? depsTree : + const depvsn = pkgdeps[key] + const dep = deps[key][depvsn] + const ass = !depsTree[key] ? depsTree : depsTree[key].version != dep.version ? obj.dependencies : null obj.requires[key] = depvsn if (ass) { - let dep_ = { ...dep, _dependencies: undefined } + const dep_ = { ...dep, _dependencies: undefined } ass[key] = dep_ addDeps(dep_, key, depvsn, { ...seen }) } }) } -let deps = {} -let depsTree = {} +const deps = {} +const depsTree = {} objKeys(yarnJson).forEach(key => { - let dep = yarnJson[key] - let [name, vsn] = splitNameVsn(key) - let [url, sha1] = dep.resolved.split("#", 2) - let integrity = dep.integrity || integrities[url] || - ssri.fromHex(sha1, "sha1").toString() + const dep = yarnJson[key] + const [name, vsn] = splitNameVsn(key) + const [url, sha1] = dep.resolved.split("#", 2) + const integrity = dep.integrity || integrities[url] || + ssri.fromHex(sha1, "sha1").toString() assert(integrity, "missing integrity for " + JSON.stringify(dep)) if (!deps[name]) deps[name] = {} - deps[name][vsn] = { version: dep.version, resolved: url, - integrity: integrity, - _dependencies: dep.dependencies } + deps[name][vsn] = { version: dep.version, resolved: url, + integrity: integrity, + _dependencies: dep.dependencies } }) objKeys(pkgDeps).forEach(key => { @@ -80,8 +82,8 @@ objKeys(pkgDeps).forEach(key => { addDeps(depsTree[key], key, pkgDeps[key], {}) }) -let lock = { name: pkgJson.name, version: pkgJson.version, - lockfileVersion: 1, requires: true, - dependencies: depsTree } +const lock = { name: pkgJson.name, version: pkgJson.version, + lockfileVersion: 1, requires: true, + dependencies: depsTree } fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 2)) From 4bdaa75cfb792ce819292c88cddbb62f70c39e6b Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 19 Oct 2018 15:52:04 +0200 Subject: [PATCH 09/21] plan B --- README.md | 29 ++++++++---------- mklock.js | 89 ------------------------------------------------------- 2 files changed, 12 insertions(+), 106 deletions(-) delete mode 100644 mklock.js diff --git a/README.md b/README.md index e443e39..8bdf31e 100644 --- a/README.md +++ b/README.md @@ -2,30 +2,25 @@ nix-npm-buildpackage - build nix packages that use npm/yarn packages -You can use `buildNpmPackage` to: -* convert a `yarn.lock` file to a `packages-lock.json` file if needed -* use a `packages-lock.json` file to: +You can use `buildNpmPackage`/`buildYarnPackage` to: +* use a `packages-lock.json`/`yarn.lock` file to: - download the dependencies to the nix store - - build an offline npm cache that uses those -* build a nix package from the npm package + - build an offline npm/yarn cache that uses those +* build a nix package from the npm/yarn package ## Examples ```nix { pkgs ? import {} }: let - buildNpmPackage = pkgs.callPackage .../nix-npm-buildpackage {}; - integreties = { # in case the yarn.lock file is missing hashes - "https://codeload.github.com/foo/bar.js/tar.gz/..." = "sha512-..."; - }; -in buildNpmPackage { - src = ./.; - useYarnLock = true; - yarnIntegreties = integreties; -} + { buildNpmPackage, buildYarnPackage } = pkgs.callPackage .../nix-npm-buildpackage {}; +in ... ``` -## TODO +```nix +buildNpmPackage { src = ./.; } +``` -* use symlinks to avoid duplication? -* make npm `bin` and `scripts` work? +```nix +buildYarnPackage { src = ./.; } # TODO +``` diff --git a/mklock.js b/mklock.js deleted file mode 100644 index 2c7aefd..0000000 --- a/mklock.js +++ /dev/null @@ -1,89 +0,0 @@ -const assert = require("assert") -const fs = require("fs") -const lockfile = require("@yarnpkg/lockfile") -const semver = require("semver") -const ssri = require("ssri") - -const USAGE = "node mklock.js package-lock.json package.json yarn.lock integrities.json" - -if (process.argv.length != USAGE.split(/\s+/).length) { - console.error("Usage:", USAGE) - process.exit(1) -} - -const [pkgLockFile, pkgFile, yarnLockFile, intFile] = process.argv.slice(2) - -const pkgJson = JSON.parse(fs.readFileSync(pkgFile, "utf8")) -const yarnJson = lockfile.parse(fs.readFileSync(yarnLockFile, "utf8")).object -const integrities = intFile ? JSON.parse(fs.readFileSync(intFile)) : {} - -const pkgDeps = { ...(pkgJson.devDependencies || {}), - ...(pkgJson.dependencies || {}) } - -function objKeys(o) { - const keys = Object.keys(o) - keys.sort() - return keys -} - -function splitNameVsn(key) { - // foo@vsn or @foo/bar@vsn - if (key[0] == "@") { - const [name, vsn] = key.slice(1).split("@") - return ["@"+name, vsn] - } else { - return key.split("@") - } -} - -function addDeps(obj, pkg, vsn, seen) { - const pkgvsn = pkg + "@" + vsn - if (seen[pkgvsn]) return // break cycle - seen[pkgvsn] = true - const pkgdeps = deps[pkg][vsn]._dependencies || {} - obj.requires = {} - obj.dependencies = {} - objKeys(pkgdeps).forEach(key => { - const depvsn = pkgdeps[key] - const dep = deps[key][depvsn] - const ass = !depsTree[key] ? depsTree : - depsTree[key].version != dep.version ? - obj.dependencies : null - obj.requires[key] = depvsn - if (ass) { - const dep_ = { ...dep, _dependencies: undefined } - ass[key] = dep_ - addDeps(dep_, key, depvsn, { ...seen }) - } - }) -} - -const deps = {} -const depsTree = {} - -objKeys(yarnJson).forEach(key => { - const dep = yarnJson[key] - const [name, vsn] = splitNameVsn(key) - const [url, sha1] = dep.resolved.split("#", 2) - const integrity = dep.integrity || integrities[url] || - ssri.fromHex(sha1, "sha1").toString() - assert(integrity, "missing integrity for " + JSON.stringify(dep)) - if (!deps[name]) deps[name] = {} - deps[name][vsn] = { version: dep.version, resolved: url, - integrity: integrity, - _dependencies: dep.dependencies } -}) - -objKeys(pkgDeps).forEach(key => { - depsTree[key] = { ...deps[key][pkgDeps[key]], _dependencies: undefined } -}) - -objKeys(pkgDeps).forEach(key => { - addDeps(depsTree[key], key, pkgDeps[key], {}) -}) - -const lock = { name: pkgJson.name, version: pkgJson.version, - lockfileVersion: 1, requires: true, - dependencies: depsTree } - -fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 2)) From 103c6482f1346d63aa4bef1fb5845266cbc5b8cc Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 19 Oct 2018 17:49:16 +0200 Subject: [PATCH 10/21] buildYarnPackage WIP --- default.nix | 134 ++++++++++++++++++++++++------------ mkcache.js => mknpmcache.js | 9 ++- mkyarncache.js | 21 ++++++ mkyarnjson.js | 43 ++++++++++++ 4 files changed, 158 insertions(+), 49 deletions(-) rename mkcache.js => mknpmcache.js (88%) create mode 100644 mkyarncache.js create mode 100644 mkyarnjson.js diff --git a/default.nix b/default.nix index 5d0b842..86c7d39 100644 --- a/default.nix +++ b/default.nix @@ -1,5 +1,5 @@ { stdenvNoCC, writeShellScriptBin, writeText, runCommand, - stdenv, fetchurl, makeWrapper, nodejs-10_x, yarn2nix }: + stdenv, fetchurl, makeWrapper, nodejs-10_x, yarn2nix, yarn }: with stdenv.lib; let inherit (builtins) fromJSON toJSON split removeAttrs; @@ -7,21 +7,21 @@ with stdenv.lib; let depFetchOwn = { resolved, integrity, ... }: let - ssri = split "-" integrity; # standard subresource integrity + ssri = split "-" integrity; hashType = head ssri; hash = elemAt ssri 2; fname = baseNameOf resolved; - in nameValuePair resolved (fetchurl { - url = resolved; - "${hashType}" = hash; - name = if hasSuffix ".tgz" fname || hasSuffix ".tar.gz" fname - then fname else fname + ".tgz"; - }); + name = if hasSuffix ".tgz" fname || hasSuffix ".tar.gz" fname + then fname else fname + ".tgz"; + in nameValuePair resolved { + name = fname; + path = fetchurl { inherit name; url = resolved; "${hashType}" = hash; }; + }; depToFetch = args @ { resolved ? null, dependencies ? {}, ... }: (optional (resolved != null) (depFetchOwn args)) ++ (depsToFetches dependencies); - npmCacheInput = lock: writeText "npm-cache-input.json" (toJSON (listToAttrs (depToFetch lock))); + cacheInput = out: in: writeText out (toJSON (listToAttrs (depToFetch in))); patchShebangs = writeShellScriptBin "patchShebangs.sh" '' set -e @@ -37,35 +37,30 @@ with stdenv.lib; let fi exec bash "$@" ''; -in - args @ { src, useYarnLock ? false, yarnIntegreties ? {}, - npmBuild ? "npm ci", npmBuildMore ? "", - buildInputs ? [], npmFlags ? [], ... }: - let - pkgJson = src + "/package.json"; - pkgLockJson = src + "/package-lock.json"; - yarnLock = src + "/yarn.lock"; - yarnIntFile = writeText "integreties.json" (toJSON yarnIntegreties); - lockfile = if useYarnLock then mkLockFileFromYarn else pkgLockJson; + npmInfo = src: rec { + pkgJson = src + "/package.json"; + info = fromJSON (readFile pkgJson); + name = "${info.name}-${info.version}"; + }; - info = fromJSON (readFile pkgJson); - lock = fromJSON (readFile lockfile); + npm = "${nodejs-10_x}/bin/npm"; + npmAlias = ''npm() { ${npm} "$@" $npmFlags; }''; + npmModules = "${nodejs-10_x}/lib/node_modules/npm/node_modules"; - name = "${info.name}-${info.version}"; - npm = "${nodejs-10_x}/bin/npm"; - npmAlias = ''npm() { ${npm} "$@" $npmFlags; }''; - npmModules = "${nodejs-10_x}/lib/node_modules/npm/node_modules"; - - mkLockFileFromYarn = runCommand "yarn-package-lock.json" {} '' - set -e - addToSearchPath NODE_PATH ${npmModules} - addToSearchPath NODE_PATH ${yarn2nix.node_modules} - ${nodejs-10_x}/bin/node ${./mklock.js} $out ${pkgJson} ${yarnLock} ${yarnIntFile} - ''; + yarn = "${yarn}/bin/yarn"; + yarnAlias = ''yarn() { ${yarn} "$@" $yarnFlags; }''; +in { + buildNpmPackage = args @ { + src, npmBuild ? "npm ci", npmBuildMore ? "", + buildInputs ? [], npmFlags ? [], ... + }: + let + info = npmInfo src; + lock = fromJSON (readFile (src + "/package-lock.json")); in stdenv.mkDerivation ({ - inherit name; - inherit (info) version; + inherit (info) name; + inherit (info.info) version; XDG_CONFIG_DIRS = "."; NO_UPDATE_NOTIFIER = true; @@ -74,12 +69,8 @@ in installJavascript = true; npmCachePhase = '' - if [ "$useYarnLock" = "1" ]; then - cp ${lockfile} ${builtins.baseNameOf pkgLockJson} - chmod u+w ${builtins.baseNameOf pkgLockJson} - fi - addToSearchPath NODE_PATH ${npmModules} - node ${./mkcache.js} ${npmCacheInput lock} + addToSearchPath NODE_PATH ${npmModules} # pacote + node ${./mknpmcache.js} ${cacheInput "npm-cache-input.json" lock} ''; buildPhase = '' @@ -98,15 +89,70 @@ in ''; # unpack the .tgz into output directory and add npm wrapper + # TODO: "cd $out" vs NIX_NPM_BUILDPACKAGE_OUT=$out? installPhase = '' mkdir -p $out/bin - tar xzvf ./${name}.tgz -C $out --strip-components=1 + tar xzvf ./${info.name}.tgz -C $out --strip-components=1 if [ "$installJavascript" = "1" ]; then cp -R node_modules $out/ makeWrapper ${npm} $out/bin/npm --run "cd $out" fi ''; - } // removeAttrs args [ "yarnIntegreties" ] // { - buildInputs = [ nodejs-10_x makeWrapper ] ++ buildInputs; + } // args // { + buildInputs = [ nodejs-10_x makeWrapper ] ++ buildInputs; # TODO: git? npmFlags = [ "--cache=./npm-cache" "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ] ++ npmFlags; - }) + }); + + buildYarnPackage = args @ { + src, yarnBuild ? "yarn", yarnBuildMore ? "", + buildInputs ? [], yarnFlags ? [], npmFlags ? [], ... + }: + let + info = npmInfo src; + deps = { dependencies = fromJSON (readFile yarnJson); }; + yarnIntFile = writeText "integreties.json" {}; # TODO + yarnJson = runCommand "yarn.json" {} '' + set -e + addToSearchPath NODE_PATH ${yarn2nix.node_modules} # @yarnpkg/lockfile + ${nodejs-10_x}/bin/node ${./mkyarnjson.js} ${yarnIntFile} > $out + ''; + in stdenv.mkDerivation ({ + inherit (info) name; + inherit (info.info) version; + + # ... TODO ... + + preBuildPhases = [ "yarnConfigPhase" "yarnCachePhase" ]; + + # TODO + yarnConfigPhase = '' + { echo yarn-offline-mirror \"$PWD/yarn-cache\" + echo script-shell \"${shellWrap}/bin/npm-shell-wrap.sh\" + } >> .yarnrc + ''; + + yarnCachePhase = '' + node ${./mkyarncache.js} ${cacheInput "yarn-cache-input.json" deps} + ''; + + # ... TODO ... + + buildPhase = '' + ${yarnAlias} + runHook preBuild + ${yarnBuild} + ${yarnBuildMore} + runHook postBuild + ''; + + # ... TODO ... + + # TODO + installPhase = '' + ''; + } // args // { + buildInputs = [ nodejs-10_x yarn ] ++ buildInputs; # TODO: git? + yarnFlags = [ "--offline" ] ++ yarnFlags; + # TODO: npmFlags + }); +} diff --git a/mkcache.js b/mknpmcache.js similarity index 88% rename from mkcache.js rename to mknpmcache.js index 9b7e0e7..212b34f 100644 --- a/mkcache.js +++ b/mknpmcache.js @@ -1,9 +1,8 @@ const assert = require("assert") const fs = require("fs") const pacote = require("pacote") -const path = require("path") -const USAGE = "node mkcache.js npm-cache-input.json" +const USAGE = "node mknpmcache.js npm-cache-input.json" if (process.argv.length != USAGE.split(/\s+/).length) { console.error("Usage:", USAGE) @@ -21,7 +20,7 @@ function traverseDeps(pkg, fn) { async function main(lockfile, nix, cache) { const promises = Object.keys(nix).map(async function (url) { - const tar = nix[url] + const tar = nix[url].path const manifest = await pacote.manifest(tar, { offline: true, cache }) return [url, manifest._integrity] }) @@ -43,8 +42,8 @@ const lock = JSON.parse(fs.readFileSync(pkgLockFile, "utf8")) const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) process.on("unhandledRejection", error => { - console.log("unhandledRejection", error.message); + console.log("unhandledRejection", error.message) process.exit(1) -}); +}) main(lock, nixPkgs, "./npm-cache/_cacache") diff --git a/mkyarncache.js b/mkyarncache.js new file mode 100644 index 0000000..b487791 --- /dev/null +++ b/mkyarncache.js @@ -0,0 +1,21 @@ +const fs = require("fs") +const path = require("path") + +const USAGE = "node mkyarncache.js yarn-cache-input.json" + +if (process.argv.length != USAGE.split(/\s+/).length) { + console.error("Usage:", USAGE) + process.exit(1) +} + +const [nixPkgsFile] = process.argv.slice(2) +const yarnLockFile = "./yarn.lock" +const yarnCacheDir = "./yarn-cache" + +const lock = JSON.parse(fs.readFileSync(yarnLockFile, "utf8")) +const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) + +Object.keys(nixPkgs).forEach(url => { + const dep = nix[url]; + fs.symlinkSync(dep.path, path.join(yarnCacheDir, dep.name)) +}) diff --git a/mkyarnjson.js b/mkyarnjson.js new file mode 100644 index 0000000..82a7a56 --- /dev/null +++ b/mkyarnjson.js @@ -0,0 +1,43 @@ +const assert = require("assert") +const fs = require("fs") +const lockfile = require("@yarnpkg/lockfile") +const ssri = require("ssri") + +const USAGE = "node mkyarnjson.js integrities.json" + +if (process.argv.length != USAGE.split(/\s+/).length) { + console.error("Usage:", USAGE) + process.exit(1) +} + +const [intFile] = process.argv.slice(2) +const yarnLockFile = "./yarn.lock" + +const yarnJson = lockfile.parse(fs.readFileSync(yarnLockFile, "utf8")).object +const integrities = JSON.parse(fs.readFileSync(intFile)) + +function splitNameVsn(key) { + // foo@vsn or @foo/bar@vsn + if (key[0] == "@") { + const [name, vsn] = key.slice(1).split("@") + return ["@"+name, vsn] + } else { + return key.split("@") + } +} + +function processDeps(deps, result = {}) { + objKeys(deps).forEach(key => { + if (key in result) return + const dep = yarnJson[key] + const [name, vsn] = splitNameVsn(key) + const [url, sha1] = dep.resolved.split("#", 2) + const integrity = dep.integrity || integrities[url] || + ssri.fromHex(sha1, "sha1").toString() + assert(integrity, "missing integrity for " + JSON.stringify(dep)) + result[key] = { resolved: url, integrity: integrity } + if (dep.dependencies) processDeps(dep.dependencies, result) + } +}) + +console.log(JSON.stringify(processDeps(yarnJson), null, 2)) From 801f9856690617719814195967a690b354d6a752 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 19 Oct 2018 17:51:54 +0200 Subject: [PATCH 11/21] fix README examples --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8bdf31e..fb596f1 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,14 @@ You can use `buildNpmPackage`/`buildYarnPackage` to: ```nix { pkgs ? import {} }: let - { buildNpmPackage, buildYarnPackage } = pkgs.callPackage .../nix-npm-buildpackage {}; + bp = pkgs.callPackage .../nix-npm-buildpackage {}; in ... ``` ```nix -buildNpmPackage { src = ./.; } +bp.buildNpmPackage { src = ./.; } ``` ```nix -buildYarnPackage { src = ./.; } # TODO +bp.buildYarnPackage { src = ./.; } # TODO ``` From c57d014e91196900acc5e4c8e0bd9dff716dd670 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 19 Oct 2018 18:55:36 +0200 Subject: [PATCH 12/21] a few fixes --- default.nix | 23 +++++++++++++---------- mknpmcache.js | 16 ++++++++-------- mkyarncache.js | 6 ++---- mkyarnjson.js | 32 +++++++++++++++----------------- 4 files changed, 38 insertions(+), 39 deletions(-) diff --git a/default.nix b/default.nix index 86c7d39..7bc14af 100644 --- a/default.nix +++ b/default.nix @@ -21,7 +21,7 @@ with stdenv.lib; let depToFetch = args @ { resolved ? null, dependencies ? {}, ... }: (optional (resolved != null) (depFetchOwn args)) ++ (depsToFetches dependencies); - cacheInput = out: in: writeText out (toJSON (listToAttrs (depToFetch in))); + cacheInput = oFile: iFile: writeText oFile (toJSON (listToAttrs (depToFetch iFile))); patchShebangs = writeShellScriptBin "patchShebangs.sh" '' set -e @@ -44,12 +44,12 @@ with stdenv.lib; let name = "${info.name}-${info.version}"; }; - npm = "${nodejs-10_x}/bin/npm"; - npmAlias = ''npm() { ${npm} "$@" $npmFlags; }''; + npmCmd = "${nodejs-10_x}/bin/npm"; + npmAlias = ''npm() { ${npmCmd} "$@" $npmFlags; }''; npmModules = "${nodejs-10_x}/lib/node_modules/npm/node_modules"; - yarn = "${yarn}/bin/yarn"; - yarnAlias = ''yarn() { ${yarn} "$@" $yarnFlags; }''; + yarnCmd = "${yarn}/bin/yarn"; + yarnAlias = ''yarn() { ${yarnCmd} "$@" $yarnFlags; }''; in { buildNpmPackage = args @ { src, npmBuild ? "npm ci", npmBuildMore ? "", @@ -95,7 +95,7 @@ in { tar xzvf ./${info.name}.tgz -C $out --strip-components=1 if [ "$installJavascript" = "1" ]; then cp -R node_modules $out/ - makeWrapper ${npm} $out/bin/npm --run "cd $out" + makeWrapper ${npmCmd} $out/bin/npm --run "cd $out" fi ''; } // args // { @@ -104,17 +104,19 @@ in { }); buildYarnPackage = args @ { - src, yarnBuild ? "yarn", yarnBuildMore ? "", + src, yarnBuild ? "yarn", yarnBuildMore ? "", integreties ? {}, buildInputs ? [], yarnFlags ? [], npmFlags ? [], ... }: let info = npmInfo src; deps = { dependencies = fromJSON (readFile yarnJson); }; - yarnIntFile = writeText "integreties.json" {}; # TODO + yarnIntFile = writeText "integreties.json" (toJSON integreties); + yarnLock = src + "/yarn.lock"; yarnJson = runCommand "yarn.json" {} '' set -e + addToSearchPath NODE_PATH ${npmModules} # ssri addToSearchPath NODE_PATH ${yarn2nix.node_modules} # @yarnpkg/lockfile - ${nodejs-10_x}/bin/node ${./mkyarnjson.js} ${yarnIntFile} > $out + ${nodejs-10_x}/bin/node ${./mkyarnjson.js} ${yarnLock} ${yarnIntFile} > $out ''; in stdenv.mkDerivation ({ inherit (info) name; @@ -132,6 +134,7 @@ in { ''; yarnCachePhase = '' + mkdir -p yarn-cache node ${./mkyarncache.js} ${cacheInput "yarn-cache-input.json" deps} ''; @@ -150,7 +153,7 @@ in { # TODO installPhase = '' ''; - } // args // { + } // removeAttrs args [ "integreties" ] // { buildInputs = [ nodejs-10_x yarn ] ++ buildInputs; # TODO: git? yarnFlags = [ "--offline" ] ++ yarnFlags; # TODO: npmFlags diff --git a/mknpmcache.js b/mknpmcache.js index 212b34f..6ea6fd9 100644 --- a/mknpmcache.js +++ b/mknpmcache.js @@ -1,8 +1,8 @@ -const assert = require("assert") -const fs = require("fs") -const pacote = require("pacote") +const assert = require("assert") +const fs = require("fs") +const pacote = require("pacote") -const USAGE = "node mknpmcache.js npm-cache-input.json" +const USAGE = "node mknpmcache.js npm-cache-input.json" if (process.argv.length != USAGE.split(/\s+/).length) { console.error("Usage:", USAGE) @@ -11,6 +11,10 @@ if (process.argv.length != USAGE.split(/\s+/).length) { const [nixPkgsFile] = process.argv.slice(2) +const pkgLockFile = "./package-lock.json" +const lock = JSON.parse(fs.readFileSync(pkgLockFile, "utf8")) +const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) + function traverseDeps(pkg, fn) { Object.values(pkg.dependencies).forEach(dep => { if (dep.resolved && dep.integrity) fn(dep) @@ -37,10 +41,6 @@ async function main(lockfile, nix, cache) { fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 2)) } -const pkgLockFile = "./package-lock.json" -const lock = JSON.parse(fs.readFileSync(pkgLockFile, "utf8")) -const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) - process.on("unhandledRejection", error => { console.log("unhandledRejection", error.message) process.exit(1) diff --git a/mkyarncache.js b/mkyarncache.js index b487791..f3b95e0 100644 --- a/mkyarncache.js +++ b/mkyarncache.js @@ -9,13 +9,11 @@ if (process.argv.length != USAGE.split(/\s+/).length) { } const [nixPkgsFile] = process.argv.slice(2) -const yarnLockFile = "./yarn.lock" -const yarnCacheDir = "./yarn-cache" -const lock = JSON.parse(fs.readFileSync(yarnLockFile, "utf8")) +const yarnCacheDir = "./yarn-cache" const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) Object.keys(nixPkgs).forEach(url => { - const dep = nix[url]; + const dep = nixPkgs[url]; fs.symlinkSync(dep.path, path.join(yarnCacheDir, dep.name)) }) diff --git a/mkyarnjson.js b/mkyarnjson.js index 82a7a56..054d91a 100644 --- a/mkyarnjson.js +++ b/mkyarnjson.js @@ -3,19 +3,18 @@ const fs = require("fs") const lockfile = require("@yarnpkg/lockfile") const ssri = require("ssri") -const USAGE = "node mkyarnjson.js integrities.json" +const USAGE = "node mkyarnjson.js yarn.lock integrities.json" if (process.argv.length != USAGE.split(/\s+/).length) { console.error("Usage:", USAGE) process.exit(1) } -const [intFile] = process.argv.slice(2) -const yarnLockFile = "./yarn.lock" +const [yarnLockFile, intFile] = process.argv.slice(2) const yarnJson = lockfile.parse(fs.readFileSync(yarnLockFile, "utf8")).object const integrities = JSON.parse(fs.readFileSync(intFile)) - + function splitNameVsn(key) { // foo@vsn or @foo/bar@vsn if (key[0] == "@") { @@ -26,18 +25,17 @@ function splitNameVsn(key) { } } -function processDeps(deps, result = {}) { - objKeys(deps).forEach(key => { - if (key in result) return - const dep = yarnJson[key] - const [name, vsn] = splitNameVsn(key) - const [url, sha1] = dep.resolved.split("#", 2) - const integrity = dep.integrity || integrities[url] || - ssri.fromHex(sha1, "sha1").toString() - assert(integrity, "missing integrity for " + JSON.stringify(dep)) - result[key] = { resolved: url, integrity: integrity } - if (dep.dependencies) processDeps(dep.dependencies, result) - } +const deps = {} + +Object.keys(yarnJson).forEach(key => { + if (key in deps) return + const dep = yarnJson[key] + const [name, vsn] = splitNameVsn(key) + const [url, sha1] = dep.resolved.split("#", 2) + const integrity = dep.integrity || integrities[url] || + (sha1 && ssri.fromHex(sha1, "sha1").toString()) + assert(integrity, "missing integrity for " + JSON.stringify(dep)) + deps[key] = { resolved: url, integrity: integrity } }) -console.log(JSON.stringify(processDeps(yarnJson), null, 2)) +console.log(JSON.stringify(deps, null, 2)) From abb6155a38349c75f0f79c52ab5678db213e00a3 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 19 Oct 2018 19:20:25 +0200 Subject: [PATCH 13/21] make yarn cache work w/ @foo/bar deps --- default.nix | 15 ++++++++------- mkyarncache.js | 6 +++++- mkyarnjson.js | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/default.nix b/default.nix index 7bc14af..a11a77a 100644 --- a/default.nix +++ b/default.nix @@ -5,17 +5,17 @@ with stdenv.lib; let depsToFetches = deps: concatMap depToFetch (attrValues deps); - depFetchOwn = { resolved, integrity, ... }: + depFetchOwn = { resolved, integrity, name ? null, ... }: let ssri = split "-" integrity; hashType = head ssri; hash = elemAt ssri 2; - fname = baseNameOf resolved; - name = if hasSuffix ".tgz" fname || hasSuffix ".tar.gz" fname - then fname else fname + ".tgz"; + bname = baseNameOf resolved; + fname = if hasSuffix ".tgz" bname || hasSuffix ".tar.gz" bname + then bname else bname + ".tgz"; in nameValuePair resolved { - name = fname; - path = fetchurl { inherit name; url = resolved; "${hashType}" = hash; }; + inherit name bname; + path = fetchurl { name = fname; url = resolved; "${hashType}" = hash; }; }; depToFetch = args @ { resolved ? null, dependencies ? {}, ... }: @@ -150,8 +150,9 @@ in { # ... TODO ... - # TODO installPhase = '' + mkdir -p $out/bin + # TODO ''; } // removeAttrs args [ "integreties" ] // { buildInputs = [ nodejs-10_x yarn ] ++ buildInputs; # TODO: git? diff --git a/mkyarncache.js b/mkyarncache.js index f3b95e0..b572364 100644 --- a/mkyarncache.js +++ b/mkyarncache.js @@ -13,7 +13,11 @@ const [nixPkgsFile] = process.argv.slice(2) const yarnCacheDir = "./yarn-cache" const nixPkgs = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8")) +function name(dep) { + return dep.name[0] == "@" ? dep.name.split("/")[0] + "-" + dep.bname : dep.bname +} + Object.keys(nixPkgs).forEach(url => { const dep = nixPkgs[url]; - fs.symlinkSync(dep.path, path.join(yarnCacheDir, dep.name)) + fs.symlinkSync(dep.path, path.join(yarnCacheDir, name(dep))) }) diff --git a/mkyarnjson.js b/mkyarnjson.js index 054d91a..11b01e0 100644 --- a/mkyarnjson.js +++ b/mkyarnjson.js @@ -35,7 +35,7 @@ Object.keys(yarnJson).forEach(key => { const integrity = dep.integrity || integrities[url] || (sha1 && ssri.fromHex(sha1, "sha1").toString()) assert(integrity, "missing integrity for " + JSON.stringify(dep)) - deps[key] = { resolved: url, integrity: integrity } + deps[key] = { name, resolved: url, integrity } }) console.log(JSON.stringify(deps, null, 2)) From b727599056658ea48d7bfbd5b82330b7f52886ed Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 19 Oct 2018 19:48:33 +0200 Subject: [PATCH 14/21] yarn: pack & install; refactor --- README.md | 2 +- default.nix | 81 +++++++++++++++++++++++++++++------------------------ 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index fb596f1..21db93d 100644 --- a/README.md +++ b/README.md @@ -22,5 +22,5 @@ bp.buildNpmPackage { src = ./.; } ``` ```nix -bp.buildYarnPackage { src = ./.; } # TODO +bp.buildYarnPackage { src = ./.; } ``` diff --git a/default.nix b/default.nix index a11a77a..8ef9177 100644 --- a/default.nix +++ b/default.nix @@ -44,12 +44,34 @@ with stdenv.lib; let name = "${info.name}-${info.version}"; }; - npmCmd = "${nodejs-10_x}/bin/npm"; - npmAlias = ''npm() { ${npmCmd} "$@" $npmFlags; }''; - npmModules = "${nodejs-10_x}/lib/node_modules/npm/node_modules"; + npmCmd = "${nodejs-10_x}/bin/npm"; + npmAlias = ''npm() { ${npmCmd} "$@" $npmFlags; }''; + npmModules = "${nodejs-10_x}/lib/node_modules/npm/node_modules"; - yarnCmd = "${yarn}/bin/yarn"; - yarnAlias = ''yarn() { ${yarnCmd} "$@" $yarnFlags; }''; + yarnCmd = "${yarn}/bin/yarn"; + yarnAlias = ''yarn() { ${yarnCmd} "$@" $yarnFlags; }''; + + npmFlagsYarn = [ "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ]; + npmFlagsNpm = [ "--cache=./npm-cache" ] + npmFlagsYarn; + + commonEnv = { + XDG_CONFIG_DIRS = "."; + NO_UPDATE_NOTIFIER = true; + installJavascript = true; + }; + + # unpack the .tgz into output directory and add npm wrapper + # TODO: "cd $out" vs NIX_NPM_BUILDPACKAGE_OUT=$out? + untarAndWrap = name: cmds: '' + mkdir -p $out/bin + tar xzvf ./${name}.tgz -C $out --strip-components=1 + if [ "$installJavascript" = "1" ]; then + cp -R node_modules $out/ + ${ concatStringsSep ";" (map (cmd: + ''makeWrapper ${cmd} $out/bin/${baseNameOf cmd} --run "cd $out"'' + ) cmds) } + fi + ''; in { buildNpmPackage = args @ { src, npmBuild ? "npm ci", npmBuildMore ? "", @@ -62,11 +84,8 @@ in { inherit (info) name; inherit (info.info) version; - XDG_CONFIG_DIRS = "."; - NO_UPDATE_NOTIFIER = true; - preBuildPhases = [ "npmCachePhase" ]; - preInstallPhases = [ "npmPackPhase" ]; - installJavascript = true; + preBuildPhases = [ "npmCachePhase" ]; + preInstallPhases = [ "npmPackPhase" ]; npmCachePhase = '' addToSearchPath NODE_PATH ${npmModules} # pacote @@ -88,19 +107,10 @@ in { npm pack --ignore-scripts ''; - # unpack the .tgz into output directory and add npm wrapper - # TODO: "cd $out" vs NIX_NPM_BUILDPACKAGE_OUT=$out? - installPhase = '' - mkdir -p $out/bin - tar xzvf ./${info.name}.tgz -C $out --strip-components=1 - if [ "$installJavascript" = "1" ]; then - cp -R node_modules $out/ - makeWrapper ${npmCmd} $out/bin/npm --run "cd $out" - fi - ''; - } // args // { + installPhase = untarAndWrap info.name [npmCmd]; + } // commonEnv // args // { buildInputs = [ nodejs-10_x makeWrapper ] ++ buildInputs; # TODO: git? - npmFlags = [ "--cache=./npm-cache" "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ] ++ npmFlags; + npmFlags = npmFlagsNpm ++ npmFlags; }); buildYarnPackage = args @ { @@ -122,9 +132,8 @@ in { inherit (info) name; inherit (info.info) version; - # ... TODO ... - - preBuildPhases = [ "yarnConfigPhase" "yarnCachePhase" ]; + preBuildPhases = [ "yarnConfigPhase" "yarnCachePhase" ]; + preInstallPhases = [ "yarnPackPhase" ]; # TODO yarnConfigPhase = '' @@ -138,9 +147,8 @@ in { node ${./mkyarncache.js} ${cacheInput "yarn-cache-input.json" deps} ''; - # ... TODO ... - buildPhase = '' + ${npmAlias} ${yarnAlias} runHook preBuild ${yarnBuild} @@ -148,15 +156,16 @@ in { runHook postBuild ''; - # ... TODO ... - - installPhase = '' - mkdir -p $out/bin - # TODO + # TODO: install --production? + yarnPackPhase = '' + ${yarnAlias} + yarn pack --ignore-scripts --filename ${info.name}.tgz ''; - } // removeAttrs args [ "integreties" ] // { - buildInputs = [ nodejs-10_x yarn ] ++ buildInputs; # TODO: git? - yarnFlags = [ "--offline" ] ++ yarnFlags; - # TODO: npmFlags + + installPhase = untarAndWrap info.name [npmCmd yarnCmd]; + } // commonEnv // removeAttrs args [ "integreties" ] // { + buildInputs = [ nodejs-10_x yarn ] ++ buildInputs; # TODO: git? + yarnFlags = [ "--offline" "--frozen-lockfile" "--non-interactive" ] ++ yarnFlags; + npmFlags = npmFlagsYarn ++ npmFlags; }); } From 73265a7bdd60041132f0a045545ecf2412415eb0 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 19 Oct 2018 21:21:33 +0200 Subject: [PATCH 15/21] refactor + fixes --- default.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index 8ef9177..565ab11 100644 --- a/default.nix +++ b/default.nix @@ -49,7 +49,7 @@ with stdenv.lib; let npmModules = "${nodejs-10_x}/lib/node_modules/npm/node_modules"; yarnCmd = "${yarn}/bin/yarn"; - yarnAlias = ''yarn() { ${yarnCmd} "$@" $yarnFlags; }''; + yarnAlias = ''yarn() { ${yarnCmd} $yarnFlags "$@"; }''; npmFlagsYarn = [ "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ]; npmFlagsNpm = [ "--cache=./npm-cache" ] + npmFlagsYarn; @@ -60,6 +60,8 @@ with stdenv.lib; let installJavascript = true; }; + commonBuildInputs = [ nodejs-10_x makeWrapper ]; # TODO: git? + # unpack the .tgz into output directory and add npm wrapper # TODO: "cd $out" vs NIX_NPM_BUILDPACKAGE_OUT=$out? untarAndWrap = name: cmds: '' @@ -109,7 +111,7 @@ in { installPhase = untarAndWrap info.name [npmCmd]; } // commonEnv // args // { - buildInputs = [ nodejs-10_x makeWrapper ] ++ buildInputs; # TODO: git? + buildInputs = commonBuildInputs ++ buildInputs; npmFlags = npmFlagsNpm ++ npmFlags; }); @@ -164,7 +166,7 @@ in { installPhase = untarAndWrap info.name [npmCmd yarnCmd]; } // commonEnv // removeAttrs args [ "integreties" ] // { - buildInputs = [ nodejs-10_x yarn ] ++ buildInputs; # TODO: git? + buildInputs = [ yarn ] ++ commonBuildInputs ++ buildInputs; yarnFlags = [ "--offline" "--frozen-lockfile" "--non-interactive" ] ++ yarnFlags; npmFlags = npmFlagsYarn ++ npmFlags; }); From cdd4e36da70129c56130bb051dee30ab3292ba40 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Sat, 20 Oct 2018 03:28:50 +0200 Subject: [PATCH 16/21] same node for yarn; set nodedir --- default.nix | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/default.nix b/default.nix index 565ab11..fee6f46 100644 --- a/default.nix +++ b/default.nix @@ -3,6 +3,9 @@ with stdenv.lib; let inherit (builtins) fromJSON toJSON split removeAttrs; + _nodejs = nodejs-10_x; + _yarn = yarn.override { nodejs = _nodejs; }; + depsToFetches = deps: concatMap depToFetch (attrValues deps); depFetchOwn = { resolved, integrity, name ? null, ... }: @@ -44,11 +47,11 @@ with stdenv.lib; let name = "${info.name}-${info.version}"; }; - npmCmd = "${nodejs-10_x}/bin/npm"; + npmCmd = "${_nodejs}/bin/npm"; npmAlias = ''npm() { ${npmCmd} "$@" $npmFlags; }''; - npmModules = "${nodejs-10_x}/lib/node_modules/npm/node_modules"; + npmModules = "${_nodejs}/lib/node_modules/npm/node_modules"; - yarnCmd = "${yarn}/bin/yarn"; + yarnCmd = "${_yarn}/bin/yarn"; yarnAlias = ''yarn() { ${yarnCmd} $yarnFlags "$@"; }''; npmFlagsYarn = [ "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ]; @@ -60,7 +63,7 @@ with stdenv.lib; let installJavascript = true; }; - commonBuildInputs = [ nodejs-10_x makeWrapper ]; # TODO: git? + commonBuildInputs = [ _nodejs makeWrapper ]; # TODO: git? # unpack the .tgz into output directory and add npm wrapper # TODO: "cd $out" vs NIX_NPM_BUILDPACKAGE_OUT=$out? @@ -128,7 +131,7 @@ in { set -e addToSearchPath NODE_PATH ${npmModules} # ssri addToSearchPath NODE_PATH ${yarn2nix.node_modules} # @yarnpkg/lockfile - ${nodejs-10_x}/bin/node ${./mkyarnjson.js} ${yarnLock} ${yarnIntFile} > $out + ${_nodejs}/bin/node ${./mkyarnjson.js} ${yarnLock} ${yarnIntFile} > $out ''; in stdenv.mkDerivation ({ inherit (info) name; @@ -139,9 +142,12 @@ in { # TODO yarnConfigPhase = '' - { echo yarn-offline-mirror \"$PWD/yarn-cache\" - echo script-shell \"${shellWrap}/bin/npm-shell-wrap.sh\" - } >> .yarnrc + cat <<-END >> .yarnrc + + yarn-offline-mirror "$PWD/yarn-cache" + script-shell "${shellWrap}/bin/npm-shell-wrap.sh" + nodedir "${_nodejs}" + END ''; yarnCachePhase = '' @@ -166,7 +172,7 @@ in { installPhase = untarAndWrap info.name [npmCmd yarnCmd]; } // commonEnv // removeAttrs args [ "integreties" ] // { - buildInputs = [ yarn ] ++ commonBuildInputs ++ buildInputs; + buildInputs = [ _yarn ] ++ commonBuildInputs ++ buildInputs; yarnFlags = [ "--offline" "--frozen-lockfile" "--non-interactive" ] ++ yarnFlags; npmFlags = npmFlagsYarn ++ npmFlags; }); From 1f607e575b6b313dc6ac7bc83313ef718d1e2184 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Mon, 22 Oct 2018 16:22:23 +0200 Subject: [PATCH 17/21] --nodedir for npm --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index fee6f46..3156819 100644 --- a/default.nix +++ b/default.nix @@ -55,7 +55,7 @@ with stdenv.lib; let yarnAlias = ''yarn() { ${yarnCmd} $yarnFlags "$@"; }''; npmFlagsYarn = [ "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ]; - npmFlagsNpm = [ "--cache=./npm-cache" ] + npmFlagsYarn; + npmFlagsNpm = [ "--cache=./npm-cache" "--nodedir=${_nodejs}" ] + npmFlagsYarn; commonEnv = { XDG_CONFIG_DIRS = "."; From faaf1afb3e9e7c54a8e2da5aa268b3f2fd0466f1 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 2 Nov 2018 16:37:46 +0100 Subject: [PATCH 18/21] use travis CI --- .travis.nix | 10 ++++++++++ .travis.yml | 3 +++ 2 files changed, 13 insertions(+) create mode 100644 .travis.nix create mode 100644 .travis.yml diff --git a/.travis.nix b/.travis.nix new file mode 100644 index 0000000..a7f2ecc --- /dev/null +++ b/.travis.nix @@ -0,0 +1,10 @@ +# TODO: replace w/ more generic test & also test npm packages +{ pkgs ? import {} }: +let + bp = pkgs.callPackage (import ./default.nix) { inherit pkgs; }; + vault-gist = fetchGit { + url = "https://github.com/obfusk/nix-vault-with-ui.git"; + rev = "664d3d9a1f30626d0277a62c520cf82df0e0a2a1"; + ref = "664d3d9-tag-you-are-it"; # TODO: "v0.1.0" + }; +in vault-gist { inherit pkgs; npm-buildpackage = bp; } diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..3115ea9 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +language: nix +nix: 2.1.3 +script: nix-build .travis.nix From 2c10f7c7bf2938f4f3ebdc098f972c49b048e251 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 2 Nov 2018 16:42:13 +0100 Subject: [PATCH 19/21] README: (hidden) travis badge --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 21db93d..3ddbe06 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ ## Description + + nix-npm-buildpackage - build nix packages that use npm/yarn packages You can use `buildNpmPackage`/`buildYarnPackage` to: From 66d51b37d678eea42af5f43a09bff8dff7b9ba6b Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 2 Nov 2018 16:51:40 +0100 Subject: [PATCH 20/21] fix .travis.nix --- .travis.nix | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.travis.nix b/.travis.nix index a7f2ecc..5b787d5 100644 --- a/.travis.nix +++ b/.travis.nix @@ -1,10 +1,14 @@ -# TODO: replace w/ more generic test & also test npm packages +# TODO: +# * replace w/ more generic test +# * also test npm packages +# * unhide badge + { pkgs ? import {} }: let - bp = pkgs.callPackage (import ./default.nix) { inherit pkgs; }; - vault-gist = fetchGit { + bp = pkgs.callPackage ./default.nix { inherit pkgs; }; +in + import (fetchGit { url = "https://github.com/obfusk/nix-vault-with-ui.git"; rev = "664d3d9a1f30626d0277a62c520cf82df0e0a2a1"; ref = "664d3d9-tag-you-are-it"; # TODO: "v0.1.0" - }; -in vault-gist { inherit pkgs; npm-buildpackage = bp; } + }) { inherit pkgs; npm-buildpackage = bp; } From 32891fbe69760950c19e7ed8a1c84f51ade16fe4 Mon Sep 17 00:00:00 2001 From: "Felix C. Stegerman" Date: Fri, 2 Nov 2018 16:53:58 +0100 Subject: [PATCH 21/21] fix .travis.nix (again) --- .travis.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.nix b/.travis.nix index 5b787d5..26ddcea 100644 --- a/.travis.nix +++ b/.travis.nix @@ -5,7 +5,7 @@ { pkgs ? import {} }: let - bp = pkgs.callPackage ./default.nix { inherit pkgs; }; + bp = pkgs.callPackage ./default.nix {}; in import (fetchGit { url = "https://github.com/obfusk/nix-vault-with-ui.git";