import, refactor, improve & yarn support

README.md

@@ -1,2 +1,26 @@
-# nix-npm-buildpackage
-Build nix packages that use npm/yarn packages.
+## Description
+
+nix-npm-buildpackage - build nix packages that use npm/yarn packages
+
+You can use `buildNpmPackage`/`buildYarnPackage` to:
+* use a `packages-lock.json`/`yarn.lock` file to:
+  - download the dependencies to the nix store
+  - build an offline npm/yarn cache that uses those
+* build a nix package from the npm/yarn package
+
+## Examples
+
+```nix
+{ pkgs ? import <nixpkgs> {} }:
+let
+  bp = pkgs.callPackage .../nix-npm-buildpackage {};
+in ...
+```
+
+```nix
+bp.buildNpmPackage { src = ./.; }
+```
+
+```nix
+bp.buildYarnPackage { src = ./.; }
+```

default.nix

@@ -0,0 +1,179 @@
+{ stdenvNoCC, writeShellScriptBin, writeText, runCommand,
+  stdenv, fetchurl, makeWrapper, nodejs-10_x, yarn2nix, yarn }:
+with stdenv.lib; let
+  inherit (builtins) fromJSON toJSON split removeAttrs;
+
+  _nodejs = nodejs-10_x;
+  _yarn   = yarn.override { nodejs = _nodejs; };
+
+  depsToFetches = deps: concatMap depToFetch (attrValues deps);
+
+  depFetchOwn = { resolved, integrity, name ? null, ... }:
+    let
+      ssri      = split "-" integrity;
+      hashType  = head ssri;
+      hash      = elemAt ssri 2;
+      bname     = baseNameOf resolved;
+      fname     = if hasSuffix ".tgz" bname || hasSuffix ".tar.gz" bname
+                  then bname else bname + ".tgz";
+    in nameValuePair resolved {
+      inherit name bname;
+      path = fetchurl { name = fname; url = resolved; "${hashType}" = hash; };
+    };
+
+  depToFetch = args @ { resolved ? null, dependencies ? {}, ... }:
+    (optional (resolved != null) (depFetchOwn args)) ++ (depsToFetches dependencies);
+
+  cacheInput = oFile: iFile: writeText oFile (toJSON (listToAttrs (depToFetch iFile)));
+
+  patchShebangs = writeShellScriptBin "patchShebangs.sh" ''
+    set -e
+    source ${stdenvNoCC}/setup
+    patchShebangs "$@"
+  '';
+
+  shellWrap = writeShellScriptBin "npm-shell-wrap.sh" ''
+    set -e
+    if [ ! -e .shebangs_patched ]; then
+      ${patchShebangs}/bin/patchShebangs.sh .
+      touch .shebangs_patched
+    fi
+    exec bash "$@"
+  '';
+
+  npmInfo = src: rec {
+    pkgJson = src + "/package.json";
+    info    = fromJSON (readFile pkgJson);
+    name    = "${info.name}-${info.version}";
+  };
+
+  npmCmd        = "${_nodejs}/bin/npm";
+  npmAlias      = ''npm() { ${npmCmd} "$@" $npmFlags; }'';
+  npmModules    = "${_nodejs}/lib/node_modules/npm/node_modules";
+
+  yarnCmd       = "${_yarn}/bin/yarn";
+  yarnAlias     = ''yarn() { ${yarnCmd} $yarnFlags "$@"; }'';
+
+  npmFlagsYarn  = [ "--offline" "--script-shell=${shellWrap}/bin/npm-shell-wrap.sh" ];
+  npmFlagsNpm   = [ "--cache=./npm-cache" "--nodedir=${_nodejs}" ] + npmFlagsYarn;
+
+  commonEnv = {
+    XDG_CONFIG_DIRS     = ".";
+    NO_UPDATE_NOTIFIER  = true;
+    installJavascript   = true;
+  };
+
+  commonBuildInputs = [ _nodejs makeWrapper ];  # TODO: git?
+
+  # unpack the .tgz into output directory and add npm wrapper
+  # TODO: "cd $out" vs NIX_NPM_BUILDPACKAGE_OUT=$out?
+  untarAndWrap = name: cmds: ''
+    mkdir -p $out/bin
+    tar xzvf ./${name}.tgz -C $out --strip-components=1
+    if [ "$installJavascript" = "1" ]; then
+      cp -R node_modules $out/
+      ${ concatStringsSep ";" (map (cmd:
+        ''makeWrapper ${cmd} $out/bin/${baseNameOf cmd} --run "cd $out"''
+      ) cmds) }
+    fi
+  '';
+in {
+  buildNpmPackage = args @ {
+    src, npmBuild ? "npm ci", npmBuildMore ? "",
+    buildInputs ? [], npmFlags ? [], ...
+  }:
+    let
+      info  = npmInfo src;
+      lock  = fromJSON (readFile (src + "/package-lock.json"));
+    in stdenv.mkDerivation ({
+      inherit (info) name;
+      inherit (info.info) version;
+
+      preBuildPhases    = [ "npmCachePhase" ];
+      preInstallPhases  = [ "npmPackPhase" ];
+
+      npmCachePhase = ''
+        addToSearchPath NODE_PATH ${npmModules}   # pacote
+        node ${./mknpmcache.js} ${cacheInput "npm-cache-input.json" lock}
+      '';
+
+      buildPhase = ''
+        ${npmAlias}
+        runHook preBuild
+        ${npmBuild}
+        ${npmBuildMore}
+        runHook postBuild
+      '';
+
+      # make a package .tgz (no way around it)
+      npmPackPhase = ''
+        ${npmAlias}
+        npm prune --production
+        npm pack --ignore-scripts
+      '';
+
+      installPhase = untarAndWrap info.name [npmCmd];
+    } // commonEnv // args // {
+      buildInputs = commonBuildInputs ++ buildInputs;
+      npmFlags    = npmFlagsNpm ++ npmFlags;
+    });
+
+  buildYarnPackage = args @ {
+    src, yarnBuild ? "yarn", yarnBuildMore ? "", integreties ? {},
+    buildInputs ? [], yarnFlags ? [], npmFlags ? [], ...
+  }:
+    let
+      info        = npmInfo src;
+      deps        = { dependencies = fromJSON (readFile yarnJson); };
+      yarnIntFile = writeText "integreties.json" (toJSON integreties);
+      yarnLock    = src + "/yarn.lock";
+      yarnJson    = runCommand "yarn.json" {} ''
+        set -e
+        addToSearchPath NODE_PATH ${npmModules}             # ssri
+        addToSearchPath NODE_PATH ${yarn2nix.node_modules}  # @yarnpkg/lockfile
+        ${_nodejs}/bin/node ${./mkyarnjson.js} ${yarnLock} ${yarnIntFile} > $out
+      '';
+    in stdenv.mkDerivation ({
+      inherit (info) name;
+      inherit (info.info) version;
+
+      preBuildPhases    = [ "yarnConfigPhase" "yarnCachePhase" ];
+      preInstallPhases  = [ "yarnPackPhase" ];
+
+      # TODO
+      yarnConfigPhase = ''
+        cat <<-END >> .yarnrc
+
+        	yarn-offline-mirror "$PWD/yarn-cache"
+        	script-shell "${shellWrap}/bin/npm-shell-wrap.sh"
+        	nodedir "${_nodejs}"
+        END
+      '';
+
+      yarnCachePhase = ''
+        mkdir -p yarn-cache
+        node ${./mkyarncache.js} ${cacheInput "yarn-cache-input.json" deps}
+      '';
+
+      buildPhase = ''
+        ${npmAlias}
+        ${yarnAlias}
+        runHook preBuild
+        ${yarnBuild}
+        ${yarnBuildMore}
+        runHook postBuild
+      '';
+
+      # TODO: install --production?
+      yarnPackPhase = ''
+        ${yarnAlias}
+        yarn pack --ignore-scripts --filename ${info.name}.tgz
+      '';
+
+      installPhase = untarAndWrap info.name [npmCmd yarnCmd];
+    } // commonEnv // removeAttrs args [ "integreties" ] // {
+      buildInputs = [ _yarn ] ++ commonBuildInputs ++ buildInputs;
+      yarnFlags   = [ "--offline" "--frozen-lockfile" "--non-interactive" ] ++ yarnFlags;
+      npmFlags    = npmFlagsYarn ++ npmFlags;
+    });
+}

mknpmcache.js

@@ -0,0 +1,49 @@
+const assert        = require("assert")
+const fs            = require("fs")
+const pacote        = require("pacote")
+
+const USAGE         = "node mknpmcache.js npm-cache-input.json"
+
+if (process.argv.length != USAGE.split(/\s+/).length) {
+  console.error("Usage:", USAGE)
+  process.exit(1)
+}
+
+const [nixPkgsFile] = process.argv.slice(2)
+
+const pkgLockFile   = "./package-lock.json"
+const lock          = JSON.parse(fs.readFileSync(pkgLockFile, "utf8"))
+const nixPkgs       = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8"))
+
+function traverseDeps(pkg, fn) {
+  Object.values(pkg.dependencies).forEach(dep => {
+    if (dep.resolved && dep.integrity) fn(dep)
+    if (dep.dependencies) traverseDeps(dep, fn)
+  })
+}
+
+async function main(lockfile, nix, cache) {
+  const promises = Object.keys(nix).map(async function (url) {
+    const tar       = nix[url].path
+    const manifest  = await pacote.manifest(tar, { offline: true, cache })
+    return [url, manifest._integrity]
+  })
+  const hashes = new Map(await Promise.all(promises))
+  traverseDeps(lockfile, dep => {
+    if (dep.integrity.startsWith("sha1-")) {
+      assert(hashes.has(dep.resolved))
+      dep.integrity = hashes.get(dep.resolved)
+    } else {
+      assert(dep.integrity == hashes.get(dep.resolved))
+    }
+  })
+  // rewrite lock file to use sha512 hashes from pacote
+  fs.writeFileSync(pkgLockFile, JSON.stringify(lock, null, 2))
+}
+
+process.on("unhandledRejection", error => {
+  console.log("unhandledRejection", error.message)
+  process.exit(1)
+})
+
+main(lock, nixPkgs, "./npm-cache/_cacache")

mkyarncache.js

@@ -0,0 +1,23 @@
+const fs            = require("fs")
+const path          = require("path")
+
+const USAGE         = "node mkyarncache.js yarn-cache-input.json"
+
+if (process.argv.length != USAGE.split(/\s+/).length) {
+  console.error("Usage:", USAGE)
+  process.exit(1)
+}
+
+const [nixPkgsFile] = process.argv.slice(2)
+
+const yarnCacheDir  = "./yarn-cache"
+const nixPkgs       = JSON.parse(fs.readFileSync(nixPkgsFile, "utf8"))
+
+function name(dep) {
+  return dep.name[0] == "@" ? dep.name.split("/")[0] + "-" + dep.bname : dep.bname
+}
+
+Object.keys(nixPkgs).forEach(url => {
+  const dep = nixPkgs[url];
+  fs.symlinkSync(dep.path, path.join(yarnCacheDir, name(dep)))
+})

mkyarnjson.js

@@ -0,0 +1,41 @@
+const assert        = require("assert")
+const fs            = require("fs")
+const lockfile      = require("@yarnpkg/lockfile")
+const ssri          = require("ssri")
+
+const USAGE         = "node mkyarnjson.js yarn.lock integrities.json"
+
+if (process.argv.length != USAGE.split(/\s+/).length) {
+  console.error("Usage:", USAGE)
+  process.exit(1)
+}
+
+const [yarnLockFile, intFile] = process.argv.slice(2)
+
+const yarnJson      = lockfile.parse(fs.readFileSync(yarnLockFile, "utf8")).object
+const integrities   = JSON.parse(fs.readFileSync(intFile))
+
+function splitNameVsn(key) {
+  // foo@vsn or @foo/bar@vsn
+  if (key[0] == "@") {
+    const [name, vsn] = key.slice(1).split("@")
+    return ["@"+name, vsn]
+  } else {
+    return key.split("@")
+  }
+}
+
+const deps = {}
+
+Object.keys(yarnJson).forEach(key => {
+  if (key in deps) return
+  const dep         = yarnJson[key]
+  const [name, vsn] = splitNameVsn(key)
+  const [url, sha1] = dep.resolved.split("#", 2)
+  const integrity   = dep.integrity || integrities[url] ||
+                      (sha1 && ssri.fromHex(sha1, "sha1").toString())
+  assert(integrity, "missing integrity for " + JSON.stringify(dep))
+  deps[key]         = { name, resolved: url, integrity }
+})
+
+console.log(JSON.stringify(deps, null, 2))