Lua generate nix packages from luarocks (#54978)

* lua: generate packages from luarocks

* luarocks-nix: update

* removed packages already available in nixpkgs

* adressing reviews

update script can now accept another csv file as input with -c

* Remove obsolete comment

maintainers/scripts/luarocks-packages.csv

@@ -0,0 +1,21 @@
+ansicolors,
+argparse,
+dkjson
+lrexlib-gnu,
+lrexlib-posix,
+ltermbox,
+lua-cmsgpack,
+lua_cliargs,
+lua-term,
+luaffi,http://luarocks.org/dev,
+luuid,
+penlight,
+say,
+luv,
+luasystem,
+mediator_lua,http://luarocks.org/manifests/teto
+mpack,http://luarocks.org/manifests/teto
+nvim-client,http://luarocks.org/manifests/teto
+busted,http://luarocks.org/manifests/teto
+luassert,http://luarocks.org/manifests/teto
+coxpcall,https://luarocks.org/manifests/hisham,1.17.0-1

maintainers/scripts/update-luarocks-packages

@@ -0,0 +1,112 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -p nix-prefetch-scripts luarocks-nix -i bash
+
+# You'll likely want to use
+# ``
+# nixpkgs $ maintainers/scripts/update-luarocks-packages pkgs/development/lua-modules/generated-packages.nix
+# ``
+# to update all libraries in that folder.
+# to debug, redirect stderr to stdout with 2>&1
+
+
+# stop the script upon C-C
+set -eu -o pipefail
+
+if [ $# -lt 1 ]; then
+	print_help
+    exit 1
+fi
+
+CSV_FILE="maintainers/scripts/luarocks-packages.csv"
+TMP_FILE="$(mktemp)"
+
+exit_trap()
+{
+  local lc="$BASH_COMMAND" rc=$?
+  test $rc -eq 0 || echo -e "*** error $rc: $lc.\nGenerated temporary file in $TMP_FILE" >&2
+}
+trap exit_trap EXIT
+
+print_help() {
+    echo "Usage: $0 <GENERATED_FILE>"
+    echo "(most likely pkgs/development/lua-modules/generated-packages.nix)"
+	echo ""
+	echo " -c <CSV_FILE> to set the list of luarocks package to generate"
+	exit 1
+}
+
+
+while getopts ":hc:" opt; do
+  case $opt in
+    h)
+      print_help
+      ;;
+    c)
+      echo "Loading package list from $OPTARG !" >&2
+      CSV_FILE="$OPTARG"
+      ;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      ;;
+  esac
+  shift $((OPTIND-1))
+done
+
+GENERATED_NIXFILE="$1"
+
+HEADER="
+/* ${GENERATED_NIXFILE} is an auto-generated file -- DO NOT EDIT!
+Regenerate it with:
+nixpkgs$ ${0} ${GENERATED_NIXFILE}
+
+These packages are manually refined in lua-overrides.nix
+*/
+{ self, lua, stdenv, fetchurl, fetchgit, pkgs, ... } @ args:
+self: super:
+with self;
+{
+"
+
+FOOTER="
+}
+/* GENERATED */
+"
+
+
+function convert_pkg () {
+    pkg="$1"
+	server=""
+	if [ ! -z "$2" ]; then
+		server=" --server=$2"
+	fi
+
+    version="${3:-}"
+
+    echo "looking at $pkg (version $version) from server [$server]" >&2
+    cmd="luarocks nix $server $pkg $version"
+    drv="$($cmd)"
+    if [ $? -ne 0 ]; then
+        echo "Failed to convert $pkg" >&2
+        echo "$drv" >&2
+    else
+        echo "$drv" | tee -a "$TMP_FILE"
+    fi
+}
+
+# params needed when called via callPackage
+echo "$HEADER" | tee "$TMP_FILE"
+
+# list of packages with format
+# name,server,version
+while IFS=, read -r pkg_name server version
+do
+	if [ -z "$pkg_name" ]; then
+		echo "Skipping empty package name" >&2
+	fi
+	convert_pkg "$pkg_name" "$server" "$version"
+done < "$CSV_FILE"
+
+# close the set
+echo "$FOOTER" | tee -a "$TMP_FILE"
+
+cp "$TMP_FILE" "$GENERATED_NIXFILE"

pkgs/development/interpreters/lua-5/build-lua-package.nix

@@ -0,0 +1,182 @@
+# Generic builder for lua packages
+{ lib
+, lua
+, stdenv
+, wrapLua
+, unzip
+, writeText
+# Whether the derivation provides a lua module or not.
+, toLuaModule
+}:
+
+{
+name ? "${attrs.pname}-${attrs.version}"
+
+, version
+
+# by default prefix `name` e.g. "lua5.2-${name}"
+, namePrefix ? "lua" + lua.luaversion + "-"
+
+# Dependencies for building the package
+, buildInputs ? []
+
+# Dependencies needed for running the checkPhase.
+# These are added to buildInputs when doCheck = true.
+, checkInputs ? []
+
+# propagate build dependencies so in case we have A -> B -> C,
+# C can import package A propagated by B
+, propagatedBuildInputs ? []
+, propagatedNativeBuildInputs ? []
+
+# used to disable derivation, useful for specific lua versions
+, disabled ? false
+
+# Additional arguments to pass to the makeWrapper function, which wraps
+# generated binaries.
+, makeWrapperArgs ? []
+, external_deps ? propagatedBuildInputs ++ buildInputs
+
+# Skip wrapping of lua programs altogether
+, dontWrapLuaPrograms ? false
+
+, meta ? {}
+
+, passthru ? {}
+, doCheck ? false
+
+# appended to the luarocks generated config
+# in peculiar variables like { EVENT_INCDIR } can be useful to work around
+# luarocks limitations, ie, luarocks consider include/lib folders to be subfolders of the same package in external_deps_dirs
+# as explained in https://github.com/luarocks/luarocks/issues/766
+, extraConfig ? ""
+
+# relative to srcRoot, path to the rockspec to use when using rocks
+, rockspecFilename ?  "../*.rockspec"
+
+# must be set for packages that don't have a rock
+, knownRockspec ? null
+
+, ... } @ attrs:
+
+
+# Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
+if disabled
+then throw "${name} not supported for interpreter ${lua}"
+else
+
+let
+
+  deps_dirs= lib.concatStringsSep ", " (
+    map (x: "\"${builtins.toString x}\"") external_deps
+  );
+
+  # TODO
+  # - add rocktrees (look at torch-distro.nix/https://github.com/luarocks/luarocks/wiki/Config-file-format)
+  # - silence warnings
+  luarocks_config = "luarocksConfig";
+  luarocks_content = ''
+    local_cache = ""
+    -- array of strings
+    external_deps_dirs = {
+    ${deps_dirs}
+    }
+    rocks_trees = {
+    }
+    ${extraConfig}
+  '';
+in
+toLuaModule ( lua.stdenv.mkDerivation (
+builtins.removeAttrs attrs ["disabled" "checkInputs"] // {
+
+  name = namePrefix + name;
+
+  buildInputs = [ wrapLua lua.pkgs.luarocks ]
+    ++ buildInputs
+    ++ lib.optionals doCheck checkInputs
+    ;
+
+  # propagate lua to active setup-hook in nix-shell
+  propagatedBuildInputs = propagatedBuildInputs ++ [ lua ];
+  doCheck = false;
+
+  # enabled only for src.rock
+  setSourceRoot= let
+    name_only=(builtins.parseDrvName name).name;
+  in
+    lib.optionalString (knownRockspec == null) ''
+    # format is rockspec_basename/source_basename
+    # rockspec can set it via spec.source.dir
+    folder=$(find . -mindepth 2 -maxdepth 2 -type d -path '*${name_only}*/*'|head -n1)
+    sourceRoot="$folder"
+  '';
+
+  configurePhase = ''
+    runHook preConfigure
+
+    cat > ${luarocks_config} <<EOF
+    ${luarocks_content}
+    EOF
+    export LUAROCKS_CONFIG=$PWD/${luarocks_config};
+  ''
+  + lib.optionalString (knownRockspec != null) ''
+
+    # prevents the following type of error:
+    # Inconsistency between rockspec filename (42fm1b3d7iv6fcbhgm9674as3jh6y2sh-luv-1.22.0-1.rockspec) and its contents (luv-1.22.0-1.rockspec)
+    rockspecFilename="$TMP/$(stripHash ''${knownRockspec})"
+    cp ''${knownRockspec} $rockspecFilename
+    runHook postConfigure
+  '';
+
+  buildPhase = ''
+    runHook preBuild
+
+    nix_debug "Using LUAROCKS_CONFIG=$LUAROCKS_CONFIG"
+
+    LUAROCKS=luarocks
+    if (( ''${NIX_DEBUG:-0} >= 1 )); then
+        LUAROCKS="$LUAROCKS --verbose"
+    fi
+
+    patchShebangs .
+
+    runHook postBuild
+  '';
+
+  postFixup = lib.optionalString (!dontWrapLuaPrograms) ''
+    wrapLuaPrograms
+  '' + attrs.postFixup or '''';
+
+  installPhase = attrs.installPhase or ''
+    runHook preInstall
+
+    # luarocks make assumes sources are available in cwd
+    # After the build is complete, it also installs the rock.
+    # If no argument is given, it looks for a rockspec in the current directory
+    # but some packages have several rockspecs in their source directory so
+    # we force the use of the upper level since it is
+    # the sole rockspec in that folder
+    # maybe we could reestablish dependency checking via passing --rock-trees
+
+    nix_debug "ROCKSPEC $rockspecFilename"
+    nix_debug "cwd: $PWD"
+    $LUAROCKS make --deps-mode=none --tree $out ''${rockspecFilename}
+
+    # to prevent collisions when creating environments
+    # also added -f as it doesn't always exist
+    # don't remove the whole directory as
+    rm -rf $out/lib/luarocks/rocks/manifest
+
+    runHook postInstall
+  '';
+
+  passthru = {
+    inherit lua; # The lua interpreter
+  } // passthru;
+
+  meta = with lib.maintainers; {
+    platforms = lua.meta.platforms;
+    # add extra maintainer(s) to every package
+    maintainers = (meta.maintainers or []) ++ [ ];
+  } // meta;
+}))

pkgs/development/interpreters/lua-5/setup-hook.sh

@@ -1,7 +1,7 @@
 # set -e
 
 nix_print() {
-    if (( "${NIX_DEBUG:-0}" >= $1 )); then
+    if [ ${NIX_DEBUG:-0} -ge $1 ]; then
         echo "$2"
     fi
 }
@@ -32,13 +32,13 @@ addToLuaPath() {
     cd "$dir"
     for pattern in @luapathsearchpaths@;
     do
-        addToLuaSearchPathWithCustomDelimiter LUA_PATH "$PWD/$pattern"
+        addToLuaSearchPathWithCustomDelimiter NIX_LUA_PATH "$PWD/$pattern"
     done
 
     # LUA_CPATH
     for pattern in @luacpathsearchpaths@;
     do
-        addToLuaSearchPathWithCustomDelimiter LUA_CPATH "$PWD/$pattern"
+        addToLuaSearchPathWithCustomDelimiter NIX_LUA_CPATH "$PWD/$pattern"
     done
     cd - >/dev/null
 }

pkgs/development/interpreters/lua-5/wrap-lua.nix

@@ -0,0 +1,19 @@
+{ lib
+, lua
+, makeSetupHook
+, makeWrapper
+}:
+
+with lib;
+
+# defined in trivial-builders.nix
+# imported as wrapLua in lua-packages.nix and passed to build-lua-derivation to be used as buildInput
+makeSetupHook {
+      deps = makeWrapper;
+      substitutions.executable = lua.interpreter;
+      substitutions.lua = lua;
+      substitutions.LuaPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
+      substitutions.LuaCPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
+
+} ./wrap.sh
+