node-gyp@5.1.0 Release Proposal

.github/workflows/Python_tests.yml

@@ -0,0 +1,40 @@
+# TODO: Line 15, enable python-version: 3.5
+# TODO: Line 36, enable pytest --doctest-modules
+
+name: Python_tests
+on: [push, pull_request]
+jobs:
+  Python_tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      max-parallel: 15
+      matrix:
+        os: [macos-latest, ubuntu-latest, windows-latest]
+        python-version: [2.7, 3.6, 3.7, 3.8]  # 3.5,
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest  # -r requirements.txt
+      - name: Lint with flake8
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest (Linux and macOS)
+        if: matrix.os != 'windows-latest'
+        run: pytest
+      - name: Test with pytest (Windows)
+        if: matrix.os == 'windows-latest'
+        shell: bash
+        run: GYP_MSVS_VERSION=2015 GYP_MSVS_OVERRIDE_PATH="C:\\Dummy" pytest
+      # - name: Run doctests with pytest
+      #   run: pytest --doctest-modules

.travis.yml

@@ -1,33 +1,17 @@
 dist: xenial
 language: python
 cache: pip
-matrix:
+addons:
+  homebrew:
+    update: true
+    packages:
+    - npm
+    - pyenv
+jobs:
   include:
     - name: "Python 2.7 on Linux"
       env: NODE_GYP_FORCE_PYTHON=python2
       python: 2.7
-    - name: "Python 2.7 on macOS"
-      os: osx
-      osx_image: xcode11.2
-      language: shell  # 'language: python' is not yet supported on macOS
-      env: NODE_GYP_FORCE_PYTHON=python2
-      before_install: HOMEBREW_NO_AUTO_UPDATE=1 brew install npm
-    - name: "Node.js 6 & Python 2.7 on Windows"
-      os: windows
-      language: node_js
-      node_js: 6  # node
-      env: >-
-        PATH=/c/Python27:/c/Python27/Scripts:$PATH
-        NODE_GYP_FORCE_PYTHON=/c/Python27/python.exe
-      before_install: choco install python2
-    - name: "Node.js 12 & Python 2.7 on Windows"
-      os: windows
-      language: node_js
-      node_js: 12  # node
-      env: >-
-        PATH=/c/Python27:/c/Python27/Scripts:$PATH
-        NODE_GYP_FORCE_PYTHON=/c/Python27/python.exe
-      before_install: choco install python2
 
     - name: "Node.js 6 & Python 3.8 on Linux"
       python: 3.8
@@ -58,12 +42,36 @@ matrix:
       env: NODE_GYP_FORCE_PYTHON=python3
       before_install: nvm install 12
 
-    - name: "Python 3.7 on macOS"
+    - name: "Python 2.7 on macOS"
       os: osx
       osx_image: xcode11.2
       language: shell  # 'language: python' is not yet supported on macOS
-      env: NODE_GYP_FORCE_PYTHON=python3
-      before_install: HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade npm || true
+      env: NODE_GYP_FORCE_PYTHON=python2 PATH=$HOME/.pyenv/shims:$PATH PYENV_VERSION=2.7.17
+      before_install: pyenv install $PYENV_VERSION
+    - name: "Python 3.8 on macOS"
+      os: osx
+      osx_image: xcode11.2
+      language: shell  # 'language: python' is not yet supported on macOS
+      env: NODE_GYP_FORCE_PYTHON=python3 PATH=$HOME/.pyenv/shims:$PATH PYENV_VERSION=3.8.0
+      before_install: pyenv install $PYENV_VERSION
+
+    - name: "Node.js 6 & Python 2.7 on Windows"
+      os: windows
+      language: node_js
+      node_js: 6  # node
+      env: >-
+        PATH=/c/Python27:/c/Python27/Scripts:$PATH
+        NODE_GYP_FORCE_PYTHON=/c/Python27/python.exe
+      before_install: choco install python2
+    - name: "Node.js 12 & Python 2.7 on Windows"
+      os: windows
+      language: node_js
+      node_js: 12  # node
+      env: >-
+        PATH=/c/Python27:/c/Python27/Scripts:$PATH
+        NODE_GYP_FORCE_PYTHON=/c/Python27/python.exe
+      before_install: choco install python2
+
     - name: "Node.js 12 & Python 3.7 on Windows"
       os: windows
       language: node_js
@@ -82,20 +90,19 @@ matrix:
       before_install: choco install python
 
 install:
-  #- pip install -r requirements.txt
-  - pip install --upgrade flake8 pytest==4.6.6  # pytest 5 no longer supports legacy Python
+  - python -m pip install --upgrade flake8 pytest==4.6.6  # pytest 5 no longer supports legacy Python
 before_script:
-  - flake8 --version
+  - python -m flake8 --version
   # stop the build if there are Python syntax errors or undefined names
-  - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+  - python -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
   # exit-zero treats all errors as warnings.  Two space indentation is OK.  The GitHub editor is 127 chars wide
-  - flake8 . --count --exit-zero --ignore=E111,E114,W503 --max-complexity=10 --max-line-length=127 --statistics
+  - python -m flake8 . --count --exit-zero --ignore=E111,E114,W503 --max-complexity=10 --max-line-length=127 --statistics
   - npm install
   - npm list
 script:
   - node -e 'require("npmlog").level="verbose"; require("./lib/find-python")(null,()=>{})'
   - npm test
-  - GYP_MSVS_VERSION=2015 GYP_MSVS_OVERRIDE_PATH="C:\\Dummy" pytest
+  - GYP_MSVS_VERSION=2015 GYP_MSVS_OVERRIDE_PATH="C:\\Dummy" python -m pytest
 notifications:
   on_success: change
   on_failure: change  # `always` will be the setting once code changes slow down

CHANGELOG.md

@@ -1,3 +1,8 @@
+v5.0.7 2019-12-16
+=================
+
+Republish of v5.0.6 with unnecessary tarball removed from pack file.
+
 v5.0.6 2019-12-16
 =================
 

README.md

@@ -1,5 +1,8 @@
 # `node-gyp` - Node.js native addon build tool
 
+[![Travis CI](https://travis-ci.com/nodejs/node-gyp.svg?branch=master)](https://travis-ci.com/nodejs/node-gyp)
+[![Build Status](https://github.com/nodejs/node-gyp/workflows/Python_tests/badge.svg)](https://github.com/nodejs/node-gyp/actions?workflow=Python_tests)
+
 `node-gyp` is a cross-platform command-line tool written in Node.js for
 compiling native addon modules for Node.js. It contains a fork of the
 [gyp](https://gyp.gsrc.io) project that was previously used by the Chromium
@@ -14,7 +17,7 @@ etc.), regardless of what version of Node.js is actually installed on your syste
 ## Features
 
  * The same build commands work on any of the supported platforms
- * Supports the targetting of different versions of Node.js
+ * Supports the targeting of different versions of Node.js
 
 ## Installation
 
@@ -191,7 +194,8 @@ Some additional resources for Node.js native addons and writing `gyp` configurat
 | `--devdir=$path`                  | SDK download directory (default is OS cache directory)
 | `--ensure`                        | Don't reinstall headers if already present
 | `--dist-url=$url`                 | Download header tarball from custom URL
-| `--proxy=$url`                    | Set HTTP proxy for downloading header tarball
+| `--proxy=$url`                    | Set HTTP(S) proxy for downloading header tarball
+| `--noproxy=$urls`                 | Set urls to ignore proxies when downloading header tarball
 | `--cafile=$cafile`                | Override default CA chain (to download tarball)
 | `--nodedir=$path`                 | Set the path to the node source code
 | `--python=$path`                  | Set path to the Python binary

lib/install.js

@@ -11,6 +11,7 @@ const request = require('request')
 const mkdir = require('mkdirp')
 const processRelease = require('./process-release')
 const win = process.platform === 'win32'
+const getProxyFromURI = require('./proxy')
 
 function install (fs, gyp, argv, callback) {
   var release = processRelease(argv, gyp, process.version, process.release)
@@ -410,10 +411,7 @@ function download (gyp, env, url) {
   }
 
   // basic support for a proxy server
-  var proxyUrl = gyp.opts.proxy ||
-              env.http_proxy ||
-              env.HTTP_PROXY ||
-              env.npm_config_proxy
+  var proxyUrl = getProxyFromURI(gyp, env, url)
   if (proxyUrl) {
     if (/^https?:\/\//i.test(proxyUrl)) {
       log.verbose('download', 'using proxy url: "%s"', proxyUrl)

lib/node-gyp.js

@@ -67,6 +67,7 @@ proto.configDefs = {
   ensure: Boolean, // 'install'
   solution: String, // 'build' (windows only)
   proxy: String, // 'install'
+  noproxy: String, // 'install'
   devdir: String, // everywhere
   nodedir: String, // 'configure'
   loglevel: String, // everywhere

lib/proxy.js

@@ -0,0 +1,92 @@
+'use strict'
+// Taken from https://github.com/request/request/blob/212570b/lib/getProxyFromURI.js
+
+const url = require('url')
+
+function formatHostname (hostname) {
+  // canonicalize the hostname, so that 'oogle.com' won't match 'google.com'
+  return hostname.replace(/^\.*/, '.').toLowerCase()
+}
+
+function parseNoProxyZone (zone) {
+  zone = zone.trim().toLowerCase()
+
+  var zoneParts = zone.split(':', 2)
+  var zoneHost = formatHostname(zoneParts[0])
+  var zonePort = zoneParts[1]
+  var hasPort = zone.indexOf(':') > -1
+
+  return { hostname: zoneHost, port: zonePort, hasPort: hasPort }
+}
+
+function uriInNoProxy (uri, noProxy) {
+  var port = uri.port || (uri.protocol === 'https:' ? '443' : '80')
+  var hostname = formatHostname(uri.hostname)
+  var noProxyList = noProxy.split(',')
+
+  // iterate through the noProxyList until it finds a match.
+  return noProxyList.map(parseNoProxyZone).some(function (noProxyZone) {
+    var isMatchedAt = hostname.indexOf(noProxyZone.hostname)
+    var hostnameMatched = (
+      isMatchedAt > -1 &&
+        (isMatchedAt === hostname.length - noProxyZone.hostname.length)
+    )
+
+    if (noProxyZone.hasPort) {
+      return (port === noProxyZone.port) && hostnameMatched
+    }
+
+    return hostnameMatched
+  })
+}
+
+function getProxyFromURI (gyp, env, uri) {
+  // If a string URI/URL was given, parse it into a URL object
+  if (typeof uri === 'string') {
+    // eslint-disable-next-line
+    uri = url.parse(uri)
+  }
+
+  // Decide the proper request proxy to use based on the request URI object and the
+  // environmental variables (NO_PROXY, HTTP_PROXY, etc.)
+  // respect NO_PROXY environment variables (see: https://lynx.invisible-island.net/lynx2.8.7/breakout/lynx_help/keystrokes/environments.html)
+
+  var noProxy = gyp.opts.noproxy || env.NO_PROXY || env.no_proxy || env.npm_config_noproxy || ''
+
+  // if the noProxy is a wildcard then return null
+
+  if (noProxy === '*') {
+    return null
+  }
+
+  // if the noProxy is not empty and the uri is found return null
+
+  if (noProxy !== '' && uriInNoProxy(uri, noProxy)) {
+    return null
+  }
+
+  // Check for HTTP or HTTPS Proxy in environment Else default to null
+
+  if (uri.protocol === 'http:') {
+    return gyp.opts.proxy ||
+      env.HTTP_PROXY ||
+      env.http_proxy ||
+      env.npm_config_proxy || null
+  }
+
+  if (uri.protocol === 'https:') {
+    return gyp.opts.proxy ||
+      env.HTTPS_PROXY ||
+      env.https_proxy ||
+      env.HTTP_PROXY ||
+      env.http_proxy ||
+      env.npm_config_proxy || null
+  }
+
+  // if none of that works, return null
+  // (What uri protocol are you using then?)
+
+  return null
+}
+
+module.exports = getProxyFromURI

macOS_Catalina.md

@@ -1,39 +1,37 @@
 # Installation notes for macOS Catalina (v10.15)
 
-_This document specifically refers to upgrades from previous versions of macOS to Catalina (10.15). It should be removed from the source repository when Catalina ceases to be the latest macOS version or updated to deal with challenges involved in upgrades to the next version of macOS._
+_This document specifically refers to upgrades from previous versions of macOS to Catalina (10.15). It should be removed from the source repository when Catalina ceases to be the latest macOS version or when future Catalina versions no longer raise these issues._
 
-Lessons learned from:
-* https://github.com/nodejs/node-gyp/issues/1779
-* https://github.com/nodejs/node-gyp/issues/1861
-* https://github.com/nodejs/node-gyp/issues/1927 and elsewhere
-
-Installing `node-gyp` on macOS can be found at https://github.com/nodejs/node-gyp#on-macos
-
-However, upgrading to macOS Catalina changes some settings that may cause normal `node-gyp` installations to fail.
+**Upgrading to macOS Catalina may cause normal `node-gyp` installations to fail.**
 
 ### Is my Mac running macOS Catalina?
-Let's make first make sure that your Mac is currently running Catalina:
-% `sw_vers`
+Let's first make sure that your Mac is running Catalina:
+```
+% sw_vers
     ProductName:	Mac OS X
     ProductVersion:	10.15
     BuildVersion:	19A602
+```
+If `ProductVersion` is less then `10.15` then this document is not for you. Normal install docs for `node-gyp` on macOS can be found at https://github.com/nodejs/node-gyp#on-macos
 
-If `ProductVersion` is less then `10.15` then this document is not really for you.
 
 ### The acid test
-Next, lets see if `Xcode Command Line Tools` are installed:
+To see if `Xcode Command Line Tools` is installed in a way that will work with `node-gyp`, run:
 1. `/usr/sbin/pkgutil --packages | grep CL`
-    * If nothing is listed, then [skip to the next section](#Two-roads).
-    * If `com.apple.pkg.CLTools_Executables` is listed then try:
+    * `com.apple.pkg.CLTools_Executables` should be listed. If it isn't, this test failed.
 2. `/usr/sbin/pkgutil --pkg-info com.apple.pkg.CLTools_Executables`
-    * If `version: 11.0.0` or later is listed then _you are done_!  Your Mac should be ready to install `node-gyp`.  Doing `clang -v` should show `Apple clang version 11.0.0` or later.
+    * `version: 11.0.0` (or later) should be listed. If it isn't, this test failed.
+
+If both tests succeeded, _you are done_! You should be ready to install `node-gyp`.
+
+If either test failed, there is a problem with your Xcode Command Line Tools installation. [Continue to Solutions](#Solutions). 
 
-As you go through the remainder of this document, at anytime you can try these `acid test` commands. If they pass then your Mac should be ready to install `node-gyp`.
+### Solutions
+There are three ways to install the Xcode libraries `node-gyp` needs on macOS. People running Catalina have had success with some but not others in a way that has been unpredictable. 
 
-### Two roads
-There are two main ways to install `node-gyp` on macOS:
 1. With the full Xcode (~7.6 GB download) from the `App Store` app.
 2. With the _much_ smaller Xcode Command Line Tools via `xcode-select --install`
+3. With the _much_ smaller Xcode Command Line Tools via manual download. **For people running the latest version of Catalina (10.15.2 at the time of this writing), this has worked when the other two solutions haven't.**
 
 ### Installing `node-gyp` using the full Xcode
 1. `xcodebuild -version` should show `Xcode 11.1` or later.
@@ -52,7 +50,7 @@ There are two main ways to install `node-gyp` on macOS:
 10. `sudo xcode-select --reset`  # Enter root password.  No output is normal.
 11. Repeat step 7 above.  Is the path different this time?  Repeat the _acid test_.
 
-### Installing `node-gyp` using the Xcode Command Line Tools
+### Installing `node-gyp` using the Xcode Command Line Tools via `xcode-select --install`
 1. If the _acid test_ has not succeeded, then try `xcode-select --install`
 2. Wait until the install process is _complete_.
 3. `softwareupdate -l`  # No listing is a good sign.
@@ -64,6 +62,11 @@ There are two main ways to install `node-gyp` on macOS:
 8. `sudo xcode-select --reset`  # Enter root password.  No output is normal.
 9. Repeat step 5 above.  Is the path different this time?  Repeat the _acid test_.
 
+### Installing `node-gyp` using the Xcode Command Line Tools via manual download
+1. Download the appropriate version of the "Command Line Tools for Xcode" for your version of Catalina from developer.apple.com/download. As of MacOS 10.15.2, that's Command_Line_Tools_for_Xcode_11.3.dmg
+2. Install the package.
+3. Run the _acid test_.
+
 ### I did all that and the acid test still does not pass :-(
 1. `sudo rm -rf $(xcode-select -print-path)`  # Enter root password.  No output is normal.
 2. `xcode-select --install`
@@ -72,3 +75,9 @@ There are two main ways to install `node-gyp` on macOS:
 5. `npm explore npm -g -- npm explore npm-lifecycle -- npm install node-gyp@latest`
 6. If the _acid test_ still does _not_ pass then...
 7. Add a comment to https://github.com/nodejs/node-gyp/issues/1927 so we can improve.
+
+Lessons learned from:
+* https://github.com/nodejs/node-gyp/issues/1779
+* https://github.com/nodejs/node-gyp/issues/1861
+* https://github.com/nodejs/node-gyp/issues/1927 and elsewhere
+* Thanks to @rrrix for discovering Solution 3