Merge branch 'master' into dependabot/add-v2-config-file

CHANGELOG.md

@@ -1,3 +1,31 @@
+# [6.15.0] / 2021-12-17
+
+### Fixed
+* Fix docker Conan and swift - [66031df9](https://github.com/pivotal/LicenseFinder/commit/66031df912c2e1e21aa794a4b897fc61c9ec6b02) 
+
+# [6.14.2] / 2021-10-27
+
+### Added
+* Zlib License - [0f004b52](https://github.com/pivotal/LicenseFinder/commit/0f004b528d436b4d53db8bd373ede0594c07d9e8) - blooper05
+
+# [6.14.1] / 2021-06-25
+
+First two commit were supposed to show up in v6.14.0, but GPG bug prevented a correct build. Therefore, a follow up patch build was made to include the GPG fix.
+
+### Changed
+* Upgrade Docker image to use Ubuntu Bionic [#178471230] [1c12588c](https://github.com/pivotal/LicenseFinder/commit/1c12588cceecb8b7350d090c85b519b24bcc6682)
+* Update the default timezone to GMT [#178471230] - [9fcab84](https://github.com/pivotal/LicenseFinder/commit/9fcab84605cda81e7f276d3c567d14409e371333)
+* Use local copy of Swift puglic GPG keys [#178674224] - [4db4b3e](https://github.com/pivotal/LicenseFinder/commit/4db4b3e5980ca52019549d74da574a2342a7846e)
+
+### Added 
+* Added --npm_options option to customize npm behavior. [b8457a62](https://github.com/pivotal/LicenseFinder/commit/b8457a62e7b531294934364d1e5f72cd78a7686a) - Alexander-Malott 
+
+### Security
+* Fix issue where commands could be injected running on Cocoapods projects. [b0a61a2d](https://github.com/pivotal/LicenseFinder/commit/b0a61a2d833921c714cc39cdda8ba80af3f33d04)
+
+  Thanks to Joern SchneeweiszStaff Security Engineer, Security Research | GitLab for raising the issue
+
+
 # [6.13.0] / 2021-04-27
 
 ### Fixed
@@ -954,3 +982,6 @@ Bugfixes:
 [6.12.1]: https://github.com/pivotal/LicenseFinder/compare/v6.12.0...v6.12.1
 [6.12.2]: https://github.com/pivotal/LicenseFinder/compare/v6.12.1...v6.12.2
 [6.13.0]: https://github.com/pivotal/LicenseFinder/compare/v6.12.2...v6.13.0
+[6.14.1]: https://github.com/pivotal/LicenseFinder/compare/v6.13.0...v6.14.1
+[6.14.2]: https://github.com/pivotal/LicenseFinder/compare/v6.14.1...v6.14.2
+[6.15.0]: https://github.com/pivotal/LicenseFinder/compare/v6.14.2...v6.15.0

Dockerfile

@@ -1,4 +1,6 @@
-FROM ubuntu:xenial
+FROM ubuntu:bionic
+
+WORKDIR /tmp
 
 # Versioning
 ENV PIP_INSTALL_VERSION 19.0.2
@@ -18,7 +20,8 @@ RUN apt-get update && apt-get install -y \
   sudo \
   unzip \
   wget \
-  gnupg2 \ 
+  gnupg2 \
+  apt-utils \
   software-properties-common \
   bzr
 
@@ -76,7 +79,6 @@ RUN mkdir -p /usr/local/share/sbt-launcher-packaging && \
     rm -f "/tmp/sbt-${SBT_VERSION}.tgz"
 
 # install gradle
-WORKDIR /tmp
 RUN curl -L -o gradle.zip https://services.gradle.org/distributions/gradle-$GRADLE_VERSION-bin.zip && \
     unzip -q gradle.zip && \
     rm gradle.zip && \
@@ -102,18 +104,24 @@ RUN mkdir /gopath && \
   go get -u github.com/rancher/trash && \
   go clean -cache
 
+WORKDIR /tmp
 # Fix the locale
 RUN apt-get install -y locales
 RUN locale-gen en_US.UTF-8
 ENV LANG=en_US.UTF-8
 ENV LANGUAGE=en_US:en
 ENV LC_ALL=en_US.UTF-8
 
+# install Cargo
+RUN curl https://sh.rustup.rs -sSf | bash -ls -- -y --profile minimal
+
 #install rvm
 RUN apt-add-repository -y ppa:rael-gc/rvm && \
     apt update && apt install -y rvm && \
     /usr/share/rvm/bin/rvm install --default $RUBY_VERSION
-ENV PATH=/usr/share/rvm/bin:$PATH
+
+# install bundler
+RUN bash -lc "gem update --system && gem install bundler"
 
 #install mix
 RUN wget https://packages.erlang-solutions.com/erlang-solutions_${MIX_VERSION}_all.deb && \
@@ -123,41 +131,40 @@ RUN wget https://packages.erlang-solutions.com/erlang-solutions_${MIX_VERSION}_a
     sudo apt-get install -y esl-erlang && \
     sudo apt-get install -y elixir
 
-# install bundler
-RUN bash -lc "gem update --system && gem install bundler"
-
 # install conan
 RUN apt-get install -y python-dev && \
 	pip install --no-cache-dir --ignore-installed six --ignore-installed colorama \
 	    --ignore-installed requests --ignore-installed chardet \
 	    --ignore-installed urllib3 \
 	    --upgrade setuptools && \
-    pip install --no-cache-dir -Iv conan==1.11.2
+    pip install --no-cache-dir -Iv conan==1.43.0 && \
+    conan config install https://github.com/conan-io/conanclientcert.git
 
-# install Cargo
-RUN curl https://sh.rustup.rs -sSf | bash -s -- -y --profile minimal
 
 # install NuGet (w. mono)
 # https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools#macoslinux
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF &&\
-  echo "deb https://download.mono-project.com/repo/ubuntu stable-xenial main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list &&\
+  echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list &&\
   apt-get update &&\
   apt-get install -y mono-complete &&\
   curl -o "/usr/local/bin/nuget.exe" "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" &&\
   curl -o "/usr/local/bin/nugetv3.5.0.exe" "https://dist.nuget.org/win-x86-commandline/v3.5.0/nuget.exe"
 
 # install dotnet core
-WORKDIR /tmp
-RUN wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb &&\
+RUN wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb &&\
   sudo dpkg -i packages-microsoft-prod.deb &&\
   rm packages-microsoft-prod.deb &&\
   sudo apt-get update &&\
   sudo apt-get install -y dotnet-runtime-2.1 dotnet-sdk-2.1 dotnet-sdk-2.2 dotnet-sdk-3.0 dotnet-sdk-3.1
 
 # install Composer
+# The ARG and ENV are for installing tzdata which is part of this installaion.
+# https://serverfault.com/questions/949991/how-to-install-tzdata-on-a-ubuntu-docker-image
+ENV TZ=GMT
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 4F4EA0AAE5267A6C &&\
-    echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu xenial main" | sudo tee /etc/apt/sources.list.d/php.list &&\
+    echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu bionic main" | sudo tee /etc/apt/sources.list.d/php.list &&\
     apt-get update &&\
+    export DEBIAN_FRONTEND=noninteractive &&\
     apt-get install -y php7.4-cli &&\
     EXPECTED_COMPOSER_INSTALLER_CHECKSUM="$(curl --silent https://composer.github.io/installer.sig)" &&\
     php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" &&\
@@ -170,7 +177,6 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 4F4EA0AAE5
 # install miniconda
 # See https://docs.conda.io/en/latest/miniconda_hashes.html
 # for latest versions and SHAs.
-WORKDIR /tmp
 RUN  \
   conda_installer=Miniconda3-py38_4.9.2-Linux-x86_64.sh &&\
   ref='1314b90489f154602fd794accfc90446111514a5a72fe1f71ab83e07de9504a7' &&\
@@ -180,10 +186,12 @@ RUN  \
   (echo; echo "yes") | sh "${conda_installer}"
 
 # install Swift Package Manager
-# Based on https://github.com/apple/swift-docker/blob/main/5.3/ubuntu/16.04/Dockerfile
+# Based on https://github.com/apple/swift-docker/blob/main/5.3/ubuntu/18.04/Dockerfile
+# The GPG download steps has been modified. Keys are now on LF repo and copied instaad of downloaded.
+# Refer to https://swift.org/download/#using-downloads in the Linux section on how to download the keys
 RUN apt-get -q install -y \
     libatomic1 \
-    libcurl3 \
+    libcurl4 \
     libxml2 \
     libedit2 \
     libsqlite3-0 \
@@ -194,32 +202,37 @@ RUN apt-get -q install -y \
     zlib1g-dev \
     libpython2.7 \
     tzdata \
+    git \
     pkg-config \
     && rm -r /var/lib/apt/lists/*
 
-# pub   4096R/ED3D1561 2019-03-22 [expires: 2021-03-21]
+# pub   4096R/ED3D1561 2019-03-22 [SC] [expires: 2023-03-23]
 #       Key fingerprint = A62A E125 BBBF BB96 A6E0  42EC 925C C1CC ED3D 1561
 # uid                  Swift 5.x Release Signing Key <swift-infrastructure@swift.org
 ARG SWIFT_SIGNING_KEY=A62AE125BBBFBB96A6E042EC925CC1CCED3D1561
-ARG SWIFT_PLATFORM=ubuntu16.04
+ARG SWIFT_PLATFORM=ubuntu18.04
 ARG SWIFT_BRANCH=swift-5.3.3-release
 ARG SWIFT_VERSION=swift-5.3.3-RELEASE
-ARG SWIFT_WEBROOT=https://swift.org/builds/
+ARG SWIFT_WEBROOT=https://download.swift.org
 
 ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \
     SWIFT_PLATFORM=$SWIFT_PLATFORM \
     SWIFT_BRANCH=$SWIFT_BRANCH \
     SWIFT_VERSION=$SWIFT_VERSION \
     SWIFT_WEBROOT=$SWIFT_WEBROOT
 
+COPY swift-all-keys.asc .
 RUN set -e; \
-    SWIFT_WEBDIR="$SWIFT_WEBROOT/$SWIFT_BRANCH/$(echo $SWIFT_PLATFORM | tr -d .)/" \
+    SWIFT_WEBDIR="$SWIFT_WEBROOT/$SWIFT_BRANCH/$(echo $SWIFT_PLATFORM | tr -d .)" \
     && SWIFT_BIN_URL="$SWIFT_WEBDIR/$SWIFT_VERSION/$SWIFT_VERSION-$SWIFT_PLATFORM.tar.gz" \
     && SWIFT_SIG_URL="$SWIFT_BIN_URL.sig" \
+    # - Grab curl here so we cache better up above
+    && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -q update && apt-get -q install -y curl && rm -rf /var/lib/apt/lists/* \
     # - Download the GPG keys, Swift toolchain, and toolchain signature, and verify.
     && export GNUPGHOME="$(mktemp -d)" \
     && curl -fsSL "$SWIFT_BIN_URL" -o swift.tar.gz "$SWIFT_SIG_URL" -o swift.tar.gz.sig \
-    && gpg --batch --quiet --keyserver ha.pool.sks-keyservers.net --recv-keys "$SWIFT_SIGNING_KEY" \
+    && gpg --import swift-all-keys.asc \
     && gpg --batch --verify swift.tar.gz.sig swift.tar.gz \
     # - Unpack the toolchain, set libs permissions, and clean up.
     && tar -xzf swift.tar.gz --directory / --strip-components=1 \

README.md

@@ -6,9 +6,9 @@ Build status
 * Ruby 2.3.8 [![Ruby 2.3.8 build status](https://norsk.cf-app.com/api/v1/teams/main/pipelines/LicenseFinder/jobs/ruby-2.3.8/badge)](https://norsk.cf-app.com/teams/main/pipelines/LicenseFinder)
 * Ruby 2.4.9 [![Ruby 2.4.9 build status](https://norsk.cf-app.com/api/v1/teams/main/pipelines/LicenseFinder/jobs/ruby-2.4.9/badge)](https://norsk.cf-app.com/teams/main/pipelines/LicenseFinder)
 * Ruby 2.5.7 [![Ruby 2.5.7 build status](https://norsk.cf-app.com/api/v1/teams/main/pipelines/LicenseFinder/jobs/ruby-2.5.7/badge)](https://norsk.cf-app.com/teams/main/pipelines/LicenseFinder)
-* Ruby 2.6.5 [![Ruby 2.6.5 build status](https://norsk.cf-app.com/api/v1/teams/main/pipelines/LicenseFinder/jobs/ruby-2.6.5/badge)](https://norsk.cf-app.com/teams/main/pipelines/LicenseFinder)
-* Ruby 2.7.1 [![Ruby 2.7.1 build status](https://norsk.cf-app.com/api/v1/teams/main/pipelines/LicenseFinder/jobs/ruby-2.7.1/badge)](https://norsk.cf-app.com/teams/main/pipelines/LicenseFinder)
-* JRuby 9.2.9.0 [![JRuby 9.2.9.0 build status](https://norsk.cf-app.com/api/v1/teams/main/pipelines/LicenseFinder/jobs/ruby-jruby-9.2.9.0/badge)](https://norsk.cf-app.com/teams/main/pipelines/LicenseFinder)
+* Ruby 2.6.9 [![Ruby 2.6.9 build status](https://norsk.cf-app.com/api/v1/teams/main/pipelines/LicenseFinder/jobs/ruby-2.6.9/badge)](https://norsk.cf-app.com/teams/main/pipelines/LicenseFinder)
+* Ruby 2.7.5 [![Ruby 2.7.5 build status](https://norsk.cf-app.com/api/v1/teams/main/pipelines/LicenseFinder/jobs/ruby-2.7.5/badge)](https://norsk.cf-app.com/teams/main/pipelines/LicenseFinder)
+* JRuby 9.3.1.0 [![JRuby 9.3.1.0 build status](https://norsk.cf-app.com/api/v1/teams/main/pipelines/LicenseFinder/jobs/ruby-jruby-9.3.1.0/badge)](https://norsk.cf-app.com/teams/main/pipelines/LicenseFinder)
 
 
 LicenseFinder works with your package managers to find dependencies,

Rakefile

@@ -54,7 +54,7 @@ task :update_pipeline, [:slack_url, :slack_channel] do |_, args|
     puts 'Warning: You should provide slack channel and url to receive slack notifications on build failures'
   end
 
-  ruby_versions = %w[2.7.1 2.6.5 2.5.7 2.4.9 2.3.8 jruby-9.2.14.0]
+  ruby_versions = %w[2.7.5 2.6.9 2.5.7 2.4.9 2.3.8 jruby-9.3.1.0]
 
   params = []
   params << "ruby_versions=#{ruby_versions.join(',')}"

VERSION

@@ -1 +1 @@
-6.13.0
+6.15.0

bin/license_finder_pip.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-
 import json
 import sys
 
@@ -21,12 +20,16 @@
 from pip._vendor import pkg_resources
 from pip._vendor.six import print_
 
+
 reqs = []
 for req in parse_requirements(sys.argv[1], session=PipSession()):
-    if req.req == None or (req.markers != None and not req.markers.evaluate()): continue
-    reqs.append(req)
-
-requirements = [pkg_resources.Requirement.parse(str(req.req)) for req in reqs]
+    try:
+        if req.req is not None and (req.markers is None or req.markers.evaluate()):
+            reqs.append(pkg_resources.Requirement.parse(str(req.req)))
+    except AttributeError:
+        # Since pip 20.1 (pip now takes care of markers at the resolve step)
+        if req.requirement is not None:
+            reqs.append(pkg_resources.Requirement.parse(str(req.requirement)))
 
 transform = lambda dist: {
         'name': dist.project_name,
@@ -35,7 +38,6 @@
         'dependencies': list(map(lambda dependency: dependency.project_name, dist.requires())),
         }
 
-packages = [transform(dist) for dist
-            in pkg_resources.working_set.resolve(requirements)]
 
+packages = [transform(dist) for dist in pkg_resources.working_set.resolve(reqs)]
 print_(json.dumps(packages))

ci/pipelines/release.yml.erb

@@ -156,7 +156,7 @@ jobs:
   plan:
   - get: lf-git
     tags: ["private-worker"]
-    passed: [<%= "#{ruby_versions.map{ |version| "ruby-#{version}" unless version == "jruby-9.2.14.0" }.compact.join(', ') }, rubocop" %>]
+    passed: [<%= "#{ruby_versions.map{ |version| "ruby-#{version}" unless version == "jruby-9.3.1.0" }.compact.join(', ') }, rubocop" %>]
   - get: semver-version
     tags: ["private-worker"]
     trigger: true

ci/scripts/run-tests.sh

@@ -7,13 +7,17 @@ PROJECT_ROOT="$( dirname "$( dirname $DIR )" )"
 
 pushd "$PROJECT_ROOT"
 
-  rvm install --default $RUBY_VERSION_UNDER_TEST
+  gem update --system
+  # Since we update the system gem, we need to ensure that RVM
+  # re-installs requested ruby version to ensure it is
+  # installed correctly again. If the ruby version is does not exist,
+  # re-install will install it.
+  rvm reinstall --default $RUBY_VERSION_UNDER_TEST
   ruby --version
 
   export GOPATH=$HOME/go
   export RUBYOPT='-E utf-8'
 
-  gem update --system
   gem install bundler
   bundle install
 

dlf

@@ -1,10 +1,14 @@
 #!/bin/bash
 if `which docker > /dev/null`; then
-	if [ $# -eq 0 ]; then
-  		docker run -v $PWD:/scan -it licensefinder/license_finder
-	else
-  		docker run -v $PWD:/scan -it licensefinder/license_finder /bin/bash -lc "cd /scan && `echo $@`"
-	fi
+  if [ $# -eq 0 ]; then
+    docker run -v $PWD:/scan -it licensefinder/license_finder
+  else
+    escaped_params=""
+    for p in "$@"; do
+      escaped_params="$escaped_params \"$p\""
+    done
+    docker run -v $PWD:/scan -it licensefinder/license_finder /bin/bash -lc "cd /scan && $escaped_params"
+  fi
 else
   echo "You do not have docker installed. Please install it:"
   echo "    https://docs.docker.com/engine/installation/"

features/features/package_managers/conan_spec.rb

@@ -8,6 +8,6 @@
   specify 'are shown in reports for a project' do
     LicenseFinder::TestingDSL::ConanProject.create
     conan_developer.run_license_finder
-    expect(conan_developer).to be_seeing_line 'range-v3, 0.3.0, MIT'
+    expect(conan_developer).to be_seeing_line 'range-v3, 0.10.0, MIT'
   end
 end

features/features/package_managers/conda_spec.rb

@@ -12,6 +12,6 @@
   specify 'are shown in reports' do
     LicenseFinder::TestingDSL::CondaProject.create
     conda_developer.run_license_finder
-    expect(conda_developer).to be_seeing_line 'zlib, 1.2.11, zlib'
+    expect(conda_developer).to be_seeing_line 'zlib, 1.2.11, "zlib/libpng license"'
   end
 end

features/fixtures/conanfile.txt

@@ -1,5 +1,5 @@
 [requires]
-range-v3/0.3.0@ericniebler/stable
+range-v3/0.10.0
 
 [generators]
 txt

lib/license_finder/cli/base.rb

@@ -42,6 +42,7 @@ def license_finder_config
           :gradle_include_groups,
           :maven_include_groups,
           :maven_options,
+          :npm_options,
           :pip_requirements_path,
           :python_version,
           :rebar_command,

lib/license_finder/cli/main.rb

@@ -30,6 +30,7 @@ class Main < Base
                           Defaults to 'gradlew' / 'gradlew.bat' if the wrapper is present, otherwise to 'gradle'."
       class_option :maven_include_groups, desc: 'Whether dependency name should include group id. Only meaningful if used with a Java/maven project. Defaults to false.'
       class_option :maven_options, desc: 'Maven options to append to command. Defaults to empty.'
+      class_option :npm_options, desc: 'npm options to append to command. Defaults to empty.'
       class_option :pip_requirements_path, desc: 'Path to python requirements file. Defaults to requirements.txt.'
       class_option :python_version, desc: 'Python version to invoke pip with. Valid versions: 2 or 3. Default: 2'
       class_option :rebar_command, desc: "Command to use when fetching rebar packages. Only meaningful if used with a Erlang/rebar project. Defaults to 'rebar'."

lib/license_finder/configuration.rb

@@ -93,6 +93,10 @@ def maven_options
       get(:maven_options)
     end
 
+    def npm_options
+      get(:npm_options)
+    end
+
     def pip_requirements_path
       get(:pip_requirements_path)
     end

lib/license_finder/core.rb

@@ -99,6 +99,7 @@ def options # rubocop:disable Metrics/AbcSize
         gradle_include_groups: config.gradle_include_groups,
         maven_include_groups: config.maven_include_groups,
         maven_options: config.maven_options,
+        npm_options: config.npm_options,
         pip_requirements_path: config.pip_requirements_path,
         python_version: config.python_version,
         rebar_command: config.rebar_command,

lib/license_finder/decisions.rb

@@ -280,7 +280,12 @@ def save!(file)
     def self.restore(persisted, result = new)
       return result unless persisted
 
-      actions = YAML.load(persisted)
+      # From https://makandracards.com/makandra/465149-ruby-the-yaml-safe_load-method-hides-some-pitfalls
+      actions = if Gem::Version.new(Psych::VERSION) >= Gem::Version.new('3.1.0.pre1')
+                  YAML.safe_load(persisted, permitted_classes: [Symbol, Time], aliases: true)
+                else
+                  YAML.safe_load(persisted, [Symbol, Time], [], true)
+                end
 
       list_of_actions = (actions || []).map(&:first)