From e739f2180c88504152c0e19477489177012f5631 Mon Sep 17 00:00:00 2001 From: Etienne Cadic Date: Wed, 26 Jan 2022 13:21:32 +0100 Subject: [PATCH] :sparkle: provide Flutter project scanning support > through pub package manager > needs to have the proper ENV variable set --- Dockerfile | 19 ++ README.md | 1 + .../features/package_managers/pub_spec.rb | 16 ++ features/fixtures/pubspec.yaml | 25 +++ features/support/testing_dsl.rb | 12 ++ lib/license_finder/package.rb | 2 + lib/license_finder/package_manager.rb | 1 + lib/license_finder/package_managers/pub.rb | 79 ++++++++ .../packages/pubspec_package.rb | 18 ++ lib/license_finder/scanner.rb | 2 +- spec/fixtures/all_pms/pubspec.lock | 168 ++++++++++++++++++ spec/fixtures/all_pms/pubspec.yaml | 25 +++ spec/fixtures/config/pub_deps.json | 40 +++++ .../package_managers/pub_package_spec.rb | 105 +++++++++++ 14 files changed, 512 insertions(+), 1 deletion(-) create mode 100644 features/features/package_managers/pub_spec.rb create mode 100644 features/fixtures/pubspec.yaml create mode 100644 lib/license_finder/package_managers/pub.rb create mode 100644 lib/license_finder/packages/pubspec_package.rb create mode 100644 spec/fixtures/all_pms/pubspec.lock create mode 100644 spec/fixtures/all_pms/pubspec.yaml create mode 100644 spec/fixtures/config/pub_deps.json create mode 100644 spec/lib/license_finder/package_managers/pub_package_spec.rb diff --git a/Dockerfile b/Dockerfile index 9aad76418..ce621803a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -206,6 +206,25 @@ RUN apt-get -q install -y \ pkg-config \ && rm -r /var/lib/apt/lists/* +#install flutter +ENV FLUTTER_HOME=/root/flutter +RUN curl -o flutter_linux_2.8.1-stable.tar.xz https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.8.1-stable.tar.xz +RUN tar xf flutter_linux_2.8.1-stable.tar.xz +RUN mv flutter ${FLUTTER_HOME} +RUN rm flutter_linux_2.8.1-stable.tar.xz + +ENV PATH=$PATH:${FLUTTER_HOME}/bin:${FLUTTER_HOME}/bin/cache/dart-sdk/bin +RUN flutter doctor -v \ + && flutter update-packages \ + && flutter precache +# Accepting all licences +RUN yes | flutter doctor --android-licenses -v +# Creating Flutter sample projects to put binaries in cache fore each template type +RUN flutter create --template=app ${TEMP}/app_sample \ + && flutter create --template=package ${TEMP}/package_sample \ + && flutter create --template=plugin ${TEMP}/plugin_sample + + # pub 4096R/ED3D1561 2019-03-22 [SC] [expires: 2023-03-23] # Key fingerprint = A62A E125 BBBF BB96 A6E0 42EC 925C C1CC ED3D 1561 # uid Swift 5.x Release Signing Key =2.12.0 <3.0.0" + +dependencies: + flutter: + sdk: flutter + device_info: ^2.0.3 + +dev_dependencies: + flutter_test: + sdk: flutter + +flutter: + generate: true + # The following line ensures that the Material Icons font is + # included with your application, so that you can use the icons in + # the material Icons class. + uses-material-design: true diff --git a/features/support/testing_dsl.rb b/features/support/testing_dsl.rb index 421a9ca8d..98989665d 100644 --- a/features/support/testing_dsl.rb +++ b/features/support/testing_dsl.rb @@ -492,6 +492,18 @@ def install end end + class FlutterProject < Project + def add_dep + install_fixture('pubspec.yaml') + end + + def install + ENV['PUB_CACHE'] = "~/flutter/.pub-cache/" + shell_out('flutter pub get') + end + end + + class ConanProject < Project def add_dep install_fixture('conanfile.txt') diff --git a/lib/license_finder/package.rb b/lib/license_finder/package.rb index deb204777..349c92a30 100644 --- a/lib/license_finder/package.rb +++ b/lib/license_finder/package.rb @@ -200,3 +200,5 @@ def log_activation(activation) require 'license_finder/packages/cargo_package' require 'license_finder/packages/composer_package' require 'license_finder/packages/conda_package' +require 'license_finder/packages/pubspec_package' + diff --git a/lib/license_finder/package_manager.rb b/lib/license_finder/package_manager.rb index 276fbd424..95c9efcd8 100644 --- a/lib/license_finder/package_manager.rb +++ b/lib/license_finder/package_manager.rb @@ -177,5 +177,6 @@ def log_to_file(prep_cmd, contents) require 'license_finder/package_managers/cargo' require 'license_finder/package_managers/composer' require 'license_finder/package_managers/conda' +require 'license_finder/package_managers/pub' require 'license_finder/package' diff --git a/lib/license_finder/package_managers/pub.rb b/lib/license_finder/package_managers/pub.rb new file mode 100644 index 000000000..6435c11b8 --- /dev/null +++ b/lib/license_finder/package_managers/pub.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require 'json' + +module LicenseFinder + class Pub < PackageManager + class PubError < RuntimeError; end + + def current_packages + unless File.exist?('pubspec.lock') + raise PubError, "No checked-out Pub packages found. + Please install your dependencies first." + end + + unless !(ENV["PUB_CACHE"].nil? || ENV["PUB_CACHE"].empty?) + raise PubError, "While PUB_CACHE environment variable is empty, retrieving package licenses is impossible. Please set the PUB_CACHE env variable (default: ~/.pub)" + end + _stdout, stderr, status = Cmd.run("flutter pub deps --json") + yaml_deps = JSON.parse(_stdout) + yaml_deps["packages"].map do |dependency| + package_name = dependency['name'] + subpath = "#{dependency['name']}-#{dependency['version']}" + package_version = dependency['version'] + homepage = nil + + project_repo = dependency["source"] == "git" ? Pathname("#{ENV["PUB_CACHE"]}/git/#{dependency["name"]}-*/") : Pathname("#{ENV["PUB_CACHE"]}/hosted/pub.dartlang.org/#{subpath}") + + PubPackage.new( + package_name, + package_version, + license_text(project_repo), + logger: logger, + install_path: project_repo, + homepage: homepage, + ) + end + end + + def possible_package_paths + [project_path.join("pubspec.lock")] + end + + def package_management_command + 'flutter' + end + + def prepare_command + 'flutter pub get' + end + + def prepare + prep_cmd = '#{prepare_command} #{production_flag}' + _stdout, stderr, status = Dir.chdir(project_path) { Cmd.run(prep_cmd) } + + return if status.success? + + log_errors stderr + raise "Prepare command '#{prep_cmd}' failed" unless @prepare_no_fail + end + + private + + + def license_text(subpath) + license_path = license_pattern(subpath).find { |f| File.exist?(f) } + license_path.nil? ? nil : IO.read(license_path) + end + + def license_pattern(subpath) + Dir.glob(subpath.join('LICENSE*'), File::FNM_CASEFOLD) + end + + def production_flag + return '' if @ignored_groups.nil? + + @ignored_groups.include?('devDependencies') ? '' : 'no-' + end + end +end diff --git a/lib/license_finder/packages/pubspec_package.rb b/lib/license_finder/packages/pubspec_package.rb new file mode 100644 index 000000000..c6a18e33a --- /dev/null +++ b/lib/license_finder/packages/pubspec_package.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module LicenseFinder + class PubPackage < Package + def initialize(name, version, license_text, options = {}) + super(name, version, options) + @license = License.find_by_text(license_text.to_s) + end + + def licenses_from_spec + [@license].compact + end + + def package_manager + 'Pub' + end + end +end diff --git a/lib/license_finder/scanner.rb b/lib/license_finder/scanner.rb index a428b210d..a0fbc196f 100644 --- a/lib/license_finder/scanner.rb +++ b/lib/license_finder/scanner.rb @@ -5,7 +5,7 @@ class Scanner PACKAGE_MANAGERS = [ GoModules, GoDep, GoWorkspace, Go15VendorExperiment, Glide, Gvt, Govendor, Trash, Dep, Bundler, NPM, Pip, Yarn, Bower, Maven, Gradle, CocoaPods, Rebar, Erlangmk, Nuget, Carthage, Mix, Conan, Sbt, Cargo, Dotnet, Composer, Pipenv, - Conda, Spm + Conda, Spm, Pub ].freeze class << self diff --git a/spec/fixtures/all_pms/pubspec.lock b/spec/fixtures/all_pms/pubspec.lock new file mode 100644 index 000000000..fee33aea6 --- /dev/null +++ b/spec/fixtures/all_pms/pubspec.lock @@ -0,0 +1,168 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + async: + dependency: transitive + description: + name: async + url: "https://pub.dartlang.org" + source: hosted + version: "2.8.1" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + url: "https://pub.dartlang.org" + source: hosted + version: "2.1.0" + characters: + dependency: transitive + description: + name: characters + url: "https://pub.dartlang.org" + source: hosted + version: "1.1.0" + charcode: + dependency: transitive + description: + name: charcode + url: "https://pub.dartlang.org" + source: hosted + version: "1.3.1" + clock: + dependency: transitive + description: + name: clock + url: "https://pub.dartlang.org" + source: hosted + version: "1.1.0" + collection: + dependency: transitive + description: + name: collection + url: "https://pub.dartlang.org" + source: hosted + version: "1.15.0" + device_info: + dependency: "direct main" + description: + name: device_info + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.3" + device_info_platform_interface: + dependency: transitive + description: + name: device_info_platform_interface + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.1" + fake_async: + dependency: transitive + description: + name: fake_async + url: "https://pub.dartlang.org" + source: hosted + version: "1.2.0" + flutter: + dependency: "direct main" + description: flutter + source: sdk + version: "0.0.0" + flutter_test: + dependency: "direct dev" + description: flutter + source: sdk + version: "0.0.0" + matcher: + dependency: transitive + description: + name: matcher + url: "https://pub.dartlang.org" + source: hosted + version: "0.12.10" + meta: + dependency: transitive + description: + name: meta + url: "https://pub.dartlang.org" + source: hosted + version: "1.7.0" + path: + dependency: transitive + description: + name: path + url: "https://pub.dartlang.org" + source: hosted + version: "1.8.0" + plugin_platform_interface: + dependency: transitive + description: + name: plugin_platform_interface + url: "https://pub.dartlang.org" + source: hosted + version: "2.1.2" + sky_engine: + dependency: transitive + description: flutter + source: sdk + version: "0.0.99" + source_span: + dependency: transitive + description: + name: source_span + url: "https://pub.dartlang.org" + source: hosted + version: "1.8.1" + stack_trace: + dependency: transitive + description: + name: stack_trace + url: "https://pub.dartlang.org" + source: hosted + version: "1.10.0" + stream_channel: + dependency: transitive + description: + name: stream_channel + url: "https://pub.dartlang.org" + source: hosted + version: "2.1.0" + string_scanner: + dependency: transitive + description: + name: string_scanner + url: "https://pub.dartlang.org" + source: hosted + version: "1.1.0" + term_glyph: + dependency: transitive + description: + name: term_glyph + url: "https://pub.dartlang.org" + source: hosted + version: "1.2.0" + test_api: + dependency: transitive + description: + name: test_api + url: "https://pub.dartlang.org" + source: hosted + version: "0.4.2" + typed_data: + dependency: transitive + description: + name: typed_data + url: "https://pub.dartlang.org" + source: hosted + version: "1.3.0" + vector_math: + dependency: transitive + description: + name: vector_math + url: "https://pub.dartlang.org" + source: hosted + version: "2.1.0" +sdks: + dart: ">=2.12.0 <3.0.0" + flutter: ">=1.12.13+hotfix.5" diff --git a/spec/fixtures/all_pms/pubspec.yaml b/spec/fixtures/all_pms/pubspec.yaml new file mode 100644 index 000000000..97944a869 --- /dev/null +++ b/spec/fixtures/all_pms/pubspec.yaml @@ -0,0 +1,25 @@ +name: test_app +description: Test app for Flutter license_finder evolution + +publish_to: 'none' # Remove this line if you wish to publish to pub.dev + +version: 1.0.0+1 + +environment: + sdk: ">=2.12.0 <3.0.0" + +dependencies: + flutter: + sdk: flutter + device_info: ^2.0.3 + +dev_dependencies: + flutter_test: + sdk: flutter + +flutter: + generate: true + # The following line ensures that the Material Icons font is + # included with your application, so that you can use the icons in + # the material Icons class. + uses-material-design: true diff --git a/spec/fixtures/config/pub_deps.json b/spec/fixtures/config/pub_deps.json new file mode 100644 index 000000000..72973ea5b --- /dev/null +++ b/spec/fixtures/config/pub_deps.json @@ -0,0 +1,40 @@ +{ + "root": "test_app", + "packages": [ + { + "name": "flutter", + "version": "0.0.0", + "kind": "direct", + "source": "sdk", + "dependencies": [ + "characters", + "collection", + "meta", + "typed_data", + "vector_math", + "sky_engine" + ] + }, + { + "name": "device_info", + "version": "2.0.3", + "kind": "direct", + "source": "hosted", + "dependencies": [ + "flutter", + "device_info_platform_interface" + ] + } + ], + "sdks": [ + { + "name": "Dart", + "version": "2.14.4" + }, + { + "name": "Flutter", + "version": "2.5.3" + } + ], + "executables": [] +} \ No newline at end of file diff --git a/spec/lib/license_finder/package_managers/pub_package_spec.rb b/spec/lib/license_finder/package_managers/pub_package_spec.rb new file mode 100644 index 000000000..ddb0ae882 --- /dev/null +++ b/spec/lib/license_finder/package_managers/pub_package_spec.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'fakefs/spec_helpers' + +module LicenseFinder + describe Pub do + let(:root) { '/fake-pub-project' } + let(:project_path) { fixture_path('all_pms') } + let(:pub) { Pub.new(project_path: project_path) } + let(:new_bsd_common_text) { "Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." } + + it_behaves_like 'a PackageManager' + + def stub_resolved(frameworks) + allow(IO).to receive(:read) + .with(project_path.join('pubspec.lock')) + .and_return(frameworks) + end + + def stub_license_md(hash = {}) + hash.each_key do |key| + license_pattern = project_path.join('.pub/hosted/pub.dartlang.org/', key, 'LICENSE*') + filename = project_path.join('.pub/hosted/pub.dartlang.org/', key, 'LICENSE') + allow(IO).to receive(:read) + .with(filename) + .and_return(hash[key]) + + allow(File).to receive(:exist?) + .with(filename) + .and_return(true) + + allow(Dir).to receive(:glob) + .with(license_pattern, File::FNM_CASEFOLD) + .and_return([filename]) + end + end + + let(:dependency_json) do + FakeFS.without do + fixture_from('pub_deps.json') + end + end + + before do + allow(IO).to receive(:read).and_call_original + allow(File).to receive(:exist?).and_return(false) + allow(Dir).to receive(:glob).and_return([]) + end + describe 'current_packages' do + context 'when PUB already ran and pubspec.lock exists' do + before do + allow(File).to receive(:exist?).and_return(true) + allow(SharedHelpers::Cmd).to receive(:run).with('flutter pub deps --json') + .and_return([dependency_json, '', cmd_success]) + allow(ENV).to receive(:[]).with('PUB_CACHE').and_return(project_path.join('.pub')) + end + + it 'lists all the current packages' do + expect(pub.current_packages.map { |p| [p.name, p.version] }).to eq [ + ["flutter", "0.0.0"], + ["device_info", "2.0.3"] + ] + end + + it 'passes the license text to the package' do + stub_license_md("device_info-2.0.3" => new_bsd_common_text) + expect(pub.current_packages.last.licenses.map(&:name)).to eq ["New BSD"] + end + + it 'handles no licenses' do + expect(pub.current_packages.first.licenses.map(&:name)).to eq ['unknown'] + end + end + + # context "when spm did not run yet" do + # it "raises an exception to explain the reason" do + # expect do + # spm.current_packages.first.licenses.map(&:name) + # end.to raise_exception(Spm::SpmError) + # end + # end + end + end +end