From 1997263161ac3a9ae659468ded5a02acd32ce11d Mon Sep 17 00:00:00 2001 From: Alexander Xydes Date: Wed, 14 Sep 2022 10:27:56 -0700 Subject: [PATCH 1/7] Updated 'inherits_from' to correct new format. --- rsc/tooling-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rsc/tooling-config.yaml b/rsc/tooling-config.yaml index d04e9ce..56b1719 100644 --- a/rsc/tooling-config.yaml +++ b/rsc/tooling-config.yaml @@ -36,7 +36,8 @@ levels: flags: "" tooling_with_pylint: - inherits_from: "tooling" + inherits_from: + - "tooling" discovery: python: flags: "" From 9c079ceb8fdd7db0a2e21e6e80b859b32988c20d Mon Sep 17 00:00:00 2001 From: Alexander Xydes Date: Wed, 14 Sep 2022 10:44:46 -0700 Subject: [PATCH 2/7] Pin flake8<5 until a bug is fixed. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 6a40c3d..b26b8a4 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,7 @@ passenv = CI setenv = PY_IGNORE_IMPORTMISMATCH = 1 deps = codecov + flake8<5 # Pin until https://github.com/tholo/pytest-flake8/issues/87 is fixed. flake8-pep3101 pycodestyle pydocstyle From f72ac93e9e1350d773f10aeab57c44f90aae69be Mon Sep 17 00:00:00 2001 From: Alexander Xydes Date: Wed, 14 Sep 2022 11:27:46 -0700 Subject: [PATCH 3/7] Pin pycodestyle<2.9.0 to match flake8 pinning. Updating tool plugins to pass tests. --- .../plugins/tool/dockerfile_lint_tool_plugin.py | 4 +++- .../plugins/tool/dockerfilelint_tool_plugin.py | 4 +++- src/statick_tooling/plugins/tool/hadolint_tool_plugin.py | 4 +++- tox.ini | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/statick_tooling/plugins/tool/dockerfile_lint_tool_plugin.py b/src/statick_tooling/plugins/tool/dockerfile_lint_tool_plugin.py index f5b2c16..5c31871 100644 --- a/src/statick_tooling/plugins/tool/dockerfile_lint_tool_plugin.py +++ b/src/statick_tooling/plugins/tool/dockerfile_lint_tool_plugin.py @@ -92,7 +92,9 @@ def add_filename(cls, output: str, src: str) -> str: logging.warning("ValueError: %s", ex) return output - def parse_output(self, total_output: List[str]) -> List[Issue]: + def parse_output( + self, total_output: List[str], package: Optional[Package] = None + ) -> List[Issue]: """Parse tool output and report issues.""" issues: List[Issue] = [] diff --git a/src/statick_tooling/plugins/tool/dockerfilelint_tool_plugin.py b/src/statick_tooling/plugins/tool/dockerfilelint_tool_plugin.py index 290ee7d..18834d0 100644 --- a/src/statick_tooling/plugins/tool/dockerfilelint_tool_plugin.py +++ b/src/statick_tooling/plugins/tool/dockerfilelint_tool_plugin.py @@ -80,7 +80,9 @@ def scan(self, package: Package, level: str) -> Optional[List[Issue]]: # pylint: enable=too-many-locals - def parse_output(self, total_output: List[str]) -> List[Issue]: + def parse_output( + self, total_output: List[str], package: Optional[Package] = None + ) -> List[Issue]: """Parse tool output and report issues.""" issues: List[Issue] = [] diff --git a/src/statick_tooling/plugins/tool/hadolint_tool_plugin.py b/src/statick_tooling/plugins/tool/hadolint_tool_plugin.py index a3f5d2f..da212ec 100644 --- a/src/statick_tooling/plugins/tool/hadolint_tool_plugin.py +++ b/src/statick_tooling/plugins/tool/hadolint_tool_plugin.py @@ -172,7 +172,9 @@ def scan_docker( logging.warning("Couldn't find %s! (%s)", tool_bin, ex) return None - def parse_output(self, total_output: List[str]) -> List[Issue]: + def parse_output( + self, total_output: List[str], package: Optional[Package] = None + ) -> List[Issue]: """Parse tool output and report issues.""" issues: List[Issue] = [] diff --git a/tox.ini b/tox.ini index b26b8a4..657ec14 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ deps = codecov flake8<5 # Pin until https://github.com/tholo/pytest-flake8/issues/87 is fixed. flake8-pep3101 - pycodestyle + pycodestyle<2.9.0 # Pin until https://github.com/tholo/pytest-flake8/issues/87 is fixed. pydocstyle pytest pytest-cov From 2e240b6f52b83084e72516a4f772933346bce0f2 Mon Sep 17 00:00:00 2001 From: Alexander Xydes Date: Wed, 14 Sep 2022 14:00:05 -0700 Subject: [PATCH 4/7] Fully updating dockerfile_lint tool plugin to new structure. --- .../tool/dockerfile_lint_tool_plugin.py | 20 ++++++++----------- .../test_dockerfile_lint_tool_plugin.py | 1 + 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/statick_tooling/plugins/tool/dockerfile_lint_tool_plugin.py b/src/statick_tooling/plugins/tool/dockerfile_lint_tool_plugin.py index 5c31871..272b0d6 100644 --- a/src/statick_tooling/plugins/tool/dockerfile_lint_tool_plugin.py +++ b/src/statick_tooling/plugins/tool/dockerfile_lint_tool_plugin.py @@ -17,8 +17,14 @@ def get_name(self) -> str: """Get name of tool.""" return "dockerfile-lint" + def get_file_types(self) -> List[str]: + """Return a list of file types the plugin can scan.""" + return ["dockerfile_src"] + # pylint: disable=too-many-locals - def scan(self, package: Package, level: str) -> Optional[List[Issue]]: + def process_files( + self, package: Package, level: str, files: List[str], user_flags: List[str] + ) -> Optional[List[str]]: """Run tool and gather output.""" tool_bin = "dockerfile_lint" @@ -34,13 +40,8 @@ def scan(self, package: Package, level: str) -> Optional[List[Issue]]: if format_file_name is not None: flags += ["-r", format_file_name] flags += ["--json"] - user_flags = self.get_user_flags(level) flags += user_flags - files: List[str] = [] - if "dockerfile_src" in package: - files += package["dockerfile_src"] - total_output: List[str] = [] for src in files: @@ -69,12 +70,7 @@ def scan(self, package: Package, level: str) -> Optional[List[Issue]]: for output in total_output: logging.debug("%s", output) - with open(self.get_name() + ".log", "w", encoding="utf8") as fid: - for output in total_output: - fid.write(output) - - issues: List[Issue] = self.parse_output(total_output) - return issues + return total_output # pylint: enable=too-many-locals diff --git a/tests/tool/dockerfile_lint_tool_plugin/test_dockerfile_lint_tool_plugin.py b/tests/tool/dockerfile_lint_tool_plugin/test_dockerfile_lint_tool_plugin.py index c73e395..e6c3830 100644 --- a/tests/tool/dockerfile_lint_tool_plugin/test_dockerfile_lint_tool_plugin.py +++ b/tests/tool/dockerfile_lint_tool_plugin/test_dockerfile_lint_tool_plugin.py @@ -37,6 +37,7 @@ def setup_dockerfilelint_tool_plugin(): ) config = Config(resources.get_file("config.yaml")) plugin_context = PluginContext(arg_parser.parse_args([]), resources, config) + plugin_context.args.output_directory = os.path.dirname(__file__) plugin = DockerfileULintToolPlugin() plugin.set_plugin_context(plugin_context) return plugin From 9ab3e42ece5fbfab84457aa30bb5efe14f88758a Mon Sep 17 00:00:00 2001 From: Alexander Xydes Date: Wed, 14 Sep 2022 14:01:51 -0700 Subject: [PATCH 5/7] Fully updating dockerfilelint tool plugin to new structure. --- .../tool/dockerfilelint_tool_plugin.py | 20 ++++++++----------- .../test_dockerfilelint_tool_plugin.py | 1 + 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/statick_tooling/plugins/tool/dockerfilelint_tool_plugin.py b/src/statick_tooling/plugins/tool/dockerfilelint_tool_plugin.py index 18834d0..256911f 100644 --- a/src/statick_tooling/plugins/tool/dockerfilelint_tool_plugin.py +++ b/src/statick_tooling/plugins/tool/dockerfilelint_tool_plugin.py @@ -18,8 +18,14 @@ def get_name(self) -> str: """Get name of tool.""" return "dockerfilelint" + def get_file_types(self) -> List[str]: + """Return a list of file types the plugin can scan.""" + return ["dockerfile_src"] + # pylint: disable=too-many-locals - def scan(self, package: Package, level: str) -> Optional[List[Issue]]: + def process_files( + self, package: Package, level: str, files: List[str], user_flags: List[str] + ) -> Optional[List[str]]: """Run tool and gather output.""" tool_bin = "dockerfilelint" @@ -36,13 +42,8 @@ def scan(self, package: Package, level: str) -> Optional[List[Issue]]: if format_file_name is not None: flags += ["-c", str(format_file_path)] flags += ["-o", "json"] - user_flags = self.get_user_flags(level) flags += user_flags - files: List[str] = [] - if "dockerfile_src" in package: - files += package["dockerfile_src"] - total_output: List[str] = [] for src in files: @@ -71,12 +72,7 @@ def scan(self, package: Package, level: str) -> Optional[List[Issue]]: for output in total_output: logging.debug("%s", output) - with open(self.get_name() + ".log", "w", encoding="utf8") as fid: - for output in total_output: - fid.write(output) - - issues: List[Issue] = self.parse_output(total_output) - return issues + return total_output # pylint: enable=too-many-locals diff --git a/tests/tool/dockerfilelint_tool_plugin/test_dockerfilelint_tool_plugin.py b/tests/tool/dockerfilelint_tool_plugin/test_dockerfilelint_tool_plugin.py index 3d25b16..e4846d0 100644 --- a/tests/tool/dockerfilelint_tool_plugin/test_dockerfilelint_tool_plugin.py +++ b/tests/tool/dockerfilelint_tool_plugin/test_dockerfilelint_tool_plugin.py @@ -37,6 +37,7 @@ def setup_dockerfilelint_tool_plugin(package="valid_package"): ) config = Config(resources.get_file("config.yaml")) plugin_context = PluginContext(arg_parser.parse_args([]), resources, config) + plugin_context.args.output_directory = os.path.dirname(__file__) plugin = DockerfileLintToolPlugin() plugin.set_plugin_context(plugin_context) return plugin From b1e721405c4d0d427de38f9dbff95293d4a68dc2 Mon Sep 17 00:00:00 2001 From: Alexander Xydes Date: Wed, 14 Sep 2022 14:04:51 -0700 Subject: [PATCH 6/7] Fully updating hadolint tool plugin to new structure. --- .../plugins/tool/hadolint_tool_plugin.py | 20 ++++++++----------- .../test_hadolint_tool_plugin.py | 1 + 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/statick_tooling/plugins/tool/hadolint_tool_plugin.py b/src/statick_tooling/plugins/tool/hadolint_tool_plugin.py index da212ec..f3e37ae 100644 --- a/src/statick_tooling/plugins/tool/hadolint_tool_plugin.py +++ b/src/statick_tooling/plugins/tool/hadolint_tool_plugin.py @@ -33,8 +33,14 @@ def gather_args(self, args: argparse.Namespace) -> None: help="Use hadolint docker image instead of binary", ) + def get_file_types(self) -> List[str]: + """Return a list of file types the plugin can scan.""" + return ["dockerfile_src"] + # pylint: disable=too-many-locals - def scan(self, package: Package, level: str) -> Optional[List[Issue]]: + def process_files( + self, package: Package, level: str, files: List[str], user_flags: List[str] + ) -> Optional[List[str]]: """Run tool and gather output.""" tool_bin = "hadolint" @@ -51,7 +57,6 @@ def scan(self, package: Package, level: str) -> Optional[List[Issue]]: config_file_path = self.plugin_context.resources.get_file(tool_config) flags: List[str] = ["-f", "json", "--no-fail"] - user_flags = self.get_user_flags(level) if "-f" in user_flags: idx = user_flags.index("-f") logging.warning( @@ -63,10 +68,6 @@ def scan(self, package: Package, level: str) -> Optional[List[Issue]]: user_flags.pop(idx) flags += user_flags - files: List[str] = [] - if "dockerfile_src" in package: - files += package["dockerfile_src"] - total_output: List[str] = [] if ( self.plugin_context @@ -87,12 +88,7 @@ def scan(self, package: Package, level: str) -> Optional[List[Issue]]: for output in total_output: logging.debug("%s", output) - with open(self.get_name() + ".log", "w", encoding="utf8") as fid: - for output in total_output: - fid.write(output) - - issues: List[Issue] = self.parse_output(total_output) - return issues + return total_output # pylint: enable=too-many-locals diff --git a/tests/tool/hadolint_tool_plugin/test_hadolint_tool_plugin.py b/tests/tool/hadolint_tool_plugin/test_hadolint_tool_plugin.py index c733d33..bfad3d0 100644 --- a/tests/tool/hadolint_tool_plugin/test_hadolint_tool_plugin.py +++ b/tests/tool/hadolint_tool_plugin/test_hadolint_tool_plugin.py @@ -46,6 +46,7 @@ def setup_hadolint_tool_plugin( ) config = Config(resources.get_file("config.yaml")) plugin_context = PluginContext(arg_parser.parse_args([]), resources, config) + plugin_context.args.output_directory = os.path.dirname(__file__) plugin = HadolintToolPlugin() if binary: plugin_context.args.hadolint_bin = binary From 44678b52c5c979f3763ff4422da47e0a98b4c394 Mon Sep 17 00:00:00 2001 From: Alexander Xydes Date: Wed, 14 Sep 2022 14:39:53 -0700 Subject: [PATCH 7/7] Update changelog. --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3b44b5..9e2f99c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## Unreleased -### Added +## Changed + +- Updated tool plugins to match new structure introduced in sscpac/statick#423. +- Update `inherits_from` usage in configuration file to match new list format. ### Fixed -### Removed +- Pin flake8<5 and pycodestyle<2.9.0 until is fixed. ## v0.1.0 - 2022-01-04