Skip to content

Commit

Permalink
Support Trivy JSON files as source for the security warnings metric. C…
Browse files Browse the repository at this point in the history 
…loses #6927.
  • Loading branch information
@fniessink
fniessink committed Sep 18, 2023
1 parent 2cce7d9 commit 0a57df4
Show file tree
Hide file tree
Showing 15 changed files with 507 additions and 68 deletions.
10 changes: 10 additions & 0 deletions components/api_server/src/example-reports/example-report.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,16 @@
"password": "",
"private_token": ""
}
},
"84eee1c2-53e5-5260-75cb-2b444a5ba336": {
"type": "trivy_json",
"parameters": {
"url": "http://testdata:8000/reports/trivy_json/trivy.json",
"landing_url": "http://localhost:8000/reports/trivy_json/trivy.json",
"username": "",
"password": "",
"private_token": ""
}
}
},
"name": null,
Expand Down
141 changes: 136 additions & 5 deletions components/collector/.vulture_ignore_list.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,136 @@
__new__ # unused function (src/source_collectors/azure_devops/source_up_to_dateness.py:60)
__new__ # unused function (src/source_collectors/gitlab/source_up_to_dateness.py:136)
scan_status # unused variable (src/source_collectors/harbor/security_warnings.py:43)
id # unused variable (src/source_collectors/jira/velocity.py:17)
id # unused variable (src/source_collectors/jira/velocity.py:31)
AnchoreSecurityWarnings # unused class (src/source_collectors/anchore/security_warnings.py:9)
AnchoreSourceUpToDateness # unused class (src/source_collectors/anchore/source_up_to_dateness.py:13)
AnchoreJenkinsPluginSecurityWarnings # unused class (src/source_collectors/anchore_jenkins_plugin/security_warnings.py:9)
AnchoreJenkinsPluginSourceUpToDateness # unused class (src/source_collectors/anchore_jenkins_plugin/source_up_to_dateness.py:6)
AxeCoreAccessibility # unused class (src/source_collectors/axe_core/accessibility.py:53)
AxeCoreSourceUpToDateness # unused class (src/source_collectors/axe_core/source_up_to_dateness.py:10)
AxeCoreSourceVersion # unused class (src/source_collectors/axe_core/source_version.py:9)
AxeCSVAccessibility # unused class (src/source_collectors/axe_csv/accessibility.py:12)
AxeHTMLReporterAccessibility # unused class (src/source_collectors/axe_html_reporter/accessibility.py:14)
AzureDevopsAverageIssueLeadTime # unused class (src/source_collectors/azure_devops/average_issue_lead_time.py:13)
AzureDevopsFailedJobs # unused class (src/source_collectors/azure_devops/failed_jobs.py:8)
AzureDevopsJobRunsWithinTimePeriod # unused class (src/source_collectors/azure_devops/job_runs_within_time_period.py:11)
AzureDevopsSourceUpToDateness # unused class (src/source_collectors/azure_devops/source_up_to_dateness.py:53)
__new__ # unused function (src/source_collectors/azure_devops/source_up_to_dateness.py:56)
AzureDevopsUnmergedBranches # unused class (src/source_collectors/azure_devops/unmerged_branches.py:15)
AzureDevopsUnusedJobs # unused class (src/source_collectors/azure_devops/unused_jobs.py:11)
AzureDevopsUserStoryPoints # unused class (src/source_collectors/azure_devops/user_story_points.py:12)
BanditSecurityWarnings # unused class (src/source_collectors/bandit/security_warnings.py:10)
BanditSourceUpToDateness # unused class (src/source_collectors/bandit/source_up_to_dateness.py:10)
CalendarSourceUpToDateness # unused class (src/source_collectors/calendar/source_up_to_dateness.py:13)
CalendarTimeRemaining # unused class (src/source_collectors/calendar/time_remaining.py:13)
CargoAuditSecurityWarnings # unused class (src/source_collectors/cargo_audit/security_warnings.py:10)
ClocLOC # unused class (src/source_collectors/cloc/loc.py:10)
ClocSourceVersion # unused class (src/source_collectors/cloc/source_version.py:9)
CoberturaSourceUpToDateness # unused class (src/source_collectors/cobertura/source_up_to_dateness.py:11)
CoberturaSourceVersion # unused class (src/source_collectors/cobertura/source_version.py:10)
CoberturaUncoveredBranches # unused class (src/source_collectors/cobertura/uncovered_branches.py:6)
CoberturaUncoveredLines # unused class (src/source_collectors/cobertura/uncovered_lines.py:6)
CoberturaJenkinsPluginSourceUpToDateness # unused class (src/source_collectors/cobertura_jenkins_plugin/source_up_to_dateness.py:8)
CoberturaJenkinsPluginUncoveredBranches # unused class (src/source_collectors/cobertura_jenkins_plugin/uncovered_branches.py:6)
CoberturaJenkinsPluginUncoveredLines # unused class (src/source_collectors/cobertura_jenkins_plugin/uncovered_lines.py:6)
ComposerDependencies # unused class (src/source_collectors/composer/dependencies.py:7)
CxSASTSecurityWarnings # unused class (src/source_collectors/cxsast/security_warnings.py:10)
CxSASTSourceUpToDateness # unused class (src/source_collectors/cxsast/source_up_to_dateness.py:10)
CxSASTSourceVersion # unused class (src/source_collectors/cxsast/source_version.py:12)
GatlingPerformanceTestDuration # unused class (src/source_collectors/gatling/performancetest_duration.py:9)
GatlingSlowTransactions # unused class (src/source_collectors/gatling/slow_transactions.py:12)
GatlingSourceUpToDateness # unused class (src/source_collectors/gatling/source_up_to_dateness.py:15)
GatlingSourceVersion # unused class (src/source_collectors/gatling/source_version.py:11)
GenericJSONSecurityWarnings # unused class (src/source_collectors/generic_json/security_warnings.py:11)
GitLabFailedJobs # unused class (src/source_collectors/gitlab/failed_jobs.py:8)
GitLabJobRunsWithinTimePeriod # unused class (src/source_collectors/gitlab/job_runs_within_time_period.py:12)
GitLabMergeRequests # unused class (src/source_collectors/gitlab/merge_requests.py:70)
GitLabSourceUpToDateness # unused class (src/source_collectors/gitlab/source_up_to_dateness.py:130)
__new__ # unused function (src/source_collectors/gitlab/source_up_to_dateness.py:133)
GitLabSourceVersion # unused class (src/source_collectors/gitlab/source_version.py:11)
GitLabUnmergedBranches # unused class (src/source_collectors/gitlab/unmerged_branches.py:15)
GitLabUnusedJobs # unused class (src/source_collectors/gitlab/unused_jobs.py:11)
scan_status # unused variable (src/source_collectors/harbor/security_warnings.py:58)
HarborSecurityWarnings # unused class (src/source_collectors/harbor/security_warnings.py:62)
JacocoSourceUpToDateness # unused class (src/source_collectors/jacoco/source_up_to_dateness.py:11)
JacocoUncoveredBranches # unused class (src/source_collectors/jacoco/uncovered_branches.py:6)
JacocoUncoveredLines # unused class (src/source_collectors/jacoco/uncovered_lines.py:6)
JacocoJenkinsPluginSourceUpToDateness # unused class (src/source_collectors/jacoco_jenkins_plugin/source_up_to_dateness.py:8)
JacocoJenkinsPluginUncoveredBranches # unused class (src/source_collectors/jacoco_jenkins_plugin/uncovered_branches.py:6)
JacocoJenkinsPluginUncoveredLines # unused class (src/source_collectors/jacoco_jenkins_plugin/uncovered_lines.py:6)
JenkinsFailedJobs # unused class (src/source_collectors/jenkins/failed_jobs.py:8)
JenkinsJobRunsWithinTimePeriod # unused class (src/source_collectors/jenkins/job_runs_within_time_period.py:12)
JenkinsSourceUpToDateness # unused class (src/source_collectors/jenkins/source_up_to_dateness.py:9)
JenkinsSourceVersion # unused class (src/source_collectors/jenkins/source_version.py:9)
JenkinsUnusedJobs # unused class (src/source_collectors/jenkins/unused_jobs.py:11)
JenkinsTestReportSourceUpToDateness # unused class (src/source_collectors/jenkins_test_report/source_up_to_dateness.py:9)
JiraAverageIssueLeadTime # unused class (src/source_collectors/jira/average_issue_lead_time.py:13)
JiraIssueStatus # unused class (src/source_collectors/jira/issue_status.py:11)
JiraManualTestDuration # unused class (src/source_collectors/jira/manual_test_duration.py:6)
JiraManualTestExecution # unused class (src/source_collectors/jira/manual_test_execution.py:14)
JiraSourceVersion # unused class (src/source_collectors/jira/source_version.py:11)
JiraUserStoryPoints # unused class (src/source_collectors/jira/user_story_points.py:6)
id # unused variable (src/source_collectors/jira/velocity.py:16)
id # unused variable (src/source_collectors/jira/velocity.py:30)
JiraVelocity # unused class (src/source_collectors/jira/velocity.py:38)
JMeterCSVPerformanceTestDuration # unused class (src/source_collectors/jmeter_csv/performancetest_duration.py:9)
JMeterCSVSlowTransactions # unused class (src/source_collectors/jmeter_csv/slow_transactions.py:11)
JMeterCSVSourceUpToDateness # unused class (src/source_collectors/jmeter_csv/source_up_to_dateness.py:15)
JMeterJSONSlowTransactions # unused class (src/source_collectors/jmeter_json/slow_transactions.py:10)
JUnitSourceUpToDateness # unused class (src/source_collectors/junit/source_up_to_dateness.py:13)
ManualNumber # unused class (src/source_collectors/manual_number/all_metrics.py:8)
NCoverSourceUpToDateness # unused class (src/source_collectors/ncover/source_up_to_dateness.py:13)
NCoverUncoveredBranches # unused class (src/source_collectors/ncover/uncovered_branches.py:6)
NCoverUncoveredLines # unused class (src/source_collectors/ncover/uncovered_lines.py:6)
NpmDependencies # unused class (src/source_collectors/npm/dependencies.py:10)
OJAuditViolations # unused class (src/source_collectors/ojaudit/violations.py:22)
OpenVASSecurityWarnings # unused class (src/source_collectors/openvas/security_warnings.py:11)
OpenVASSourceUpToDateness # unused class (src/source_collectors/openvas/source_up_to_dateness.py:11)
OpenVASSourceVersion # unused class (src/source_collectors/openvas/source_version.py:10)
OWASPDependencyCheckSecurityWarnings # unused class (src/source_collectors/owasp_dependency_check/security_warnings.py:11)
OWASPDependencyCheckSourceUpToDateness # unused class (src/source_collectors/owasp_dependency_check/source_up_to_dateness.py:13)
OWASPDependencyCheckSourceVersion # unused class (src/source_collectors/owasp_dependency_check/source_version.py:12)
OWASPZAPSecurityWarnings # unused class (src/source_collectors/owasp_zap/security_warnings.py:13)
OWASPZAPSourceUpToDateness # unused class (src/source_collectors/owasp_zap/source_up_to_dateness.py:11)
OWASPZAPSourceVersion # unused class (src/source_collectors/owasp_zap/source_version.py:10)
PerformanceTestRunnerPerformanceTestDuration # unused class (src/source_collectors/performancetest_runner/performancetest_duration.py:13)
PerformanceTestRunnerScalability # unused class (src/source_collectors/performancetest_runner/performancetest_scalability.py:13)
PerformanceTestRunnerPerformanceTestStability # unused class (src/source_collectors/performancetest_runner/performancetest_stability.py:13)
PerformanceTestRunnerSlowTransactions # unused class (src/source_collectors/performancetest_runner/slow_transactions.py:11)
PerformanceTestRunnerSoftwareVersion # unused class (src/source_collectors/performancetest_runner/software_version.py:14)
PerformanceTestRunnerSourceUpToDateness # unused class (src/source_collectors/performancetest_runner/source_up_to_dateness.py:15)
PipDependencies # unused class (src/source_collectors/pip/dependencies.py:10)
PyupioSafetySecurityWarnings # unused class (src/source_collectors/pyupio_safety/security_warnings.py:12)
QualityTimeMetrics # unused class (src/source_collectors/quality_time/metrics.py:19)
QualityTimeMissingMetrics # unused class (src/source_collectors/quality_time/missing_metrics.py:13)
QualityTimeSourceUpToDateness # unused class (src/source_collectors/quality_time/source_up_to_dateness.py:13)
QualityTimeSourceVersion # unused class (src/source_collectors/quality_time/source_version.py:9)
RobotFrameworkSourceUpToDateness # unused class (src/source_collectors/robot_framework/source_up_to_dateness.py:13)
RobotFrameworkSourceVersion # unused class (src/source_collectors/robot_framework/source_version.py:12)
RobotFrameworkJenkinsPluginSourceUpToDateness # unused class (src/source_collectors/robot_framework_jenkins_plugin/source_up_to_dateness.py:8)
SARIFJSONSecurityWarnings # unused class (src/source_collectors/sarif/security_warnings.py:6)
SARIFJSONViolations # unused class (src/source_collectors/sarif/violations.py:6)
SnykSecurityWarnings # unused class (src/source_collectors/snyk/security_warnings.py:13)
SonarQubeCommentedOutCode # unused class (src/source_collectors/sonarqube/commented_out_code.py:6)
SonarQubeComplexUnits # unused class (src/source_collectors/sonarqube/complex_units.py:6)
SonarQubeDuplicatedLines # unused class (src/source_collectors/sonarqube/duplicated_lines.py:6)
SonarQubeLOC # unused class (src/source_collectors/sonarqube/loc.py:11)
SonarQubeLongUnits # unused class (src/source_collectors/sonarqube/long_units.py:6)
SonarQubeManyParameters # unused class (src/source_collectors/sonarqube/many_parameters.py:6)
SonarQubeRemediationEffort # unused class (src/source_collectors/sonarqube/remediation_effort.py:12)
SonarQubeSecurityWarnings # unused class (src/source_collectors/sonarqube/security_warnings.py:11)
SonarQubeSoftwareVersion # unused class (src/source_collectors/sonarqube/software_version.py:11)
SonarQubeSourceUpToDateness # unused class (src/source_collectors/sonarqube/source_up_to_dateness.py:12)
SonarQubeSourceVersion # unused class (src/source_collectors/sonarqube/source_version.py:9)
SonarQubeSuppressedViolations # unused class (src/source_collectors/sonarqube/suppressed_violations.py:10)
SonarQubeUncoveredBranches # unused class (src/source_collectors/sonarqube/uncovered_branches.py:6)
SonarQubeUncoveredLines # unused class (src/source_collectors/sonarqube/uncovered_lines.py:6)
TrelloIssues # unused class (src/source_collectors/trello/issues.py:13)
TrelloSourceUpToDateness # unused class (src/source_collectors/trello/source_up_to_dateness.py:12)
VulnerabilityID # unused variable (src/source_collectors/trivy/security_warnings.py:13)
Title # unused variable (src/source_collectors/trivy/security_warnings.py:14)
Description # unused variable (src/source_collectors/trivy/security_warnings.py:15)
PkgName # unused variable (src/source_collectors/trivy/security_warnings.py:17)
InstalledVersion # unused variable (src/source_collectors/trivy/security_warnings.py:18)
FixedVersion # unused variable (src/source_collectors/trivy/security_warnings.py:19)
References # unused variable (src/source_collectors/trivy/security_warnings.py:20)
Target # unused variable (src/source_collectors/trivy/security_warnings.py:26)
Vulnerabilities # unused variable (src/source_collectors/trivy/security_warnings.py:27)
TrivyJSONSecurityWarnings # unused class (src/source_collectors/trivy/security_warnings.py:32)
RobotFrameworkSourceVersion # unused class (tests/source_collectors/robot_framework/test_source_version.py:6)
1 change: 1 addition & 0 deletions components/collector/src/source_collectors/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,3 +142,4 @@
from .testng.tests import TestNGTests
from .trello.issues import TrelloIssues
from .trello.source_up_to_dateness import TrelloSourceUpToDateness
from .trivy.security_warnings import TrivyJSONSecurityWarnings
Empty file added components/collector/src/source_collectors/trivy/__init__.py
View file
Empty file.
62 changes: 62 additions & 0 deletions components/collector/src/source_collectors/trivy/security_warnings.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""Trivy JSON collector."""

from typing import TypedDict, cast

from base_collectors import JSONFileSourceCollector
from collector_utilities.type import JSON
from model import Entities, Entity


class TrivyJSONVulnerability(TypedDict):
"""Trivy JSON for one vulnerability."""

VulnerabilityID: str
Title: str
Description: str
Severity: str
PkgName: str
InstalledVersion: str
FixedVersion: str
References: list[str]


class TrivyJSONDependencyRepository(TypedDict):
"""Trivy JSON for one dependency repository."""

Target: str
Vulnerabilities: list[TrivyJSONVulnerability] | None


TrivyJSON = list[TrivyJSONDependencyRepository]


class TrivyJSONSecurityWarnings(JSONFileSourceCollector):
"""Trivy JSON collector for security warnings."""

def _parse_json(self, json: JSON, filename: str) -> Entities:
"""Override to parse the analysis results from the Trivy JSON."""
entities = Entities()
for dependency_repository in cast(TrivyJSON, json):
target = dependency_repository["Target"]
for vulnerability in dependency_repository.get("Vulnerabilities") or []:
vulnerability_id = vulnerability["VulnerabilityID"]
package_name = vulnerability["PkgName"]
entities.append(
Entity(
key=f"{vulnerability_id}@{package_name}@{target}",
vulnerability_id=vulnerability_id,
title=vulnerability["Title"],
description=vulnerability["Description"],
level=vulnerability["Severity"],
package_name=package_name,
installed_version=vulnerability["InstalledVersion"],
fixed_version=vulnerability.get("FixedVersion", "none"),
url=vulnerability["References"][0],
),
)
return entities

def _include_entity(self, entity: Entity) -> bool:
"""Return whether to include the entity in the measurement."""
levels = self._parameter("levels")
return entity["level"].lower() in levels
Empty file added components/collector/tests/source_collectors/trivy/__init__.py
View file
Empty file.
Loading

0 comments on commit 0a57df4

Please sign in to comment.