Support Trivy JSON files as source for the security warnings metric. C…

…loses #6927.

components/api_server/src/example-reports/example-report.json

@@ -472,6 +472,16 @@
                                 "password": "",
                                 "private_token": ""
                             }
+                        },
+                        "84eee1c2-53e5-5260-75cb-2b444a5ba336": {
+                            "type": "trivy_json",
+                            "parameters": {
+                                "url": "http://testdata:8000/reports/trivy_json/trivy.json",
+                                "landing_url": "http://localhost:8000/reports/trivy_json/trivy.json",
+                                "username": "",
+                                "password": "",
+                                "private_token": ""
+                            }
                         }
                     },
                     "name": null,

components/collector/.vulture_ignore_list.py

@@ -1,5 +1,14 @@
-__new__  # unused function (src/source_collectors/azure_devops/source_up_to_dateness.py:60)
-__new__  # unused function (src/source_collectors/gitlab/source_up_to_dateness.py:136)
-scan_status  # unused variable (src/source_collectors/harbor/security_warnings.py:43)
-id  # unused variable (src/source_collectors/jira/velocity.py:17)
-id  # unused variable (src/source_collectors/jira/velocity.py:31)
+__new__  # unused function (src/source_collectors/azure_devops/source_up_to_dateness.py:56)
+__new__  # unused function (src/source_collectors/gitlab/source_up_to_dateness.py:133)
+scan_status  # unused variable (src/source_collectors/harbor/security_warnings.py:58)
+id  # unused variable (src/source_collectors/jira/velocity.py:16)
+id  # unused variable (src/source_collectors/jira/velocity.py:30)
+VulnerabilityID  # unused variable (src/source_collectors/trivy/security_warnings.py:13)
+Title  # unused variable (src/source_collectors/trivy/security_warnings.py:14)
+Description  # unused variable (src/source_collectors/trivy/security_warnings.py:15)
+PkgName  # unused variable (src/source_collectors/trivy/security_warnings.py:17)
+InstalledVersion  # unused variable (src/source_collectors/trivy/security_warnings.py:18)
+FixedVersion  # unused variable (src/source_collectors/trivy/security_warnings.py:19)
+References  # unused variable (src/source_collectors/trivy/security_warnings.py:20)
+Target  # unused variable (src/source_collectors/trivy/security_warnings.py:26)
+Vulnerabilities  # unused variable (src/source_collectors/trivy/security_warnings.py:27)

components/collector/ci/quality.sh

@@ -20,7 +20,7 @@ run pipx run `spec safety` check --bare -r requirements/requirements.txt -r requ
 run pipx run `spec bandit` --quiet --recursive src/
 
 # Vulture
-NAMES_TO_IGNORE='Anchore*,Axe*,AzureDevops*,Bandit*,Calendar*,CargoAudit*,Cloc*,Cobertura*,Composer*,CxSAST*,Gatling*,Generic*,GitLab*,Harbor*,Jacoco*,Jenkins*,Jira*,JMeter*,JUnit*,ManualNumber*,NCover*,Npm*,OJAudit*,OpenVAS*,OWASPDependencyCheck*,OWASPZAP*,PerformanceTestRunner*,Pip*,PyupioSafety*,QualityTime*,RobotFramework*,SARIF*,Snyk*,SonarQube*,Trello*'
+NAMES_TO_IGNORE='Anchore*,Axe*,AzureDevops*,Bandit*,Calendar*,CargoAudit*,Cloc*,Cobertura*,Composer*,CxSAST*,Gatling*,Generic*,GitLab*,Harbor*,Jacoco*,Jenkins*,Jira*,JMeter*,JUnit*,ManualNumber*,NCover*,Npm*,OJAudit*,OpenVAS*,OWASPDependencyCheck*,OWASPZAP*,PerformanceTestRunner*,Pip*,PyupioSafety*,QualityTime*,RobotFramework*,SARIF*,Snyk*,SonarQube*,Trello*,TrivyJSON*'
 run pipx run `spec vulture` --min-confidence 0 --ignore-names $NAMES_TO_IGNORE src/ tests/ .vulture_ignore_list.py
 
 # Black

components/collector/src/model/entity.py

@@ -18,7 +18,10 @@ def __init__(self, key: str, **attributes) -> None:
 
     @staticmethod
     def safe_entity_key(key: str) -> str:
-        """Return an escaped version of the key that is safe in URLs and as Mongo document key."""
+        """Return an escaped version of the key that is safe in URLs and as Mongo document key.
+
+        See https://www.mongodb.com/docs/manual/core/document/#field-names.
+        """
         return re.sub(QUOTED_SLASH, "-", str(key).replace("/", "-").replace(".", "_"))
 
 

components/collector/src/source_collectors/__init__.py

@@ -142,3 +142,4 @@
 from .testng.tests import TestNGTests
 from .trello.issues import TrelloIssues
 from .trello.source_up_to_dateness import TrelloSourceUpToDateness
+from .trivy.security_warnings import TrivyJSONSecurityWarnings

components/collector/src/source_collectors/trivy/security_warnings.py

@@ -0,0 +1,68 @@
+"""Trivy JSON collector."""
+
+from typing import TypedDict, cast
+
+from base_collectors import JSONFileSourceCollector
+from collector_utilities.type import JSON
+from model import Entities, Entity
+
+# The types below are based on https://aquasecurity.github.io/trivy/v0.45/docs/configuration/reporting/#json.
+# That documentation says: "VulnerabilityID, PkgName, InstalledVersion, and Severity in Vulnerabilities are always
+# filled with values, but other fields might be empty." This unfortunately does not tell us whether empty means
+# an empty string or null. It's also unclear whether keys may be missing. For now we assume all keys are always
+# present and missing values are empty strings.
+
+
+class TrivyJSONVulnerability(TypedDict):
+    """Trivy JSON for one vulnerability."""
+
+    VulnerabilityID: str
+    Title: str
+    Description: str
+    Severity: str
+    PkgName: str
+    InstalledVersion: str
+    FixedVersion: str
+    References: list[str]
+
+
+class TrivyJSONDependencyRepository(TypedDict):
+    """Trivy JSON for one dependency repository."""
+
+    Target: str
+    Vulnerabilities: list[TrivyJSONVulnerability] | None  # The examples in the Trivy docs show this key can be null
+
+
+TrivyJSON = list[TrivyJSONDependencyRepository]
+
+
+class TrivyJSONSecurityWarnings(JSONFileSourceCollector):
+    """Trivy JSON collector for security warnings."""
+
+    def _parse_json(self, json: JSON, filename: str) -> Entities:
+        """Override to parse the vulnerabilities from the Trivy JSON."""
+        entities = Entities()
+        for dependency_repository in cast(TrivyJSON, json):
+            target = dependency_repository["Target"]
+            for vulnerability in dependency_repository.get("Vulnerabilities") or []:
+                vulnerability_id = vulnerability["VulnerabilityID"]
+                package_name = vulnerability["PkgName"]
+                entities.append(
+                    Entity(
+                        key=f"{vulnerability_id}@{package_name}@{target}",
+                        vulnerability_id=vulnerability_id,
+                        title=vulnerability["Title"],
+                        description=vulnerability["Description"],
+                        level=vulnerability["Severity"],
+                        package_name=package_name,
+                        installed_version=vulnerability["InstalledVersion"],
+                        fixed_version=vulnerability["FixedVersion"],
+                        url=vulnerability["References"][0],  # Assume the 1st link is at least as relevant as the others
+                    ),
+                )
+        return entities
+
+    def _include_entity(self, entity: Entity) -> bool:
+        """Return whether to include the entity in the measurement."""
+        levels = self._parameter("levels")
+        return entity["level"].lower() in levels

components/collector/tests/source_collectors/trivy/test_security_warnings.py

@@ -0,0 +1,85 @@
+"""Unit tests for the Trivy JSON security warnings collector."""
+
+from typing import ClassVar
+
+from source_collectors.trivy.security_warnings import TrivyJSON
+
+from tests.source_collectors.source_collector_test_case import SourceCollectorTestCase
+
+
+class TrivyJSONSecurityWarningsTest(SourceCollectorTestCase):
+    """Unit tests for the security warning metric."""
+
+    SOURCE_TYPE = "trivy_json"
+    METRIC_TYPE = "security_warnings"
+    VULNERABILITIES_JSON: ClassVar[TrivyJSON] = [
+        {
+            "Target": "php-app/composer.lock",
+            "Vulnerabilities": None,
+        },
+        {
+            "Target": "trivy-ci-test (alpine 3.7.1)",
+            "Vulnerabilities": [
+                {
+                    "VulnerabilityID": "CVE-2018-16840",
+                    "PkgName": "curl",
+                    "InstalledVersion": "7.61.0-r0",
+                    "FixedVersion": "7.61.1-r1",
+                    "Title": 'curl: Use-after-free when closing "easy" handle in Curl_close()',
+                    "Description": "A heap use-after-free flaw was found in curl versions from 7.59.0 through ...",
+                    "Severity": "HIGH",
+                    "References": [
+                        "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-16840",
+                    ],
+                },
+                {
+                    "VulnerabilityID": "CVE-2019-3822",
+                    "PkgName": "curl",
+                    "InstalledVersion": "7.61.1-r0",
+                    "FixedVersion": "7.61.2-r2",
+                    "Title": "curl: NTLMv2 type-3 header stack buffer overflow",
+                    "Description": "libcurl versions from 7.36.0 to before 7.64.0 are vulnerable to ...",
+                    "Severity": "MEDIUM",
+                    "References": [
+                        "https://curl.haxx.se/docs/CVE-2019-3822.html",
+                        "https://lists.apache.org/thread.html",
+                    ],
+                },
+            ],
+        },
+    ]
+    EXPECTED_ENTITIES: ClassVar[list[dict[str, str]]] = [
+        {
+            "key": "CVE-2018-16840@curl@trivy-ci-test (alpine 3_7_1)",
+            "vulnerability_id": "CVE-2018-16840",
+            "title": 'curl: Use-after-free when closing "easy" handle in Curl_close()',
+            "description": "A heap use-after-free flaw was found in curl versions from 7.59.0 through ...",
+            "level": "HIGH",
+            "package_name": "curl",
+            "installed_version": "7.61.0-r0",
+            "fixed_version": "7.61.1-r1",
+            "url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-16840",
+        },
+        {
+            "key": "CVE-2019-3822@curl@trivy-ci-test (alpine 3_7_1)",
+            "vulnerability_id": "CVE-2019-3822",
+            "title": "curl: NTLMv2 type-3 header stack buffer overflow",
+            "description": "libcurl versions from 7.36.0 to before 7.64.0 are vulnerable to ...",
+            "level": "MEDIUM",
+            "package_name": "curl",
+            "installed_version": "7.61.1-r0",
+            "fixed_version": "7.61.2-r2",
+            "url": "https://curl.haxx.se/docs/CVE-2019-3822.html",
+        },
+    ]
+
+    async def test_warnings(self):
+        """Test the number of security warnings."""
+        response = await self.collect(get_request_json_return_value=self.VULNERABILITIES_JSON)
+        self.assert_measurement(response, value="2", entities=self.EXPECTED_ENTITIES)
+
+    async def test_warning_levels(self):
+        """Test the number of security warnings when specifying a level."""
+        self.set_source_parameter("levels", ["high", "critical"])
+        response = await self.collect(get_request_json_return_value=self.VULNERABILITIES_JSON)
+        self.assert_measurement(response, value="1", entities=[self.EXPECTED_ENTITIES[0]])

components/shared_code/src/shared_data_model/metrics.py

@@ -327,6 +327,50 @@
         sources=["manual_number", "performancetest_runner"],
         tags=[Tag.PERFORMANCE],
     ),
+    "security_warnings": Metric(
+        name="Security warnings",
+        description="The number of security warnings about the software.",
+        rationale="Monitor security warnings about the software, its source code, dependencies, or "
+        "infrastructure so vulnerabilities can be fixed before they end up in production.",
+        unit=Unit.SECURITY_WARNINGS,
+        near_target="5",
+        sources=[
+            "anchore",
+            "anchore_jenkins_plugin",
+            "bandit",
+            "cargo_audit",
+            "cxsast",
+            "generic_json",
+            "harbor",
+            "manual_number",
+            "openvas",
+            "owasp_dependency_check",
+            "owasp_zap",
+            "pyupio_safety",
+            "sarif_json",
+            "snyk",
+            "sonarqube",
+            "trivy_json",
+        ],
+        tags=[Tag.SECURITY],
+    ),
+    "sentiment": Metric(
+        name="Sentiment",
+        description="How are the team members feeling?",
+        rationale="Satisfaction is how fulfilled developers feel with their work, team, tools, or culture; "
+        "well-being is how healthy and happy they are, and how their work impacts it. Measuring satisfaction "
+        "and well-being can be beneficial for understanding productivity and perhaps even for predicting it. "
+        "For example, productivity and satisfaction are correlated, and it is possible that satisfaction could "
+        "serve as a leading indicator for productivity; a decline in satisfaction and engagement could signal "
+        "upcoming burnout and reduced productivity.",
+        rationale_urls=["https://queue.acm.org/detail.cfm?id=3454124"],
+        unit=Unit.NONE,
+        addition=Addition.MIN,
+        direction=Direction.MORE_IS_BETTER,
+        target="10",
+        near_target="8",
+        sources=["manual_number"],
+    ),
     "slow_transactions": Metric(
         name="Slow transactions",
         description="The number of transactions slower than their target response time.",
@@ -337,6 +381,24 @@
         sources=["gatling", "manual_number", "jmeter_csv", "jmeter_json", "performancetest_runner"],
         tags=[Tag.PERFORMANCE],
     ),
+    "software_version": Metric(
+        name="Software version",
+        description="The version number of the software as analyzed by the source.",
+        rationale="Monitor that the version of the software is at least a specific version or get notified when "
+        "the software version becomes higher than a specific version.",
+        explanation=VERSION_NUMBER_EXPLANATION,
+        explanation_urls=VERSION_NUMBER_EXPLANATION_URLS,
+        scales=["version_number"],
+        addition=Addition.MIN,
+        direction=Direction.MORE_IS_BETTER,
+        target="1.0",
+        near_target="0.9",
+        sources=[
+            "performancetest_runner",
+            "sonarqube",
+        ],
+        tags=[Tag.CI],
+    ),
     "source_up_to_dateness": Metric(
         name="Source up-to-dateness",
         description="The number of days since the source was last updated.",
@@ -378,24 +440,6 @@
         ],
         tags=[Tag.CI],
     ),
-    "software_version": Metric(
-        name="Software version",
-        description="The version number of the software as analyzed by the source.",
-        rationale="Monitor that the version of the software is at least a specific version or get notified when "
-        "the software version becomes higher than a specific version.",
-        explanation=VERSION_NUMBER_EXPLANATION,
-        explanation_urls=VERSION_NUMBER_EXPLANATION_URLS,
-        scales=["version_number"],
-        addition=Addition.MIN,
-        direction=Direction.MORE_IS_BETTER,
-        target="1.0",
-        near_target="0.9",
-        sources=[
-            "performancetest_runner",
-            "sonarqube",
-        ],
-        tags=[Tag.CI],
-    ),
     "source_version": Metric(
         name="Source version",
         description="The version number of the source.",
@@ -438,49 +482,6 @@
         ],
         tags=[Tag.CI],
     ),
-    "security_warnings": Metric(
-        name="Security warnings",
-        description="The number of security warnings about the software.",
-        rationale="Monitor security warnings about the software, its source code, dependencies, or "
-        "infrastructure so vulnerabilities can be fixed before they end up in production.",
-        unit=Unit.SECURITY_WARNINGS,
-        near_target="5",
-        sources=[
-            "anchore",
-            "anchore_jenkins_plugin",
-            "bandit",
-            "cargo_audit",
-            "cxsast",
-            "generic_json",
-            "harbor",
-            "manual_number",
-            "openvas",
-            "owasp_dependency_check",
-            "owasp_zap",
-            "pyupio_safety",
-            "sarif_json",
-            "snyk",
-            "sonarqube",
-        ],
-        tags=[Tag.SECURITY],
-    ),
-    "sentiment": Metric(
-        name="Sentiment",
-        description="How are the team members feeling?",
-        rationale="Satisfaction is how fulfilled developers feel with their work, team, tools, or culture; "
-        "well-being is how healthy and happy they are, and how their work impacts it. Measuring satisfaction "
-        "and well-being can be beneficial for understanding productivity and perhaps even for predicting it. "
-        "For example, productivity and satisfaction are correlated, and it is possible that satisfaction could "
-        "serve as a leading indicator for productivity; a decline in satisfaction and engagement could signal "
-        "upcoming burnout and reduced productivity.",
-        rationale_urls=["https://queue.acm.org/detail.cfm?id=3454124"],
-        unit=Unit.NONE,
-        addition=Addition.MIN,
-        direction=Direction.MORE_IS_BETTER,
-        target="10",
-        near_target="8",
-        sources=["manual_number"],
-    ),
     "suppressed_violations": Metric(
         name="Suppressed violations",
         description="The number of violations suppressed in the source.",

components/shared_code/src/shared_data_model/sources/__init__.py

@@ -36,6 +36,7 @@
 from .sonarqube import SONARQUBE
 from .testng import TESTNG
 from .trello import TRELLO
+from .trivy import TRIVY_JSON
 
 SOURCES = {
     "anchore": ANCHORE,
@@ -81,5 +82,6 @@
     "snyk": SNYK,
     "sonarqube": SONARQUBE,
     "testng": TESTNG,
+    "trivy_json": TRIVY_JSON,
     "trello": TRELLO,
 }