diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f8fadac8059..c858a899b76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,6 +63,7 @@ jobs: env: MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + NVD_API_KEY: ${{ secrets.NVD_API_KEY }} run: mvn -s settings.xml -Prelease clean package verify source:jar javadoc:jar gpg:sign deploy -DreleaseTesting --no-transfer-progress --batch-mode -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} - name: SARIF Multitool uses: microsoft/sarif-actions@v0.1 diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml index 040a88a81ee..2f8080ae2b9 100644 --- a/.github/workflows/pull_requests.yml +++ b/.github/workflows/pull_requests.yml @@ -73,6 +73,8 @@ jobs: version: 6.0.2 - name: Regression Test Maven Plugin id: build + env: + NVD_API_KEY: ${{ secrets.NVD_API_KEY }} run: | mvn -s settings.xml -pl utils,core,maven -am compile verify -DtestMavenPlugin -DreleaseTesting --no-transfer-progress --batch-mode - name: Archive IT test logs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ab974ffa4f7..a465dc5b09b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,6 +74,7 @@ jobs: env: MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + NVD_API_KEY: ${{ secrets.NVD_API_KEY }} run: | mvn -s settings.xml -Prelease "-DnexusUrl=https://oss.sonatype.org/" clean package source:jar javadoc:jar gpg:sign deploy site site:stage -DreleaseTesting --no-transfer-progress --batch-mode -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} - name: Archive code coverage results diff --git a/README.md b/README.md index fd4629bee44..9f035e17e94 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,25 @@ Dependency-Check is a Software Composition Analysis (SCA) tool that attempts to Documentation and links to production binary releases can be found on the [github pages](http://jeremylong.github.io/DependencyCheck/). Additionally, more information about the architecture and ways to extend dependency-check can be found on the [wiki]. -## 8.0.0 Upgrade Notice +## 9.0.0 Upgrade Notice -8.0.0 contains breaking changes which requires updates to the database. If using +### NVD API Key Highly Recommended + +With 9.0.0 dependency-check has moved from using the NVD data-feed to the NVD API. +Users of dependency-check are **highly** encouraged to obtain an NVD API Key; see https://nvd.nist.gov/developers/request-an-api-key +Without an NVD API Key dependency-check's updates will be **extremely slow**. +Please see the documentation for the cli, maven, gradle, or ant integrations on +how to set the NVD API key. + +### Breaking Changes + +9.0.0 contains breaking changes which requires updates to the database. If using an externally hosted database the schema will need to be updated. When using the -embedded H2 database the schema should be upgraded automatically. However, if +embedded H2 database, the schema should be upgraded automatically. However, if issues arise you may need to purge the database: - gradle: `./gradlew dependencyCheckPurge` -- maven: `mvn org.owasp:dependency-check-maven:8.0.0:purge` +- maven: `mvn org.owasp:dependency-check-maven:9.0.0:purge` - cli: `dependency-check.sh --purge` ## Requirements diff --git a/ant/pom.xml b/ant/pom.xml index b52eec3c430..a975bebfaa4 100644 --- a/ant/pom.xml +++ b/ant/pom.xml @@ -20,7 +20,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved. org.owasp dependency-check-parent - 8.4.4-SNAPSHOT + 9.0.0-SNAPSHOT dependency-check-ant diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java index fbfe1c05b86..f267e311155 100644 --- a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java +++ b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java @@ -41,8 +41,8 @@ import org.owasp.dependencycheck.utils.Settings; import org.owasp.dependencycheck.utils.SeverityUtil; import org.slf4j.impl.StaticLoggerBinder; -//CSOFF: MethodCount +//CSOFF: MethodCount /** * An Ant task definition to execute dependency-check during an Ant build. * @@ -882,7 +882,7 @@ public Boolean isNuspecAnalyzerEnabled() { */ public Boolean isNugetconfAnalyzerEnabled() { return nugetconfAnalyzerEnabled; - } + } /** * Sets whether or not the analyzer is enabled. @@ -2217,8 +2217,8 @@ private void checkForFailure(Dependency[] dependencies) throws BuildException { for (Dependency d : dependencies) { boolean addName = true; for (Vulnerability v : d.getVulnerabilities()) { - if ((v.getCvssV2() != null && v.getCvssV2().getScore() >= failBuildOnCVSS) - || (v.getCvssV3() != null && v.getCvssV3().getBaseScore() >= failBuildOnCVSS) + if ((v.getCvssV2() != null && v.getCvssV2().getCvssData().getBaseScore() >= failBuildOnCVSS) + || (v.getCvssV3() != null && v.getCvssV3().getCvssData().getBaseScore() >= failBuildOnCVSS) || (v.getUnscoredSeverity() != null && SeverityUtil.estimateCvssV2(v.getUnscoredSeverity()) >= failBuildOnCVSS) //safety net to fail on any if for some reason the above misses on 0 || (failBuildOnCVSS <= 0.0f)) { diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java index b0ea013aa77..152b883e078 100644 --- a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java +++ b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java @@ -133,10 +133,13 @@ public void setHostedSuppressionsUrl(final String hostedSuppressionsUrl) { } /** - * Sets the {@link Thread#getContextClassLoader() Thread Context Class Loader} to the one for this class, - * and then calls {@link #executeWithContextClassloader()}. This is done because the JCS cache needs to have - * the Thread Context Class Loader set to something that can resolve it's classes. Other build tools do this - * by default but Ant does not. + * Sets the + * {@link Thread#getContextClassLoader() Thread Context Class Loader} to the + * one for this class, and then calls + * {@link #executeWithContextClassloader()}. This is done because the JCS + * cache needs to have the Thread Context Class Loader set to something that + * can resolve it's classes. Other build tools do this by default but Ant + * does not. * * @throws BuildException throws if there is a problem. See * {@link #executeWithContextClassloader()} for details diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java index 3cf35c381a8..ed128d7c45c 100644 --- a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java +++ b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java @@ -17,14 +17,11 @@ */ package org.owasp.dependencycheck.taskdefs; -import java.util.Optional; - import org.apache.tools.ant.BuildException; import org.apache.tools.ant.Project; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.utils.CveUrlParser; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.impl.StaticLoggerBinder; @@ -40,6 +37,31 @@ @SuppressWarnings("common-java:DuplicatedBlocks") public class Update extends Purge { + /** + * The NVD API Key. + */ + private String nvdApiKey; + /** + * The number of hours to wait before checking for new updates from the NVD. + */ + private Integer nvdValidForHours; + /** + * The NVD API Data Feed URL. + */ + private String nvdDatafeedUrl; + /** + * The username for basic auth to the NVD Data Feed. + */ + private String nvdUser; + /** + * The password for basic auth to the NVD Data Feed. + */ + private String nvdPassword; + /** + * The time in milliseconds to wait between downloading NVD API data. + */ + private int nvdApiDelay = 0; + /** * The Proxy Server. */ @@ -89,47 +111,136 @@ public class Update extends Purge { */ private String databasePassword; /** - * The URL for the modified NVD CVE JSON file. + * The number of hours to wait before re-checking hosted suppressions file + * for updates. */ - private String cveUrlModified; + private Integer hostedSuppressionsValidForHours; /** - * Base Data Mirror URL for CVE JSON files. + * Whether the hosted suppressions file will be updated regardless of the + * `autoupdate` settings. Defaults to false. */ - private String cveUrlBase; + private Boolean hostedSuppressionsForceUpdate; /** - * The wait time in milliseconds between downloads from the NVD. + * Whether the hosted suppressions file will be used. Defaults to true. */ - private String cveWaitTime; + private Boolean hostedSuppressionsEnabled; + /** - * The number of hours to wait before re-checking for updates. + * Construct a new UpdateTask. */ - private Integer cveValidForHours; + public Update() { + super(); + // Call this before Dependency Check Core starts logging anything - this way, all SLF4J messages from + // core end up coming through this tasks logger + StaticLoggerBinder.getSingleton().setTask(this); + } + /** - * The number of hours to wait before re-checking hosted suppressions file for updates. + * Get the value of nvdApiKey. + * + * @return the value of nvdApiKey */ - private Integer hostedSuppressionsValidForHours; + public String getNvdApiKey() { + return nvdApiKey; + } + /** - * Whether the hosted suppressions file will be updated regardless of the `autoupdate` settings. Defaults to false. + * Set the value of nvdApiKey. + * + * @param nvdApiKey new value of nvdApiKey */ - private Boolean hostedSuppressionsForceUpdate; + public void setNvdApiKey(String nvdApiKey) { + this.nvdApiKey = nvdApiKey; + } + /** - * Whether the hosted suppressions file will be used. Defaults to true. + * Get the value of nvdValidForHours. + * + * @return the value of nvdValidForHours */ - private Boolean hostedSuppressionsEnabled; + public int getNvdValidForHours() { + return nvdValidForHours; + } /** - * Specify the first year of NVD CVE data to download; default is 2002. + * Set the value of nvdValidForHours. + * + * @param nvdValidForHours new value of nvdValidForHours */ - private Integer cveStartYear; + public void setNvdValidForHours(int nvdValidForHours) { + this.nvdValidForHours = nvdValidForHours; + } /** - * Construct a new UpdateTask. + * Get the value of nvdDatafeedUrl. + * + * @return the value of nvdDatafeedUrl */ - public Update() { - super(); - // Call this before Dependency Check Core starts logging anything - this way, all SLF4J messages from - // core end up coming through this tasks logger - StaticLoggerBinder.getSingleton().setTask(this); + public String getNvdDatafeedUrl() { + return nvdDatafeedUrl; + } + + /** + * Set the value of nvdDatafeedUrl. + * + * @param nvdDatafeedUrl new value of nvdDatafeedUrl + */ + public void setNvdDatafeedUrl(String nvdDatafeedUrl) { + this.nvdDatafeedUrl = nvdDatafeedUrl; + } + + /** + * Get the value of nvdUser. + * + * @return the value of nvdUser + */ + public String getNvdUser() { + return nvdUser; + } + + /** + * Set the value of nvdUser. + * + * @param nvdUser new value of nvdUser + */ + public void setNvdUser(String nvdUser) { + this.nvdUser = nvdUser; + } + + /** + * Get the value of nvdPassword. + * + * @return the value of nvdPassword + */ + public String getNvdPassword() { + return nvdPassword; + } + + /** + * Set the value of nvdPassword. + * + * @param nvdPassword new value of nvdPassword + */ + public void setNvdPassword(String nvdPassword) { + this.nvdPassword = nvdPassword; + } + + /** + * Get the value of nvdApiDelay. + * + * @return the value of nvdApiDelay + */ + public int getNvdApiDelay() { + return nvdApiDelay; + } + + /** + * Set the value of nvdApiDelay. + * + * @param nvdApiDelay new value of nvdApiDelay + */ + public void setNvdApiDelay(int nvdApiDelay) { + this.nvdApiDelay = nvdApiDelay; } /** @@ -348,101 +459,6 @@ public void setDatabasePassword(String databasePassword) { this.databasePassword = databasePassword; } - /** - * Set the value of cveUrlModified. - * - * @param cveUrlModified new value of cveUrlModified - */ - public void setCveUrlModified(String cveUrlModified) { - this.cveUrlModified = cveUrlModified; - } - - /** - * Get the value of cveUrlModified. - * - * @return the value of cveUrlModified - */ - public String getCveUrlModified() { - return cveUrlModified; - } - - /** - * Get the value of cveUrlBase. - * - * @return the value of cveUrlBase - */ - public String getCveUrlBase() { - return cveUrlBase; - } - - /** - * Set the value of cveUrlBase. - * - * @param cveUrlBase new value of cveUrlBase - */ - public void setCveUrlBase(String cveUrlBase) { - this.cveUrlBase = cveUrlBase; - } - - /** - * Get the value of cveUrlBase. - * - * @return the value of cveUrlBase - */ - public String getCveWaitTime() { - return cveWaitTime; - } - - /** - * Set the value of cveWaitTime. - * - * @param cveWaitTime new value of cveWaitTime - */ - public void setCveWaitTime(String cveWaitTime) { - this.cveWaitTime = cveWaitTime; - } - - /** - * Get the value of cveValidForHours. - * - * @return the value of cveValidForHours - */ - public Integer getCveValidForHours() { - return cveValidForHours; - } - - /** - * Set the value of cveValidForHours. - * - * @param cveValidForHours new value of cveValidForHours - */ - public void setCveValidForHours(Integer cveValidForHours) { - this.cveValidForHours = cveValidForHours; - } - - /** - * Get the value of cveStartYear. - * - * @return the value of cveStartYear - */ - public Integer getCveStartYear() { - return cveStartYear; - } - - /** - * Set the value of cveStartYear. - * - * @param cveStartYear new value of cveStartYear - */ - public void setCveStartYear(Integer cveStartYear) { - if (cveStartYear != null && cveStartYear < 2002) { - log("Invalid Configuration: cveStartYear must be 2002 or greater", Project.MSG_ERR); - this.cveStartYear = 2002; - } else { - this.cveStartYear = cveStartYear; - } - } - /** * Get the value of hostedSuppressionsValidForHours. * @@ -455,7 +471,8 @@ public Integer getHostedSuppressionsValidForHours() { /** * Set the value of hostedSuppressionsValidForHours. * - * @param hostedSuppressionsValidForHours new value of hostedSuppressionsValidForHours + * @param hostedSuppressionsValidForHours new value of + * hostedSuppressionsValidForHours */ public void setHostedSuppressionsValidForHours(final Integer hostedSuppressionsValidForHours) { this.hostedSuppressionsValidForHours = hostedSuppressionsValidForHours; @@ -473,7 +490,8 @@ public Boolean isHostedSuppressionsForceUpdate() { /** * Set the value of hostedSuppressionsForceUpdate. * - * @param hostedSuppressionsForceUpdate new value of hostedSuppressionsForceUpdate + * @param hostedSuppressionsForceUpdate new value of + * hostedSuppressionsForceUpdate */ public void setHostedSuppressionsForceUpdate(final Boolean hostedSuppressionsForceUpdate) { this.hostedSuppressionsForceUpdate = hostedSuppressionsForceUpdate; @@ -487,6 +505,7 @@ public void setHostedSuppressionsForceUpdate(final Boolean hostedSuppressionsFor public Boolean isHostedSuppressionsEnabled() { return hostedSuppressionsEnabled; } + /** * Set the value of hostedSuppressionsEnabled. * @@ -550,28 +569,21 @@ protected void populateSettings() throws BuildException { getSettings().setStringIfNotEmpty(Settings.KEYS.DB_CONNECTION_STRING, connectionString); getSettings().setStringIfNotEmpty(Settings.KEYS.DB_USER, databaseUser); getSettings().setStringIfNotEmpty(Settings.KEYS.DB_PASSWORD, databasePassword); - - final String cveModifiedJson = Optional.ofNullable(cveUrlModified) - .filter(url -> !url.isEmpty()) - .orElseGet(this::getDefaultCveUrlModified); - getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, cveModifiedJson); - getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, cveUrlBase); - getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, cveWaitTime); - getSettings().setIntIfNotNull(Settings.KEYS.CVE_START_YEAR, cveStartYear); getSettings().setIntIfNotNull(Settings.KEYS.HOSTED_SUPPRESSIONS_VALID_FOR_HOURS, hostedSuppressionsValidForHours); getSettings().setBooleanIfNotNull(Settings.KEYS.HOSTED_SUPPRESSIONS_FORCEUPDATE, hostedSuppressionsForceUpdate); getSettings().setBooleanIfNotNull(Settings.KEYS.HOSTED_SUPPRESSIONS_ENABLED, hostedSuppressionsEnabled); - if (cveValidForHours != null) { - if (cveValidForHours >= 0) { - getSettings().setInt(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours); + + getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_KEY, nvdApiKey); + getSettings().setIntIfNotNull(Settings.KEYS.NVD_API_DELAY, nvdApiDelay); + getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_URL, nvdDatafeedUrl); + getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_USER, nvdUser); + getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_PASSWORD, nvdPassword); + if (nvdValidForHours != null) { + if (nvdValidForHours >= 0) { + getSettings().setInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, nvdValidForHours); } else { - throw new BuildException("Invalid setting: `cpeValidForHours` must be 0 or greater"); + throw new BuildException("Invalid setting: `nvdValidForHours` must be 0 or greater"); } } } - - private String getDefaultCveUrlModified() { - return CveUrlParser.newInstance(getSettings()) - .getDefaultCveUrlModified(cveUrlBase); - } } diff --git a/ant/src/main/resources/task.properties b/ant/src/main/resources/task.properties index 03c72cdc675..c35921991a8 100644 --- a/ant/src/main/resources/task.properties +++ b/ant/src/main/resources/task.properties @@ -1,2 +1,2 @@ # the path to the data directory -data.directory=data/7.0 +data.directory=data/9.0 diff --git a/ant/src/site/markdown/config-update.md b/ant/src/site/markdown/config-update.md index c3b776a0ec3..3cb7b0d7bdf 100644 --- a/ant/src/site/markdown/config-update.md +++ b/ant/src/site/markdown/config-update.md @@ -30,15 +30,16 @@ failOnError | Whether the build should fail if there is an error execu Advanced Configuration ==================== -The following properties can be configured in the plugin. However, they are less frequently changed. One exception -may be the cvedUrl properties, which can be used to host a mirror of the NVD within an enterprise environment. +The following properties can be configured in the plugin. However, they are less frequently changed. Property | Description | Default Value ---------------------|----------------------------------------------------------------------------------------------------------------------|------------------ -cveUrlModified | URL for the modified CVE JSON data feed. When mirroring the NVD you must mirror the *.json.gz and the *.meta files. Optional if your custom cveUrlBase is just a domain name change. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz -cveUrlBase | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz -cveWaitTime | The time in milliseconds to wait between downloads from the NVD. | 4000 -cveStartYear | The first year of NVD CVE data to download from the NVD. | 2002 +nvdApiKey | The API Key to access the NVD API; obtained from https://nvd.nist.gov/developers/request-an-api-key |   +nvdApiDelay | The number of milliseconds to wait between calls to the NVD API. |   +nvdDatafeedUrl | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data - example value `https://internal.server/cache/nvdcve-{0}.json.gz` |   +nvdUser | Credentials used for basic authentication for the NVD API Data feed. |   +nvdPassword | Credentials used for basic authentication for the NVD API Data feed. |   +nvdValidForHours | The number of hours to wait before checking for new updates from the NVD. The default is 4 hours. | 4 dataDirectory | Data directory that is used to store the local copy of the NVD. This should generally not be changed. | data databaseDriverName | The name of the database driver. Example: org.h2.Driver. |   databaseDriverPath | The path to the database driver JAR file; only used if the driver is not in the class path. |   diff --git a/ant/src/site/markdown/configuration.md b/ant/src/site/markdown/configuration.md index 590ba015646..7910d1d76e9 100644 --- a/ant/src/site/markdown/configuration.md +++ b/ant/src/site/markdown/configuration.md @@ -33,7 +33,6 @@ The following properties can be set on the dependency-check task. Property | Description | Default Value ----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------- autoUpdate | Sets whether auto-updating of the NVD CVE/CPE data is enabled. It is not recommended that this be turned to false. | true -cveValidForHours | Sets the number of hours to wait before checking for new updates from the NVD | 4 failOnError | Whether the build should fail if there is an error executing the dependency-check analysis | true failBuildOnCVSS | Specifies if the build should be failed if a CVSS score equal to or above a specified level is identified. The default is 11 which means since the CVSS scores are 0-10, by default the build will never fail. More information on CVSS scores can be found at the [NVD](https://nvd.nist.gov/vuln-metrics/cvss)| 11 junitFailOnCVSS | If using the JUNIT report format the junitFailOnCVSS sets the CVSS score threshold that is considered a failure. | 0 @@ -140,15 +139,16 @@ pathToGo | The path to `go`. Advanced Configuration ==================== -The following properties can be configured in the plugin. However, they are less frequently changed. One exception -may be the cvedUrl properties, which can be used to host a mirror of the NVD within an enterprise environment. +The following properties can be configured in the plugin. However, they are less frequently changed. -Property | Description | Default Value ----------------------|--------------------------------------------------------------------------|------------------ -cveUrlModified | URL for the modified CVE JSON data feed. When mirroring the NVD you must mirror the *.json.gz and the *.meta files. Optional if your custom cveUrlBase is just a domain name change. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz -cveUrlBase | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz -cveWaitTime | The time in milliseconds to wait between downloads from the NVD. | 4000 -cveStartYear | The first year of NVD CVE data to download from the NVD. | 2002 +Property | Description | Default Value +---------------------|--------------------------------------------------------------------------------------------------------------|------------------ +nvdApiKey | The API Key to access the NVD API; obtained from https://nvd.nist.gov/developers/request-an-api-key |   +nvdApiDelay | The number of milliseconds to wait between calls to the NVD API. |   +nvdDatafeedUrl | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data - example value `https://internal.server/cache/nvdcve-{0}.json.gz` |   +nvdUser | Credentials used for basic authentication for the NVD API Data feed. |   +nvdPassword | Credentials used for basic authentication for the NVD API Data feed. |   +nvdValidForHours | The number of hours to wait before checking for new updates from the NVD. The default is 4 hours. | 4 dataDirectory | Data directory that is used to store the local copy of the NVD. This should generally not be changed. | data databaseDriverName | The name of the database driver. Example: org.h2.Driver. |   databaseDriverPath | The path to the database driver JAR file; only used if the driver is not in the class path. |   diff --git a/ant/src/test/java/org/owasp/dependencycheck/taskdefs/UpdateTest.java b/ant/src/test/java/org/owasp/dependencycheck/taskdefs/UpdateTest.java deleted file mode 100644 index 27b77fc031c..00000000000 --- a/ant/src/test/java/org/owasp/dependencycheck/taskdefs/UpdateTest.java +++ /dev/null @@ -1,90 +0,0 @@ -package org.owasp.dependencycheck.taskdefs; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.Is.is; - -import org.junit.Test; -import org.owasp.dependencycheck.BaseTest; -import org.owasp.dependencycheck.utils.Settings; - -public class UpdateTest extends BaseTest { - - @Test - public void testPopulateSettingsShouldSetDefaultValueToCveUrlModified() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Update update = new Update(); - update.setCveUrlModified(null); - update.setCveUrlBase("https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz"); - - // When - update.populateSettings(); - - // Then - String output = update.getSettings().getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to a default of the same model", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldSetDefaultValueToCveUrlModifiedWhenCveUrlModifiedIsEmpty() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Update update = new Update(); - update.setCveUrlModified(""); - update.setCveUrlBase("https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz"); - - // When - update.populateSettings(); - - // Then - String output = update.getSettings().getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to a default of the same model when arg is empty", output, - is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldNotSetDefaultValueToCveUrlModifiedWhenValueIsExplicitelySet() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Update update = new Update(); - update.setCveUrlModified("https://another-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"); - update.setCveUrlBase("https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/some-unusual-file-name-%d.json.gz"); - - // When - update.populateSettings(); - - // Then - String output = update.getSettings().getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://another-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to the specified value", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldNotSetDefaultValueToCveUrlModifiedWhenUnknownValueIsSet() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Update update = new Update(); - update.setCveUrlModified(null); - update.setCveUrlBase("https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/some-unusual-file-name-%d.json.gz"); - - // When - update.populateSettings(); - - // Then - String output = update.getSettings().getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must not be set when name is not the same as from the nvd datasource", output, - is(expectedOutput)); - } - -} diff --git a/archetype/pom.xml b/archetype/pom.xml index edec64cde8e..74938e08934 100644 --- a/archetype/pom.xml +++ b/archetype/pom.xml @@ -20,7 +20,7 @@ Copyright (c) 2017 Jeremy Long. All Rights Reserved. org.owasp dependency-check-parent - 8.4.4-SNAPSHOT + 9.0.0-SNAPSHOT dependency-check-plugin Dependency-Check Plugin Archetype diff --git a/cli/pom.xml b/cli/pom.xml index bae7e1aa4ff..60508c9b68f 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -20,7 +20,7 @@ Copyright (c) 2012 - Jeremy Long. All Rights Reserved. org.owasp dependency-check-parent - 8.4.4-SNAPSHOT + 9.0.0-SNAPSHOT dependency-check-cli diff --git a/cli/src/main/java/org/owasp/dependencycheck/App.java b/cli/src/main/java/org/owasp/dependencycheck/App.java index a326d91bc58..7011a077c86 100644 --- a/cli/src/main/java/org/owasp/dependencycheck/App.java +++ b/cli/src/main/java/org/owasp/dependencycheck/App.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.Set; import org.apache.commons.cli.ParseException; @@ -34,7 +33,6 @@ import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.owasp.dependencycheck.exception.ExceptionCollection; import org.owasp.dependencycheck.exception.ReportException; -import org.owasp.dependencycheck.utils.CveUrlParser; import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; @@ -306,21 +304,23 @@ private int determineReturnCode(Engine engine, float cvssFailScore) { for (Dependency d : engine.getDependencies()) { boolean addName = true; for (Vulnerability v : d.getVulnerabilities()) { - final float cvssV2 = v.getCvssV2() != null ? v.getCvssV2().getScore() : -1; - final float cvssV3 = v.getCvssV3() != null ? v.getCvssV3().getBaseScore() : -1; - final float unscoredCvss = v.getUnscoredSeverity() != null ? SeverityUtil.estimateCvssV2(v.getUnscoredSeverity()) : -1; + final Double cvssV2 = v.getCvssV2() != null && v.getCvssV2().getCvssData() != null + && v.getCvssV2().getCvssData().getBaseScore() != null ? v.getCvssV2().getCvssData().getBaseScore() : -1; + final Double cvssV3 = v.getCvssV3() != null && v.getCvssV3().getCvssData() != null + && v.getCvssV3().getCvssData().getBaseScore() != null ? v.getCvssV3().getCvssData().getBaseScore() : -1; + final Double unscoredCvss = v.getUnscoredSeverity() != null ? SeverityUtil.estimateCvssV2(v.getUnscoredSeverity()) : -1; if (cvssV2 >= cvssFailScore || cvssV3 >= cvssFailScore || unscoredCvss >= cvssFailScore //safety net to fail on any if for some reason the above misses on 0 || (cvssFailScore <= 0.0f)) { - float score = 0.0f; - if (cvssV3 >= 0.0f) { + double score = 0.0; + if (cvssV3 >= 0.0) { score = cvssV3; - } else if (cvssV2 >= 0.0f) { + } else if (cvssV2 >= 0.0) { score = cvssV2; - } else if (unscoredCvss >= 0.0f) { + } else if (unscoredCvss >= 0.0) { score = unscoredCvss; } if (addName) { @@ -478,10 +478,6 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException { cli.getStringArgument(CliParser.ARGUMENT.CONNECTION_READ_TIMEOUT)); settings.setStringIfNotEmpty(Settings.KEYS.HINTS_FILE, cli.getStringArgument(CliParser.ARGUMENT.HINTS_FILE)); - settings.setIntIfNotNull(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, - cli.getIntegerValue(CliParser.ARGUMENT.CVE_VALID_FOR_HOURS)); - settings.setIntIfNotNull(Settings.KEYS.CVE_START_YEAR, - cli.getIntegerValue(CliParser.ARGUMENT.CVE_START_YEAR)); settings.setArrayIfNotEmpty(Settings.KEYS.SUPPRESSION_FILE, cli.getStringArguments(CliParser.ARGUMENT.SUPPRESSION_FILES)); //File Type Analyzer Settings @@ -650,21 +646,13 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException { cli.getStringArgument(CliParser.ARGUMENT.ADDITIONAL_ZIP_EXTENSIONS)); settings.setStringIfNotEmpty(Settings.KEYS.ANALYZER_ASSEMBLY_DOTNET_PATH, cli.getStringArgument(CliParser.ARGUMENT.PATH_TO_CORE)); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, - cli.getStringArgument(CliParser.ARGUMENT.CVE_BASE_URL)); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, - cli.getStringArgument(CliParser.ARGUMENT.CVE_DOWNLOAD_WAIT_TIME)); - final String cveModifiedJson = Optional.ofNullable(cli.getStringArgument(CliParser.ARGUMENT.CVE_MODIFIED_URL)) - .filter(arg -> !arg.isEmpty()) - .orElseGet(() -> getDefaultCveUrlModified(cli)); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, - cveModifiedJson); - - settings.setStringIfNotEmpty(Settings.KEYS.CVE_USER, - cli.getStringArgument(CliParser.ARGUMENT.CVE_USER)); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_PASSWORD, - cli.getStringArgument(CliParser.ARGUMENT.CVE_PASSWORD, Settings.KEYS.CVE_PASSWORD)); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_KEY, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_KEY)); + settings.setIntIfNotNull(Settings.KEYS.NVD_API_DELAY, cli.getIntegerValue(CliParser.ARGUMENT.NVD_API_DELAY)); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_URL, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_DATAFEED_URL)); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_USER, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_DATAFEED_USER)); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_PASSWORD, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_DATAFEED_PASSWORD)); + settings.setIntIfNotNull(Settings.KEYS.NVD_API_VALID_FOR_HOURS, cli.getIntegerValue(CliParser.ARGUMENT.NVD_API_VALID_FOR_HOURS)); settings.setStringIfNotNull(Settings.KEYS.HOSTED_SUPPRESSIONS_URL, cli.getStringArgument(CliParser.ARGUMENT.HOSTED_SUPPRESSIONS_URL)); @@ -676,11 +664,6 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException { cli.getIntegerValue(CliParser.ARGUMENT.HOSTED_SUPPRESSIONS_VALID_FOR_HOURS)); } - private String getDefaultCveUrlModified(CliParser cli) { - return CveUrlParser.newInstance(settings) - .getDefaultCveUrlModified(cli.getStringArgument(CliParser.ARGUMENT.CVE_BASE_URL)); - } - //CSON: MethodLength /** * Creates a file appender and adds it to logback. diff --git a/cli/src/main/java/org/owasp/dependencycheck/CliParser.java b/cli/src/main/java/org/owasp/dependencycheck/CliParser.java index 2a88f067330..50d55702145 100644 --- a/cli/src/main/java/org/owasp/dependencycheck/CliParser.java +++ b/cli/src/main/java/org/owasp/dependencycheck/CliParser.java @@ -114,26 +114,26 @@ private CommandLine parseArgs(String[] args) throws ParseException { */ private void validateArgs() throws FileNotFoundException, ParseException { if (isUpdateOnly() || isRunScan()) { - String value = line.getOptionValue(ARGUMENT.CVE_VALID_FOR_HOURS); + String value = line.getOptionValue(ARGUMENT.NVD_API_VALID_FOR_HOURS); if (value != null) { try { final int i = Integer.parseInt(value); if (i < 0) { - throw new ParseException("Invalid Setting: cveValidForHours must be a number greater than or equal to 0."); + throw new ParseException("Invalid Setting: nvdValidForHours must be a number greater than or equal to 0."); } } catch (NumberFormatException ex) { - throw new ParseException("Invalid Setting: cveValidForHours must be a number greater than or equal to 0."); + throw new ParseException("Invalid Setting: nvdValidForHours must be a number greater than or equal to 0."); } } - value = line.getOptionValue(ARGUMENT.CVE_START_YEAR); + value = line.getOptionValue(ARGUMENT.NVD_API_DELAY); if (value != null) { try { final int i = Integer.parseInt(value); - if (i < 2002) { - throw new ParseException("Invalid Setting: cveStartYear must be a number greater than or equal to 2002."); + if (i < 0) { + throw new ParseException("Invalid Setting: nvdApiDelay must be a number greater than or equal to 0."); } } catch (NumberFormatException ex) { - throw new ParseException("Invalid Setting: cveStartYear must be a number greater than or equal to 2002."); + throw new ParseException("Invalid Setting: nvdApiDelay must be a number greater than or equal to 0."); } } } @@ -155,12 +155,6 @@ private void validateArgs() throws FileNotFoundException, ParseException { } } } - final String base = getStringArgument(ARGUMENT.CVE_BASE_URL); - final String modified = getStringArgument(ARGUMENT.CVE_MODIFIED_URL); - if ((base != null && modified == null) || (base == null && modified != null)) { - final String msg = "If one of the CVE URLs is specified they must all be specified; please add the missing CVE URL."; - throw new ParseException(msg); - } if (line.hasOption(ARGUMENT.SYM_LINK_DEPTH)) { try { final int i = Integer.parseInt(line.getOptionValue(ARGUMENT.SYM_LINK_DEPTH)); @@ -325,6 +319,7 @@ private void addStandardOptions(final Options options) { .addOptionGroup(newOptionGroup(newOptionWithArg(ARGUMENT.SUPPRESSION_FILES, "file", "The file path to the suppression XML file. This can be specified more then once to utilize multiple suppression files"))) .addOption(newOption(ARGUMENT.EXPERIMENTAL, "Enables the experimental analyzers.")) + .addOption(newOptionWithArg(ARGUMENT.NVD_API_KEY, "apiKey", "The API Key to access the NVD API.")) .addOption(newOptionWithArg(ARGUMENT.FAIL_ON_CVSS, "score", "Specifies if the build should be failed if a CVSS score above a specified level is identified. The default is 11; " + "since the CVSS scores are 0-10, by default the build will never fail.")) @@ -344,16 +339,16 @@ private void addAdvancedOptions(final Options options) { options .addOption(newOption(ARGUMENT.UPDATE_ONLY, "Only update the local NVD data cache; no scan will be executed.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_BASE_URL, "url", - "Base URL for each year’s CVE files (json.gz), the %d will be replaced with the year.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_MODIFIED_URL, "url", - "URL for the modified CVE (json.gz).")) - .addOption(newOptionWithArg(ARGUMENT.CVE_DOWNLOAD_WAIT_TIME, "milliseconds", + .addOption(newOptionWithArg(ARGUMENT.NVD_API_DELAY, "milliseconds", "Time in milliseconds to wait between downloading from the NVD.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_USER, "user", - "Credentials for basic authentication to the CVE data.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_PASSWORD, "password", - "Credentials for basic authentication to the CVE data.")) + .addOption(newOptionWithArg(ARGUMENT.NVD_API_DATAFEED_URL, "url", + "The URL to the NVD API Datafeed.")) + .addOption(newOptionWithArg(ARGUMENT.NVD_API_DATAFEED_USER, "user", + "Credentials for basic authentication to the NVD API Datafeed.")) + .addOption(newOptionWithArg(ARGUMENT.NVD_API_DATAFEED_PASSWORD, "password", + "Credentials for basic authentication to the NVD API Datafeed.")) + .addOption(newOptionWithArg(ARGUMENT.NVD_API_VALID_FOR_HOURS, "hours", + "The number of hours to wait before checking for new updates from the NVD.")) .addOption(newOptionWithArg(ARGUMENT.PROXY_PORT, "port", "The proxy port to use when downloading resources.")) .addOption(newOptionWithArg(ARGUMENT.PROXY_SERVER, "server", @@ -429,10 +424,6 @@ private void addAdvancedOptions(final Options options) { "The path to the `yarn` executable.")) .addOption(newOptionWithArg(ARGUMENT.PATH_TO_PNPM, "path", "The path to the `pnpm` executable.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_VALID_FOR_HOURS, "hours", - "The number of hours to wait before checking for new updates from the NVD.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_START_YEAR, "year", - "The first year to retrieve NVD CVE data for; default is 2002.")) .addOption(newOptionWithArg(ARGUMENT.RETIREJS_FILTERS, "pattern", "Specify Retire JS content filter used to exclude files from analysis based on their content; " + "most commonly used to exclude based on your applications own copyright line. This " @@ -503,11 +494,11 @@ private void addAdvancedOptions(final Options options) { .addOption(newOption(ARGUMENT.PURGE_NVD, "Purges the local NVD data cache")) .addOption(newOption(ARGUMENT.DISABLE_HOSTED_SUPPRESSIONS, "Disable the usage of the hosted suppressions file")) .addOption(newOption(ARGUMENT.HOSTED_SUPPRESSIONS_FORCEUPDATE, "Force the hosted suppressions file to update even" - + " if autoupdate is disabled")) + + " if autoupdate is disabled")) .addOption(newOptionWithArg(ARGUMENT.HOSTED_SUPPRESSIONS_VALID_FOR_HOURS, "hours", - "The number of hours to wait before checking for new updates of the the hosted suppressions file.")) + "The number of hours to wait before checking for new updates of the the hosted suppressions file.")) .addOption(newOptionWithArg(ARGUMENT.HOSTED_SUPPRESSIONS_URL, "url", - "The URL for a mirrored hosted suppressions file")); + "The URL for a mirrored hosted suppressions file")); } @@ -1125,15 +1116,28 @@ public static class ARGUMENT { /** * The CLI argument name for setting the URL for the CVE Data Files. */ - public static final String CVE_MODIFIED_URL = "cveUrlModified"; + public static final String NVD_API_KEY = "nvdApiKey"; /** - * The CLI argument name for setting the URL for the CVE Data Files. + * The CLI argument name for setting the number of hours to wait before + * checking for new updates from the NVD. */ - public static final String CVE_BASE_URL = "cveUrlBase"; + public static final String NVD_API_VALID_FOR_HOURS = "nvdValidForHours"; /** - * The time in milliseconds to wait between downloading NVD CVE data. + * The CLI argument name for the NVD API Data Feed URL. */ - public static final String CVE_DOWNLOAD_WAIT_TIME = "cveDownloadWait"; + public static final String NVD_API_DATAFEED_URL = "nvdDatafeed"; + /** + * The username for basic auth to the CVE data. + */ + public static final String NVD_API_DATAFEED_USER = "nvdUser"; + /** + * The password for basic auth to the CVE data. + */ + public static final String NVD_API_DATAFEED_PASSWORD = "nvdPassword"; + /** + * The time in milliseconds to wait between downloading NVD API data. + */ + public static final String NVD_API_DELAY = "nvdApiDelay"; /** * The short CLI argument name for setting the location of the data * directory. @@ -1162,24 +1166,6 @@ public static class ARGUMENT { * The CLI argument name for setting the location of the hint file. */ public static final String HINTS_FILE = "hints"; - /** - * The CLI argument name for setting the number of hours to wait before - * checking for new updates from the NVD. - */ - public static final String CVE_VALID_FOR_HOURS = "cveValidForHours"; - /** - * The CLI argument name for setting the first year to retrieve NVD - * data. - */ - public static final String CVE_START_YEAR = "cveStartYear"; - /** - * The username for basic auth to the CVE data. - */ - public static final String CVE_USER = "cveUser"; - /** - * The password for basic auth to the CVE data. - */ - public static final String CVE_PASSWORD = "cvePassword"; /** * Disables the Jar Analyzer. */ @@ -1529,20 +1515,23 @@ public static class ARGUMENT { */ public static final String FAIL_JUNIT_ON_CVSS = "junitFailOnCVSS"; /** - * The CLI argument to set the number of hours to wait before re-checking hosted suppressions file for updates. + * The CLI argument to set the number of hours to wait before + * re-checking hosted suppressions file for updates. */ public static final String DISABLE_HOSTED_SUPPRESSIONS = "disableHostedSuppressions"; /** - * The CLI argument to set the number of hours to wait before re-checking hosted suppressions file for updates. + * The CLI argument to set the number of hours to wait before + * re-checking hosted suppressions file for updates. */ public static final String HOSTED_SUPPRESSIONS_VALID_FOR_HOURS = "hostedSuppressionsValidForHours"; /** - * The CLI argument to set Whether the hosted suppressions file will update regardless of the `noupdate` argument. + * The CLI argument to set Whether the hosted suppressions file will + * update regardless of the `noupdate` argument. */ public static final String HOSTED_SUPPRESSIONS_FORCEUPDATE = "hostedSuppressionsForceUpdate"; /** - * The CLI argument to set the location of a mirrored hosted suppressions - * file . + * The CLI argument to set the location of a mirrored hosted + * suppressions file . */ public static final String HOSTED_SUPPRESSIONS_URL = "hostedSuppressionsUrl"; } diff --git a/cli/src/main/resources/completion-for-dependency-check.sh b/cli/src/main/resources/completion-for-dependency-check.sh index 76af9d41d9a..1f94fd5d0a3 100755 --- a/cli/src/main/resources/completion-for-dependency-check.sh +++ b/cli/src/main/resources/completion-for-dependency-check.sh @@ -19,13 +19,6 @@ _odc_completions() --bundleAuditWorkingDirectory -c --connectiontimeout --connectionString - --cveUrlBase - --cveUrlModified - --cveValidForHours - --cveStartYear - --cveUser - --cvePassword - --cveDownloadWait -d --data --dbDriverName --dbDriverPath @@ -94,6 +87,12 @@ _odc_completions() --nodeAuditSkipDevDependencies --nodePackageSkipDevDependencies --nonProxyHosts + --nvdApiKey + --nvdDatafeed + --nvdUser + --nvdPassword + --nvdApiDelay + --nvdValidForHours -o --out --ossIndexPassword --ossIndexUsername diff --git a/cli/src/site/markdown/arguments.md b/cli/src/site/markdown/arguments.md index b91953e1f4c..ffebc003254 100644 --- a/cli/src/site/markdown/arguments.md +++ b/cli/src/site/markdown/arguments.md @@ -20,7 +20,6 @@ The following table lists the command line arguments: | \-h | \-\-help | | Print the help message. | Optional | | | \-\-advancedHelp | | Print the advanced help message. | Optional | | \-v | \-\-version | | Print the version information. | Optional | -| | \-\-cveValidForHours | \ | The number of hours to wait before checking for new updates from the NVD. The default is 4 hours. | Optional | | | \-\-enableExperimental | | Enable the [experimental analyzers](../analyzers/index.html). If not set the analyzers marked as experimental below will not be loaded or used. | Optional | | | \-\-enableRetired | | Enable the [retired analyzers](../analyzers/index.html). If not set the analyzers marked as retired below will not be loaded or used. | Optional | @@ -28,12 +27,12 @@ Advanced Options ================ | Short | Argument Name | Parameter | Description | Default Value | |-------|---------------------------------------|-----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------| -| | \-\-cveUrlModified | \ | URL for the modified CVE JSON data feed. When mirroring the NVD you must mirror the *.json.gz and the *.meta files. Optional if your custom cveUrlBase is just a domain name change. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz | -| | \-\-cveUrlBase | \ | Base URL for each year's CVE JSON data feed, the %d will be replaced with the year. | https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz | -| | \-\-cveUser | \ | Credentials used for basic authentication for the CVE data. |   | -| | \-\-cvePassword | \ | Credentials used for basic authentication for the CVE data. |   | -| | \-\-cveStartYear | \ | The first year of NVD CVE data to retrieve. | 2002 | -| | \-\-cveDownloadWait | \| The number of milliseconds to wait between NVD CVE download. | 4000 | +| | \-\-nvdApiKey | \ | The API Key to access the NVD API; obtained from https://nvd.nist.gov/developers/request-an-api-key |   | +| | \-\-nvdApiDelay | \| The number of milliseconds to wait between calls to the NVD API. |   | +| | \-\-nvdDatafeed | \ | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data - example value `https://internal.server/cache/nvdcve-{0}.json.gz` |   | +| | \-\-nvdUser | \ | Credentials used for basic authentication for the NVD API Data feed. |   | +| | \-\-nvdPassword | \ | Credentials used for basic authentication for the NVD API Data feed. |   | +| | \-\-nvdValidForHours | \ | The number of hours to wait before checking for new updates from the NVD. The default is 4 hours. | 4 | | | \-\-hints | \ | The file path to the XML hints file \- used to resolve [false negatives](../general/hints.html) |   | | \-P | \-\-propertyfile | \ | Specifies a file that contains properties to use instead of application defaults. The key values used in the properties file are not the same as the arguments listed on this page; use the keys here: https://github.com/jeremylong/DependencyCheck/blob/main/core/src/main/resources/dependencycheck.properties |   | | | \-\-updateonly | | If set only the update phase of dependency-check will be executed; no scan will be executed and no report will be generated. |   | diff --git a/cli/src/test/java/org/owasp/dependencycheck/AppTest.java b/cli/src/test/java/org/owasp/dependencycheck/AppTest.java index 4e8ca751be4..ea41abb957e 100644 --- a/cli/src/test/java/org/owasp/dependencycheck/AppTest.java +++ b/cli/src/test/java/org/owasp/dependencycheck/AppTest.java @@ -171,93 +171,6 @@ public void testPopulatingSuppressionSettingsWithMultipleFiles() throws Exceptio assertThat("Expected the suppression files to be set in the Settings with a separator", getSettings().getString(KEYS.SUPPRESSION_FILE), is("[\"first-file.xml\",\"another-file.xml\"]")); } - @Test - public void testPopulateSettingsShouldSetDefaultValueToCveUrlModified() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Settings settings = getSettings(); - final App app = new App(settings); - - String[] args = {"--cveUrlBase", "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz"}; - final CliParser parser = new CliParser(settings); - parser.parse(args); - - // When - app.populateSettings(parser); - - // Then - String output = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to a default of the same model", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldSetDefaultValueToCveUrlModifiedWhenCveUrlModifiedIsEmpty() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Settings settings = getSettings(); - final App app = new App(settings); - - String[] args = {"--cveUrlBase", "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz", "--cveUrlModified", ""}; - final CliParser parser = new CliParser(settings); - parser.parse(args); - - // When - app.populateSettings(parser); - - // Then - String output = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to a default of the same model when arg is empty", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldNotSetDefaultValueToCveUrlModifiedWhenValueIsExplicitelySet() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Settings settings = getSettings(); - final App app = new App(settings); - - String[] args = {"--cveUrlBase", "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/some-unusual-file-name-%d.json.gz", "--cveUrlModified", "https://another-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"}; - final CliParser parser = new CliParser(settings); - parser.parse(args); - - // When - app.populateSettings(parser); - - // Then - String output = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://another-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to the specified value", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldNotSetDefaultValueToCveUrlModifiedWhenUnknownValueIsSet() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Settings settings = getSettings(); - final App app = new App(settings); - - String[] args = {"--cveUrlBase", "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/some-unusual-file-name-%d.json.gz"}; - final CliParser parser = new CliParser(settings); - parser.parse(args); - - // When - app.populateSettings(parser); - - // Then - String output = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must not be set when name is not the same as from the nvd datasource", output, is(expectedOutput)); - } private boolean testBooleanProperties(String[] args, Map expected) throws URISyntaxException, FileNotFoundException, ParseException, InvalidSettingException { this.reloadSettings(); diff --git a/core/pom.xml b/core/pom.xml index a02b0e40d11..4cb89ef9cf1 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,7 +20,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved. org.owasp dependency-check-parent - 8.4.4-SNAPSHOT + 9.0.0-SNAPSHOT dependency-check-core @@ -105,7 +105,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved. jsonschema2pojo-maven-plugin - + generate-knownexploited generate-sources @@ -184,6 +184,10 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved. + + io.github.jeremylong + open-vulnerability-clients + org.anarres.jdiagnostics jdiagnostics @@ -289,6 +293,10 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved. com.fasterxml.jackson.core jackson-databind + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + com.fasterxml.jackson.module jackson-module-afterburner @@ -380,20 +388,11 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved. org.apache.maven.plugins maven-failsafe-plugin - - - data.driver_path - ${driver_path} - - - data.driver_name - ${driver_name} - - - data.connection_string - ${connection_string} - - + + ${driver_path} + ${driver_name} + ${connection_string} + **/*MySqlIT.java @@ -430,20 +429,11 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved. org.apache.maven.plugins maven-failsafe-plugin - - - data.driver_path - ${driver_path} - - - data.driver_name - ${driver_name} - - - data.connection_string - ${connection_string} - - + + ${driver_path} + ${driver_name} + ${connection_string} + **/*MySqlIT.java diff --git a/core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java b/core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java index b7f893fb85d..2d357f2148d 100644 --- a/core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java +++ b/core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java @@ -101,12 +101,17 @@ public class DependencyCheckScanAgent { * to 11. The valid range for the fail build on CVSS is 0 to 11, where * anything above 10 will not cause the build to fail. */ - private float failBuildOnCVSS = 11; + private Double failBuildOnCVSS = 11.0; /** * Sets whether auto-updating of the NVD CVE/CPE data is enabled. It is not * recommended that this be turned to false. Default is true. */ private boolean autoUpdate = true; + /** + * The NVD API key. + */ + private String nvdApiKey; + /** * Sets whether the data directory should be updated without performing a * scan. Default is false. @@ -208,14 +213,6 @@ public class DependencyCheckScanAgent { * comma-separated list of file extensions to treat like ZIP files. */ private String zipExtensions; - /** - * The URL for the modified NVD CVE JSON. - */ - private String cveUrlModified; - /** - * The base URL for the NVD CVE JSON data feeds. - */ - private String cveUrlBase; /** * The path to dotnet core for .NET assembly analysis. */ @@ -251,6 +248,24 @@ public void setApplicationName(String applicationName) { this.applicationName = applicationName; } + /** + * Get the value of nvdApiKey. + * + * @return the value of nvdApiKey + */ + public String getNvdApiKey() { + return nvdApiKey; + } + + /** + * Set the value of nvdApiKey. + * + * @param nvdApiKey new value of nvdApiKey + */ + public void setNvdApiKey(String nvdApiKey) { + this.nvdApiKey = nvdApiKey; + } + /** * Returns a list of pre-determined dependencies. * @@ -310,7 +325,7 @@ public void setReportOutputDirectory(String reportOutputDirectory) { * * @return the value of failBuildOnCVSS */ - public float getFailBuildOnCVSS() { + public Double getFailBuildOnCVSS() { return failBuildOnCVSS; } @@ -319,7 +334,7 @@ public float getFailBuildOnCVSS() { * * @param failBuildOnCVSS new value of failBuildOnCVSS */ - public void setFailBuildOnCVSS(float failBuildOnCVSS) { + public void setFailBuildOnCVSS(Double failBuildOnCVSS) { this.failBuildOnCVSS = failBuildOnCVSS; } @@ -801,42 +816,6 @@ public void setZipExtensions(String zipExtensions) { this.zipExtensions = zipExtensions; } - /** - * Get the value of cveUrlModified. - * - * @return the value of cveUrlModified - */ - public String getCveUrlModified() { - return cveUrlModified; - } - - /** - * Set the value of cveUrlModified. - * - * @param cveUrlModified new value of cveUrlModified - */ - public void setCveUrlModified(String cveUrlModified) { - this.cveUrlModified = cveUrlModified; - } - - /** - * Get the value of cveUrlBase. - * - * @return the value of cveUrlBase - */ - public String getCveUrlBase() { - return cveUrlBase; - } - - /** - * Set the value of cveUrlBase. - * - * @param cveUrlBase new value of cveUrlBase - */ - public void setCveUrlBase(String cveUrlBase) { - this.cveUrlBase = cveUrlBase; - } - /** * Get the value of pathToCore. * @@ -970,8 +949,7 @@ private void populateSettings() { settings.setStringIfNotEmpty(Settings.KEYS.DB_USER, databaseUser); settings.setStringIfNotEmpty(Settings.KEYS.DB_PASSWORD, databasePassword); settings.setStringIfNotEmpty(Settings.KEYS.ADDITIONAL_ZIP_EXTENSIONS, zipExtensions); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, cveUrlModified); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, cveUrlBase); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_KEY, nvdApiKey); settings.setStringIfNotEmpty(Settings.KEYS.ANALYZER_ASSEMBLY_DOTNET_PATH, pathToCore); } @@ -993,7 +971,7 @@ public Engine execute() throws ScanAgentException { if (this.showSummary) { showSummary(engine.getDependencies()); } - if (this.failBuildOnCVSS <= 10) { + if (this.failBuildOnCVSS <= 10.0) { checkForFailure(engine.getDependencies()); } } @@ -1025,8 +1003,8 @@ private void checkForFailure(Dependency[] dependencies) throws ScanAgentExceptio for (Dependency d : dependencies) { boolean addName = true; for (Vulnerability v : d.getVulnerabilities()) { - if ((v.getCvssV2() != null && v.getCvssV2().getScore() >= failBuildOnCVSS) - || (v.getCvssV3() != null && v.getCvssV3().getBaseScore() >= failBuildOnCVSS) + if ((v.getCvssV2() != null && v.getCvssV2().getCvssData().getBaseScore() >= failBuildOnCVSS) + || (v.getCvssV3() != null && v.getCvssV3().getCvssData().getBaseScore() >= failBuildOnCVSS) || (v.getUnscoredSeverity() != null && SeverityUtil.estimateCvssV2(v.getUnscoredSeverity()) >= failBuildOnCVSS) //safety net to fail on any if for some reason the above misses on 0 || (failBuildOnCVSS <= 0.0f)) { diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java index 593b6d42500..3a1338f2914 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java @@ -40,7 +40,6 @@ import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/LibmanAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/LibmanAnalyzer.java index 7d95c108768..b952c3f904e 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/LibmanAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/LibmanAnalyzer.java @@ -183,7 +183,7 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy final JsonArray libraries = json.getJsonArray("libraries"); libraries.forEach(e -> { - JsonObject reference = (JsonObject) e; + final JsonObject reference = (JsonObject) e; final String provider = reference.getString("provider", defaultProvider); final String library = reference.getString("library"); @@ -244,4 +244,4 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy throw new AnalysisException("Problem occurred while reading dependency file", e); } } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/NodePackageAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/NodePackageAnalyzer.java index 68e2eca3016..34e2795c3e1 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/NodePackageAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/NodePackageAnalyzer.java @@ -93,7 +93,7 @@ public class NodePackageAnalyzer extends AbstractNpmAnalyzer { */ public static final String SHRINKWRAP_JSON = "npm-shrinkwrap.json"; /** - * The name of the directory that contains node modules + * The name of the directory that contains node modules. */ public static final String NODE_MODULES_DIRNAME = "node_modules"; /** diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/OssIndexAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/OssIndexAnalyzer.java index be16d18cc8c..a8566dcaeaa 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/OssIndexAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/OssIndexAnalyzer.java @@ -17,20 +17,19 @@ */ package org.owasp.dependencycheck.analyzer; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2Data; import org.sonatype.ossindex.service.api.componentreport.ComponentReport; import org.sonatype.ossindex.service.api.componentreport.ComponentReportVulnerability; import org.sonatype.ossindex.service.api.cvss.Cvss2Severity; import org.sonatype.ossindex.service.api.cvss.Cvss2Vector; -import org.sonatype.ossindex.service.api.cvss.Cvss3Severity; -import org.sonatype.ossindex.service.api.cvss.Cvss3Vector; import org.sonatype.ossindex.service.api.cvss.CvssVector; import org.sonatype.ossindex.service.api.cvss.CvssVectorFactory; import org.sonatype.ossindex.service.client.OssindexClient; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.analyzer.exception.AnalysisException; import org.owasp.dependencycheck.data.ossindex.OssindexClientFactory; -import org.owasp.dependencycheck.dependency.CvssV2; -import org.owasp.dependencycheck.dependency.CvssV3; + import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.dependency.Vulnerability; import org.owasp.dependencycheck.dependency.VulnerableSoftware; @@ -58,6 +57,7 @@ import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; +import org.owasp.dependencycheck.utils.CvssUtil; import org.sonatype.goodies.packageurl.InvalidException; import org.sonatype.ossindex.service.client.transport.Transport.TransportException; @@ -318,34 +318,61 @@ private Vulnerability transform(final ComponentReport report, final ComponentRep result.setDescription(source.getDescription()); result.addCwe(source.getCwe()); - final float cvssScore = source.getCvssScore() != null ? source.getCvssScore() : -1; + final double cvssScore = source.getCvssScore() != null ? source.getCvssScore().doubleValue() : -1; if (source.getCvssVector() != null) { if (source.getCvssVector().startsWith("CVSS:3")) { - result.setCvssV3(new CvssV3(source.getCvssVector(), cvssScore)); + result.setCvssV3(CvssUtil.vectorToCvssV3(source.getCvssVector(), cvssScore)); } else { // convert cvss details final CvssVector cvssVector = CvssVectorFactory.create(source.getCvssVector()); final Map metrics = cvssVector.getMetrics(); if (cvssVector instanceof Cvss2Vector) { - result.setCvssV2(new CvssV2( - cvssScore, - metrics.get(Cvss2Vector.ACCESS_VECTOR), - metrics.get(Cvss2Vector.ACCESS_COMPLEXITY), - metrics.get(Cvss2Vector.AUTHENTICATION), - metrics.get(Cvss2Vector.CONFIDENTIALITY_IMPACT), - metrics.get(Cvss2Vector.INTEGRITY_IMPACT), - metrics.get(Cvss2Vector.AVAILABILITY_IMPACT), - Cvss2Severity.of(cvssScore).name() - )); + String tmp = metrics.get(Cvss2Vector.ACCESS_VECTOR); + CvssV2Data.AccessVectorType accessVector = null; + if (tmp != null) { + accessVector = CvssV2Data.AccessVectorType.fromValue(tmp); + } + tmp = metrics.get(Cvss2Vector.ACCESS_COMPLEXITY); + CvssV2Data.AccessComplexityType accessComplexity = null; + if (tmp != null) { + accessComplexity = CvssV2Data.AccessComplexityType.fromValue(tmp); + } + tmp = metrics.get(Cvss2Vector.AUTHENTICATION); + CvssV2Data.AuthenticationType authentication = null; + if (tmp != null) { + authentication = CvssV2Data.AuthenticationType.fromValue(tmp); + } + tmp = metrics.get(Cvss2Vector.CONFIDENTIALITY_IMPACT); + CvssV2Data.CiaType confidentialityImpact = null; + if (tmp != null) { + confidentialityImpact = CvssV2Data.CiaType.fromValue(tmp); + } + tmp = metrics.get(Cvss2Vector.INTEGRITY_IMPACT); + CvssV2Data.CiaType integrityImpact = null; + if (tmp != null) { + integrityImpact = CvssV2Data.CiaType.fromValue(tmp); + } + tmp = metrics.get(Cvss2Vector.AVAILABILITY_IMPACT); + CvssV2Data.CiaType availabilityImpact = null; + if (tmp != null) { + availabilityImpact = CvssV2Data.CiaType.fromValue(tmp); + } + final String severity = Cvss2Severity.of((float) cvssScore).name().toUpperCase(); + final CvssV2Data cvssData = new CvssV2Data("2.0", source.getCvssVector(), accessVector, + accessComplexity, authentication, confidentialityImpact, + integrityImpact, availabilityImpact, cvssScore, + severity, null, null, null, null, null, null, null, null, null, null); + final CvssV2 cvssV2 = new CvssV2(null, null, cvssData, severity, null, null, null, null, null, null, null); + result.setCvssV2(cvssV2); } else { LOG.warn("Unsupported CVSS vector: {}", cvssVector); - result.setUnscoredSeverity(Float.toString(cvssScore)); + result.setUnscoredSeverity(Double.toString(cvssScore)); } } } else { LOG.debug("OSS has no vector for {}", result.getName()); - result.setUnscoredSeverity(Float.toString(cvssScore)); + result.setUnscoredSeverity(Double.toString(cvssScore)); } // generate a reference to the vulnerability details on OSS Index result.addReference(REFERENCE_TYPE, source.getTitle(), source.getReference().toString()); diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/RetireJsAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/RetireJsAnalyzer.java index 7961f8f77c5..75402ea7f5b 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/RetireJsAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/RetireJsAnalyzer.java @@ -182,7 +182,7 @@ public void initialize(Settings settings) { protected void prepareFileTypeAnalyzer(Engine engine) throws InitializationException { // RetireJS outputs a bunch of repeated output like the following for // vulnerable dependencies, with little context: - // + // // INFO: Vulnerability found: jquery below 1.6.3 // // This logging is suppressed because it isn't particularly useful, and diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/Advisory.java b/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/Advisory.java index a6f044f9dbe..97fab27f268 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/Advisory.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/Advisory.java @@ -17,8 +17,9 @@ */ package org.owasp.dependencycheck.data.nodeaudit; -import org.owasp.dependencycheck.dependency.CvssV3; + +import io.github.jeremylong.openvulnerability.client.nvd.CvssV3; import java.io.Serializable; import java.util.List; import javax.annotation.concurrent.ThreadSafe; diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NpmAuditParser.java b/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NpmAuditParser.java index 5f0cf87d155..7c442c9f4fe 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NpmAuditParser.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NpmAuditParser.java @@ -17,15 +17,17 @@ */ package org.owasp.dependencycheck.data.nodeaudit; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV3; import org.json.JSONArray; import org.json.JSONObject; -import org.owasp.dependencycheck.dependency.CvssV3; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.json.JSONException; +import org.owasp.dependencycheck.utils.CvssUtil; /** * Parser for NPM Audit API response. This parser is derived from: @@ -118,7 +120,7 @@ private Advisory parseAdvisory(JSONObject object) throws JSONException { } final JSONObject jsonCvss = object.optJSONObject("cvss"); if (jsonCvss != null) { - float baseScore = -1.0f; + double baseScore = -1.0; final String score = jsonCvss.optString("score"); if (score != null) { try { @@ -133,7 +135,7 @@ private Advisory parseAdvisory(JSONObject object) throws JSONException { if (vector != null) { if (vector.startsWith("CVSS:3") && baseScore >= 0.0) { try { - final CvssV3 cvss = new CvssV3(vector, baseScore); + final CvssV3 cvss = CvssUtil.vectorToCvssV3(vector, baseScore); advisory.setCvssV3(cvss); } catch (IllegalArgumentException iae) { LOGGER.warn("Invalid CVSS vector format encountered in NPM Audit results '{}' ", vector, iae); diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NpmPayloadBuilder.java b/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NpmPayloadBuilder.java index 505425cb059..d61e102c726 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NpmPayloadBuilder.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nodeaudit/NpmPayloadBuilder.java @@ -108,7 +108,8 @@ public static JsonObject build(JsonObject lockJson, JsonObject packageJson, } if (dependencies != null) { - dependencies.forEach((key, value) -> { + dependencies.forEach((k, value) -> { + String key = k; final int indexOfNodeModule = key.lastIndexOf(NodePackageAnalyzer.NODE_MODULES_DIRNAME + "/"); if (indexOfNodeModule >= 0) { key = key.substring(indexOfNodeModule + NodePackageAnalyzer.NODE_MODULES_DIRNAME.length() + 1); @@ -249,7 +250,7 @@ private static JsonObject buildDependencies(JsonObject dep, MultiValuedMapnull */ private boolean hasMultipleVendorProductConfigurations(DefCveItem cve) { - final List cpeEntries = cve.getConfigurations().getNodes().stream() - .collect(NodeFlatteningCollector.getInstance()) - .collect(CpeMatchStreamCollector.getInstance()) - .filter(defCpeMatch -> defCpeMatch.getCpe23Uri() != null) - .collect(Collectors.toList()); - if (!cpeEntries.isEmpty() && cpeEntries.size() > 1) { - final DefCpeMatch firstMatch = cpeEntries.get(0); - final String uri = firstMatch.getCpe23Uri(); - final int pos = uri.indexOf(":", uri.indexOf(":", 10) + 1); - final String match = uri.substring(0, pos + 1); - return !cpeEntries.stream().allMatch(e -> e.getCpe23Uri().startsWith(match)); + if (cve.getCve().getConfigurations() != null) { + final List cpeEntries = cve.getCve().getConfigurations().stream() + .map(Config::getNodes) + .flatMap(List::stream) + .map(Node::getCpeMatch) + .flatMap(List::stream) + .filter(match -> match.getCriteria() != null) + .collect(Collectors.toList()); + if (!cpeEntries.isEmpty() && cpeEntries.size() > 1) { + final CpeMatch firstMatch = cpeEntries.get(0); + final String uri = firstMatch.getCriteria(); + final int pos = uri.indexOf(":", uri.indexOf(":", 10) + 1); + final String match = uri.substring(0, pos + 1); + return !cpeEntries.stream().allMatch(e -> e.getCriteria().startsWith(match)); + } } return false; } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvd/ecosystem/DescriptionEcosystemMapper.java b/core/src/main/java/org/owasp/dependencycheck/data/nvd/ecosystem/DescriptionEcosystemMapper.java index d4dae989047..1d325ec8bff 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvd/ecosystem/DescriptionEcosystemMapper.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nvd/ecosystem/DescriptionEcosystemMapper.java @@ -18,7 +18,7 @@ package org.owasp.dependencycheck.data.nvd.ecosystem; import org.apache.commons.lang3.StringUtils; -import org.owasp.dependencycheck.data.nvd.json.DefCveItem; +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; import java.util.HashMap; import java.util.Map; @@ -179,7 +179,7 @@ protected void increment(int i, int[] ecosystemMap) { */ public String getEcosystem(DefCveItem cve) { final int[] ecosystemMap = new int[ECOSYSTEMS.length]; - cve.getCve().getDescription().getDescriptionData().stream() + cve.getCve().getDescriptions().stream() .filter((langString) -> (langString.getLang().equals("en"))) .forEachOrdered((langString) -> search(langString.getValue(), ecosystemMap)); return getResult(ecosystemMap); diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvd/ecosystem/UrlEcosystemMapper.java b/core/src/main/java/org/owasp/dependencycheck/data/nvd/ecosystem/UrlEcosystemMapper.java index 8721ed54b1c..bd48613e95f 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvd/ecosystem/UrlEcosystemMapper.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nvd/ecosystem/UrlEcosystemMapper.java @@ -23,13 +23,13 @@ import javax.annotation.concurrent.NotThreadSafe; -import org.owasp.dependencycheck.data.nvd.json.CVEJSON40Min11; -import org.owasp.dependencycheck.data.nvd.json.DefCveItem; -import org.owasp.dependencycheck.data.nvd.json.Reference; -import org.owasp.dependencycheck.data.nvd.json.References; +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; +import io.github.jeremylong.openvulnerability.client.nvd.Reference; import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie; import com.hankcs.algorithm.AhoCorasickDoubleArrayTrie.Hit; +import io.github.jeremylong.openvulnerability.client.nvd.CveItem; +import java.util.List; @NotThreadSafe public class UrlEcosystemMapper { @@ -69,14 +69,13 @@ public UrlEcosystemMapper() { * @return the ecosystem */ public String getEcosystem(DefCveItem cve) { - final References references = Optional.ofNullable(cve) + final List references = Optional.ofNullable(cve) .map(DefCveItem::getCve) - .map(CVEJSON40Min11::getReferences) + .map(CveItem::getReferences) .orElse(null); if (Objects.nonNull(references)) { - for (Reference r : references.getReferenceData()) { - + for (Reference r : references) { final Hit ecosystem = search.findFirst(r.getUrl()); if (ecosystem != null) { return ecosystem.value; diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/CpeMatchStreamCollector.java b/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/CpeMatchStreamCollector.java deleted file mode 100644 index 652e45a2ef2..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/CpeMatchStreamCollector.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2018 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.nvd.json; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.EnumSet; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.BinaryOperator; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collector; -import java.util.stream.Stream; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * - * @author Jeremy Long - * - */ -@ThreadSafe -public final class CpeMatchStreamCollector implements Collector, Stream> { - - /** - * The singleton instance. - */ - private static final CpeMatchStreamCollector INSTANCE; - - static { - INSTANCE = new CpeMatchStreamCollector(); - } - - public static CpeMatchStreamCollector getInstance() { - return INSTANCE; - } - - private CpeMatchStreamCollector() { - } - - @Override - public Supplier> supplier() { - return ArrayList::new; - } - - @Override - public BiConsumer, DefNode> accumulator() { - return (match, nodes) -> match.addAll(nodes.getCpeMatch()); - } - - @Override - public BinaryOperator> combiner() { - return (map, other) -> { - map.addAll(other); - return map; - }; - } - - @Override - public Function, Stream> finisher() { - return Collection::stream; - } - - @Override - public Set characteristics() { - return EnumSet.of(Characteristics.UNORDERED); - } - -} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/MetaProperties.java b/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/MetaProperties.java deleted file mode 100644 index 322ba5c02da..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/MetaProperties.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2019 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.nvd.json; - -import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.Properties; -import org.owasp.dependencycheck.data.update.exception.InvalidDataException; - -/** - * Meta properties object to hold information about the NVD CVE data. - * - * @author Jeremy Long - */ -public class MetaProperties { - - /** - * The SHA256 of the NVD file. - */ - private final String sha256; - /** - * The last modified date of the NVD file in epoch time. - */ - private final long lastModifiedDate; - /** - * The size of the NVD file. - */ - private final long size; - /** - * The size of the zipped NVD file. - */ - private final long zipSize; - /** - * The size of the gzipped NVD file. - */ - private final long gzSize; - - /** - * Get the value of gzSize. - * - * @return the value of gzSize - */ - public long getGzSize() { - return gzSize; - } - - /** - * Get the value of zipSize. - * - * @return the value of zipSize - */ - public long getZipSize() { - return zipSize; - } - - /** - * Get the value of size. - * - * @return the value of size - */ - public long getSize() { - return size; - } - - /** - * Get the value of lastModifiedDate in epoch time. - * - * @return the value of lastModifiedDate - */ - public long getLastModifiedDate() { - return lastModifiedDate; - } - - /** - * Get the value of SHA256. - * - * @return the value of SHA256 - */ - public String getSha256() { - return sha256; - } - - /** - * Constructs a new MetaProperties object to hold information about the NVD - * data. - * - * @param contents the contents of the meta file - * @throws InvalidDataException thrown if the meta file contents cannot be - * parsed - */ - public MetaProperties(String contents) throws InvalidDataException { - final Properties properties = new Properties(); - try (Reader r = new StringReader(contents)) { - properties.load(r); - } catch (IOException ex) { - throw new InvalidDataException("Unable to parse meta file data", ex); - } - this.sha256 = properties.getProperty("sha256"); - try { - final String date = properties.getProperty("lastModifiedDate"); - if (date == null) { - throw new InvalidDataException("lastModifiedDate not found in meta file"); - } - this.lastModifiedDate = ZonedDateTime.parse(date, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toEpochSecond(); - } catch (DateTimeParseException ex) { - throw new InvalidDataException("Meta file lastModifiedDate cannot be parsed: " - + properties.getProperty("lastModifiedDate"), ex); - } - try { - this.zipSize = Long.parseLong(properties.getProperty("zipSize")); - } catch (NumberFormatException ex) { - throw new InvalidDataException("Meta file zip size cannot be parsed: " - + properties.getProperty("zipSize"), ex); - } - try { - this.gzSize = Long.parseLong(properties.getProperty("gzSize")); - } catch (NumberFormatException ex) { - throw new InvalidDataException("Meta file gz size cannot be parsed: " - + properties.getProperty("gzSize"), ex); - } - try { - this.size = Long.parseLong(properties.getProperty("size")); - } catch (NumberFormatException ex) { - throw new InvalidDataException("Meta file size cannot be parsed: " - + properties.getProperty("size"), ex); - } - } -} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/NodeFlatteningCollector.java b/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/NodeFlatteningCollector.java deleted file mode 100644 index 7dfe7e291ff..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/NodeFlatteningCollector.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2018 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.nvd.json; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.EnumSet; -import java.util.List; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.BinaryOperator; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collector; -import java.util.stream.Stream; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * Used to flatten a hierarchical list of nodes with children. - * - * @author Jeremy Long - */ -@ThreadSafe -public final class NodeFlatteningCollector implements Collector, Stream> { - - /** - * Singleton instance variable. - */ - private static final NodeFlatteningCollector INSTANCE; - - static { - INSTANCE = new NodeFlatteningCollector(); - } - - public static NodeFlatteningCollector getInstance() { - return INSTANCE; - } - - private NodeFlatteningCollector() { - } - - /** - * Flattens the hierarchical list of nodes. - * - * @param node the node with children to flatten - * @return the flattened list of nodes - */ - private List flatten(DefNode node) { - final List result = new ArrayList<>(); - result.add(node); - return flatten(result, node.getChildren()); - } - - /** - * Flattens the hierarchical list of nodes. - * - * @param result the results - * @param nodes the nodes - * @return the flattened list of nodes - */ - private List flatten(List result, List nodes) { - nodes.forEach(n -> { - flatten(result, n.getChildren()); - result.add(n); - }); - return result; - } - - @Override - public Supplier> supplier() { - return ArrayList::new; - } - - @Override - public BiConsumer, DefNode> accumulator() { - return (nodes, n) -> nodes.addAll(flatten(n)); - } - - @Override - public BinaryOperator> combiner() { - return (map, other) -> { - map.addAll(other); - return map; - }; - } - - @Override - public Function, Stream> finisher() { - return Collection::stream; - } - - @Override - public Set characteristics() { - return EnumSet.of(Characteristics.UNORDERED); - } -} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/package-info.java b/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/package-info.java deleted file mode 100644 index cd090a6f6c0..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvd/json/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -/** - * Contains utility classes used to work with the generated code from the - * NVD CVE JSON data. - */ -package org.owasp.dependencycheck.data.nvd.json; diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index 752597623ef..2ad22166f66 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -20,6 +20,8 @@ import com.google.common.io.Resources; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.github.jeremylong.openvulnerability.client.nvd.Config; +import io.github.jeremylong.openvulnerability.client.nvd.CpeMatch; import org.apache.commons.collections.map.ReferenceMap; import org.owasp.dependencycheck.dependency.Vulnerability; import org.owasp.dependencycheck.dependency.VulnerableSoftware; @@ -46,20 +48,18 @@ import static org.apache.commons.collections.map.AbstractReferenceMap.SOFT; import org.owasp.dependencycheck.analyzer.exception.LambdaExceptionWrapper; import org.owasp.dependencycheck.analyzer.exception.UnexpectedAnalysisException; -import org.owasp.dependencycheck.data.nvd.json.BaseMetricV2; -import org.owasp.dependencycheck.data.nvd.json.BaseMetricV3; -import org.owasp.dependencycheck.data.nvd.json.CpeMatchStreamCollector; -import org.owasp.dependencycheck.data.nvd.json.DefCpeMatch; -import org.owasp.dependencycheck.data.nvd.json.DefCveItem; -import org.owasp.dependencycheck.data.nvd.json.LangString; -import org.owasp.dependencycheck.data.nvd.json.NodeFlatteningCollector; -import org.owasp.dependencycheck.data.nvd.json.ProblemtypeDatum; -import org.owasp.dependencycheck.data.nvd.json.Reference; +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb.*; import org.owasp.dependencycheck.data.update.cpe.CpeEcosystemCache; import org.owasp.dependencycheck.data.update.cpe.CpePlus; -import org.owasp.dependencycheck.dependency.CvssV2; -import org.owasp.dependencycheck.dependency.CvssV3; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2Data; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV3; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV3Data; +import io.github.jeremylong.openvulnerability.client.nvd.LangString; +import io.github.jeremylong.openvulnerability.client.nvd.Node; +import io.github.jeremylong.openvulnerability.client.nvd.Reference; +import io.github.jeremylong.openvulnerability.client.nvd.Weakness; import org.owasp.dependencycheck.dependency.VulnerableSoftwareBuilder; import us.springett.parsers.cpe.Cpe; import us.springett.parsers.cpe.CpeBuilder; @@ -143,8 +143,7 @@ public int updateEcosystemCache() { final URL url = Resources.getResource(DB_ECOSYSTEM_CACHE); final List sql = Resources.readLines(url, StandardCharsets.UTF_8); - try (Connection conn = databaseManager.getConnection(); - Statement statement = conn.createStatement()) { + try (Connection conn = databaseManager.getConnection(); Statement statement = conn.createStatement()) { for (String single : sql) { updateCount += statement.executeUpdate(single); } @@ -461,8 +460,7 @@ DatabaseProperties reloadProperties() { */ public Set getCPEs(String vendor, String product) { final Set cpe = new HashSet<>(); - try (Connection conn = databaseManager.getConnection(); - PreparedStatement ps = getPreparedStatement(conn, SELECT_CPE_ENTRIES)) { + try (Connection conn = databaseManager.getConnection(); PreparedStatement ps = getPreparedStatement(conn, SELECT_CPE_ENTRIES)) { //part, vendor, product, version, update_version, edition, //lang, sw_edition, target_sw, target_hw, other, ecosystem ps.setString(1, vendor); @@ -567,8 +565,7 @@ public Properties getProperties() { */ public void saveProperty(String key, String value) { clearCache(); - try (Connection conn = databaseManager.getConnection(); - PreparedStatement mergeProperty = getPreparedStatement(conn, MERGE_PROPERTY)) { + try (Connection conn = databaseManager.getConnection(); PreparedStatement mergeProperty = getPreparedStatement(conn, MERGE_PROPERTY)) { if (mergeProperty != null) { mergeProperty.setString(1, key); mergeProperty.setString(2, value); @@ -623,8 +620,7 @@ public List getVulnerabilities(Cpe cpe) throws DatabaseException } final List vulnerabilities = new ArrayList<>(); - try (Connection conn = databaseManager.getConnection(); - PreparedStatement ps = getPreparedStatement(conn, SELECT_CVE_FROM_SOFTWARE)) { + try (Connection conn = databaseManager.getConnection(); PreparedStatement ps = getPreparedStatement(conn, SELECT_CVE_FROM_SOFTWARE)) { ps.setString(1, cpe.getVendor()); ps.setString(2, cpe.getProduct()); try (ResultSet rs = ps.executeQuery()) { @@ -714,8 +710,7 @@ public Vulnerability getVulnerability(String cve, Connection conn) throws Databa final VulnerableSoftwareBuilder vulnerableSoftwareBuilder = new VulnerableSoftwareBuilder(); Vulnerability vuln = null; try { - try (PreparedStatement psV = getPreparedStatement(conn, SELECT_VULNERABILITY, cve); - ResultSet rsV = psV.executeQuery()) { + try (PreparedStatement psV = getPreparedStatement(conn, SELECT_VULNERABILITY, cve); ResultSet rsV = psV.executeQuery()) { if (rsV.next()) { //1.id, 2.description, cveId = rsV.getInt(1); @@ -729,23 +724,77 @@ public Vulnerability getVulnerability(String cve, Connection conn) throws Databa //12.v2AccessVector, 13.v2AccessComplexity, 14.v2Authentication, 15.v2ConfidentialityImpact, //16.v2IntegrityImpact, 17.v2AvailabilityImpact, 18.v2Version, if (rsV.getObject(11) != null) { - final CvssV2 cvss = new CvssV2(rsV.getFloat(11), rsV.getString(12), - rsV.getString(13), rsV.getString(14), rsV.getString(15), - rsV.getString(16), rsV.getString(17), rsV.getString(3), - getFloatValue(rsV, 4), getFloatValue(rsV, 5), - getBooleanValue(rsV, 6), getBooleanValue(rsV, 7), getBooleanValue(rsV, 8), - getBooleanValue(rsV, 9), getBooleanValue(rsV, 10), rsV.getString(18)); + + final CvssV2Data.AccessVectorType accessVector = CvssV2Data.AccessVectorType.fromValue(rsV.getString(12)); + final CvssV2Data.AccessComplexityType accessComplexity = CvssV2Data.AccessComplexityType.fromValue(rsV.getString(13)); + final CvssV2Data.AuthenticationType authentication = CvssV2Data.AuthenticationType.fromValue(rsV.getString(14)); + final CvssV2Data.CiaType confidentialityImpact = CvssV2Data.CiaType.fromValue(rsV.getString(15)); + final CvssV2Data.CiaType integrityImpact = CvssV2Data.CiaType.fromValue(rsV.getString(16)); + final CvssV2Data.CiaType availabilityImpact = CvssV2Data.CiaType.fromValue(rsV.getString(17)); + final String vector = String.format("/AV:%s/AC:%s/Au:%s/C:%s/I:%s/A:%s", + accessVector == null ? "" : accessVector.value().substring(0, 1), + accessComplexity == null ? "" : accessComplexity.value().substring(0, 1), + authentication == null ? "" : authentication.value().substring(0, 1), + confidentialityImpact == null ? "" : confidentialityImpact.value().substring(0, 1), + integrityImpact == null ? "" : integrityImpact.value().substring(0, 1), + availabilityImpact == null ? "" : availabilityImpact.value().substring(0, 1)); + + //some older test data may not correctly have the version set. + String cveVersion = "2.0"; + if (rsV.getString(18) != null) { + cveVersion = rsV.getString(18); + } + final CvssV2Data cvssData = new CvssV2Data(cveVersion, vector, accessVector, + accessComplexity, authentication, confidentialityImpact, + integrityImpact, availabilityImpact, rsV.getDouble(11), rsV.getString(3), + null, null, null, null, null, null, null, null, null, null); + final CvssV2 cvss = new CvssV2(null, CvssV2.Type.PRIMARY, cvssData, rsV.getString(3), + rsV.getDouble(4), rsV.getDouble(5), rsV.getBoolean(6), rsV.getBoolean(7), + rsV.getBoolean(8), rsV.getBoolean(9), rsV.getBoolean(10)); vuln.setCvssV2(cvss); } //19.v3ExploitabilityScore, 20.v3ImpactScore, 21.v3AttackVector, 22.v3AttackComplexity, 23.v3PrivilegesRequired, //24.v3UserInteraction, 25.v3Scope, 26.v3ConfidentialityImpact, 27.v3IntegrityImpact, 28.v3AvailabilityImpact, - //29.v3BaseScore, 30.v3BaseSeverity, 21.v3Version + //29.v3BaseScore, 30.v3BaseSeverity, 31.v3Version if (rsV.getObject(21) != null) { - final CvssV3 cvss = new CvssV3(rsV.getString(21), rsV.getString(22), - rsV.getString(23), rsV.getString(24), rsV.getString(25), - rsV.getString(26), rsV.getString(27), rsV.getString(28), - rsV.getFloat(29), rsV.getString(30), getFloatValue(rsV, 19), - getFloatValue(rsV, 20), rsV.getString(31)); + //some older test data may not correctly have the version set. + String cveVersion = "3.1"; + if (rsV.getString(31) != null) { + cveVersion = rsV.getString(31); + } + final CvssV3Data.Version version = CvssV3Data.Version.fromValue(cveVersion); + final CvssV3Data.AttackVectorType attackVector = CvssV3Data.AttackVectorType.fromValue(rsV.getString(21)); + final CvssV3Data.AttackComplexityType attackComplexity = CvssV3Data.AttackComplexityType.fromValue(rsV.getString(22)); + final CvssV3Data.PrivilegesRequiredType privilegesRequired = CvssV3Data.PrivilegesRequiredType.fromValue(rsV.getString(23)); + final CvssV3Data.UserInteractionType userInteraction = CvssV3Data.UserInteractionType.fromValue(rsV.getString(24)); + final CvssV3Data.ScopeType scope = CvssV3Data.ScopeType.fromValue(rsV.getString(25)); + final CvssV3Data.CiaType confidentialityImpact = CvssV3Data.CiaType.fromValue(rsV.getString(26)); + final CvssV3Data.CiaType integrityImpact = CvssV3Data.CiaType.fromValue(rsV.getString(27)); + final CvssV3Data.CiaType availabilityImpact = CvssV3Data.CiaType.fromValue(rsV.getString(28)); + final CvssV3Data.SeverityType baseSeverity = CvssV3Data.SeverityType.fromValue(rsV.getString(30)); + final String vector = String.format("CVSS:%s/AV:%s/AC:%s/PR:%s/UI:%s/S:%s/C:%s/I:%s/A:%s", + version == null ? "" : version, + attackVector == null ? "" : attackVector.value().substring(0, 1), + attackComplexity == null ? "" : attackComplexity.value().substring(0, 1), + privilegesRequired == null ? "" : privilegesRequired.value().substring(0, 1), + userInteraction == null ? "" : userInteraction.value().substring(0, 1), + scope == null ? "" : scope.value().substring(0, 1), + confidentialityImpact == null ? "" : confidentialityImpact.value().substring(0, 1), + integrityImpact == null ? "" : integrityImpact.value().substring(0, 1), + availabilityImpact == null ? "" : availabilityImpact.value().substring(0, 1)); + + final CvssV3Data cvssData = new CvssV3Data(version, vector, attackVector, attackComplexity, privilegesRequired, + userInteraction, scope, confidentialityImpact, integrityImpact, availabilityImpact, + rsV.getDouble(29), baseSeverity, CvssV3Data.ExploitCodeMaturityType.PROOF_OF_CONCEPT, + CvssV3Data.RemediationLevelType.NOT_DEFINED, CvssV3Data.ConfidenceType.REASONABLE, 0.0, + CvssV3Data.SeverityType.MEDIUM, CvssV3Data.CiaRequirementType.NOT_DEFINED, + CvssV3Data.CiaRequirementType.NOT_DEFINED, CvssV3Data.CiaRequirementType.NOT_DEFINED, + CvssV3Data.ModifiedAttackVectorType.ADJACENT_NETWORK, CvssV3Data.ModifiedAttackComplexityType.NOT_DEFINED, + CvssV3Data.ModifiedPrivilegesRequiredType.NOT_DEFINED, CvssV3Data.ModifiedUserInteractionType.NOT_DEFINED, + CvssV3Data.ModifiedScopeType.NOT_DEFINED, CvssV3Data.ModifiedCiaType.NOT_DEFINED, + CvssV3Data.ModifiedCiaType.NOT_DEFINED, CvssV3Data.ModifiedCiaType.NOT_DEFINED, 1.0, + CvssV3Data.SeverityType.NONE); + final CvssV3 cvss = new CvssV3(null, null, cvssData, rsV.getDouble(19), rsV.getDouble(20)); vuln.setCvssV3(cvss); } } else { @@ -753,20 +802,17 @@ public Vulnerability getVulnerability(String cve, Connection conn) throws Databa return null; } } - try (PreparedStatement psCWE = getPreparedStatement(conn, SELECT_VULNERABILITY_CWE, cveId); - ResultSet rsC = psCWE.executeQuery()) { + try (PreparedStatement psCWE = getPreparedStatement(conn, SELECT_VULNERABILITY_CWE, cveId); ResultSet rsC = psCWE.executeQuery()) { while (rsC.next()) { vuln.addCwe(rsC.getString(1)); } } - try (PreparedStatement psR = getPreparedStatement(conn, SELECT_REFERENCES, cveId); - ResultSet rsR = psR.executeQuery()) { + try (PreparedStatement psR = getPreparedStatement(conn, SELECT_REFERENCES, cveId); ResultSet rsR = psR.executeQuery()) { while (rsR.next()) { vuln.addReference(rsR.getString(1), rsR.getString(2), rsR.getString(3)); } } - try (PreparedStatement psS = getPreparedStatement(conn, SELECT_SOFTWARE, cveId); - ResultSet rsS = psS.executeQuery()) { + try (PreparedStatement psS = getPreparedStatement(conn, SELECT_SOFTWARE, cveId); ResultSet rsS = psS.executeQuery()) { //1 part, 2 vendor, 3 product, 4 version, 5 update_version, 6 edition, 7 lang, //8 sw_edition, 9 target_sw, 10 target_hw, 11 other, 12 versionEndExcluding, //13 versionEndIncluding, 14 versionStartExcluding, 15 versionStartIncluding, 16 vulnerable @@ -810,13 +856,13 @@ public Vulnerability getVulnerability(String cve, Connection conn) throws Databa */ public void updateVulnerability(DefCveItem cve, String baseEcosystem) { clearCache(); - final String cveId = cve.getCve().getCVEDataMeta().getId(); + final String cveId = cve.getCve().getId(); try { - final String description = cveItemConverter.extractDescription(cve); - if (cveItemConverter.isRejected(description)) { + if (cve.getCve().getVulnStatus().toUpperCase().startsWith("REJECT")) { deleteVulnerability(cveId); } else { if (cveItemConverter.testCveCpeStartWithFilter(cve)) { + final String description = cveItemConverter.extractDescription(cve); final int vulnerabilityId = updateOrInsertVulnerability(cve, description); updateVulnerabilityInsertCwe(vulnerabilityId, cve); updateVulnerabilityInsertReferences(vulnerabilityId, cve); @@ -825,7 +871,6 @@ public void updateVulnerability(DefCveItem cve, String baseEcosystem) { updateVulnerabilityInsertSoftware(vulnerabilityId, cveId, software, baseEcosystem); } } - } catch (SQLException ex) { final String msg = String.format("Error updating '%s'", cveId); LOGGER.debug(msg, ex); @@ -858,8 +903,7 @@ private void loadCpeEcosystemCache() { private void saveCpeEcosystemCache() { final Map, String> map = CpeEcosystemCache.getChanged(); if (map != null && !map.isEmpty()) { - try (Connection conn = databaseManager.getConnection(); - PreparedStatement ps = getPreparedStatement(conn, MERGE_CPE_ECOSYSTEM)) { + try (Connection conn = databaseManager.getConnection(); PreparedStatement ps = getPreparedStatement(conn, MERGE_CPE_ECOSYSTEM)) { for (Map.Entry, String> entry : map.entrySet()) { ps.setString(1, entry.getKey().getLeft()); ps.setString(2, entry.getKey().getRight()); @@ -894,8 +938,7 @@ private int updateOrInsertVulnerability(DefCveItem cve, String description) { loadCpeEcosystemCache(); } final int vulnerabilityId; - try (Connection conn = databaseManager.getConnection(); - PreparedStatement callUpdate = getPreparedStatement(conn, UPDATE_VULNERABILITY)) { + try (Connection conn = databaseManager.getConnection(); PreparedStatement callUpdate = getPreparedStatement(conn, UPDATE_VULNERABILITY)) { // String 1.cve, String 2.description, String 3.v2Severity, Float 4.v2ExploitabilityScore, // Float 5.v2ImpactScore, Boolean 6.v2AcInsufInfo, Boolean 7.v2ObtainAllPrivilege, // Boolean 8.v2ObtainUserPrivilege, Boolean 9.v2ObtainOtherPrivilege, Boolean 10.v2UserInteractionRequired, @@ -906,79 +949,91 @@ private int updateOrInsertVulnerability(DefCveItem cve, String description) { // String 23.v3PrivilegesRequired, String 24.v3UserInteraction, String 25.v3Scope, // String 26.v3ConfidentialityImpact, String 27.v3IntegrityImpact, String 28.v3AvailabilityImpact, // Float 29.v3BaseScore, String 30.v3BaseSeverity, String 31.v3Version - callUpdate.setString(1, cve.getCve().getCVEDataMeta().getId()); + callUpdate.setString(1, cve.getCve().getId()); callUpdate.setString(2, description); - if (cve.getImpact().getBaseMetricV2() != null) { - final BaseMetricV2 cvssv2 = cve.getImpact().getBaseMetricV2(); - Map props = cvssv2.getAdditionalProperties(); - callUpdate.setString(3, cvssv2.getSeverity()); - setFloatValue(callUpdate, 4, props, "exploitabilityScore"); - setFloatValue(callUpdate, 5, props, "impactScore"); - setBooleanValue(callUpdate, 6, props, "acInsufInfo"); - setBooleanValue(callUpdate, 7, props, "obtainAllPrivilege"); - setBooleanValue(callUpdate, 8, props, "obtainUserPrivilege"); - setBooleanValue(callUpdate, 9, props, "obtainOtherPrivilege"); - setBooleanValue(callUpdate, 10, props, "userInteractionRequired"); - callUpdate.setFloat(11, cvssv2.getCvssV2().getBaseScore().floatValue()); - callUpdate.setString(12, cvssv2.getCvssV2().getAccessVector().value()); - callUpdate.setString(13, cvssv2.getCvssV2().getAccessComplexity().value()); - callUpdate.setString(14, cvssv2.getCvssV2().getAuthentication().value()); - callUpdate.setString(15, cvssv2.getCvssV2().getConfidentialityImpact().value()); - callUpdate.setString(16, cvssv2.getCvssV2().getIntegrityImpact().value()); - callUpdate.setString(17, cvssv2.getCvssV2().getAvailabilityImpact().value()); - props = cvssv2.getCvssV2().getAdditionalProperties(); - setStringValue(callUpdate, 18, props, "version"); + Optional optCvssv2 = null; + if (cve.getCve().getMetrics() != null && cve.getCve().getMetrics().getCvssMetricV2() != null) { + optCvssv2 = cve.getCve().getMetrics().getCvssMetricV2().stream().sorted(Comparator.comparing(CvssV2::getType)).findFirst(); + } + if (optCvssv2 != null && optCvssv2.isPresent()) { + final CvssV2 cvssv2 = optCvssv2.get(); + setUpdateColumn(callUpdate, 3, cvssv2.getBaseSeverity()); + setUpdateColumn(callUpdate, 4, cvssv2.getExploitabilityScore()); + setUpdateColumn(callUpdate, 5, cvssv2.getImpactScore()); + setUpdateColumn(callUpdate, 6, cvssv2.getAcInsufInfo()); + setUpdateColumn(callUpdate, 7, cvssv2.getObtainAllPrivilege()); + setUpdateColumn(callUpdate, 8, cvssv2.getObtainUserPrivilege()); + setUpdateColumn(callUpdate, 9, cvssv2.getObtainOtherPrivilege()); + setUpdateColumn(callUpdate, 10, cvssv2.getUserInteractionRequired()); + setUpdateColumn(callUpdate, 11, cvssv2.getCvssData().getBaseScore()); + setUpdateColumn(callUpdate, 12, cvssv2.getCvssData().getAccessVector()); + setUpdateColumn(callUpdate, 13, cvssv2.getCvssData().getAccessComplexity()); + setUpdateColumn(callUpdate, 14, cvssv2.getCvssData().getAuthentication()); + setUpdateColumn(callUpdate, 15, cvssv2.getCvssData().getConfidentialityImpact()); + setUpdateColumn(callUpdate, 16, cvssv2.getCvssData().getIntegrityImpact()); + setUpdateColumn(callUpdate, 17, cvssv2.getCvssData().getAvailabilityImpact()); + setUpdateColumn(callUpdate, 18, cvssv2.getCvssData().getVersion()); } else { - callUpdate.setNull(3, java.sql.Types.NULL); - callUpdate.setNull(4, java.sql.Types.NULL); - callUpdate.setNull(5, java.sql.Types.NULL); - callUpdate.setNull(6, java.sql.Types.NULL); - callUpdate.setNull(7, java.sql.Types.NULL); - callUpdate.setNull(8, java.sql.Types.NULL); - callUpdate.setNull(9, java.sql.Types.NULL); - callUpdate.setNull(10, java.sql.Types.NULL); - callUpdate.setNull(11, java.sql.Types.NULL); - callUpdate.setNull(12, java.sql.Types.NULL); - callUpdate.setNull(13, java.sql.Types.NULL); - callUpdate.setNull(14, java.sql.Types.NULL); - callUpdate.setNull(15, java.sql.Types.NULL); - callUpdate.setNull(16, java.sql.Types.NULL); - callUpdate.setNull(17, java.sql.Types.NULL); - callUpdate.setNull(18, java.sql.Types.NULL); + callUpdate.setNull(3, java.sql.Types.VARCHAR); + callUpdate.setNull(4, java.sql.Types.DOUBLE); + callUpdate.setNull(5, java.sql.Types.DOUBLE); + callUpdate.setNull(6, java.sql.Types.VARCHAR); + callUpdate.setNull(7, java.sql.Types.BOOLEAN); + callUpdate.setNull(8, java.sql.Types.BOOLEAN); + callUpdate.setNull(9, java.sql.Types.BOOLEAN); + callUpdate.setNull(10, java.sql.Types.BOOLEAN); + callUpdate.setNull(11, java.sql.Types.DOUBLE); + callUpdate.setNull(12, java.sql.Types.VARCHAR); + callUpdate.setNull(13, java.sql.Types.VARCHAR); + callUpdate.setNull(14, java.sql.Types.VARCHAR); + callUpdate.setNull(15, java.sql.Types.VARCHAR); + callUpdate.setNull(16, java.sql.Types.VARCHAR); + callUpdate.setNull(17, java.sql.Types.VARCHAR); + callUpdate.setNull(18, java.sql.Types.VARCHAR); + } + Optional optCvssv30 = Optional.empty(); + if (cve.getCve().getMetrics() != null && cve.getCve().getMetrics().getCvssMetricV30() != null) { + optCvssv30 = cve.getCve().getMetrics().getCvssMetricV30().stream().sorted(Comparator.comparing(CvssV3::getType)).findFirst(); + } + Optional optCvssv31 = Optional.empty(); + if (cve.getCve().getMetrics() != null && cve.getCve().getMetrics().getCvssMetricV31() != null) { + optCvssv31 = cve.getCve().getMetrics().getCvssMetricV31().stream().sorted(Comparator.comparing(CvssV3::getType)).findFirst(); } - if (cve.getImpact().getBaseMetricV3() != null) { - final BaseMetricV3 cvssv3 = cve.getImpact().getBaseMetricV3(); - Map props = cvssv3.getAdditionalProperties(); - setFloatValue(callUpdate, 19, props, "exploitabilityScore"); - setFloatValue(callUpdate, 20, props, "impactScore"); - - callUpdate.setString(21, cvssv3.getCvssV3().getAttackVector().value()); - callUpdate.setString(22, cvssv3.getCvssV3().getAttackComplexity().value()); - callUpdate.setString(23, cvssv3.getCvssV3().getPrivilegesRequired().value()); - callUpdate.setString(24, cvssv3.getCvssV3().getUserInteraction().value()); - callUpdate.setString(25, cvssv3.getCvssV3().getScope().value()); - callUpdate.setString(26, cvssv3.getCvssV3().getConfidentialityImpact().value()); - callUpdate.setString(27, cvssv3.getCvssV3().getIntegrityImpact().value()); - callUpdate.setString(28, cvssv3.getCvssV3().getAvailabilityImpact().value()); - callUpdate.setFloat(29, cvssv3.getCvssV3().getBaseScore().floatValue()); - callUpdate.setString(30, cvssv3.getCvssV3().getBaseSeverity().value()); - - props = cvssv3.getCvssV3().getAdditionalProperties(); - setStringValue(callUpdate, 31, props, "version"); + + CvssV3 cvssv3 = null; + if (optCvssv31.isPresent()) { + cvssv3 = optCvssv31.get(); + } else if (optCvssv30.isPresent()) { + cvssv3 = optCvssv30.get(); + } + if (cvssv3 != null) { + setUpdateColumn(callUpdate, 19, cvssv3.getExploitabilityScore()); + setUpdateColumn(callUpdate, 20, cvssv3.getImpactScore()); + setUpdateColumn(callUpdate, 21, cvssv3.getCvssData().getAttackVector()); + setUpdateColumn(callUpdate, 22, cvssv3.getCvssData().getAttackComplexity()); + setUpdateColumn(callUpdate, 23, cvssv3.getCvssData().getPrivilegesRequired()); + setUpdateColumn(callUpdate, 24, cvssv3.getCvssData().getUserInteraction()); + setUpdateColumn(callUpdate, 25, cvssv3.getCvssData().getScope()); + setUpdateColumn(callUpdate, 26, cvssv3.getCvssData().getConfidentialityImpact()); + setUpdateColumn(callUpdate, 27, cvssv3.getCvssData().getIntegrityImpact()); + setUpdateColumn(callUpdate, 28, cvssv3.getCvssData().getAvailabilityImpact()); + setUpdateColumn(callUpdate, 29, cvssv3.getCvssData().getBaseScore()); + setUpdateColumn(callUpdate, 30, cvssv3.getCvssData().getBaseSeverity()); + setUpdateColumn(callUpdate, 31, cvssv3.getCvssData().getVersion()); } else { - callUpdate.setNull(19, java.sql.Types.NULL); - callUpdate.setNull(20, java.sql.Types.NULL); - callUpdate.setNull(21, java.sql.Types.NULL); - callUpdate.setNull(22, java.sql.Types.NULL); - callUpdate.setNull(23, java.sql.Types.NULL); - callUpdate.setNull(24, java.sql.Types.NULL); - callUpdate.setNull(25, java.sql.Types.NULL); - callUpdate.setNull(26, java.sql.Types.NULL); - callUpdate.setNull(27, java.sql.Types.NULL); - callUpdate.setNull(28, java.sql.Types.NULL); - callUpdate.setNull(29, java.sql.Types.NULL); - callUpdate.setNull(30, java.sql.Types.NULL); - callUpdate.setNull(31, java.sql.Types.NULL); + callUpdate.setNull(19, java.sql.Types.DOUBLE); + callUpdate.setNull(20, java.sql.Types.DOUBLE); + callUpdate.setNull(21, java.sql.Types.VARCHAR); + callUpdate.setNull(22, java.sql.Types.VARCHAR); + callUpdate.setNull(23, java.sql.Types.VARCHAR); + callUpdate.setNull(24, java.sql.Types.VARCHAR); + callUpdate.setNull(25, java.sql.Types.VARCHAR); + callUpdate.setNull(26, java.sql.Types.VARCHAR); + callUpdate.setNull(27, java.sql.Types.VARCHAR); + callUpdate.setNull(28, java.sql.Types.VARCHAR); + callUpdate.setNull(29, java.sql.Types.DOUBLE); + callUpdate.setNull(30, java.sql.Types.VARCHAR); + callUpdate.setNull(31, java.sql.Types.VARCHAR); } if (isOracle) { try { @@ -987,7 +1042,7 @@ private int updateOrInsertVulnerability(DefCveItem cve, String description) { cs.executeUpdate(); vulnerabilityId = cs.getInt(32); } catch (SQLException ex) { - final String msg = String.format("Unable to retrieve id for new vulnerability for '%s'", cve.getCve().getCVEDataMeta().getId()); + final String msg = String.format("Unable to retrieve id for new vulnerability for '%s'", cve.getCve().getId()); throw new DatabaseException(msg, ex); } } else { @@ -995,7 +1050,7 @@ private int updateOrInsertVulnerability(DefCveItem cve, String description) { rs.next(); vulnerabilityId = rs.getInt(1); } catch (SQLException ex) { - final String msg = String.format("Unable to retrieve id for new vulnerability for '%s'", cve.getCve().getCVEDataMeta().getId()); + final String msg = String.format("Unable to retrieve id for new vulnerability for '%s'", cve.getCve().getId()); throw new DatabaseException(msg, ex); } } @@ -1013,22 +1068,24 @@ private int updateOrInsertVulnerability(DefCveItem cve, String description) { * @throws SQLException thrown if there is an error inserting the data */ private void updateVulnerabilityInsertCwe(int vulnerabilityId, DefCveItem cve) throws SQLException { - try (Connection conn = databaseManager.getConnection(); - PreparedStatement insertCWE = getPreparedStatement(conn, INSERT_CWE, vulnerabilityId)) { - for (ProblemtypeDatum datum : cve.getCve().getProblemtype().getProblemtypeData()) { - for (LangString desc : datum.getDescription()) { - if ("en".equals(desc.getLang())) { - insertCWE.setString(2, desc.getValue()); - if (isBatchInsertEnabled()) { - insertCWE.addBatch(); - } else { - insertCWE.execute(); + if (cve.getCve() != null && cve.getCve().getWeaknesses() != null) { + try (Connection conn = databaseManager.getConnection(); + PreparedStatement insertCWE = getPreparedStatement(conn, INSERT_CWE, vulnerabilityId)) { + for (Weakness weakness : cve.getCve().getWeaknesses()) { + for (LangString desc : weakness.getDescription()) { + if ("en".equals(desc.getLang())) { + insertCWE.setString(2, desc.getValue()); + if (isBatchInsertEnabled()) { + insertCWE.addBatch(); + } else { + insertCWE.execute(); + } } } } - } - if (isBatchInsertEnabled()) { - insertCWE.executeBatch(); + if (isBatchInsertEnabled()) { + insertCWE.executeBatch(); + } } } } @@ -1112,8 +1169,7 @@ public void updateKnownExploitedVulnerabilities( private void updateVulnerabilityInsertSoftware(int vulnerabilityId, String cveId, List software, String baseEcosystem) throws DatabaseException, SQLException { - try (Connection conn = databaseManager.getConnection(); - PreparedStatement insertSoftware = getPreparedStatement(conn, INSERT_SOFTWARE)) { + try (Connection conn = databaseManager.getConnection(); PreparedStatement insertSoftware = getPreparedStatement(conn, INSERT_SOFTWARE)) { for (VulnerableSoftware parsedCpe : software) { insertSoftware.setInt(1, vulnerabilityId); insertSoftware.setString(2, parsedCpe.getPart().getAbbreviation()); @@ -1167,14 +1223,29 @@ private void updateVulnerabilityInsertSoftware(int vulnerabilityId, String cveId * @throws SQLException thrown if there is an error inserting the data */ private void updateVulnerabilityInsertReferences(int vulnerabilityId, DefCveItem cve) throws SQLException { - try (Connection conn = databaseManager.getConnection(); - PreparedStatement insertReference = getPreparedStatement(conn, INSERT_REFERENCE)) { + try (Connection conn = databaseManager.getConnection(); PreparedStatement insertReference = getPreparedStatement(conn, INSERT_REFERENCE)) { if (cve.getCve().getReferences() != null) { - for (Reference r : cve.getCve().getReferences().getReferenceData()) { + for (Reference r : cve.getCve().getReferences()) { insertReference.setInt(1, vulnerabilityId); - insertReference.setString(2, r.getName()); - insertReference.setString(3, r.getUrl()); - insertReference.setString(4, r.getRefsource()); + Optional name = Optional.empty(); + if (r.getTags() != null) { + name = r.getTags().stream().sorted().findFirst(); + } + if (name.isPresent()) { + insertReference.setString(2, name.get()); + } else { + insertReference.setNull(2, java.sql.Types.VARCHAR); + } + if (r.getUrl() != null && !r.getUrl().isEmpty()) { + insertReference.setString(3, r.getUrl()); + } else { + insertReference.setNull(3, java.sql.Types.VARCHAR); + } + if (r.getSource() != null && !r.getSource().isEmpty()) { + insertReference.setString(4, r.getSource()); + } else { + insertReference.setNull(4, java.sql.Types.VARCHAR); + } if (isBatchInsertEnabled()) { insertReference.addBatch(); } else { @@ -1198,20 +1269,23 @@ private void updateVulnerabilityInsertReferences(int vulnerabilityId, DefCveItem */ private List parseCpes(DefCveItem cve) throws CpeValidationException { final List software = new ArrayList<>(); - final List cpeEntries = cve.getConfigurations().getNodes().stream() - .collect(NodeFlatteningCollector.getInstance()) - .collect(CpeMatchStreamCollector.getInstance()) - .filter(predicate -> predicate.getCpe23Uri() != null) - .filter(predicate -> predicate.getCpe23Uri().startsWith(cpeStartsWithFilter)) + + final List cpeEntries = cve.getCve().getConfigurations().stream() + .map(Config::getNodes) + .flatMap(List::stream) + .map(Node::getCpeMatch) + .flatMap(List::stream) + .filter(predicate -> predicate.getCriteria() != null) + .filter(predicate -> predicate.getCriteria().startsWith(cpeStartsWithFilter)) //this single CPE entry causes nearly 100% FP - so filtering it at the source. - .filter(entry -> !("CVE-2009-0754".equals(cve.getCve().getCVEDataMeta().getId()) - && "cpe:2.3:a:apache:apache:*:*:*:*:*:*:*:*".equals(entry.getCpe23Uri()))) + .filter(entry -> !("CVE-2009-0754".equals(cve.getCve().getId()) + && "cpe:2.3:a:apache:apache:*:*:*:*:*:*:*:*".equals(entry.getCriteria()))) .collect(Collectors.toList()); final VulnerableSoftwareBuilder builder = new VulnerableSoftwareBuilder(); try { cpeEntries.forEach(entry -> { - builder.cpe(parseCpe(entry, cve.getCve().getCVEDataMeta().getId())) + builder.cpe(parseCpe(entry, cve.getCve().getId())) .versionEndExcluding(entry.getVersionEndExcluding()) .versionStartExcluding(entry.getVersionStartExcluding()) .versionEndIncluding(entry.getVersionEndIncluding()) @@ -1240,22 +1314,14 @@ private List parseCpes(DefCveItem cve) throws CpeValidationE * @throws DatabaseException thrown if there is an error converting the * CpeMatch into a CPE object */ - private Cpe parseCpe(DefCpeMatch cpe, String cveId) throws DatabaseException { - Cpe parsedCpe; + private Cpe parseCpe(CpeMatch cpe, String cveId) throws DatabaseException { + final Cpe parsedCpe; try { //the replace is a hack as the NVD does not properly escape backslashes in their JSON - parsedCpe = CpeParser.parse(cpe.getCpe23Uri(), true); + parsedCpe = CpeParser.parse(cpe.getCriteria(), true); } catch (CpeParsingException ex) { - LOGGER.debug("NVD (" + cveId + ") contain an invalid 2.3 CPE: " + cpe.getCpe23Uri()); - if (cpe.getCpe22Uri() != null && !cpe.getCpe22Uri().isEmpty()) { - try { - parsedCpe = CpeParser.parse(cpe.getCpe22Uri(), true); - } catch (CpeParsingException ex2) { - throw new DatabaseException("Unable to parse CPE: " + cpe.getCpe23Uri(), ex); - } - } else { - throw new DatabaseException("Unable to parse CPE: " + cpe.getCpe23Uri(), ex); - } + LOGGER.debug("NVD (" + cveId + ") contain an invalid 2.3 CPE: " + cpe.getCriteria()); + throw new DatabaseException("Unable to parse CPE: " + cpe.getCriteria(), ex); } return parsedCpe; } @@ -1399,8 +1465,7 @@ public void persistEcosystemCache() { public void defrag() { if (isH2) { final long start = System.currentTimeMillis(); - try (Connection conn = databaseManager.getConnection(); - CallableStatement psCompaxt = conn.prepareCall("SHUTDOWN DEFRAG")) { + try (Connection conn = databaseManager.getConnection(); CallableStatement psCompaxt = conn.prepareCall("SHUTDOWN DEFRAG")) { LOGGER.info("Begin database defrag"); psCompaxt.execute(); final long millis = System.currentTimeMillis() - start; @@ -1447,8 +1512,7 @@ VulnerableSoftware getMatchingSoftware(Cpe cpe, Set vulnerab */ public void deleteUnusedCpe() { clearCache(); - try (Connection conn = databaseManager.getConnection(); - PreparedStatement ps = getPreparedStatement(conn, DELETE_UNUSED_DICT_CPE)) { + try (Connection conn = databaseManager.getConnection(); PreparedStatement ps = getPreparedStatement(conn, DELETE_UNUSED_DICT_CPE)) { ps.executeUpdate(); } catch (SQLException ex) { LOGGER.error("Unable to delete CPE dictionary entries", ex); @@ -1468,8 +1532,7 @@ public void deleteUnusedCpe() { */ public void addCpe(String cpe, String vendor, String product) { clearCache(); - try (Connection conn = databaseManager.getConnection(); - PreparedStatement ps = getPreparedStatement(conn, ADD_DICT_CPE)) { + try (Connection conn = databaseManager.getConnection(); PreparedStatement ps = getPreparedStatement(conn, ADD_DICT_CPE)) { ps.setString(1, cpe); ps.setString(2, vendor); ps.setString(3, product); @@ -1528,6 +1591,134 @@ private void addNullableStringParameter(PreparedStatement ps, int pos, String va } } + private void setUpdateColumn(PreparedStatement ps, int i, Double value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.DOUBLE); + } else { + ps.setDouble(i, value); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV2Data.AuthenticationType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV2Data.CiaType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV2Data.Version value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV2Data.AccessComplexityType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV2Data.AccessVectorType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, String value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, Boolean value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.BOOLEAN); + } else { + ps.setBoolean(i, value); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV3Data.AttackVectorType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV3Data.AttackComplexityType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV3Data.PrivilegesRequiredType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV3Data.UserInteractionType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV3Data.ScopeType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV3Data.SeverityType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV3Data.CiaType value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + + private void setUpdateColumn(PreparedStatement ps, int i, CvssV3Data.Version value) throws SQLException { + if (value == null) { + ps.setNull(i, java.sql.Types.VARCHAR); + } else { + ps.setString(i, value.value()); + } + } + /** * Sets the float parameter on a prepared statement from a properties map. * @@ -1542,10 +1733,10 @@ private void setFloatValue(PreparedStatement ps, int i, Map prop try { ps.setFloat(i, Float.parseFloat(props.get(key).toString())); } catch (NumberFormatException nfe) { - ps.setNull(i, java.sql.Types.NULL); + ps.setNull(i, java.sql.Types.FLOAT); } } else { - ps.setNull(i, java.sql.Types.NULL); + ps.setNull(i, java.sql.Types.FLOAT); } } @@ -1562,7 +1753,7 @@ private void setStringValue(PreparedStatement ps, int i, Map pro if (props != null && props.containsKey(key)) { ps.setString(i, props.get(key).toString()); } else { - ps.setNull(i, java.sql.Types.NULL); + ps.setNull(i, java.sql.Types.VARCHAR); } } @@ -1579,7 +1770,7 @@ private void setBooleanValue(PreparedStatement ps, int i, Map pr if (props != null && props.containsKey(key)) { ps.setBoolean(i, Boolean.parseBoolean(props.get(key).toString())); } else { - ps.setNull(i, java.sql.Types.NULL); + ps.setNull(i, java.sql.Types.BOOLEAN); } } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveItemOperator.java b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveItemOperator.java index 87866186e72..19278a20867 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveItemOperator.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveItemOperator.java @@ -17,13 +17,14 @@ */ package org.owasp.dependencycheck.data.nvdcve; +import io.github.jeremylong.openvulnerability.client.nvd.Config; import java.util.stream.Collectors; import org.owasp.dependencycheck.data.nvd.ecosystem.Ecosystem; -import org.owasp.dependencycheck.data.nvd.json.CpeMatchStreamCollector; -import org.owasp.dependencycheck.data.nvd.json.DefCveItem; -import org.owasp.dependencycheck.data.nvd.json.LangString; -import org.owasp.dependencycheck.data.nvd.json.NodeFlatteningCollector; +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; +import io.github.jeremylong.openvulnerability.client.nvd.LangString; +import io.github.jeremylong.openvulnerability.client.nvd.Node; +import java.util.List; import org.owasp.dependencycheck.dependency.VulnerableSoftware; /** @@ -57,7 +58,7 @@ public CveItemOperator(String cpeStartsWithFilter) { * @return the English descriptions from the CVE object */ public String extractDescription(DefCveItem cve) { - return cve.getCve().getDescription().getDescriptionData().stream().filter((desc) + return cve.getCve().getDescriptions().stream().filter((desc) -> "en".equals(desc.getLang())).map(LangString::getValue).collect(Collectors.joining(" ")); } @@ -216,11 +217,16 @@ public boolean isRejected(String description) { * configured CPE Starts with filter */ protected boolean testCveCpeStartWithFilter(final DefCveItem cve) { - //cycle through to see if this is a CPE we care about (use the CPE filters - return cve.getConfigurations().getNodes().stream() - .collect(NodeFlatteningCollector.getInstance()) - .collect(CpeMatchStreamCollector.getInstance()) - .filter(cpe -> cpe.getCpe23Uri() != null) - .anyMatch(cpe -> cpe.getCpe23Uri().startsWith(cpeStartsWithFilter)); + if (cve.getCve().getConfigurations() != null) { + //cycle through to see if this is a CPE we care about (use the CPE filters + return cve.getCve().getConfigurations().stream() + .map(Config::getNodes) + .flatMap(List::stream) + .map(Node::getCpeMatch) + .flatMap(List::stream) + .filter(cpe -> cpe.getCriteria() != null) + .anyMatch(cpe -> cpe.getCriteria().startsWith(cpeStartsWithFilter)); + } + return false; } } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java index f1ef704d3f1..8733dc453ee 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java @@ -27,7 +27,6 @@ import java.util.TreeMap; import javax.annotation.concurrent.ThreadSafe; -import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo; import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,26 +45,21 @@ public class DatabaseProperties { */ private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseProperties.class); /** - * Modified key word, used as a key to store information about the modified - * file (i.e. the containing the last 8 days of updates).. + * The last modified request data for the NVD API. */ - public static final String MODIFIED = "Modified"; + public static final String NVD_API_LAST_MODIFIED = "nvd.api.last.modified"; /** - * The properties file key for the last checked field - used to store the - * last check time of the Modified NVD CVE xml file. + * The date the NVD API was last checked for an update. */ - public static final String LAST_CHECKED = "NVD CVE Checked"; + public static final String NVD_API_LAST_CHECKED = "nvd.api.last.checked"; /** - * The properties file key for the last updated field - used to store the - * last updated time of the Modified NVD CVE xml file. + * The date the NVD cache was last checked for an update. */ - public static final String LAST_UPDATED = "NVD CVE Modified"; + public static final String NVD_CACHE_LAST_CHECKED = "nvd.cache.last.checked"; /** - * Stores the last updated time for each of the NVD CVE files. These - * timestamps should be updated if we process the modified file within 7 - * days of the last update. + * The date the NVD cache data was last modified/updated. */ - public static final String LAST_UPDATED_BASE = "NVD CVE "; + public static final String NVD_CACHE_LAST_MODIFIED = "nvd.cache.last.modified"; /** * The key for the last time the CPE data was updated. */ @@ -74,7 +68,6 @@ public class DatabaseProperties { * The key for the database schema version. */ public static final String VERSION = "version"; - /** * The key for the last check time for the Known Exploited Vulnerabilities. */ @@ -83,7 +76,6 @@ public class DatabaseProperties { * The key for the version the Known Exploited Vulnerabilities. */ public static final String KEV_VERSION = "kev.version"; - /** * A collection of properties about the data. */ @@ -112,19 +104,6 @@ public synchronized boolean isEmpty() { return properties == null || properties.isEmpty(); } - /** - * Saves the last updated information to the properties file. - * - * @param updatedValue the updated NVD CVE entry - * @throws UpdateException is thrown if there is an update exception - */ - public synchronized void save(NvdCveInfo updatedValue) throws UpdateException { - if (updatedValue == null) { - return; - } - save(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp())); - } - /** * Saves the key value pair to the properties store. * @@ -198,4 +177,43 @@ public synchronized Map getMetaData() { } return map; } + + /** + * Retrieves a zoned date time. + * + * @param key the property key + * @return the zoned date time + */ + public ZonedDateTime getTimestamp(String key) { + return DatabaseProperties.getTimestamp(properties, key); + } + + /** + * Stores a timestamp. + * + * @param key the property key + * @param timestamp the zoned date time + */ + public void save(String key, ZonedDateTime timestamp) throws UpdateException { + final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX"); + save(key, dtf.format(timestamp)); + } + + /** + * Retrieves a zoned date time. + * + * @param properties the properties file containing the date time + * @param key the property key + * @return the zoned date time + */ + public static ZonedDateTime getTimestamp(Properties properties, String key) { + final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX"); + final String val = properties.getProperty(key); + if (val != null) { + final String value = properties.getProperty(key); + return ZonedDateTime.parse(value, dtf); + } + return null; + } + } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java b/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java index c81d8c7a740..4fffe1a5373 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java @@ -123,14 +123,14 @@ public boolean update(Engine engine) throws UpdateException { final CveDB db = engine.getDatabase(); final boolean autoupdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE, true); final boolean enabled = settings.getBoolean(Settings.KEYS.UPDATE_VERSION_CHECK_ENABLED, true); - final String original = settings.getString(Settings.KEYS.CVE_ORIGINAL_JSON); - final String current = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); + final String datafeed = settings.getString(Settings.KEYS.NVD_API_DATAFEED_URL); /* * Only update if auto-update is enabled, the engine check is - * enabled, and the NVD CVE URLs have not been modified (i.e. the - * user has not configured them to point to an internal source). + * enabled, and the NVD DataFeed is being used (i.e. the user + * is likely on a private network). This check is not really needed + * so we are okay skipping it. */ - if (enabled && autoupdate && original != null && original.equals(current)) { + if (enabled && autoupdate && datafeed != null) { LOGGER.debug("Begin Engine Version Check"); final DatabaseProperties properties = db.getDatabaseProperties(); diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java index 459af715d94..dea02961c39 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java @@ -98,7 +98,7 @@ public boolean update(Engine engine) throws UpdateException { @Override public boolean purge(Engine engine) { - //do nothing - covered by the NvdCveUpdater data source. + //do nothing - covered by the NvdApiDataSource. return true; } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java new file mode 100644 index 00000000000..09fee6f9799 --- /dev/null +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java @@ -0,0 +1,549 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2023 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; +import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClient; +import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.net.MalformedURLException; +import java.net.URL; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.owasp.dependencycheck.Engine; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; +import org.owasp.dependencycheck.data.update.exception.UpdateException; +import org.owasp.dependencycheck.data.update.nvd.api.DownloadTask; +import org.owasp.dependencycheck.data.update.nvd.api.NvdApiProcessor; +import org.owasp.dependencycheck.utils.DateUtil; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.ResourceNotFoundException; +import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.utils.TooManyRequestsException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * @author Jeremy Long + */ +public class NvdApiDataSource implements CachedWebDataSource { + + /** + * The logger. + */ + private static final Logger LOGGER = LoggerFactory.getLogger(NvdApiDataSource.class); + /** + * The thread pool size to use for CPU-intense tasks. + */ + private static final int PROCESSING_THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors(); + /** + * The configured settings. + */ + private Settings settings; + /** + * Reference to the DAO. + */ + private CveDB cveDb = null; + /** + * The properties obtained from the database. + */ + private DatabaseProperties dbProperties = null; + /** + * The key for the NVD API cache properties file's last modified date. + */ + private static final String NVD_API_CACHE_MODIFIED_DATE = "lastModifiedDate"; + /** + * The number of results per page from the NVD API. The default is 2000; we + * are setting the value to be explicit. + */ + private static final int RESULTS_PER_PAGE = 2000; + + @Override + public boolean update(Engine engine) throws UpdateException { + this.settings = engine.getSettings(); + this.cveDb = engine.getDatabase(); + if (isUpdateConfiguredFalse()) { + return false; + } + dbProperties = cveDb.getDatabaseProperties(); + + final String nvdDataFeedUrl = settings.getString(Settings.KEYS.NVD_API_DATAFEED_URL); + if (nvdDataFeedUrl != null) { + return processDatafeed(nvdDataFeedUrl); + } + return processApi(); + } + + private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException { + boolean updatesMade = false; + try { + dbProperties = cveDb.getDatabaseProperties(); + if (checkUpdate()) { + String url; + String pattern = null; + if (nvdDataFeedUrl.endsWith(".json.gz")) { + final int lio = nvdDataFeedUrl.lastIndexOf("/"); + pattern = nvdDataFeedUrl.substring(lio); + url = nvdDataFeedUrl.substring(0, lio); + } else { + url = nvdDataFeedUrl; + } + if (!url.endsWith("/")) { + url += "/"; + } + final Properties cacheProperties = getRemoteCacheProperties(url); + if (pattern == null) { + final String prefix = cacheProperties.getProperty("prefix", "nvdcve-"); + pattern = prefix + "{0}.json.gz"; + } + + final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); + final Map updateable = getUpdatesNeeded(url, pattern, cacheProperties, now); + if (!updateable.isEmpty()) { + final int downloadPoolSize; + final int max = settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 1); + downloadPoolSize = Math.min(Runtime.getRuntime().availableProcessors(), max); + + ExecutorService processingExecutorService = null; + ExecutorService downloadExecutorService = null; + try { + downloadExecutorService = Executors.newFixedThreadPool(downloadPoolSize); + processingExecutorService = Executors.newFixedThreadPool(PROCESSING_THREAD_POOL_SIZE); + + DownloadTask runLast = null; + final Set>> downloadFutures = new HashSet<>(updateable.size()); + runLast = startDownloads(updateable, processingExecutorService, runLast, downloadFutures, downloadExecutorService); + + //complete downloads + final Set> processFutures = new HashSet<>(updateable.size()); + for (Future> future : downloadFutures) { + processDownload(future, processFutures); + } + //process the data + processFuture(processFutures); + processFutures.clear(); + + //download and process the modified as the last entry + if (runLast != null) { + final Future> modified = downloadExecutorService.submit(runLast); + processDownload(modified, processFutures); + processFuture(processFutures); + } + + } finally { + if (processingExecutorService != null) { + processingExecutorService.shutdownNow(); + } + if (downloadExecutorService != null) { + downloadExecutorService.shutdownNow(); + } + } + updatesMade = true; + } + storeLastModifiedDates(now, cacheProperties, updateable); + if (updatesMade) { + cveDb.persistEcosystemCache(); + } + final int updateCount = cveDb.updateEcosystemCache(); + LOGGER.debug("Corrected the ecosystem for {} ecoSystemCache entries", updateCount); + if (updatesMade || updateCount > 0) { + cveDb.cleanupDatabase(); + } + } + } catch (UpdateException ex) { + if (ex.getCause() != null && ex.getCause() instanceof DownloadFailedException) { + final String jre = System.getProperty("java.version"); + if (jre == null || jre.startsWith("1.4") || jre.startsWith("1.5") || jre.startsWith("1.6") || jre.startsWith("1.7")) { + LOGGER.error("An old JRE is being used ({} {}), and likely does not have the correct root certificates or algorithms " + + "to connect to the NVD - consider upgrading your JRE.", System.getProperty("java.vendor"), jre); + } + } + throw ex; + } catch (DatabaseException ex) { + throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex); + } + return updatesMade; + } + + private void storeLastModifiedDates(final ZonedDateTime now, final Properties cacheProperties, + final Map updateable) throws UpdateException { + dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_CHECKED, now); + dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED, DatabaseProperties.getTimestamp(cacheProperties, + NVD_API_CACHE_MODIFIED_DATE + ".modified")); + for (String entry : updateable.keySet()) { + final ZonedDateTime date = DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE + "." + entry); + dbProperties.save(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + entry, date); + } + } + + private DownloadTask startDownloads(final Map updateable, ExecutorService processingExecutorService, DownloadTask runLast, + final Set>> downloadFutures, ExecutorService downloadExecutorService) throws UpdateException { + DownloadTask lastCall = runLast; + for (Map.Entry cve : updateable.entrySet()) { + final DownloadTask call = new DownloadTask(cve.getValue(), processingExecutorService, cveDb, settings); + if (call.isModified()) { + lastCall = call; + } else { + final boolean added = downloadFutures.add(downloadExecutorService.submit(call)); + if (!added) { + throw new UpdateException("Unable to add the download task for " + cve); + } + } + } + return lastCall; + } + + private void processFuture(final Set> processFutures) throws UpdateException { + //complete processing + for (Future future : processFutures) { + try { + final NvdApiProcessor task = future.get(); + } catch (InterruptedException ex) { + LOGGER.debug("Thread was interrupted during processing", ex); + Thread.currentThread().interrupt(); + throw new UpdateException(ex); + } catch (ExecutionException ex) { + LOGGER.debug("Execution Exception during process", ex); + throw new UpdateException(ex); + } + } + } + + private void processDownload(Future> future, final Set> processFutures) throws UpdateException { + final Future task; + try { + task = future.get(); + if (task != null) { + processFutures.add(task); + } + } catch (InterruptedException ex) { + LOGGER.debug("Thread was interrupted during download", ex); + Thread.currentThread().interrupt(); + throw new UpdateException("The download was interrupted", ex); + } catch (ExecutionException ex) { + LOGGER.debug("Thread was interrupted during download execution", ex); + throw new UpdateException("The execution of the download was interrupted", ex); + } + } + + private boolean processApi() throws UpdateException { + final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_CHECKED); + if (cveDb.dataExists() && lastChecked != null) { + final ZonedDateTime thirtyMinutesAgo = ZonedDateTime.now().minusMinutes(30); + if (thirtyMinutesAgo.isBefore(lastChecked)) { + LOGGER.info("Skipping the NVD API Update as it was completed within the last 30 minutes"); + return true; + } + } + + ZonedDateTime lastModifiedRequest = dbProperties.getTimestamp(DatabaseProperties.NVD_API_LAST_MODIFIED); + final NvdCveClientBuilder builder = NvdCveClientBuilder.aNvdCveApi(); + if (lastModifiedRequest != null) { + final ZonedDateTime end = lastModifiedRequest.minusDays(-120); + builder.withLastModifiedFilter(lastModifiedRequest, end); + } + final String key = settings.getString(Settings.KEYS.NVD_API_KEY); + if (key != null) { + //using a higher delay as the system may not be able to process these faster. + builder.withApiKey(key) + .withDelay(2000) + .withThreadCount(4); + } else { + LOGGER.warn("An NVD API Key was not provided - it is highly recommended to use " + + "an NVD API key as the update can take a VERY long time without an API Key"); + builder.withDelay(8000); + } + builder.withResultsPerPage(RESULTS_PER_PAGE); + final String virtualMatch = settings.getString(Settings.KEYS.CVE_CPE_STARTS_WITH_FILTER); + if (virtualMatch != null) { + builder.withVirtualMatchString(virtualMatch); + } + long delay = 0; + try { + delay = settings.getLong(Settings.KEYS.NVD_API_DELAY); + } catch (InvalidSettingException ex) { + LOGGER.debug("Invalid setting `NVD_API_DELAY`?"); + } + if (delay > 0) { + builder.withDelay(delay); + } + + ExecutorService processingExecutorService = null; + try { + processingExecutorService = Executors.newFixedThreadPool(PROCESSING_THREAD_POOL_SIZE); + final List> submitted = new ArrayList<>(); + int max = -1; + int ctr = 0; + try (NvdCveClient api = builder.build()) { + while (api.hasNext()) { + final Collection items = api.next(); + max = api.getTotalAvailable(); + if (ctr == 0) { + LOGGER.info(String.format("NVD API has %,d records in this update", max)); + } + if (items != null && !items.isEmpty()) { + final Future f = processingExecutorService.submit(new NvdApiProcessor(cveDb, items)); + submitted.add(f); + ctr += 1; + if ((ctr % 10) == 0) { + final double percent = (double) (ctr * RESULTS_PER_PAGE) / max * 100; + LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", ctr * RESULTS_PER_PAGE, max, percent)); + } + } + final ZonedDateTime last = api.getLastUpdated(); + if (last != null && (lastModifiedRequest == null || lastModifiedRequest.compareTo(last) < 0)) { + lastModifiedRequest = last; + } + } + + } catch (Exception e) { + throw new UpdateException("Error updating the NVD Data", e); + } + LOGGER.info(String.format("Downloaded %,d/%,d (%.0f%%)", max, max, 100f)); + max = submitted.size(); + final boolean updated = max > 0; + ctr = 0; + for (Future f : submitted) { + try { + final NvdApiProcessor proc = f.get(); + ctr += 1; + final double percent = (double) ctr / max * 100; + LOGGER.info(String.format("Completed processing batch %d/%d (%.0f%%) in %,dms", ctr, max, percent, proc.getDurationMillis())); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new RuntimeException(ex); + } catch (ExecutionException ex) { + LOGGER.error("Exception processing NVD API Results", ex); + throw new RuntimeException(ex); + } + } + if (lastModifiedRequest != null) { + dbProperties.save(DatabaseProperties.NVD_API_LAST_CHECKED, ZonedDateTime.now()); + dbProperties.save(DatabaseProperties.NVD_API_LAST_MODIFIED, lastModifiedRequest); + } + return updated; + } finally { + if (processingExecutorService != null) { + processingExecutorService.shutdownNow(); + } + } + } + + /** + * Checks if the system is configured NOT to update. + * + * @return false if the system is configured to perform an update; otherwise + * true + */ + private boolean isUpdateConfiguredFalse() { + if (!settings.getBoolean(Settings.KEYS.UPDATE_NVDCVE_ENABLED, true)) { + return true; + } + boolean autoUpdate = true; + try { + autoUpdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE); + } catch (InvalidSettingException ex) { + LOGGER.debug("Invalid setting for auto-update; using true."); + } + return !autoUpdate; + } + + @Override + public boolean purge(Engine engine) { + boolean result = true; + try { + final File dataDir = engine.getSettings().getDataDirectory(); + final File db = new File(dataDir, engine.getSettings().getString(Settings.KEYS.DB_FILE_NAME, "odc.mv.db")); + if (db.exists()) { + if (db.delete()) { + LOGGER.info("Database file purged; local copy of the NVD has been removed"); + } else { + LOGGER.error("Unable to delete '{}'; please delete the file manually", db.getAbsolutePath()); + result = false; + } + } else { + LOGGER.info("Unable to purge database; the database file does not exist: {}", db.getAbsolutePath()); + result = false; + } + final File traceFile = new File(dataDir, "odc.trace.db"); + if (traceFile.exists() && !traceFile.delete()) { + LOGGER.error("Unable to delete '{}'; please delete the file manually", traceFile.getAbsolutePath()); + result = false; + } + final File lockFile = new File(dataDir, "odc.update.lock"); + if (lockFile.exists() && !lockFile.delete()) { + LOGGER.error("Unable to delete '{}'; please delete the file manually", lockFile.getAbsolutePath()); + result = false; + } + } catch (IOException ex) { + final String msg = "Unable to delete the database"; + LOGGER.error(msg, ex); + result = false; + } + return result; + } + + /** + * Checks if the NVD API Cache JSON files were last checked recently. As an + * optimization, we can avoid repetitive checks against the NVD cache. + * + * @return true to proceed with the check, or false to skip + * @throws UpdateException thrown when there is an issue checking for + * updates + */ + private boolean checkUpdate() throws UpdateException { + boolean proceed = true; + // If the valid setting has not been specified, then we proceed to check... + final int validForHours = settings.getInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, 0); + if (dataExists() && 0 < validForHours) { + // ms Valid = valid (hours) x 60 min/hour x 60 sec/min x 1000 ms/sec + final long validForSeconds = validForHours * 60L * 60L; + final ZonedDateTime lastChecked = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_CHECKED); + final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); + final Duration duration = Duration.between(now, lastChecked); + final long difference = duration.getSeconds(); + proceed = difference > validForSeconds; + if (!proceed) { + LOGGER.info("Skipping NVD API Cache check since last check was within {} hours.", validForHours); + LOGGER.debug("Last NVD API was at {}, and now {} is within {} s.", lastChecked, now, validForSeconds); + } + } + return proceed; + } + + /** + * Checks the CVE Index to ensure data exists and analysis can continue. + * + * @return true if the database contains data + */ + private boolean dataExists() { + return cveDb.dataExists(); + } + + /** + * Determines if the index needs to be updated. This is done by fetching the + * NVD CVE meta data and checking the last update date. If the data needs to + * be refreshed this method will return the NvdCveUrl for the files that + * need to be updated. + * + * @param url the URL of the NVD API cache + * @param filePattern the string format pattern for the cached files (e.g. + * "nvdcve-{0}.json.gz") + * @param cacheProperties the properties from the remote NVD API cache + * @param now the start time of the update process + * @return the map of key to URLs - where the key is the year or `modified` + * @throws UpdateException Is thrown if there is an issue with the last + * updated properties file + */ + protected final Map getUpdatesNeeded(String url, String filePattern, + Properties cacheProperties, ZonedDateTime now) throws UpdateException { + LOGGER.debug("starting getUpdatesNeeded() ..."); + final Map updates = new HashMap<>(); + if (dbProperties != null && !dbProperties.isEmpty()) { + final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002); + // for establishing the current year use the timezone where the new year starts first + // as from that moment on CNAs might start assigning CVEs with the new year depending + // on the CNA's timezone + final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear(); + boolean needsFullUpdate = false; + for (int y = startYear; y <= endYear; y++) { + final ZonedDateTime val = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + y); + if (val == null) { + needsFullUpdate = true; + break; + } + } + final ZonedDateTime lastUpdated = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED); + final int days = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_VALID_FOR_DAYS, 7); + + if (!needsFullUpdate && lastUpdated.equals(DatabaseProperties.getTimestamp(cacheProperties, NVD_API_CACHE_MODIFIED_DATE))) { + return updates; + } else { + updates.put("modified", url + MessageFormat.format(filePattern, "modified")); + if (needsFullUpdate) { + for (int i = startYear; i < endYear; i++) { + if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) { + updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i))); + } + } + } else if (!DateUtil.withinDateRange(lastUpdated, now, days)) { + for (int i = startYear; i <= endYear; i++) { + if (cacheProperties.containsKey(NVD_API_CACHE_MODIFIED_DATE + "." + i)) { + final ZonedDateTime lastModifiedCache = DatabaseProperties.getTimestamp(cacheProperties, + NVD_API_CACHE_MODIFIED_DATE + "." + i); + final ZonedDateTime lastModifiedDB = dbProperties.getTimestamp(DatabaseProperties.NVD_CACHE_LAST_MODIFIED + "." + i); + if (lastModifiedDB == null || lastModifiedCache.compareTo(lastModifiedDB) > 0) { + updates.put(String.valueOf(i), url + MessageFormat.format(filePattern, String.valueOf(i))); + } + } + } + } + } + } + if (updates.size() > 3) { + LOGGER.info("NVD API Cache requires several updates; this could take a couple of minutes."); + } + return updates; + } + + /** + * Downloads the metadata properties of the NVD API cache. + * + * @param url the URL to the NVD API cache + * @return the cache properties + * @throws UpdateException thrown if the properties file could not be + * downloaded + */ + protected final Properties getRemoteCacheProperties(String url) throws UpdateException { + try { + final URL u = new URL(url + "cache.properties"); + final Downloader d = new Downloader(settings); + final String content = d.fetchContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + final Properties properties = new Properties(); + properties.load(new StringReader(content)); + return properties; + } catch (MalformedURLException ex) { + throw new UpdateException("Invalid NVD Cache URL", ex); + } catch (DownloadFailedException | TooManyRequestsException | ResourceNotFoundException ex) { + throw new UpdateException("Unable to download the NVD API cache.properties", ex); + } catch (IOException ex) { + throw new UpdateException("Invalid NVD Cache Properties file contents", ex); + } + } +} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java deleted file mode 100644 index cbd2939b423..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java +++ /dev/null @@ -1,585 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.MalformedURLException; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.util.HashSet; -import java.util.Set; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.stream.Collectors; -import javax.annotation.concurrent.ThreadSafe; -import org.apache.commons.io.FileUtils; - -import org.owasp.dependencycheck.Engine; -import org.owasp.dependencycheck.data.nvd.json.MetaProperties; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.nvdcve.DatabaseException; -import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; - -import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED; - -import org.owasp.dependencycheck.data.update.exception.InvalidDataException; -import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.data.update.nvd.DownloadTask; -import org.owasp.dependencycheck.data.update.nvd.NvdCache; -import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo; -import org.owasp.dependencycheck.data.update.nvd.ProcessTask; -import org.owasp.dependencycheck.utils.DateUtil; -import org.owasp.dependencycheck.utils.DownloadFailedException; -import org.owasp.dependencycheck.utils.Downloader; -import org.owasp.dependencycheck.utils.InvalidSettingException; -import org.owasp.dependencycheck.utils.ResourceNotFoundException; -import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.TooManyRequestsException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Class responsible for updating the NVD CVE data. - * - * @author Jeremy Long - */ -@ThreadSafe -public class NvdCveUpdater implements CachedWebDataSource { - - /** - * The logger. - */ - private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class); - /** - * The thread pool size to use for CPU-intense tasks. - */ - private static final int PROCESSING_THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors(); - /** - * The thread pool size to use when downloading files. - */ - private static final int DOWNLOAD_THREAD_POOL_SIZE = Math.round(1.5f * Runtime.getRuntime().availableProcessors()); - /** - * ExecutorService for CPU-intense processing tasks. - */ - private ExecutorService processingExecutorService = null; - /** - * ExecutorService for tasks that involve blocking activities and are not - * very CPU-intense, e.g. downloading files. - */ - private ExecutorService downloadExecutorService = null; - /** - * The configured settings. - */ - private Settings settings; - /** - * Reference to the DAO. - */ - private CveDB cveDb = null; - /** - * The properties obtained from the database. - */ - private DatabaseProperties dbProperties = null; - - /** - * Downloads the latest NVD CVE XML file from the web and imports it into - * the current CVE Database. A lock on a file is obtained in an attempt to - * prevent more then one thread/JVM from updating the database at the same - * time. This method may sleep upto 5 minutes. - * - * @param engine a reference to the dependency-check engine - * @return whether or not an update was made to the CveDB - * @throws UpdateException is thrown if there is an error updating the - * database - */ - @Override - public synchronized boolean update(Engine engine) throws UpdateException { - this.settings = engine.getSettings(); - this.cveDb = engine.getDatabase(); - if (isUpdateConfiguredFalse()) { - return false; - } - boolean updatesMade = false; - try { - dbProperties = cveDb.getDatabaseProperties(); - if (checkUpdate()) { - final List updateable = getUpdatesNeeded(); - if (!updateable.isEmpty()) { - initializeExecutorServices(); - performUpdate(updateable); - updatesMade = true; - } - //all dates in the db are now stored in seconds as opposed to previously milliseconds. - dbProperties.save(DatabaseProperties.LAST_CHECKED, Long.toString(System.currentTimeMillis() / 1000)); - if (updatesMade) { - cveDb.persistEcosystemCache(); - } - final int updateCount = cveDb.updateEcosystemCache(); - LOGGER.debug("Corrected the ecosystem for {} ecoSystemCache entries", updateCount); - if (updatesMade || updateCount > 0) { - cveDb.cleanupDatabase(); - } - } - } catch (UpdateException ex) { - if (ex.getCause() != null && ex.getCause() instanceof DownloadFailedException) { - final String jre = System.getProperty("java.version"); - if (jre == null || jre.startsWith("1.4") || jre.startsWith("1.5") || jre.startsWith("1.6") || jre.startsWith("1.7")) { - LOGGER.error("An old JRE is being used ({} {}), and likely does not have the correct root certificates or algorithms " - + "to connect to the NVD - consider upgrading your JRE.", System.getProperty("java.vendor"), jre); - } - } - throw ex; - } catch (DatabaseException ex) { - throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex); - } finally { - shutdownExecutorServices(); - } - return updatesMade; - } - - /** - * Checks if the system is configured NOT to update. - * - * @return false if the system is configured to perform an update; otherwise - * true - */ - private boolean isUpdateConfiguredFalse() { - if (!settings.getBoolean(Settings.KEYS.UPDATE_NVDCVE_ENABLED, true)) { - return true; - } - boolean autoUpdate = true; - try { - autoUpdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE); - } catch (InvalidSettingException ex) { - LOGGER.debug("Invalid setting for auto-update; using true."); - } - return !autoUpdate; - } - - /** - * Initialize the executor services for download and processing of the NVD - * CVE XML data. - */ - protected void initializeExecutorServices() { - final int downloadPoolSize; - final int max = settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 1); - downloadPoolSize = Math.min(DOWNLOAD_THREAD_POOL_SIZE, max); - downloadExecutorService = Executors.newFixedThreadPool(downloadPoolSize); - processingExecutorService = Executors.newFixedThreadPool(PROCESSING_THREAD_POOL_SIZE); - LOGGER.debug("#download threads: {}", downloadPoolSize); - LOGGER.debug("#processing threads: {}", PROCESSING_THREAD_POOL_SIZE); - } - - /** - * Shutdown and cleanup of resources used by the executor services. - */ - private void shutdownExecutorServices() { - if (processingExecutorService != null) { - processingExecutorService.shutdownNow(); - } - if (downloadExecutorService != null) { - downloadExecutorService.shutdownNow(); - } - } - - /** - * Checks if the NVD CVE XML files were last checked recently. As an - * optimization, we can avoid repetitive checks against the NVD. Setting - * CVE_CHECK_VALID_FOR_HOURS determines the duration since last check before - * checking again. A database property stores the timestamp of the last - * check. - * - * @return true to proceed with the check, or false to skip - * @throws UpdateException thrown when there is an issue checking for - * updates - */ - private boolean checkUpdate() throws UpdateException { - boolean proceed = true; - // If the valid setting has not been specified, then we proceed to check... - final int validForHours = settings.getInt(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, 0); - if (dataExists() && 0 < validForHours) { - // ms Valid = valid (hours) x 60 min/hour x 60 sec/min x 1000 ms/sec - final long validForSeconds = validForHours * 60L * 60L; - final long lastChecked = getPropertyInSeconds(DatabaseProperties.LAST_CHECKED); - final long now = System.currentTimeMillis() / 1000; - proceed = (now - lastChecked) > validForSeconds; - if (!proceed) { - LOGGER.info("Skipping NVD check since last check was within {} hours.", validForHours); - LOGGER.debug("Last NVD was at {}, and now {} is within {} s.", lastChecked, now, validForSeconds); - } - } - return proceed; - } - - /** - * Checks the CVE Index to ensure data exists and analysis can continue. - * - * @return true if the database contains data - */ - private boolean dataExists() { - return cveDb.dataExists(); - } - - /** - * Downloads the latest NVD CVE XML file from the web and imports it into - * the current CVE Database. - * - * @param updateable a collection of NVD CVE data file references that need - * to be downloaded and processed to update the database - * @throws UpdateException is thrown if there is an error updating the - * database - */ - @SuppressWarnings("FutureReturnValueIgnored") - private void performUpdate(List updateable) throws UpdateException { - if (updateable.isEmpty()) { - return; - } - if (updateable.size() > 3) { - LOGGER.info("NVD CVE requires several updates; this could take a couple of minutes."); - } - - DownloadTask runLast = null; - final Set>> downloadFutures = new HashSet<>(updateable.size()); - for (NvdCveInfo cve : updateable) { - final DownloadTask call = new DownloadTask(cve, processingExecutorService, cveDb, settings); - if (call.isModified()) { - runLast = call; - } else { - final boolean added = downloadFutures.add(downloadExecutorService.submit(call)); - if (!added) { - throw new UpdateException("Unable to add the download task for " + cve.getId()); - } - } - } - - //next, move the future future processTasks to just future processTasks and check for errors. - final Set> processFutures = new HashSet<>(updateable.size()); - for (Future> future : downloadFutures) { - final Future task; - try { - task = future.get(); - if (task != null) { - processFutures.add(task); - } - } catch (InterruptedException ex) { - LOGGER.debug("Thread was interrupted during download", ex); - Thread.currentThread().interrupt(); - throw new UpdateException("The download was interrupted", ex); - } catch (ExecutionException ex) { - LOGGER.debug("Thread was interrupted during download execution", ex); - throw new UpdateException("The execution of the download was interrupted", ex); - } - } - - for (Future future : processFutures) { - try { - final ProcessTask task = future.get(); - if (task.getException() != null) { - throw task.getException(); - } - } catch (InterruptedException ex) { - LOGGER.debug("Thread was interrupted during processing", ex); - Thread.currentThread().interrupt(); - throw new UpdateException(ex); - } catch (ExecutionException ex) { - LOGGER.debug("Execution Exception during process", ex); - throw new UpdateException(ex); - } - } - - if (runLast != null) { - final Future> modified = downloadExecutorService.submit(runLast); - final Future task; - try { - task = modified.get(); - if (task != null) { - final ProcessTask last = task.get(); - if (last.getException() != null) { - throw last.getException(); - } - } - } catch (InterruptedException ex) { - LOGGER.debug("Thread was interrupted during download", ex); - Thread.currentThread().interrupt(); - throw new UpdateException("The download was interrupted", ex); - } catch (ExecutionException ex) { - LOGGER.debug("Thread was interrupted during download execution", ex); - throw new UpdateException("The execution of the download was interrupted", ex); - } - } - - } - - /** - * Downloads the NVD CVE Meta file properties. - * - * @param url the URL to the NVD CVE JSON file - * @return the meta file properties - * @throws UpdateException thrown if the meta file could not be downloaded - */ - protected final MetaProperties getMetaFile(String url) throws UpdateException { - try { - final long waitTime = settings.getInt(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, 4000); - MetaProperties retVal = doMetaDownload(url, false); - - final int downloadAttempts = 4; - for (int x = 2; retVal == null && x <= downloadAttempts; x++) { - Thread.sleep(waitTime * (x / 2)); - retVal = doMetaDownload(url, x == downloadAttempts); - } - return retVal; - } catch (InterruptedException ex) { - Thread.interrupted(); - throw new UpdateException("Download interupted", ex); - } - } - - /** - * Downloads the NVD CVE Meta file properties. - * - * @param url the URL to the NVD CVE JSON file - * @param throwErrors if true and an error occurs, the error - * will be thrown; otherwise the error will be suppressed - * @return the meta file properties - * @throws UpdateException thrown if the meta file could not be downloaded - */ - private MetaProperties doMetaDownload(String url, boolean throwErrors) throws UpdateException { - final String metaUrl = url.substring(0, url.length() - 7) + "meta"; - final NvdCache cache = new NvdCache(settings); - try { - final URL u = new URL(metaUrl); - final File tmp = settings.getTempFile("nvd", "meta"); - if (cache.notInCache(u, tmp)) { - final Downloader d = new Downloader(settings); - final String content = d.fetchContent(u, true, Settings.KEYS.CVE_USER, Settings.KEYS.CVE_PASSWORD); - try (FileOutputStream fos = new FileOutputStream(tmp); - OutputStreamWriter osw = new OutputStreamWriter(fos, StandardCharsets.UTF_8); - BufferedWriter writer = new BufferedWriter(osw)) { - writer.write(content); - } - cache.storeInCache(u, tmp); - FileUtils.deleteQuietly(tmp); - return new MetaProperties(content); - } else { - final String content; - try (FileInputStream fis = new FileInputStream(tmp); - InputStreamReader isr = new InputStreamReader(fis, StandardCharsets.UTF_8); - BufferedReader reader = new BufferedReader(isr)) { - content = reader.lines().collect(Collectors.joining("\n")); - } - FileUtils.deleteQuietly(tmp); - return new MetaProperties(content); - } - } catch (MalformedURLException ex) { - if (throwErrors) { - throw new UpdateException("Meta file url is invalid: " + metaUrl, ex); - } - } catch (InvalidDataException ex) { - if (throwErrors) { - throw new UpdateException("Meta file content is invalid: " + metaUrl, ex); - } - } catch (DownloadFailedException ex) { - if (throwErrors) { - throw new UpdateException("Unable to download meta file: " + metaUrl, ex); - } - } catch (TooManyRequestsException ex) { - if (throwErrors) { - throw new UpdateException("Unable to download meta file: " + metaUrl + "; received 429 -- too many requests", ex); - } - } catch (ResourceNotFoundException ex) { - if (throwErrors) { - throw new UpdateException("Unable to download meta file: " + metaUrl + "; received 404 -- resource not found", ex); - } - } catch (IOException ex) { - if (throwErrors) { - throw new RuntimeException(ex); - } - } - return null; - } - - /** - * Determines if the index needs to be updated. This is done by fetching the - * NVD CVE meta data and checking the last update date. If the data needs to - * be refreshed this method will return the NvdCveUrl for the files that - * need to be updated. - * - * @return the collection of files that need to be updated - * @throws UpdateException Is thrown if there is an issue with the last - * updated properties file - */ - protected final List getUpdatesNeeded() throws UpdateException { - LOGGER.debug("starting getUpdatesNeeded() ..."); - final List updates = new ArrayList<>(); - if (dbProperties != null && !dbProperties.isEmpty()) { - try { - final int startYear = settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002); - // for establishing the current year use the timezone where the new year starts first - // as from that moment on CNAs might start assigning CVEs with the new year depending - // on the CNA's timezone - final ZonedDateTime today = ZonedDateTime.now().withZoneSameInstant(ZoneOffset.ofHours(14)); - final int endYear = today.getYear(); - final int dayOfEndYear = today.getDayOfYear(); - boolean needsFullUpdate = false; - for (int y = startYear; y <= endYear; y++) { - final long val = Long.parseLong(dbProperties.getProperty(DatabaseProperties.LAST_UPDATED_BASE + y, "0")); - if (val == 0) { - needsFullUpdate = true; - break; - } - } - final long lastUpdated = getPropertyInSeconds(DatabaseProperties.LAST_UPDATED); - final long now = System.currentTimeMillis() / 1000; - final int days = settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7); - - String url = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); - final MetaProperties modified = getMetaFile(url); - - if (!needsFullUpdate && lastUpdated == modified.getLastModifiedDate()) { - return updates; - } else { - final String baseUrl = settings.getString(Settings.KEYS.CVE_BASE_JSON); - final NvdCveInfo item = new NvdCveInfo(MODIFIED, url, modified.getLastModifiedDate()); - updates.add(item); - if (needsFullUpdate) { - // no need to download each one, just use the modified timestamp - for (int i = startYear; i < endYear; i++) { - url = String.format(baseUrl, i); - final NvdCveInfo entry = new NvdCveInfo(Integer.toString(i), url, modified.getLastModifiedDate()); - updates.add(entry); - } - // for endyear check metadata availability to determine inclusion when still in grace period - if (dayOfEndYear < settings.getInt(Settings.KEYS.NVD_NEW_YEAR_GRACE_PERIOD, 10)) { - try { - url = String.format(baseUrl, endYear); - final MetaProperties meta = getMetaFile(url); - } catch (UpdateException ue) { - if (ue.getCause() instanceof ResourceNotFoundException) { - LOGGER.warn("NVD Data for {} has not been published yet.", endYear); - } else { - throw ue; - } - } - } else { - url = String.format(baseUrl, endYear); - final NvdCveInfo entry = new NvdCveInfo(Integer.toString(endYear), url, modified.getLastModifiedDate()); - updates.add(entry); - } - } else if (!DateUtil.withinDateRange(lastUpdated, now, days)) { - final long waitTime = settings.getInt(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, 4000); - for (int i = startYear; i <= endYear; i++) { - try { - url = String.format(baseUrl, i); - Thread.sleep(waitTime); - final MetaProperties meta = getMetaFile(url); - final long currentTimestamp = getPropertyInSeconds(DatabaseProperties.LAST_UPDATED_BASE + i); - - if (currentTimestamp < meta.getLastModifiedDate()) { - final NvdCveInfo entry = new NvdCveInfo(Integer.toString(i), url, meta.getLastModifiedDate()); - updates.add(entry); - } - } catch (UpdateException ex) { - final int grace = settings.getInt(Settings.KEYS.NVD_NEW_YEAR_GRACE_PERIOD, 10); - if (ex.getCause() instanceof ResourceNotFoundException - && i == endYear && dayOfEndYear < grace) { - LOGGER.warn("NVD Data for {} has not been published yet.", endYear); - } else { - throw ex; - } - } catch (InterruptedException ex) { - Thread.interrupted(); - throw new UpdateException("The download of the meta file was interupted: " + url, ex); - } - } - } - } - } catch (NumberFormatException ex) { - LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file."); - LOGGER.debug("", ex); - } - } - return updates; - } - - /** - * Returns the database property value in seconds. - * - * @param key the key to the property - * @return the property value in seconds - */ - private long getPropertyInSeconds(String key) { - final String value = dbProperties.getProperty(key, "0"); - return DateUtil.getEpochValueInSeconds(value); - } - - /** - * Sets the settings object; this is used during testing. - * - * @param settings the configured settings - */ - protected synchronized void setSettings(Settings settings) { - this.settings = settings; - } - - @Override - public boolean purge(Engine engine) { - boolean result = true; - try { - final File dataDir = engine.getSettings().getDataDirectory(); - final File db = new File(dataDir, engine.getSettings().getString(Settings.KEYS.DB_FILE_NAME, "odc.mv.db")); - if (db.exists()) { - if (db.delete()) { - LOGGER.info("Database file purged; local copy of the NVD has been removed"); - } else { - LOGGER.error("Unable to delete '{}'; please delete the file manually", db.getAbsolutePath()); - result = false; - } - } else { - LOGGER.info("Unable to purge database; the database file does not exist: {}", db.getAbsolutePath()); - result = false; - } - final File traceFile = new File(dataDir, "odc.trace.db"); - if (traceFile.exists() && !traceFile.delete()) { - LOGGER.error("Unable to delete '{}'; please delete the file manually", traceFile.getAbsolutePath()); - result = false; - } - final File lockFile = new File(dataDir, "odc.update.lock"); - if (lockFile.exists() && !lockFile.delete()) { - LOGGER.error("Unable to delete '{}'; please delete the file manually", lockFile.getAbsolutePath()); - result = false; - } - } catch (IOException ex) { - final String msg = "Unable to delete the database"; - LOGGER.error(msg, ex); - result = false; - } - return result; - } -} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CpeEcosystemCache.java b/core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CpeEcosystemCache.java index 671b7bd009b..71d61cf2185 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CpeEcosystemCache.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CpeEcosystemCache.java @@ -20,7 +20,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.commons.lang3.StringUtils; -import org.owasp.dependencycheck.data.update.nvd.NvdCveParser; + import org.owasp.dependencycheck.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +50,7 @@ private CpeEcosystemCache() { /** * The logger. */ - private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveParser.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CpeEcosystemCache.class); //CSOFF: EmptyBlock /** diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java deleted file mode 100644 index a0d8f0bf9ce..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2013 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update.nvd; - -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import javax.annotation.concurrent.ThreadSafe; -import org.apache.commons.lang3.StringUtils; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.utils.DownloadFailedException; -import org.owasp.dependencycheck.utils.Downloader; -import org.owasp.dependencycheck.utils.ResourceNotFoundException; -import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.TooManyRequestsException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A callable object to download two files. - * - * @author Jeremy Long - */ -@ThreadSafe -public class DownloadTask implements Callable> { - - /** - * The Logger. - */ - private static final Logger LOGGER = LoggerFactory.getLogger(DownloadTask.class); - /** - * The CVE DB to use when processing the files. - */ - private final CveDB cveDB; - /** - * The processor service to pass the results of the download to. - */ - private final ExecutorService processorService; - /** - * The NVD CVE Meta Data. - */ - private NvdCveInfo nvdCveInfo; - /** - * A reference to the global settings object. - */ - private final Settings settings; - /** - * a file. - */ - private final File file; - - /** - * Simple constructor for the callable download task. - * - * @param nvdCveInfo the NVD CVE info - * @param processor the processor service to submit the downloaded files to - * @param cveDB the CVE DB to use to store the vulnerability data - * @param settings a reference to the global settings object; this is - * necessary so that when the thread is started the dependencies have a - * correct reference to the global settings. - * @throws UpdateException thrown if temporary files could not be created - */ - public DownloadTask(NvdCveInfo nvdCveInfo, ExecutorService processor, CveDB cveDB, Settings settings) throws UpdateException { - this.nvdCveInfo = nvdCveInfo; - this.processorService = processor; - this.cveDB = cveDB; - this.settings = settings; - - try { - this.file = File.createTempFile("cve" + nvdCveInfo.getId() + '_', ".json.gz", settings.getTempDirectory()); - } catch (IOException ex) { - throw new UpdateException("Unable to create temporary files", ex); - } - } - - /** - * Get the value of nvdCveInfo. - * - * @return the value of nvdCveInfo - */ - public NvdCveInfo getNvdCveInfo() { - return nvdCveInfo; - } - - /** - * Set the value of nvdCveInfo. - * - * @param nvdCveInfo new value of nvdCveInfo - */ - public void setNvdCveInfo(NvdCveInfo nvdCveInfo) { - this.nvdCveInfo = nvdCveInfo; - } - - /** - * Get the value of file. - * - * @return the value of file - */ - public File getFile() { - return file; - } - - @SuppressWarnings("BusyWait") - @Override - public Future call() throws Exception { - final long waitTime = settings.getInt(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, 4000); - long startDownload = 0; - final NvdCache cache = new NvdCache(settings); - try { - final URL url1 = new URL(nvdCveInfo.getUrl()); - if (cache.notInCache(url1, file)) { - Thread.sleep(waitTime); - LOGGER.info("Download Started for NVD CVE - {}", nvdCveInfo.getId()); - startDownload = System.currentTimeMillis(); - final int downloadAttempts = 5; - for (int x = 2; x <= downloadAttempts && !attemptDownload(url1, x == downloadAttempts); x++) { - LOGGER.info("Download Attempt {} for NVD CVE - {}", x, nvdCveInfo.getId()); - Thread.sleep(waitTime * (x / 2)); - } - if (file.isFile() && file.length() > 0) { - LOGGER.info("Download Complete for NVD CVE - {} ({} ms)", nvdCveInfo.getId(), - System.currentTimeMillis() - startDownload); - cache.storeInCache(url1, file); - } else { - throw new DownloadFailedException("Unable to download NVD CVE " + nvdCveInfo.getId()); - } - } - if (this.processorService == null) { - return null; - } - final ProcessTask task = new ProcessTask(cveDB, this, settings); - final Future val = this.processorService.submit(task); - return val; - - } catch (Throwable ex) { - LOGGER.error("Error downloading NVD CVE - {} Reason: {}", nvdCveInfo.getId(), ex.getMessage()); - throw ex; - } finally { - settings.cleanup(false); - } - } - - private boolean attemptDownload(final URL url1, boolean showLog) throws TooManyRequestsException, ResourceNotFoundException { - try { - final Downloader downloader = new Downloader(settings); - downloader.fetchFile(url1, file, Settings.KEYS.CVE_USER, Settings.KEYS.CVE_PASSWORD); - } catch (DownloadFailedException ex) { - if (showLog) { - LOGGER.error("Download Failed for NVD CVE - {}\nSome CVEs may not be reported. Reason: {}", - nvdCveInfo.getId(), ex.getMessage()); - if (settings.getString(Settings.KEYS.PROXY_SERVER) == null) { - LOGGER.error("If you are behind a proxy you may need to configure dependency-check to use the proxy."); - } - LOGGER.debug("", ex); - } - return false; - } - return true; - } - - /** - * Attempts to delete the files that were downloaded. - */ - public void cleanup() { - if (file != null && file.exists() && !file.delete()) { - LOGGER.debug("Failed to delete first temporary file {}", file); - file.deleteOnExit(); - } - } - - /** - * Attempts to delete the files that were downloaded. - */ - public void evictCorruptFileFromCache() { - final NvdCache cache = new NvdCache(settings); - try { - final URL url1 = new URL(nvdCveInfo.getUrl()); - cache.evictFromCache(url1); - } catch (MalformedURLException e) { - LOGGER.debug("Ignoring Cache-eviction request for an invalid URL"); - } - } - - /** - * Returns true if the process task is for the modified json file from the - * NVD. - * - * @return true if the process task is for the modified data; - * otherwise false - */ - public boolean isModified() { - return StringUtils.containsIgnoreCase(file.toString(), "modified"); - } -} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCache.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCache.java deleted file mode 100644 index 2471e7ce0f3..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCache.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2021 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update.nvd; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Files; -import java.time.Instant; -import org.apache.commons.io.FileUtils; -import org.owasp.dependencycheck.utils.Settings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Simple four hour cache for files. - * - * @author Jeremy Long - */ -public class NvdCache { - - /** - * The Logger. - */ - private static final Logger LOGGER = LoggerFactory.getLogger(DownloadTask.class); - /** - * The settings. - */ - private final Settings settings; - - /** - * Creates a new cache for the NVD files. - * - * @param settings ODC settings - */ - public NvdCache(Settings settings) { - this.settings = settings; - } - - /** - * Checks if the file is in the cache and within four hours. If found and - * viable, the cached data will be copied to the given file. - * - * @param url the URL of the file cached - * @param file the path of the file to restore from the cache - * @return true if the URL file is not in the cache; otherwise - * false - */ - public boolean notInCache(URL url, File file) { - try { - //valid for up to four hours. - final long validEpoch = Instant.now().toEpochMilli() - 14400000; - final File tmp = new File(url.getPath()); - final String filename = tmp.getName(); - final File cache = new File(settings.getDataDirectory(), "nvdcache"); - if (!cache.isDirectory()) { - return true; - } - final File nvdFile = new File(cache, filename); - if (nvdFile.isFile() && nvdFile.lastModified() > validEpoch) { - LOGGER.debug("Copying {} from cache", url); - FileUtils.copyFile(nvdFile, file); - return false; - } - return true; - } catch (IOException ex) { - LOGGER.debug("Error checking for nvd file in cache", ex); - return true; - } - } - - /** - * Stores a file in the cache. - * - * @param url the URL of the file to cache - * @param file the file to cache - */ - public void storeInCache(URL url, File file) { - if (file.isFile()) { - try { - final File tmp = new File(url.getPath()); - final String filename = tmp.getName(); - final File cache = new File(settings.getDataDirectory(), "nvdcache"); - if (!cache.isDirectory() && !cache.mkdir()) { - return; - } - final File nvdFile = new File(cache, filename); - FileUtils.copyFile(file, nvdFile); - if (!nvdFile.setLastModified(Instant.now().toEpochMilli())) { - LOGGER.debug("Unable to set last modified date on {}", nvdFile); - } - } catch (IOException ex) { - LOGGER.debug("Error storing nvd file in cache", ex); - } - } - } - - /** - * Evict a file corresponding to a URL from the cache. - *
- * Used to clear files from the cache that are found to be a corrupted download. - * - * @param url - * the origin URL of the file that is to be evicted from the cache - */ - public void evictFromCache(URL url) { - try { - final File tmp = new File(url.getPath()); - final String filename = tmp.getName(); - final File cache = new File(settings.getDataDirectory(), "nvdcache"); - if (!cache.isDirectory()) { - return; - } - LOGGER.error("Removing file from cache for {} as a corrupted download is detected", url); - final File nvdFile = new File(cache, filename); - Files.delete(nvdFile.toPath()); - } catch (IOException ex) { - LOGGER.warn("Error evicting corrupt nvd file from cache", ex); - } - } -} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfo.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfo.java deleted file mode 100644 index 57009240d7a..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfo.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2013 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update.nvd; - -import javax.annotation.concurrent.ThreadSafe; - -/** - * A pojo that contains the Url and timestamp of the current NvdCve JSON files. - * - * @author Jeremy Long - */ -@ThreadSafe -public class NvdCveInfo { - - /** - * The identifier for the CVE data file. - */ - private final String id; - /** - * The URL to download the file. - */ - private final String url; - /** - * The timestamp of the file - epoch time. - */ - private final long timestamp; - - /** - * Construct a new NVD CVE Info object. - * - * @param id the id - * @param url the url - * @param timestamp the timestamp - */ - public NvdCveInfo(String id, String url, long timestamp) { - this.id = id; - this.url = url; - this.timestamp = timestamp; - } - - /** - * Get the value of id. - * - * @return the value of id - */ - public String getId() { - return id; - } - - /** - * Get the value of URL. - * - * @return the value of URL - */ - public String getUrl() { - return url; - } - - /** - * Get the value of timestamp - epoch time. - * - * @return the value of timestamp - epoch time - */ - public long getTimestamp() { - return timestamp; - } -} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveParser.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveParser.java deleted file mode 100644 index a1451eaf919..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveParser.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2018 Steve Springett. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update.nvd; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.Module; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.databind.json.JsonMapper; -import com.fasterxml.jackson.module.blackbird.BlackbirdModule; -import com.fasterxml.jackson.module.afterburner.AfterburnerModule; - -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import static java.nio.charset.StandardCharsets.UTF_8; -import java.util.zip.GZIPInputStream; -import java.util.zip.ZipException; - -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.update.exception.CorruptedDatastreamException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.owasp.dependencycheck.data.nvd.json.DefCveItem; -import org.owasp.dependencycheck.data.nvd.ecosystem.CveEcosystemMapper; -import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.Utils; - -/** - * Parser and processor of NVD CVE JSON data feeds. - * - * @author Jeremy Long - */ -public final class NvdCveParser { - - /** - * The logger. - */ - private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveParser.class); - /** - * A reference to the CVE DB. - */ - private final CveDB cveDB; - /** - * A reference to the ODC settings. - */ - private final Settings settings; - - /** - * Creates a new NVD CVE JSON Parser. - * - * @param settings the dependency-check settings - * @param db a reference to the database - */ - public NvdCveParser(Settings settings, CveDB db) { - this.settings = settings; - this.cveDB = db; - } - - /** - * Parses the NVD JSON file and inserts/updates data into the database. - * - * @param file the NVD JSON file to parse - * @throws UpdateException thrown if the file could not be read - * @throws CorruptedDatastreamException thrown if the file was found to be a - * corrupted download (ZipException or premature EOF) - */ - public void parse(File file) throws UpdateException, CorruptedDatastreamException { - LOGGER.debug("Parsing " + file.getName()); - - final Module module; - if (Utils.getJavaVersion() <= 8) { - module = new AfterburnerModule(); - } else { - module = new BlackbirdModule(); - } - final ObjectMapper objectMapper = JsonMapper.builder() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .addModule(module) - .build(); - - final ObjectReader objectReader = objectMapper.readerFor(DefCveItem.class); - - try (InputStream fin = new FileInputStream(file); - InputStream in = new GZIPInputStream(fin); - InputStreamReader isr = new InputStreamReader(in, UTF_8); - JsonParser parser = objectReader.getFactory().createParser(isr)) { - - final CveEcosystemMapper mapper = new CveEcosystemMapper(); - init(parser); - while (parser.nextToken() == JsonToken.START_OBJECT) { - final DefCveItem cve = objectReader.readValue(parser); - cveDB.updateVulnerability(cve, mapper.getEcosystem(cve)); - } - } catch (FileNotFoundException ex) { - LOGGER.error(ex.getMessage()); - throw new UpdateException("Unable to find the NVD CVE file, `" + file + "`, to parse", ex); - } catch (ZipException | EOFException ex) { - throw new CorruptedDatastreamException("Error parsing NVD CVE file", ex); - } catch (IOException ex) { - LOGGER.error("Error reading NVD JSON data: {}", file); - LOGGER.debug("Error extracting the NVD JSON data from: " + file, ex); - throw new UpdateException("Unable to find the NVD CVE file to parse", ex); - } - } - - void init(JsonParser parser) throws IOException { - JsonToken nextToken = parser.nextToken(); - if (nextToken != JsonToken.START_OBJECT) { - throw new IOException("Expected " + JsonToken.START_OBJECT + ", got " + nextToken); - } - - do { - nextToken = parser.nextToken(); - if (nextToken == null) { - break; - } - - if (nextToken.isStructStart()) { - if (nextToken == JsonToken.START_ARRAY) { - break; - } else { - parser.skipChildren(); - } - } - } while (true); - } -} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/ProcessTask.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/ProcessTask.java deleted file mode 100644 index a3f2e70f1d4..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/ProcessTask.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2013 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update.nvd; - -import java.io.File; -import java.io.IOException; -import java.sql.SQLException; -import java.util.concurrent.Callable; -import javax.annotation.concurrent.ThreadSafe; -import javax.xml.parsers.ParserConfigurationException; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.nvdcve.DatabaseException; -import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; -import org.owasp.dependencycheck.data.update.exception.CorruptedDatastreamException; -import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.utils.Settings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A callable task that will process a given set of NVD CVE xml files and update - * the Cve Database accordingly. - * - * @author Jeremy Long - */ -@ThreadSafe -public class ProcessTask implements Callable { - - /** - * The logger. - */ - private static final Logger LOGGER = LoggerFactory.getLogger(ProcessTask.class); - /** - * A field to store any update exceptions that occur during the "call". - */ - private UpdateException exception = null; - /** - * A reference to the CveDB. - */ - private final CveDB cveDB; - /** - * A reference to the callable download task. - */ - private final DownloadTask downloadTask; - /** - * A reference to the properties. - */ - private final DatabaseProperties properties; - /** - * A reference to the global settings object. - */ - private final Settings settings; - - /** - * Get the value of exception. - * - * @return the value of exception - */ - public UpdateException getException() { - return exception; - } - - /** - * Set the value of exception. - * - * @param exception new value of exception - */ - public void setException(UpdateException exception) { - this.exception = exception; - } - - /** - * Constructs a new ProcessTask used to process an NVD CVE update. - * - * @param cveDB the data store object - * @param downloadTask the download task that contains the URL references to - * download - * @param settings a reference to the global settings object; this is - * necessary so that when the thread is started the dependencies have a - * correct reference to the global settings. - */ - public ProcessTask(final CveDB cveDB, final DownloadTask downloadTask, Settings settings) { - this.cveDB = cveDB; - this.downloadTask = downloadTask; - this.properties = cveDB.getDatabaseProperties(); - this.settings = settings; - } - - /** - * Implements the callable interface. - * - * @return this object - * @throws Exception thrown if there is an exception; note that any - * UpdateExceptions are simply added to the tasks exception collection - */ - @Override - public ProcessTask call() throws Exception { - try { - processFiles(); - } catch (UpdateException ex) { - this.exception = ex; - } finally { - settings.cleanup(false); - } - return this; - } - - /** - * Imports the NVD CVE JSON File into the database. - * - * @param file the file containing the NVD CVE JSON - * @throws ParserConfigurationException is thrown if there is a parser - * configuration exception - * @throws IOException is thrown if there is a IO Exception - * @throws SQLException is thrown if there is a SQL exception - * @throws DatabaseException is thrown if there is a database exception - * @throws ClassNotFoundException thrown if the h2 database driver cannot be - * loaded - * @throws UpdateException thrown if the file could not be found - * @throws CorruptedDatastreamException thrown if the file was found to be a corrupted download - */ - protected void importJSON(File file) throws ParserConfigurationException, IOException, SQLException, DatabaseException, - ClassNotFoundException, UpdateException, CorruptedDatastreamException { - - final NvdCveParser parser = new NvdCveParser(settings, cveDB); - parser.parse(file); - } - - /** - * Processes the NVD CVE XML file and imports the data into the DB. - * - * @throws UpdateException thrown if there is an error loading the data into - * the database - */ - private void processFiles() throws UpdateException { - LOGGER.info("Processing Started for NVD CVE - {}", downloadTask.getNvdCveInfo().getId()); - final long startProcessing = System.currentTimeMillis(); - try { - importJSON(downloadTask.getFile()); - properties.save(downloadTask.getNvdCveInfo()); - } catch (ParserConfigurationException | SQLException | DatabaseException | ClassNotFoundException | IOException ex) { - throw new UpdateException(ex); - } catch (CorruptedDatastreamException ex) { - downloadTask.evictCorruptFileFromCache(); - throw new UpdateException(ex); - } finally { - downloadTask.cleanup(); - } - LOGGER.info("Processing Complete for NVD CVE - {} ({} ms)", downloadTask.getNvdCveInfo().getId(), - System.currentTimeMillis() - startProcessing); - } -} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java new file mode 100644 index 00000000000..1bbbb89790f --- /dev/null +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java @@ -0,0 +1,121 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update.nvd.api; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20; +import java.net.URL; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import javax.annotation.concurrent.ThreadSafe; +import org.apache.commons.lang3.StringUtils; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.data.update.exception.UpdateException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.Settings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A callable object to download the NVD API cache files and start the + * NvdApiProcessor. + * + * @author Jeremy Long + */ +@ThreadSafe +public class DownloadTask implements Callable> { + + /** + * The Logger. + */ + private static final Logger LOGGER = LoggerFactory.getLogger(DownloadTask.class); + /** + * The CVE DB to use when processing the files. + */ + private final CveDB cveDB; + /** + * The processor service to pass the results of the download to. + */ + private final ExecutorService processorService; + /** + * The NVD API Cache file URL. + */ + private final String url; + /** + * A reference to the global settings object. + */ + private final Settings settings; + + /** + * Simple constructor for the callable download task. + * + * @param url the file to download + * @param processor the processor service to submit the downloaded files to + * @param cveDB the CVE DB to use to store the vulnerability data + * @param settings a reference to the global settings object; this is + * necessary so that when the thread is started the dependencies have a + * correct reference to the global settings. + * @throws UpdateException thrown if temporary files could not be created + */ + public DownloadTask(String url, ExecutorService processor, CveDB cveDB, Settings settings) { + this.url = url; + this.processorService = processor; + this.cveDB = cveDB; + this.settings = settings; + } + + @SuppressWarnings("BusyWait") + @Override + public Future call() throws Exception { + try { + final URL u = new URL(url); + LOGGER.info("Download Started for NVD Cache - {}", url); + final long startDownload = System.currentTimeMillis(); + final Downloader d = new Downloader(settings); + final String content = d.fetchGzContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + final ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + final CveApiJson20 data = objectMapper.readValue(content, CveApiJson20.class); + + if (this.processorService == null) { + return null; + } + final NvdApiProcessor task = new NvdApiProcessor(cveDB, data.getVulnerabilities(), startDownload); + final Future val = this.processorService.submit(task); + return val; + } catch (Throwable ex) { + LOGGER.error("Error downloading NVD CVE - {} Reason: {}", url, ex.getMessage()); + throw ex; + } finally { + settings.cleanup(false); + } + } + + /** + * Returns true if the process task is for the modified json file from the + * NVD API Cache. + * + * @return true if the process task is for the modified data; + * otherwise false + */ + public boolean isModified() { + return StringUtils.containsIgnoreCase(url, "modified"); + } +} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java new file mode 100644 index 00000000000..c4304fe4b64 --- /dev/null +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java @@ -0,0 +1,105 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2023 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update.nvd.api; + +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; +import java.util.Collection; +import java.util.concurrent.Callable; +import org.owasp.dependencycheck.data.nvd.ecosystem.CveEcosystemMapper; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Stores a collection of NVD CVE Data from the NVD API into the database. + * + * @author Jeremy Long + */ +public class NvdApiProcessor implements Callable { + + /** + * The Logger for use throughout the class. + */ + private static final Logger LOGGER = LoggerFactory.getLogger(NvdApiProcessor.class); + /** + * A reference to the database. + */ + private final CveDB cveDB; + /** + * The collection of NVD API data to add to the database. + */ + private Collection data; + /** + * Reference to the CVE Ecosystem Mapper object. + */ + private final CveEcosystemMapper mapper = new CveEcosystemMapper(); + /** + * The start time. + */ + private final long startTime; + /** + * The end time. + */ + private long endTime = 0; + + /** + * Create a new processor to put the NVD data into the database. + * + * @param cveDB a reference to the database + * @param data the data to add to the database + * @param startTime the start time of the update process. + */ + public NvdApiProcessor(final CveDB cveDB, Collection data, long startTime) { + this.cveDB = cveDB; + this.data = data; + this.startTime = startTime; + } + + /** + * Create a new processor to put the NVD data into the database. + * + * @param cveDB a reference to the database + * @param data the data to add to the database + */ + public NvdApiProcessor(final CveDB cveDB, Collection data) { + this(cveDB, data, System.currentTimeMillis()); + } + + @Override + public NvdApiProcessor call() throws Exception { + for (DefCveItem entry : data) { + try { + cveDB.updateVulnerability(entry, mapper.getEcosystem(entry)); + } catch (Exception ex) { + LOGGER.error("Failed to process " + entry.getCve().getId(), ex); + } + } + endTime = System.currentTimeMillis(); + data = null; + return this; + } + + /** + * Calculates how long the update process took. + * + * @return the number of milliseconds that the update process took + */ + public long getDurationMillis() { + return endTime - startTime; + } +} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/package-info.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/package-info.java new file mode 100644 index 00000000000..b2d1baeaac8 --- /dev/null +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/package-info.java @@ -0,0 +1,4 @@ +/** + * Contains classes used to download, parse, and load the NVD API CVE data from NIST into the local database.

+ */ +package org.owasp.dependencycheck.data.update.nvd.api; diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/package-info.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/package-info.java deleted file mode 100644 index 6b29278aca0..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Contains classes used to download, parse, and load the NVD CVE data from NIST into the local database.

- */ -package org.owasp.dependencycheck.data.update.nvd; diff --git a/core/src/main/java/org/owasp/dependencycheck/dependency/CvssV2.java b/core/src/main/java/org/owasp/dependencycheck/dependency/CvssV2.java deleted file mode 100644 index e82bcaf864a..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/dependency/CvssV2.java +++ /dev/null @@ -1,318 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2018 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.dependency; - -import java.io.Serializable; - -/** - * CVSS V2 scoring information. - * - * @author Jeremy Long - */ -public class CvssV2 implements Serializable { - - /** - * Serial version UID. - */ - private static final long serialVersionUID = -2203955879356702367L; - - /** - * CVSS Score. - */ - private final float score; - /** - * CVSS Access Vector. - */ - private final String accessVector; - /** - * CVSS Access Complexity. - */ - private final String accessComplexity; - /** - * CVSS Authentication. - */ - private final String authentication; - /** - * CVSS Confidentiality Impact. - */ - private final String confidentialityImpact; - /** - * CVSS Integrity Impact. - */ - private final String integrityImpact; - /** - * CVSS Availability Impact. - */ - private final String availabilityImpact; - /** - * CVSS version. - */ - private final String version; - - /** - * CVSSv2 Base Metric severity. - */ - private final String severity; - /** - * CVSSv2 Base Metric exploitability score. - */ - private final Float exploitabilityScore; - /** - * CVSSv2 Base Metric impact score. - */ - private final Float impactScore; - /** - * CVSSv2 Base Metric acInsufInfo. - */ - private final Boolean acInsufInfo; - /** - * CVSSv2 Base Metric obtain all privilege. - */ - private final Boolean obtainAllPrivilege; - /** - * CVSSv2 Base Metric obtain user privilege. - */ - private final Boolean obtainUserPrivilege; - /** - * CVSSv2 Base Metric obtain other privilege. - */ - private final Boolean obtainOtherPrivilege; - /** - * CVSSv2 Base Metric user interaction required. - */ - private final Boolean userInteractionRequired; - - /** - * Constructs a new CVSS V2 object. - * - * @param score the score - * @param accessVector the access vector - * @param accessComplexity the access complexity - * @param authentication the authentication - * @param confidentialityImpact the confidentiality impact - * @param integrityImpact the integrity impact - * @param availabilityImpact the availability impact - * @param severity the severity - */ - //CSOFF: ParameterNumber - public CvssV2(float score, String accessVector, String accessComplexity, String authentication, - String confidentialityImpact, String integrityImpact, String availabilityImpact, String severity) { - this(score, accessVector, accessComplexity, authentication, confidentialityImpact, - integrityImpact, availabilityImpact, severity, null, null, null, null, null, null, null, null); - } - - /** - * Constructs a new CVSS V2 object. - * - * @param score the score - * @param accessVector the access vector - * @param accessComplexity the access complexity - * @param authentication the authentication - * @param confidentialityImpact the confidentiality impact - * @param integrityImpact the integrity impact - * @param availabilityImpact the availability impact - * @param severity the severity - * @param exploitabilityScore the exploitability score - * @param impactScore the impact score - * @param acInsufInfo the acInsufInfo - * @param obtainAllPrivilege whether or not the vulnerability allows one to obtain all privileges - * @param obtainUserPrivilege whether or not the vulnerability allows one to obtain user privileges - * @param obtainOtherPrivilege whether or not the vulnerability allows one to obtain other privileges - * @param userInteractionRequired whether or not user interaction is required - * @param version the CVSS version - */ - //CSOFF: ParameterNumber - public CvssV2(float score, String accessVector, String accessComplexity, String authentication, - String confidentialityImpact, String integrityImpact, String availabilityImpact, String severity, - Float exploitabilityScore, Float impactScore, Boolean acInsufInfo, Boolean obtainAllPrivilege, - Boolean obtainUserPrivilege, Boolean obtainOtherPrivilege, Boolean userInteractionRequired, String version) { - this.score = score; - this.accessVector = accessVector; - this.accessComplexity = accessComplexity; - this.authentication = authentication; - this.confidentialityImpact = confidentialityImpact; - this.integrityImpact = integrityImpact; - this.availabilityImpact = availabilityImpact; - - this.severity = severity; - this.exploitabilityScore = exploitabilityScore; - this.impactScore = impactScore; - this.acInsufInfo = acInsufInfo; - this.obtainAllPrivilege = obtainAllPrivilege; - this.obtainUserPrivilege = obtainUserPrivilege; - this.obtainOtherPrivilege = obtainOtherPrivilege; - this.userInteractionRequired = userInteractionRequired; - this.version = version; - } - //CSON: ParameterNumber - - /** - * Get the value of score. - * - * @return the value of score - */ - public float getScore() { - return score; - } - - /** - * Get the value of accessVector. - * - * @return the value of accessVector - */ - public String getAccessVector() { - return accessVector; - } - - /** - * Get the value of accessComplexity. - * - * @return the value of accessComplexity - */ - public String getAccessComplexity() { - return accessComplexity; - } - - /** - * Get the value of authentication. - * - * @return the value of authentication - */ - public String getAuthentication() { - return authentication; - } - - /** - * Get the value of confidentialityImpact. - * - * @return the value of confidentialityImpact - */ - public String getConfidentialityImpact() { - return confidentialityImpact; - } - - /** - * Get the value of integrityImpact. - * - * @return the value of integrityImpact - */ - public String getIntegrityImpact() { - return integrityImpact; - } - - /** - * Get the value of availabilityImpact. - * - * @return the value of availabilityImpact - */ - public String getAvailabilityImpact() { - return availabilityImpact; - } - - /** - * Get the value of version. - * - * @return the value of version - */ - public String getVersion() { - return version; - } - - /** - * Returns the severity for the vulnerability. - * - * @return the severity - */ - public String getSeverity() { - return severity; - } - - /** - * Returns the exploitabilityScore for the vulnerability. - * - * @return the exploitabilityScore - */ - public Float getExploitabilityScore() { - return exploitabilityScore; - } - - /** - * Returns the impactScore for the vulnerability. - * - * @return the impactScore - */ - public Float getImpactScore() { - return impactScore; - } - - /** - * Returns the acInsufInfo for the vulnerability. - * - * @return the acInsufInfo - */ - public Boolean isAcInsufInfo() { - return acInsufInfo; - } - - /** - * Returns the obtainAllPrivilege for the vulnerability. - * - * @return the obtainAllPrivilege - */ - public Boolean isObtainAllPrivilege() { - return obtainAllPrivilege; - } - - /** - * Returns the obtainUserPrivilege for the vulnerability. - * - * @return the obtainUserPrivilege - */ - public Boolean isObtainUserPrivilege() { - return obtainUserPrivilege; - } - - /** - * Returns the obtainOtherPrivilege for the vulnerability. - * - * @return the obtainOtherPrivilege - */ - public Boolean isObtainOtherPrivilege() { - return obtainOtherPrivilege; - } - - /** - * Returns the userInteractionRequired for the vulnerability. - * - * @return the userInteractionRequired - */ - public Boolean isUserInteractionRequired() { - return userInteractionRequired; - } - - @Override - public String toString() { - return String.format("/AV:%s/AC:%s/Au:%s/C:%s/I:%s/A:%s", - accessVector == null ? "" : accessVector.substring(0, 1), - accessComplexity == null ? "" : accessComplexity.substring(0, 1), - authentication == null ? "" : authentication.substring(0, 1), - confidentialityImpact == null ? "" : confidentialityImpact.substring(0, 1), - integrityImpact == null ? "" : integrityImpact.substring(0, 1), - availabilityImpact == null ? "" : availabilityImpact.substring(0, 1)); - } -} diff --git a/core/src/main/java/org/owasp/dependencycheck/dependency/CvssV3.java b/core/src/main/java/org/owasp/dependencycheck/dependency/CvssV3.java deleted file mode 100644 index bf6c0814c2e..00000000000 --- a/core/src/main/java/org/owasp/dependencycheck/dependency/CvssV3.java +++ /dev/null @@ -1,330 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2018 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.dependency; - -import org.sonatype.ossindex.service.api.cvss.Cvss3Severity; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -/** - * CVSS V3 scoring information. - * - * @author Jeremy Long - */ -public class CvssV3 implements Serializable { - - /** - * Serial version UID. - */ - private static final long serialVersionUID = -315810090425928920L; - - /** - * The CVSS v3 Base Metrics (that are required by the spec for any CVSS v3 Vector String) - */ - private static final List BASE_METRICS = Arrays.asList("AV", "AC", "PR", "UI", "S", "C", "I", "A"); - - /** - * CVSS Availability Impact. - */ - private final String attackVector; - /** - * CVSS Availability Impact. - */ - private final String attackComplexity; - /** - * CVSS Availability Impact. - */ - private final String privilegesRequired; - /** - * CVSS Availability Impact. - */ - private final String userInteraction; - /** - * CVSS Availability Impact. - */ - private final String scope; - /** - * CVSS Availability Impact. - */ - private final String confidentialityImpact; - /** - * CVSS Availability Impact. - */ - private final String integrityImpact; - /** - * CVSS Availability Impact. - */ - private final String availabilityImpact; - /** - * CVSS Base Score. - */ - private final float baseScore; - /** - * CVSS Base Severity. - */ - private final String baseSeverity; - /** - * CVSSv3 Base Metric exploitability score. - */ - private final Float exploitabilityScore; - /** - * CVSSv3 Base Metric impact score. - */ - private final Float impactScore; - /** - * CVSS version. - */ - private final String version; - - /** - * Constructs a new CVSS V3 object. - * - * @param attackVector the attack vector value - * @param attackComplexity the attack complexity value - * @param privilegesRequired the privileges required value - * @param userInteraction the user interaction value - * @param scope the scope value - * @param confidentialityImpact the confidentiality impact value - * @param integrityImpact the integrity impact value - * @param availabilityImpact the availability impact value - * @param baseScore the base score - * @param baseSeverity the base severity - */ - //CSOFF: ParameterNumber - public CvssV3(String attackVector, String attackComplexity, String privilegesRequired, - String userInteraction, String scope, String confidentialityImpact, String integrityImpact, - String availabilityImpact, float baseScore, String baseSeverity) { - this(attackVector, attackComplexity, privilegesRequired, userInteraction, scope, confidentialityImpact, - integrityImpact, availabilityImpact, baseScore, baseSeverity, null, null, null); - } - - /** - * Constructs a new CVSS V3 object. - * - * @param attackVector the attack vector value - * @param attackComplexity the attack complexity value - * @param privilegesRequired the privileges required value - * @param userInteraction the user interaction value - * @param scope the scope value - * @param confidentialityImpact the confidentiality impact value - * @param integrityImpact the integrity impact value - * @param availabilityImpact the availability impact value - * @param baseScore the base score - * @param baseSeverity the base severity - * @param exploitabilityScore the exploitability score - * @param impactScore the impact score - * @param version the CVSS version - */ - public CvssV3(String attackVector, String attackComplexity, String privilegesRequired, - String userInteraction, String scope, String confidentialityImpact, String integrityImpact, - String availabilityImpact, float baseScore, String baseSeverity, Float exploitabilityScore, Float impactScore, String version) { - this.attackVector = attackVector; - this.attackComplexity = attackComplexity; - this.privilegesRequired = privilegesRequired; - this.userInteraction = userInteraction; - this.scope = scope; - this.confidentialityImpact = confidentialityImpact; - this.integrityImpact = integrityImpact; - this.availabilityImpact = availabilityImpact; - this.baseScore = baseScore; - this.baseSeverity = baseSeverity; - this.exploitabilityScore = exploitabilityScore; - this.impactScore = impactScore; - this.version = version; - } - //CSON: ParameterNumber - - /** - * Constructs a new CVSS V3 object from a CVSS v3.x Vector String representation and - * a CVSS V3 Base score. - * - * @param vectorString a CVSS v3 Vector String - * @param baseScore the CVSS v3 base score - * @throws IllegalArgumentException when the provided vectorString is not a valid - * CVSS v3.x vector string. - */ - public CvssV3(String vectorString, float baseScore) { - if (!vectorString.startsWith("CVSS:3")) { - throw new IllegalArgumentException("Not a valid CVSSv3 vector string: " + vectorString); - } - this.version = vectorString.substring(5, vectorString.indexOf('/')); - final String[] metricStrings = vectorString.substring(vectorString.indexOf('/') + 1).split("/"); - final HashMap metrics = new HashMap<>(); - for (int i = 0; i < metricStrings.length; i++) { - final String[] metricKeyVal = metricStrings[i].split(":"); - if (metricKeyVal.length != 2) { - throw new IllegalArgumentException( - String.format("Not a valid CVSSv3 vector string '%s', invalid metric component '%s'", - vectorString, metricStrings[i])); - } - metrics.put(metricKeyVal[0], metricKeyVal[1]); - } - if (!metrics.keySet().containsAll(BASE_METRICS)) { - throw new IllegalArgumentException( - String.format("Not a valid CVSSv3 vector string '%s'; missing one or more required Base Metrics;", - vectorString)); - } - this.attackVector = metrics.get("AV"); - this.attackComplexity = metrics.get("AC"); - this.privilegesRequired = metrics.get("PR"); - this.userInteraction = metrics.get("UI"); - this.scope = metrics.get("S"); - this.confidentialityImpact = metrics.get("C"); - this.integrityImpact = metrics.get("I"); - this.availabilityImpact = metrics.get("A"); - this.baseScore = baseScore; - this.baseSeverity = Cvss3Severity.of(baseScore).name(); - this.exploitabilityScore = null; - this.impactScore = null; - } - - /** - * Get the value of attackVector. - * - * @return the value of attackVector - */ - public String getAttackVector() { - return attackVector; - } - - /** - * Get the value of attackComplexity. - * - * @return the value of attackComplexity - */ - public String getAttackComplexity() { - return attackComplexity; - } - - /** - * Get the value of privilegesRequired. - * - * @return the value of privilegesRequired - */ - public String getPrivilegesRequired() { - return privilegesRequired; - } - - /** - * Get the value of userInteraction. - * - * @return the value of userInteraction - */ - public String getUserInteraction() { - return userInteraction; - } - - /** - * Get the value of scope. - * - * @return the value of scope - */ - public String getScope() { - return scope; - } - - /** - * Get the value of confidentialityImpact. - * - * @return the value of confidentialityImpact - */ - public String getConfidentialityImpact() { - return confidentialityImpact; - } - - /** - * Get the value of integrityImpact. - * - * @return the value of integrityImpact - */ - public String getIntegrityImpact() { - return integrityImpact; - } - - /** - * Get the value of availabilityImpact. - * - * @return the value of availabilityImpact - */ - public String getAvailabilityImpact() { - return availabilityImpact; - } - - /** - * Get the value of baseScore. - * - * @return the value of baseScore - */ - public float getBaseScore() { - return baseScore; - } - - /** - * Get the value of baseSeverity. - * - * @return the value of baseSeverity - */ - public String getBaseSeverity() { - return baseSeverity; - } - - /** - * Get the value of version. - * - * @return the value of version - */ - public String getVersion() { - return version; - } - - /** - * Returns the exploitabilityScore for the vulnerability. - * - * @return the exploitabilityScore - */ - public Float getexploitabilityScore() { - return exploitabilityScore; - } - - /** - * Returns the impactScore for the vulnerability. - * - * @return the impactScore - */ - public Float getimpactScore() { - return impactScore; - } - - @Override - public String toString() { - return String.format("CVSS:%s/AV:%s/AC:%s/PR:%s/UI:%s/S:%s/C:%s/I:%s/A:%s", - version == null ? "" : version, - attackVector == null ? "" : attackVector.substring(0, 1), - attackComplexity == null ? "" : attackComplexity.substring(0, 1), - privilegesRequired == null ? "" : privilegesRequired.substring(0, 1), - userInteraction == null ? "" : userInteraction.substring(0, 1), - scope == null ? "" : scope.substring(0, 1), - confidentialityImpact == null ? "" : confidentialityImpact.substring(0, 1), - integrityImpact == null ? "" : integrityImpact.substring(0, 1), - availabilityImpact == null ? "" : availabilityImpact.substring(0, 1)); - } - -} diff --git a/core/src/main/java/org/owasp/dependencycheck/dependency/Vulnerability.java b/core/src/main/java/org/owasp/dependencycheck/dependency/Vulnerability.java index c9e1c8a71fc..e13a8257270 100644 --- a/core/src/main/java/org/owasp/dependencycheck/dependency/Vulnerability.java +++ b/core/src/main/java/org/owasp/dependencycheck/dependency/Vulnerability.java @@ -17,6 +17,8 @@ */ package org.owasp.dependencycheck.dependency; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV3; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; @@ -481,7 +483,7 @@ public int compareTo(@NotNull Vulnerability o) { * be reported as the highest severity after sorting on descending severity. *
* For vulnerabilities not scored with a CVSS score we estimate a score from - * the severity text. For textual severities assumed or sementically + * the severity text. For textual severities assumed or semantically * confirmed to be of a critical nature we assign a value in between the * highest CVSSv2 HIGH and the lowest CVSSv3 CRITICAL severity level. * @@ -490,12 +492,12 @@ public int compareTo(@NotNull Vulnerability o) { * @return A float value that allows for best-effort sorting on * vulnerability severity */ - private float bestEffortSeverityLevelForSorting() { + private Double bestEffortSeverityLevelForSorting() { if (this.cvssV3 != null) { - return SeverityUtil.sortAdjustedCVSSv3BaseScore(this.cvssV3.getBaseScore()); + return SeverityUtil.sortAdjustedCVSSv3BaseScore(this.cvssV3.getCvssData().getBaseScore()); } if (this.cvssV2 != null) { - return this.cvssV2.getScore(); + return this.cvssV2.getCvssData().getBaseScore(); } return SeverityUtil.estimatedSortAdjustedCVSSv3(this.unscoredSeverity); } @@ -509,10 +511,10 @@ private float bestEffortSeverityLevelForSorting() { */ public String getHighestSeverityText() { if (this.cvssV3 != null) { - return this.cvssV3.getBaseSeverity().toUpperCase(); + return this.cvssV3.getCvssData().getBaseSeverity().value().toUpperCase(); } if (this.cvssV2 != null) { - return this.cvssV2.getSeverity().toUpperCase(); + return this.cvssV2.getCvssData().getBaseSeverity().toUpperCase(); } return SeverityUtil.unscoredToSeveritytext(this.unscoredSeverity).toUpperCase(); } diff --git a/core/src/main/java/org/owasp/dependencycheck/processing/BundlerAuditProcessor.java b/core/src/main/java/org/owasp/dependencycheck/processing/BundlerAuditProcessor.java index 973a5711df4..7a391d17749 100644 --- a/core/src/main/java/org/owasp/dependencycheck/processing/BundlerAuditProcessor.java +++ b/core/src/main/java/org/owasp/dependencycheck/processing/BundlerAuditProcessor.java @@ -20,6 +20,8 @@ import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; import com.github.packageurl.PackageURLBuilder; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2Data; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -40,7 +42,6 @@ import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.dependency.Confidence; -import org.owasp.dependencycheck.dependency.CvssV2; import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.dependency.EvidenceType; import org.owasp.dependencycheck.dependency.Reference; @@ -124,8 +125,7 @@ public void run() { final Map map = new HashMap<>(); boolean appendToDescription = false; - try (InputStreamReader ir = new InputStreamReader(getInput(), StandardCharsets.UTF_8); - BufferedReader br = new BufferedReader(ir)) { + try (InputStreamReader ir = new InputStreamReader(getInput(), StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(ir)) { String nextLine; while ((nextLine = br.readLine()) != null) { @@ -216,7 +216,7 @@ private void addReferenceToVulnerability(String parentName, Vulnerability vulner private void addCriticalityToVulnerability(String parentName, Vulnerability vulnerability, String nextLine) { if (null != vulnerability) { final String criticality = nextLine.substring(CRITICALITY.length()).trim(); - float score = -1.0f; + Double score = -1.0; Vulnerability v = null; final CveDB cvedb = engine.getDatabase(); if (cvedb != null) { @@ -235,13 +235,17 @@ private void addCriticalityToVulnerability(String parentName, Vulnerability vuln } } else { if ("High".equalsIgnoreCase(criticality)) { - score = 8.5f; + score = 8.5; } else if ("Medium".equalsIgnoreCase(criticality)) { - score = 5.5f; + score = 5.5; } else if ("Low".equalsIgnoreCase(criticality)) { - score = 2.0f; + score = 2.0; } - vulnerability.setCvssV2(new CvssV2(score, "-", "-", "-", "-", "-", "-", criticality)); + final CvssV2Data cvssData = new CvssV2Data(null, null, null, null, null, null, null, null, score, criticality.toUpperCase(), + null, null, null, null, null, null, null, null, null, null); + final CvssV2 cvssV2 = new CvssV2(null, null, cvssData, criticality.toUpperCase(), null, null, null, null, null, null, null); + vulnerability.setCvssV2(cvssV2); + vulnerability.setUnscoredSeverity(null); } } LOGGER.debug("bundle-audit ({}): {}", parentName, nextLine); @@ -289,7 +293,7 @@ private Vulnerability createVulnerability(String parentName, Dependency dependen .version(version).build(); vulnerability.addVulnerableSoftware(vs); vulnerability.setMatchedVulnerableSoftware(vs); - vulnerability.setCvssV2(new CvssV2(-1, "-", "-", "-", "-", "-", "-", "unknown")); + vulnerability.setUnscoredSeverity("UNKNOWN"); } LOGGER.debug("bundle-audit ({}): {}", parentName, nextLine); return vulnerability; diff --git a/core/src/main/java/org/owasp/dependencycheck/processing/MixAuditProcessor.java b/core/src/main/java/org/owasp/dependencycheck/processing/MixAuditProcessor.java index c4661c36894..3dda31c7b61 100644 --- a/core/src/main/java/org/owasp/dependencycheck/processing/MixAuditProcessor.java +++ b/core/src/main/java/org/owasp/dependencycheck/processing/MixAuditProcessor.java @@ -31,7 +31,7 @@ import org.owasp.dependencycheck.data.elixir.MixAuditJsonParser; import org.owasp.dependencycheck.data.elixir.MixAuditResult; import org.owasp.dependencycheck.dependency.Confidence; -import org.owasp.dependencycheck.dependency.CvssV2; + import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.dependency.EvidenceType; import org.owasp.dependencycheck.dependency.Vulnerability; @@ -197,7 +197,7 @@ private Vulnerability createVulnerability(MixAuditResult result) throws CpeValid vulnerability.addVulnerableSoftware(vs); vulnerability.setMatchedVulnerableSoftware(vs); - vulnerability.setCvssV2(new CvssV2(-1, "-", "-", "-", "-", "-", "-", "unknown")); + vulnerability.setUnscoredSeverity("UNKOWN"); vulnerability.setDescription(result.getDescription()); vulnerability.setName(result.getCve()); diff --git a/core/src/main/java/org/owasp/dependencycheck/reporting/ReportGenerator.java b/core/src/main/java/org/owasp/dependencycheck/reporting/ReportGenerator.java index 0f08ce1f362..c845347bb16 100644 --- a/core/src/main/java/org/owasp/dependencycheck/reporting/ReportGenerator.java +++ b/core/src/main/java/org/owasp/dependencycheck/reporting/ReportGenerator.java @@ -118,7 +118,8 @@ public enum Format { */ SARIF, /** - * Generate HTML report without script or non-vulnerable libraries for Jenkins. + * Generate HTML report without script or non-vulnerable libraries for + * Jenkins. */ JENKINS, /** @@ -327,8 +328,8 @@ public void write(String outputLocation, String format) throws ReportException { } else { File out = getReportFile(outputLocation, null); if (out.isDirectory()) { - out = new File(out, FilenameUtils.getBaseName(format)); - LOGGER.warn("Writing non-standard VSL output to a directory using template name as file name."); + out = new File(out, FilenameUtils.getBaseName(format)); + LOGGER.warn("Writing non-standard VSL output to a directory using template name as file name."); } processTemplate(format, out); } @@ -561,8 +562,7 @@ private void pretifyJson(String pathToJson) throws ReportException { final JsonFactory factory = new JsonFactory(); - try (InputStream is = new FileInputStream(in); - OutputStream os = new FileOutputStream(out)) { + try (InputStream is = new FileInputStream(in); OutputStream os = new FileOutputStream(out)) { final JsonParser parser = factory.createParser(is); final JsonGenerator generator = factory.createGenerator(os); diff --git a/core/src/main/java/org/owasp/dependencycheck/reporting/ReportTool.java b/core/src/main/java/org/owasp/dependencycheck/reporting/ReportTool.java index 6247764b83c..7f44c766bbe 100644 --- a/core/src/main/java/org/owasp/dependencycheck/reporting/ReportTool.java +++ b/core/src/main/java/org/owasp/dependencycheck/reporting/ReportTool.java @@ -79,7 +79,7 @@ public String identifierToSuppressionId(Identifier id) { * @param severity the text representation of a score * @return the estimated score */ - public float estimateSeverity(String severity) { + public Double estimateSeverity(String severity) { return SeverityUtil.estimateCvssV2(severity); } @@ -110,33 +110,33 @@ public Collection convertToSarifRules(List dependencies) private String determineScore(Vulnerability vuln) { if (vuln.getUnscoredSeverity() != null) { if ("0.0".equals(vuln.getUnscoredSeverity())) { - return "Unknown"; + return "unknown"; } else { return normalizeSeverity(vuln.getUnscoredSeverity().toLowerCase()); } - } else if (vuln.getCvssV3() != null && vuln.getCvssV3().getBaseSeverity() != null) { - return normalizeSeverity(vuln.getCvssV3().getBaseSeverity().toLowerCase()); - } else if (vuln.getCvssV2() != null && vuln.getCvssV2().getSeverity() != null) { - return normalizeSeverity(vuln.getCvssV2().getSeverity()); + } else if (vuln.getCvssV3() != null && vuln.getCvssV3().getCvssData().getBaseSeverity() != null) { + return normalizeSeverity(vuln.getCvssV3().getCvssData().getBaseSeverity().value().toLowerCase()); + } else if (vuln.getCvssV2() != null && vuln.getCvssV2().getCvssData().getBaseSeverity() != null) { + return normalizeSeverity(vuln.getCvssV2().getCvssData().getBaseSeverity()); } - return "Unknown"; + return "unknown"; } - public String normalizeSeverity(String sev) { - switch (sev) { + private String normalizeSeverity(String sev) { + switch (sev.toLowerCase()) { case "critical": - return "Critical"; + return "critical"; case "high": - return "High"; + return "high"; case "medium": case "moderate": - return "Medium"; + return "medium"; case "low": case "informational": case "info": - return "Low"; + return "low"; default: - return "Unknown"; + return "unknown"; } } diff --git a/core/src/main/java/org/owasp/dependencycheck/reporting/SarifRule.java b/core/src/main/java/org/owasp/dependencycheck/reporting/SarifRule.java index 781f5158ce9..57a0f339ae1 100644 --- a/core/src/main/java/org/owasp/dependencycheck/reporting/SarifRule.java +++ b/core/src/main/java/org/owasp/dependencycheck/reporting/SarifRule.java @@ -17,8 +17,8 @@ */ package org.owasp.dependencycheck.reporting; -import org.owasp.dependencycheck.dependency.CvssV2; -import org.owasp.dependencycheck.dependency.CvssV3; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV3; /** * @@ -61,7 +61,7 @@ public class SarifRule { /** * CVSS V2 field. */ - private String cvssv2ConfidentialImpact; + private String cvssv2ConfidentialityImpact; /** * CVSS V2 field. */ @@ -161,40 +161,76 @@ public SarifRule(String name, String shortDescription, String fullDescription, this.fullDescription = fullDescription; this.source = source; if (cvssV2 != null) { - this.cvssv2Score = Float.toString(cvssV2.getScore()); - this.cvssv2AccessVector = cvssV2.getAccessVector(); - this.cvssv2AccessComplexity = cvssV2.getAccessComplexity(); - this.cvssv2Authentication = cvssV2.getAuthentication(); - this.cvssv2ConfidentialImpact = cvssV2.getConfidentialityImpact(); - this.cvssv2IntegrityImpact = cvssV2.getIntegrityImpact(); - this.cvssv2AvailabilityImpact = cvssV2.getAvailabilityImpact(); - this.cvssv2Severity = cvssV2.getSeverity(); - this.cvssv2Version = cvssV2.getVersion(); + if (cvssV2.getCvssData().getBaseScore() != null) { + this.cvssv2Score = cvssV2.getCvssData().getBaseScore().toString(); + } + if (cvssV2.getCvssData().getAccessVector() != null) { + this.cvssv2AccessVector = cvssV2.getCvssData().getAccessVector().name(); + } + if (cvssV2.getCvssData().getAccessComplexity() != null) { + this.cvssv2AccessComplexity = cvssV2.getCvssData().getAccessComplexity().name(); + } + if (cvssV2.getCvssData().getAuthentication() != null) { + this.cvssv2Authentication = cvssV2.getCvssData().getAuthentication().name(); + } + if (cvssV2.getCvssData().getConfidentialityImpact() != null) { + this.cvssv2ConfidentialityImpact = cvssV2.getCvssData().getConfidentialityImpact().name(); + } + if (cvssV2.getCvssData().getIntegrityImpact() != null) { + this.cvssv2IntegrityImpact = cvssV2.getCvssData().getIntegrityImpact().name(); + } + if (cvssV2.getCvssData().getAvailabilityImpact() != null) { + this.cvssv2AvailabilityImpact = cvssV2.getCvssData().getAvailabilityImpact().name(); + } + this.cvssv2Severity = cvssV2.getCvssData().getBaseSeverity(); + if (cvssV2.getCvssData().getVersion() != null) { + this.cvssv2Version = cvssV2.getCvssData().getVersion().name(); + } if (cvssV2.getExploitabilityScore() != null) { - this.cvssv2ExploitabilityScore = Float.toString(cvssV2.getExploitabilityScore()); + this.cvssv2ExploitabilityScore = cvssV2.getExploitabilityScore().toString(); } if (cvssV2.getImpactScore() != null) { - this.cvssv2ImpactScore = Float.toString(cvssV2.getImpactScore()); + this.cvssv2ImpactScore = cvssV2.getImpactScore().toString(); } } if (cvssV3 != null) { - this.cvssv3BaseScore = Float.toString(cvssV3.getBaseScore()); - this.cvssv3AttackVector = cvssV3.getAttackVector(); - this.cvssv3AttackComplexity = cvssV3.getAttackComplexity(); - this.cvssv3PrivilegesRequired = cvssV3.getPrivilegesRequired(); - this.cvssv3UserInteraction = cvssV3.getUserInteraction(); - this.cvssv3Scope = cvssV3.getScope(); - this.cvssv3ConfidentialityImpact = cvssV3.getConfidentialityImpact(); - this.cvssv3IntegrityImpact = cvssV3.getIntegrityImpact(); - this.cvssv3AvailabilityImpact = cvssV3.getAvailabilityImpact(); - this.cvssv3BaseSeverity = cvssV3.getBaseSeverity(); - if (cvssV3.getexploitabilityScore() != null) { - this.cvssv3ExploitabilityScore = Float.toString(cvssV3.getexploitabilityScore()); + if (cvssV3.getCvssData().getBaseScore() != null) { + this.cvssv3BaseScore = cvssV3.getCvssData().getBaseScore().toString(); + } + if (cvssV3.getCvssData().getAttackVector() != null) { + this.cvssv3AttackVector = cvssV3.getCvssData().getAttackVector().name(); + } + if (cvssV3.getCvssData().getAttackComplexity() != null) { + this.cvssv3AttackComplexity = cvssV3.getCvssData().getAttackComplexity().name(); + } + if (cvssV3.getCvssData().getPrivilegesRequired() != null) { + this.cvssv3PrivilegesRequired = cvssV3.getCvssData().getPrivilegesRequired().name(); + } + if (cvssV3.getCvssData().getUserInteraction() != null) { + this.cvssv3UserInteraction = cvssV3.getCvssData().getUserInteraction().name(); + } + if (cvssV3.getCvssData().getScope() != null) { + this.cvssv3Scope = cvssV3.getCvssData().getScope().name(); + } + if (cvssV3.getCvssData().getConfidentialityImpact() != null) { + this.cvssv3ConfidentialityImpact = cvssV3.getCvssData().getConfidentialityImpact().name(); + } + if (cvssV3.getCvssData().getIntegrityImpact() != null) { + this.cvssv3IntegrityImpact = cvssV3.getCvssData().getIntegrityImpact().name(); + } + if (cvssV3.getCvssData().getAvailabilityImpact() != null) { + this.cvssv3AvailabilityImpact = cvssV3.getCvssData().getAvailabilityImpact().name(); + } + if (cvssV3.getCvssData().getBaseSeverity() != null) { + this.cvssv3BaseSeverity = cvssV3.getCvssData().getBaseSeverity().name(); + } + if (cvssV3.getExploitabilityScore() != null) { + this.cvssv3ExploitabilityScore = cvssV3.getExploitabilityScore().toString(); } - if (cvssV3.getimpactScore() != null) { - this.cvssv3ImpactScore = Float.toString(cvssV3.getimpactScore()); + if (cvssV3.getImpactScore() != null) { + this.cvssv3ImpactScore = cvssV3.getImpactScore().toString(); } - this.cvssv3Version = cvssV3.getVersion(); + this.cvssv3Version = cvssV3.getCvssData().getVersion().name(); } } @@ -560,21 +596,21 @@ public void setCvssv2IntegrityImpact(String cvssv2IntegrityImpact) { } /** - * Get the value of CVSS2 Confidential Impact. + * Get the value of CVSS2 Confidentiality Impact. * - * @return the value of CVSS2 Confidential Impact + * @return the value of CVSS2 Confidentiality Impact */ - public String getCvssv2ConfidentialImpact() { - return cvssv2ConfidentialImpact; + public String getCvssv2ConfidentialityImpact() { + return cvssv2ConfidentialityImpact; } /** - * Set the value of CVSS2 Confidential Impact. + * Set the value of CVSS2 Confidentiality Impact. * - * @param cvssv2ConfidentialImpact new value of CVSS2 Confidential Impact + * @param cvssv2ConfidentialityImpact new value of CVSS2 Confidentiality Impact */ - public void setCvssv2ConfidentialImpact(String cvssv2ConfidentialImpact) { - this.cvssv2ConfidentialImpact = cvssv2ConfidentialImpact; + public void setCvssv2ConfidentialityImpact(String cvssv2ConfidentialityImpact) { + this.cvssv2ConfidentialityImpact = cvssv2ConfidentialityImpact; } /** diff --git a/core/src/main/java/org/owasp/dependencycheck/utils/CvssUtil.java b/core/src/main/java/org/owasp/dependencycheck/utils/CvssUtil.java new file mode 100644 index 00000000000..a1b690da3a2 --- /dev/null +++ b/core/src/main/java/org/owasp/dependencycheck/utils/CvssUtil.java @@ -0,0 +1,227 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2023 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.utils; + +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV2Data; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV3; +import io.github.jeremylong.openvulnerability.client.nvd.CvssV3Data; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import org.sonatype.ossindex.service.api.cvss.Cvss3Severity; + +/** + * Utility class to create CVSS Objects. + * + * @author Jeremy Long + */ +public final class CvssUtil { + + private CvssUtil() { + //empty constructor for utility class. + } + /** + * The CVSS v3 Base Metrics (that are required by the spec for any CVSS v3 + * Vector String) + */ + private static final List BASE_METRICS_V3 = Arrays.asList("AV", "AC", "PR", "UI", "S", "C", "I", "A"); + + /** + * The CVSS V2 Metrics required for the vector string to be complete. + */ + private static final List BASE_METRICS_V2 = Arrays.asList("AV", "AC", "Au", "C", "I", "A"); + /** + * ZERO. + */ + private static final Double ZERO = 0.0; + /** + * ONE. + */ + private static final Double ONE = 1.0; + /** + * FOUR. + */ + private static final Double FOUR = 4.0; + /** + * SEVEN. + */ + private static final Double SEVEN = 7.0; + /** + * NINE. + */ + private static final Double NINE = 9.0; + /** + * TEN. + */ + private static final Double TEN = 10.0; + /** + * UNKNOWN. + */ + private static final String UNKNOWN = "UNKNOWN"; + /** + * HIGH. + */ + private static final String HIGH = "HIGH"; + /** + * MEDIUM. + */ + private static final String MEDIUM = "MEDIUM"; + /** + * LOW. + */ + private static final String LOW = "LOW"; + + /** + * Convert a CVSSv2 vector String into a CvssV3 Object. + * + * @param vectorString the vector string + * @param baseScore the base score + * @return the CVSSv2 object + */ + public static CvssV2 vectorToCvssV2(String vectorString, Double baseScore) { + if (vectorString.startsWith("CVSS:")) { + throw new IllegalArgumentException("Not a valid CVSSv2 vector string: " + vectorString); + } + final String[] metricStrings = vectorString.substring(vectorString.indexOf('/') + 1).split("/"); + final HashMap metrics = new HashMap<>(); + for (int i = 0; i < metricStrings.length; i++) { + final String[] metricKeyVal = metricStrings[i].split(":"); + if (metricKeyVal.length != 2) { + throw new IllegalArgumentException( + String.format("Not a valid CVSSv2 vector string '%s', invalid metric component '%s'", + vectorString, metricStrings[i])); + } + metrics.put(metricKeyVal[0], metricKeyVal[1]); + } + if (!metrics.keySet().containsAll(BASE_METRICS_V2)) { + throw new IllegalArgumentException( + String.format("Not a valid CVSSv2 vector string '%s'; missing one or more required Metrics;", + vectorString)); + } + + final String version = CvssV2Data.Version._2_0.value(); + //"AV:L/AC:L/Au:N/C:N/I:N/A:C" + final CvssV2Data.AccessVectorType accessVector = CvssV2Data.AccessVectorType.fromValue(metrics.get("AV")); + final CvssV2Data.AccessComplexityType attackComplexity = CvssV2Data.AccessComplexityType.fromValue(metrics.get("AC")); + final CvssV2Data.AuthenticationType authentication = CvssV2Data.AuthenticationType.fromValue(metrics.get("Au")); + final CvssV2Data.CiaType confidentialityImpact = CvssV2Data.CiaType.fromValue(metrics.get("C")); + final CvssV2Data.CiaType integrityImpact = CvssV2Data.CiaType.fromValue(metrics.get("I")); + final CvssV2Data.CiaType availabilityImpact = CvssV2Data.CiaType.fromValue(metrics.get("A")); + + final String baseSeverity = cvssV2ScoreToSeverity(baseScore); + final CvssV2Data data = new CvssV2Data(version, vectorString, accessVector, attackComplexity, + authentication, confidentialityImpact, integrityImpact, availabilityImpact, baseScore, baseSeverity, + null, null, null, null, null, null, null, null, null, null); + final CvssV2 cvss = new CvssV2(null, null, data, baseSeverity, null, null, null, null, null, null, null); + return cvss; + + } + + /** + * Determines the severity from the score. + * + * @param score the score + * @return the severity + */ + public static String cvssV2ScoreToSeverity(Double score) { + if (score != null) { + if (ZERO.compareTo(score) <= 0 && FOUR.compareTo(score) > 0) { + return LOW; + } else if (FOUR.compareTo(score) <= 0 && SEVEN.compareTo(score) > 0) { + return MEDIUM; + } else if (SEVEN.compareTo(score) <= 0 && TEN.compareTo(score) >= 0) { + return HIGH; + } + } + return UNKNOWN; + } + + /** + * Determines the severity from the score. + * + * @param score the score + * @return the severity + */ + public static CvssV3Data.SeverityType cvssV3ScoreToSeverity(Double score) { + if (score != null) { + if (ZERO.compareTo(score) == 0) { + return CvssV3Data.SeverityType.NONE; + } else if (ZERO.compareTo(score) <= 0 && FOUR.compareTo(score) > 0) { + return CvssV3Data.SeverityType.LOW; + } else if (FOUR.compareTo(score) <= 0 && SEVEN.compareTo(score) > 0) { + return CvssV3Data.SeverityType.MEDIUM; + } else if (SEVEN.compareTo(score) <= 0 && NINE.compareTo(score) > 0) { + return CvssV3Data.SeverityType.HIGH; + } else if (NINE.compareTo(score) <= 0 && TEN.compareTo(score) >= 0) { + return CvssV3Data.SeverityType.CRITICAL; + } + } + return null; + } + + /** + * Convert a CVSSv3 vector String into a CvssV3 Object. + * + * @param vectorString the vector string + * @param baseScore the base score + * @return the CVSSv3 object + */ + public static CvssV3 vectorToCvssV3(String vectorString, Double baseScore) { + if (!vectorString.startsWith("CVSS:3")) { + throw new IllegalArgumentException("Not a valid CVSSv3 vector string: " + vectorString); + } + final String versionString = vectorString.substring(5, vectorString.indexOf('/')); + final String[] metricStrings = vectorString.substring(vectorString.indexOf('/') + 1).split("/"); + final HashMap metrics = new HashMap<>(); + for (int i = 0; i < metricStrings.length; i++) { + final String[] metricKeyVal = metricStrings[i].split(":"); + if (metricKeyVal.length != 2) { + throw new IllegalArgumentException( + String.format("Not a valid CVSSv3 vector string '%s', invalid metric component '%s'", + vectorString, metricStrings[i])); + } + metrics.put(metricKeyVal[0], metricKeyVal[1]); + } + if (!metrics.keySet().containsAll(BASE_METRICS_V3)) { + throw new IllegalArgumentException( + String.format("Not a valid CVSSv3 vector string '%s'; missing one or more required Base Metrics;", + vectorString)); + } + + final CvssV3Data.Version version = CvssV3Data.Version.fromValue(versionString); + //"CVSS:3.1\/AV:L\/AC:L\/PR:L\/UI:N\/S:U\/C:N\/I:N\/A:H" + final CvssV3Data.AttackVectorType attackVector = CvssV3Data.AttackVectorType.fromValue(metrics.get("AV")); + final CvssV3Data.AttackComplexityType attackComplexity = CvssV3Data.AttackComplexityType.fromValue(metrics.get("AC")); + final CvssV3Data.PrivilegesRequiredType privilegesRequired = CvssV3Data.PrivilegesRequiredType.fromValue(metrics.get("PR")); + final CvssV3Data.UserInteractionType userInteraction = CvssV3Data.UserInteractionType.fromValue(metrics.get("UI")); + final CvssV3Data.ScopeType scope = CvssV3Data.ScopeType.fromValue(metrics.get("S")); + final CvssV3Data.CiaType confidentialityImpact = CvssV3Data.CiaType.fromValue(metrics.get("C")); + final CvssV3Data.CiaType integrityImpact = CvssV3Data.CiaType.fromValue(metrics.get("I")); + final CvssV3Data.CiaType availabilityImpact = CvssV3Data.CiaType.fromValue(metrics.get("A")); + + final String baseSeverityString = Cvss3Severity.of(baseScore.floatValue()).name(); + final CvssV3Data.SeverityType baseSeverity = CvssV3Data.SeverityType.fromValue(baseSeverityString); + final CvssV3Data data = new CvssV3Data(version, vectorString, attackVector, attackComplexity, + privilegesRequired, userInteraction, scope, confidentialityImpact, integrityImpact, availabilityImpact, baseScore, + baseSeverity, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); + final CvssV3 cvss = new CvssV3(null, null, data, null, null); + return cvss; + + } +} diff --git a/core/src/main/java/org/owasp/dependencycheck/utils/DateUtil.java b/core/src/main/java/org/owasp/dependencycheck/utils/DateUtil.java index b385892fa2c..36e3d775856 100644 --- a/core/src/main/java/org/owasp/dependencycheck/utils/DateUtil.java +++ b/core/src/main/java/org/owasp/dependencycheck/utils/DateUtil.java @@ -17,6 +17,7 @@ */ package org.owasp.dependencycheck.utils; +import java.time.ZonedDateTime; import org.owasp.dependencycheck.exception.ParseException; import java.util.Calendar; @@ -80,6 +81,19 @@ public static boolean withinDateRange(long date, long compareTo, int dayRange) { return (compareTo - date) < msRange; } + /** + * Determines if the date is within the given day range. + * + * @param date the date + * @param compareTo the date to compare against + * @param dayRange the number of days allowed + * @return if the date is within the dayRange of compareTo + */ + public static boolean withinDateRange(ZonedDateTime date, ZonedDateTime compareTo, int dayRange) { + final ZonedDateTime rangeEnd = date.plusDays(dayRange); + return compareTo.isAfter(date) && compareTo.isBefore(rangeEnd); + } + /** * Returns the string value converted to an epoch seconds. Note, in some * cases the value provided may be in milliseconds. diff --git a/core/src/main/java/org/owasp/dependencycheck/utils/SeverityUtil.java b/core/src/main/java/org/owasp/dependencycheck/utils/SeverityUtil.java index ed3283bfe37..e3c76c6597f 100644 --- a/core/src/main/java/org/owasp/dependencycheck/utils/SeverityUtil.java +++ b/core/src/main/java/org/owasp/dependencycheck/utils/SeverityUtil.java @@ -27,6 +27,22 @@ */ public final class SeverityUtil { + /** + * The cutoff for high severity. + */ + private static final Double HIGH = 10.0; + /** + * The cutoff for medium severity. + */ + private static final Double MEDIUM = 6.9; + /** + * The cutoff for low severity. + */ + private static final Double LOW = 3.9; + /** + * The cutoff for none severity. + */ + private static final Double NONE = 0.0; /** * The logger. */ @@ -47,31 +63,33 @@ private SeverityUtil() { * @param severity the severity text (e.g. "medium") * @return a score from 0 to 10 */ - public static float estimateCvssV2(String severity) { + public static Double estimateCvssV2(String severity) { switch (severity == null ? "none" : severity.toLowerCase()) { case "critical": case "high": - return 10.0f; + return HIGH; case "moderate": case "medium": - return 6.9f; + return MEDIUM; case "info": + case "none": case "informational": - return 0.0f; + return NONE; case "low": case "unknown": - case "none": default: - return 3.9f; + return LOW; } } /** - * Converts a textual severity to the text that should be used to signal - * it in a report. + * Converts a textual severity to the text that should be used to signal it + * in a report. + * * @param severity The textual unscored severity - * @return The severity when properly recognized, otherwise the severity extended with a remark that it was not recognized and - * assumed to represent a critical severity. + * @return The severity when properly recognized, otherwise the severity + * extended with a remark that it was not recognized and assumed to + * represent a critical severity. */ public static String unscoredToSeveritytext(final String severity) { switch (Severity.forUnscored(severity)) { @@ -94,108 +112,120 @@ public static String unscoredToSeveritytext(final String severity) { } /** - * Creates an estimated sort-adjusted CVSSv3 score for an unscored textual severity. - * For recognized severities below critical it returns a value at the lower bound of the CVSSv3 baseScore for that severity. - * For recognized critical severities it returns a score in-between the upper bound of the HIGH CVSSv2 score and the lowest - * sort-adjusted CVSSv3 critical score, so that unscored critical vulnerabilties are ordered in between CRITICAL scored CVSSv3 - * rated vulnerabilities and HIGH-scored CVSSv2 rated vulnerabilities. - * For unrecognized severities it returns a score in-between the top HIGH CVSSv2 score and the estimatedSortAdjustedCVSSv3 - * score for an unscored severity recognized as critical, so that recognized critical will win over unrecognized severities - * while unrecognized severities are assumed to be of a critical nature. + * Creates an estimated sort-adjusted CVSSv3 score for an unscored textual + * severity. For recognized severities below critical it returns a value at + * the lower bound of the CVSSv3 baseScore for that severity. For recognized + * critical severities it returns a score in-between the upper bound of the + * HIGH CVSSv2 score and the lowest sort-adjusted CVSSv3 critical score, so + * that unscored critical vulnerabilties are ordered in between CRITICAL + * scored CVSSv3 rated vulnerabilities and HIGH-scored CVSSv2 rated + * vulnerabilities. For unrecognized severities it returns a score + * in-between the top HIGH CVSSv2 score and the estimatedSortAdjustedCVSSv3 + * score for an unscored severity recognized as critical, so that recognized + * critical will win over unrecognized severities while unrecognized + * severities are assumed to be of a critical nature. * * @param severity The textual severity, may be null - * @return A float that can be used to numerically sort vulnerabilities in approximated severity (highest float represents - * highest severity). + * @return A float that can be used to numerically sort vulnerabilities in + * approximated severity (highest float represents highest severity). * @see #sortAdjustedCVSSv3BaseScore(float) */ - public static float estimatedSortAdjustedCVSSv3(final String severity) { + public static Double estimatedSortAdjustedCVSSv3(final String severity) { switch (Severity.forUnscored(severity)) { case CRITICAL: - return 10.2f; + return 10.2; case HIGH: - return 7.0f; + return 7.0; case MEDIUM: - return 4.0f; + return 4.0; case LOW: - return 0.1f; + return 0.1; case INFO: - return 0.0f; + return 0.0; case ASSUMED_CRITICAL: default: SeverityUtil.LOGGER.debug("Unrecognized unscored textual severity: {}, assuming critical score as worst-case " - + "estimate for sorting", - severity); - return 10.1f; + + "estimate for sorting", + severity); + return 10.1; } } /** - * Compute an adjusted CVSSv3 baseScore that ensures that CRITICAL CVSSv3 scores will win over HIGH CVSSv2 and CRITICAL - * unscored severities to allow for a best-effort sorting that enables the report to list a reliable 'highest severity' - * in the report. + * Compute an adjusted CVSSv3 baseScore that ensures that CRITICAL CVSSv3 + * scores will win over HIGH CVSSv2 and CRITICAL unscored severities to + * allow for a best-effort sorting that enables the report to list a + * reliable 'highest severity' in the report. * * @param cvssV3BaseScore The cvssV3 baseScore severity of a vulnerability - * @return The cvssV3 baseScore, adjusted if necessary in order to guarantee that CVSSv3 CRITICAL scores will rate higher than - * CVSSv2 HIGH, unscored critical severities and unscored unrecognized severities (which are assumed for sorting to be of a - * critical nature) + * @return The cvssV3 baseScore, adjusted if necessary in order to guarantee + * that CVSSv3 CRITICAL scores will rate higher than CVSSv2 HIGH, unscored + * critical severities and unscored unrecognized severities (which are + * assumed for sorting to be of a critical nature) * @see #estimatedSortAdjustedCVSSv3(String) */ - public static float sortAdjustedCVSSv3BaseScore(final float cvssV3BaseScore) { - if (cvssV3BaseScore >= 9.0f) { - return cvssV3BaseScore + 1.3f; + public static Double sortAdjustedCVSSv3BaseScore(final Double cvssV3BaseScore) { + if (cvssV3BaseScore.floatValue() >= 9.0f) { + return cvssV3BaseScore + 1.3; } return cvssV3BaseScore; } /** - * An enum to translate unscored severity texts to a severity level of a defined set of severities. - * Allows for re-use of the text-to-severity mapping in multiple helper methods. + * An enum to translate unscored severity texts to a severity level of a + * defined set of severities. Allows for re-use of the text-to-severity + * mapping in multiple helper methods. */ private enum Severity { /** - * A severity level for textual values that should be regarded as accompanying a critical severity vulnerability + * A severity level for textual values that should be regarded as + * accompanying a critical severity vulnerability */ CRITICAL, /** - * A severity level for textual values that are not recognized and therefor assumed to be accompanying a critical severity - * vulnerability + * A severity level for textual values that are not recognized and + * therefor assumed to be accompanying a critical severity vulnerability */ ASSUMED_CRITICAL, /** - * A severity level for textual values that should be regarded as accompanying a high severity vulnerability + * A severity level for textual values that should be regarded as + * accompanying a high severity vulnerability */ HIGH, /** - * A severity level for textual values that should be regarded as accompanying a medium severity vulnerability + * A severity level for textual values that should be regarded as + * accompanying a medium severity vulnerability */ MEDIUM, /** - * A severity level for textual values that should be regarded as accompanying a low severity vulnerability + * A severity level for textual values that should be regarded as + * accompanying a low severity vulnerability */ LOW, /** - * A severity level for textual values that should be regarded as accompanying a vulnerability of informational nature + * A severity level for textual values that should be regarded as + * accompanying a vulnerability of informational nature */ INFO; public static Severity forUnscored(String value) { switch (value == null ? "none" : value.toLowerCase()) { - case "critical": - return CRITICAL; - case "high": - return HIGH; - case "moderate": - case "medium": - return MEDIUM; - case "info": - case "informational": - return INFO; - case "low": - case "unknown": - case "none": - return LOW; - default: - return ASSUMED_CRITICAL; + case "critical": + return CRITICAL; + case "high": + return HIGH; + case "moderate": + case "medium": + return MEDIUM; + case "info": + case "informational": + return INFO; + case "low": + case "unknown": + case "none": + return LOW; + default: + return ASSUMED_CRITICAL; } } } diff --git a/core/src/main/java/org/owasp/dependencycheck/xml/suppression/SuppressionHandler.java b/core/src/main/java/org/owasp/dependencycheck/xml/suppression/SuppressionHandler.java index 3ea7aebbf3c..50f4df32a9f 100644 --- a/core/src/main/java/org/owasp/dependencycheck/xml/suppression/SuppressionHandler.java +++ b/core/src/main/java/org/owasp/dependencycheck/xml/suppression/SuppressionHandler.java @@ -191,10 +191,10 @@ public void endElement(String uri, String localName, String qName) throws SAXExc rule.addVulnerabilityName(processPropertyType()); break; case NOTES: - rule.addNotes(currentText.toString().trim()); + rule.setNotes(currentText.toString().trim()); break; case CVSS_BELOW: - final float cvss = Float.parseFloat(currentText.toString().trim()); + final Double cvss = Double.valueOf(currentText.toString().trim()); rule.addCvssBelow(cvss); break; default: diff --git a/core/src/main/java/org/owasp/dependencycheck/xml/suppression/SuppressionRule.java b/core/src/main/java/org/owasp/dependencycheck/xml/suppression/SuppressionRule.java index 3fd508ab05e..9b4a0b662d8 100644 --- a/core/src/main/java/org/owasp/dependencycheck/xml/suppression/SuppressionRule.java +++ b/core/src/main/java/org/owasp/dependencycheck/xml/suppression/SuppressionRule.java @@ -62,7 +62,7 @@ public class SuppressionRule { /** * The list of cvssBelow scores. */ - private List cvssBelow = new ArrayList<>(); + private List cvssBelow = new ArrayList<>(); /** * The list of CWE entries to suppress. */ @@ -230,7 +230,7 @@ public boolean hasCpe() { * * @return the value of cvssBelow */ - public List getCvssBelow() { + public List getCvssBelow() { return cvssBelow; } @@ -239,7 +239,7 @@ public List getCvssBelow() { * * @param cvssBelow new value of cvssBelow */ - public void setCvssBelow(List cvssBelow) { + public void setCvssBelow(List cvssBelow) { this.cvssBelow = cvssBelow; } @@ -248,14 +248,14 @@ public void setCvssBelow(List cvssBelow) { * * @param cvss the CVSS to add */ - public void addCvssBelow(Float cvss) { + public void addCvssBelow(Double cvss) { this.cvssBelow.add(cvss); } /** - * Returns whether or not this suppression rule has CVSS suppressions. + * Returns whether or not this suppression rule has CVSS suppression criteria. * - * @return whether or not this suppression rule has CVSS suppressions + * @return whether or not this suppression rule has CVSS suppression criteria. */ public boolean hasCvssBelow() { return !cvssBelow.isEmpty(); @@ -273,28 +273,19 @@ public String getNotes() { /** * Set the value of notes. * - * @param notes new value of cve + * @param notes new value of notes */ public void setNotes(String notes) { this.notes = notes; } - /** - * Adds the notes to the cve list. - * - * @param notes the cve to add - */ - public void addNotes(String notes) { - this.notes = notes; - } - /** * Returns whether this suppression rule has notes entries. * * @return whether this suppression rule has notes entries */ public boolean hasNotes() { - return !cve.isEmpty(); + return !notes.isEmpty(); } /** @@ -534,13 +525,14 @@ public void process(Dependency dependency) { } } if (!remove) { - for (float cvss : this.cvssBelow) { - if (v.getCvssV2() != null && v.getCvssV2().getScore() < cvss) { + for (Double cvss : this.cvssBelow) { + //TODO validate this comparison + if (v.getCvssV2() != null && v.getCvssV2().getCvssData().getBaseScore().compareTo(cvss) < 0) { remove = true; removeVulns.add(v); break; } - if (v.getCvssV3() != null && v.getCvssV3().getBaseScore() < cvss) { + if (v.getCvssV3() != null && v.getCvssV3().getCvssData().getBaseScore().compareTo(cvss) < 0) { remove = true; removeVulns.add(v); break; diff --git a/core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource b/core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource index 8e10a07f8c9..06b30b96fc5 100644 --- a/core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource +++ b/core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource @@ -1,4 +1,4 @@ -org.owasp.dependencycheck.data.update.NvdCveUpdater +org.owasp.dependencycheck.data.update.NvdApiDataSource org.owasp.dependencycheck.data.update.EngineVersionCheck org.owasp.dependencycheck.data.update.RetireJSDataSource org.owasp.dependencycheck.data.update.HostedSuppressionsDataSource diff --git a/core/src/main/resources/dependencycheck.properties b/core/src/main/resources/dependencycheck.properties index 64fbdbebddc..deef2848f9c 100644 --- a/core/src/main/resources/dependencycheck.properties +++ b/core/src/main/resources/dependencycheck.properties @@ -14,7 +14,7 @@ engine.version.url=https://jeremylong.github.io/DependencyCheck/current.txt # to be supplied. If you are using another database (MySQL, Oracle, etc.) this property # will not be used. The data.directory will be resolved and if the connection string # below contains a %s then the data.directory will replace the %s. -data.directory=[JAR]/data/7.0 +data.directory=[JAR]/data/9.0 #if the filename has a %s it will be replaced with the current expected version data.file_name=odc.mv.db @@ -51,25 +51,14 @@ data.driver_name=org.h2.Driver data.writelock.shutdownhook=org.owasp.dependencycheck.utils.WriteLockCleanupHook proxy.disableSchemas=true -# the number of days that the modified nvd cve data holds data for. We don't need -# to update the other files if we are within this timespan. Per NIST this file -# holds 8 days of updates, we are using 7 just to be safe. -cve.url.modified.validfordays=7 -# the number of hours to wait before checking if updates are available from the NVD. -cve.check.validforhours=4 -#first year to pull data from the URLs below -cve.startyear=2002 -# the time in milliseconds to wait between downloads from the NVD -cve.download.waittime=4000 -#the original URL and modified URL should be the same; this is used to detect if we are using an internal NVD CVE copy -cve.url.original=https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz -cve.url.modified=https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz -cve.url.modified.defaultFilename=nvdcve-1.1-modified.json.gz -cve.url.base=https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz -cve.url.base.defaultFilename=nvdcve-1.1-%d.json.gz -cve.cpe.startswith.filter=cpe:2.3:a: -nvd.newyear.grace.period=10 +nvd.api.check.validforhours=4 +nvd.api.datafeed.validfordays=7 +#nvd.api.datafeed.url=https://example.com/nvd-cache/ +#nvd.api.datafeed.user= +#nvd.api.datafeed.password= + +cve.cpe.startswith.filter=cpe:2.3:a: max.download.threads=1 #Known Exploited Vulnerabilities diff --git a/core/src/main/resources/schema/dependency-check.4.0.xsd b/core/src/main/resources/schema/dependency-check.4.0.xsd new file mode 100644 index 00000000000..e885b0b866e --- /dev/null +++ b/core/src/main/resources/schema/dependency-check.4.0.xsd @@ -0,0 +1,285 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/main/resources/templates/csvReport.vsl b/core/src/main/resources/templates/csvReport.vsl index 891b1572b4a..e421db6b8b9 100644 --- a/core/src/main/resources/templates/csvReport.vsl +++ b/core/src/main/resources/templates/csvReport.vsl @@ -19,5 +19,5 @@ Copyright (c) 2017 Jeremy Long. All Rights Reserved. @version 1 *### "Project","ScanDate","DependencyName","DependencyPath","Description","License","Md5","Sha1","Identifiers","CPE","CVE","CWE","Vulnerability","Source","CVSSv2_Severity","CVSSv2_Score","CVSSv2","CVSSv3_BaseSeverity","CVSSv3_BaseScore","CVSSv3","CPE Confidence","Evidence Count","VendorProject","Product","Name","DateAdded","ShortDescription","RequiredAction","DueDate","Notes"#[[ ]]##foreach($dependency in $dependencies)#if($dependency.getVulnerabilities().size()>0)#foreach($vuln in $dependency.getVulnerabilities(true)) -$enc.csv($applicationName),$enc.csv($scanDate),$enc.csv($dependency.DisplayFileName),$enc.csv($dependency.FilePath),$enc.csv($dependency.description),$enc.csv($dependency.license),#if(!$dependency.isVirtual())$enc.csv($dependency.Md5sum)#else""#end,#if(!$dependency.isVirtual())$enc.csv($dependency.Sha1sum)#else""#end,$enc.csvIdentifiers($dependency.softwareIdentifiers),$enc.csvIdentifiers($dependency.vulnerableSoftwareIdentifiers),$enc.csv($vuln.name),$enc.csv($vuln.getCwes().toString()),$enc.csv($vuln.description),$enc.csv($vuln.getSource().name()),#if($vuln.cvssV2)$enc.csv($vuln.cvssV2.severity)#else""#end,#if($vuln.cvssV2)$enc.csv($vuln.cvssV2.score)#else""#end,#if($vuln.cvssV2)$enc.csv($vuln.cvssV2.toString())#else""#end,#if($vuln.cvssV3)$enc.csv($vuln.cvssV3.baseSeverity)#else""#end,#if($vuln.cvssV3)$enc.csv($vuln.cvssV3.baseScore)#else""#end,#if($vuln.cvssV3)$enc.csv($vuln.cvssV3.toString())#else""#end,$enc.csvCpeConfidence($dependency.softwareIdentifiers),$dependency.size(),#if($vuln.getKnownExploitedVulnerability())$enc.csv($vuln.getKnownExploitedVulnerability().getVendorProject()),$enc.csv($vuln.getKnownExploitedVulnerability().getProduct()),$enc.csv($vuln.getKnownExploitedVulnerability().getVulnerabilityName()),$enc.csv($vuln.getKnownExploitedVulnerability().getDateAdded()),$enc.csv($vuln.getKnownExploitedVulnerability().getShortDescription()),$enc.csv($vuln.getKnownExploitedVulnerability().getRequiredAction()),$enc.csv($vuln.getKnownExploitedVulnerability().getDueDate()),$enc.csv($vuln.getKnownExploitedVulnerability().getNotes())#else"","","","","","","",""#end +$enc.csv($applicationName),$enc.csv($scanDate),$enc.csv($dependency.DisplayFileName),$enc.csv($dependency.FilePath),$enc.csv($dependency.description),$enc.csv($dependency.license),#if(!$dependency.isVirtual())$enc.csv($dependency.Md5sum)#else""#end,#if(!$dependency.isVirtual())$enc.csv($dependency.Sha1sum)#else""#end,$enc.csvIdentifiers($dependency.softwareIdentifiers),$enc.csvIdentifiers($dependency.vulnerableSoftwareIdentifiers),$enc.csv($vuln.name),$enc.csv($vuln.getCwes().toString()),$enc.csv($vuln.description),$enc.csv($vuln.getSource().name()),#if($vuln.cvssV2)$enc.csv($vuln.cvssV2.cvssData.baseSeverity)#else""#end,#if($vuln.cvssV2)$enc.csv($vuln.cvssV2.cvssData.baseScore)#else""#end,#if($vuln.cvssV2)$enc.csv($vuln.cvssV2.toString())#else""#end,#if($vuln.cvssV3)$enc.csv($vuln.cvssV3.cvssData.baseSeverity)#else""#end,#if($vuln.cvssV3)$enc.csv($vuln.cvssV3.cvssData.baseScore)#else""#end,#if($vuln.cvssV3)$enc.csv($vuln.cvssV3.toString())#else""#end,$enc.csvCpeConfidence($dependency.softwareIdentifiers),$dependency.size(),#if($vuln.getKnownExploitedVulnerability())$enc.csv($vuln.getKnownExploitedVulnerability().getVendorProject()),$enc.csv($vuln.getKnownExploitedVulnerability().getProduct()),$enc.csv($vuln.getKnownExploitedVulnerability().getVulnerabilityName()),$enc.csv($vuln.getKnownExploitedVulnerability().getDateAdded()),$enc.csv($vuln.getKnownExploitedVulnerability().getShortDescription()),$enc.csv($vuln.getKnownExploitedVulnerability().getRequiredAction()),$enc.csv($vuln.getKnownExploitedVulnerability().getDueDate()),$enc.csv($vuln.getKnownExploitedVulnerability().getNotes())#else"","","","","","","",""#end #end#end#end \ No newline at end of file diff --git a/core/src/main/resources/templates/htmlReport.vsl b/core/src/main/resources/templates/htmlReport.vsl index f1857f9ad52..784b3c66b32 100644 --- a/core/src/main/resources/templates/htmlReport.vsl +++ b/core/src/main/resources/templates/htmlReport.vsl @@ -826,9 +826,9 @@ Getting Help:
  • Base Score: $enc.html($vuln.getCvssV2().getSeverity()) ($vuln.getCvssV2().getScore())
    • +
    • Base Score: $enc.html($vuln.getCvssV2().getCvssData().getBaseSeverity()) ($vuln.getCvssV2().getCvssData().getBaseScore())
    • Vector: $enc.html($vuln.getCvssV2().toString())
    #end #if($vuln.getCvssV3()) CVSSv3: -