From bdce8db50e59d56143bee4af1b18e7a0fbbb0d78 Mon Sep 17 00:00:00 2001 From: Philipp Nanz Date: Mon, 11 Dec 2023 14:10:38 +0100 Subject: [PATCH] fix: stream NVD data via Jackson to reduce memory footprint --- .../nvd/api/CveApiJson20CveItemSource.java | 88 +++++++++++++++++++ .../data/update/nvd/api/CveItemSource.java | 29 ++++++ .../nvd/api/JsonArrayCveItemSource.java | 81 +++++++++++++++++ .../data/update/nvd/api/NvdApiProcessor.java | 41 +++------ 4 files changed, 208 insertions(+), 31 deletions(-) create mode 100644 core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveApiJson20CveItemSource.java create mode 100644 core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveItemSource.java create mode 100644 core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/JsonArrayCveItemSource.java diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveApiJson20CveItemSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveApiJson20CveItemSource.java new file mode 100644 index 00000000000..382094a1385 --- /dev/null +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveApiJson20CveItemSource.java @@ -0,0 +1,88 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update.nvd.api; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.zip.GZIPInputStream; + +public class CveApiJson20CveItemSource implements CveItemSource { + + private final File jsonFile; + private final ObjectMapper mapper; + private final InputStream inputStream; + private final JsonParser jsonParser; + private DefCveItem currentItem; + private DefCveItem nextItem; + + public CveApiJson20CveItemSource(File jsonFile) throws IOException { + this.jsonFile = jsonFile; + mapper = new ObjectMapper(); + mapper.registerModule(new JavaTimeModule()); + inputStream = jsonFile.getName().endsWith(".gz") ? + new GZIPInputStream(new BufferedInputStream(Files.newInputStream(jsonFile.toPath()))) : + new BufferedInputStream(Files.newInputStream(jsonFile.toPath())); + jsonParser = mapper.getFactory().createParser(inputStream); + + JsonToken token = null; + do { + token = jsonParser.nextToken(); + if (token == JsonToken.FIELD_NAME) { + String fieldName = jsonParser.getCurrentName(); + if (fieldName.equals("vulnerabilities") && (jsonParser.nextToken() == JsonToken.START_ARRAY)) { + nextItem = readItem(jsonParser); + } + } + } while (token != null && nextItem == null); + } + + @Override + public void close() throws Exception { + jsonParser.close(); + inputStream.close(); + Files.delete(jsonFile.toPath()); + } + + @Override + public boolean hasNext() { + return nextItem != null; + } + + @Override + public DefCveItem next() throws IOException { + currentItem = nextItem; + nextItem = readItem(jsonParser); + return currentItem; + } + + private DefCveItem readItem(JsonParser jsonParser) throws IOException { + if (jsonParser.nextToken() == JsonToken.START_OBJECT) { + return mapper.readValue(jsonParser, DefCveItem.class); + } + return null; + } +} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveItemSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveItemSource.java new file mode 100644 index 00000000000..c347d63edfe --- /dev/null +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/CveItemSource.java @@ -0,0 +1,29 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update.nvd.api; + +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; + +import java.io.IOException; + +public interface CveItemSource extends AutoCloseable { + + boolean hasNext(); + + T next() throws IOException; +} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/JsonArrayCveItemSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/JsonArrayCveItemSource.java new file mode 100644 index 00000000000..14e30b806ef --- /dev/null +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/JsonArrayCveItemSource.java @@ -0,0 +1,81 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update.nvd.api; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.zip.GZIPInputStream; + +public class JsonArrayCveItemSource implements CveItemSource { + + private final File jsonFile; + private final ObjectMapper mapper; + private final InputStream inputStream; + private final JsonParser jsonParser; + private DefCveItem currentItem; + private DefCveItem nextItem; + + public JsonArrayCveItemSource(File jsonFile) throws IOException { + this.jsonFile = jsonFile; + mapper = new ObjectMapper(); + mapper.registerModule(new JavaTimeModule()); + inputStream = jsonFile.getName().endsWith(".gz") ? + new GZIPInputStream(new BufferedInputStream(Files.newInputStream(jsonFile.toPath()))) : + new BufferedInputStream(Files.newInputStream(jsonFile.toPath())); + jsonParser = mapper.getFactory().createParser(inputStream); + + if (jsonParser.nextToken() == JsonToken.START_ARRAY) { + nextItem = readItem(jsonParser); + } + } + + @Override + public void close() throws Exception { + jsonParser.close(); + inputStream.close(); + Files.delete(jsonFile.toPath()); + } + + @Override + public boolean hasNext() { + return nextItem != null; + } + + @Override + public DefCveItem next() throws IOException { + currentItem = nextItem; + nextItem = readItem(jsonParser); + return currentItem; + } + + private DefCveItem readItem(JsonParser jsonParser) throws IOException { + if (jsonParser.nextToken() == JsonToken.START_OBJECT) { + return mapper.readValue(jsonParser, DefCveItem.class); + } + return null; + } +} diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java index 1aee82bb4d9..dd41664fd05 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java @@ -17,20 +17,11 @@ */ package org.owasp.dependencycheck.data.update.nvd.api; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20; import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Collection; import java.util.concurrent.Callable; -import java.util.zip.GZIPInputStream; import org.owasp.dependencycheck.data.nvd.ecosystem.CveEcosystemMapper; import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,38 +82,26 @@ public NvdApiProcessor(final CveDB cveDB, File jsonFile) { @Override public NvdApiProcessor call() throws Exception { - final ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new JavaTimeModule()); - Collection data = null; + CveItemSource itemSource = null; - if (jsonFile.getName().endsWith(".jsonarray.gz")) { - try (FileInputStream fileInputStream = new FileInputStream(jsonFile); - GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) { - data = objectMapper.readValue(gzipInputStream, new TypeReference>(){}); - } catch (IOException exception) { - throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception); - } + if (jsonFile.getName().endsWith(".jsonarray.gz")) { + itemSource = new JsonArrayCveItemSource(jsonFile); } else if (jsonFile.getName().endsWith(".gz")) { - try (FileInputStream fileInputStream = new FileInputStream(jsonFile); - GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) { - CveApiJson20 cveData = objectMapper.readValue(gzipInputStream, CveApiJson20.class); - if (cveData != null) { - data = cveData.getVulnerabilities(); - } - } catch (IOException exception) { - throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception); - } + itemSource = new CveApiJson20CveItemSource(jsonFile); } else { - data = objectMapper.readValue(jsonFile, new TypeReference>(){}); + itemSource = new JsonArrayCveItemSource(jsonFile); } - if (data != null ) { - for (DefCveItem entry : data) { + try { + while (itemSource.hasNext()) { + DefCveItem entry = itemSource.next(); try { cveDB.updateVulnerability(entry, mapper.getEcosystem(entry)); } catch (Exception ex) { LOGGER.error("Failed to process " + entry.getCve().getId(), ex); } } + } finally { + itemSource.close(); } endTime = System.currentTimeMillis(); return this;