feat(model): Require size when writing to ProvenanceFileStorage

Add a `size` argument to the `putData` function of the `ProvenanceFileStorage` which represents the size of the input stream to be written. This is currently a requirement to implement the `ProvenanceFileStorage` for the ORT Server. The ORT Server uses the PostgreSQL large object storage [1] to store binary data and the integration of that feature requires that the size of the input stream is known. It will be investigated if this is a hard technical requirement, and if not, the introduced `size` argument can be removed again. The current implementations of `ProvenanceFileStorage` do not require the size of the input stream, but the size is easily available for all callers of the `putData` function. [1]: https://www.postgresql.org/docs/current/lo-interfaces.html Signed-off-by: Martin Nonnenmacher <martin.nonnenmacher@bosch.io>
oss-review-toolkit · Jul 14, 2023 · 76c0f8e · 76c0f8e
1 parent 0c52217
commit 76c0f8e
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 12 deletions.
diff --git a/model/src/funTest/kotlin/utils/PostgresProvenanceFileStorageFunTest.kt b/model/src/funTest/kotlin/utils/PostgresProvenanceFileStorageFunTest.kt
@@ -22,6 +22,7 @@ package org.ossreviewtoolkit.model.utils
 import io.kotest.core.spec.style.WordSpec
 import io.kotest.matchers.shouldBe
 
+import java.io.ByteArrayInputStream
 import java.io.InputStream
 
 import org.ossreviewtoolkit.model.ArtifactProvenance
@@ -60,16 +61,23 @@ class PostgresProvenanceFileStorageFunTest : WordSpec({
         }
 
         "return true when data for the given provenance has been added" {
-            storage.putData(VCS_PROVENANCE, InputStream.nullInputStream())
+            storage.putData(VCS_PROVENANCE, InputStream.nullInputStream(), 0L)
 
             storage.hasData(VCS_PROVENANCE) shouldBe true
         }
     }
 
     "putData()" should {
         "return the data corresponding to the given provenance given such data has been added" {
-            storage.putData(VCS_PROVENANCE, "VCS".byteInputStream())
-            storage.putData(SOURCE_ARTIFACT_PROVENANCE, "source artifact".byteInputStream())
+            val vcsByteArray = "VCS".toByteArray()
+            val sourceArtifactByteArray = "source artifact".toByteArray()
+
+            storage.putData(VCS_PROVENANCE, ByteArrayInputStream(vcsByteArray), vcsByteArray.size.toLong())
+            storage.putData(
+                SOURCE_ARTIFACT_PROVENANCE,
+                ByteArrayInputStream(sourceArtifactByteArray),
+                sourceArtifactByteArray.size.toLong()
+            )
 
             storage.getData(VCS_PROVENANCE) shouldNotBeNull { String(use { readBytes() }) shouldBe "VCS" }
             storage.getData(SOURCE_ARTIFACT_PROVENANCE) shouldNotBeNull {
@@ -78,8 +86,19 @@ class PostgresProvenanceFileStorageFunTest : WordSpec({
         }
 
         "return the overwritten file corresponding to the given provenance" {
-            storage.putData(SOURCE_ARTIFACT_PROVENANCE, "source artifact".byteInputStream())
-            storage.putData(SOURCE_ARTIFACT_PROVENANCE, "source artifact updated".byteInputStream())
+            val sourceArtifactByteArray = "source artifact".toByteArray()
+            val sourceArtifactUpdatedByteArray = "source artifact updated".toByteArray()
+
+            storage.putData(
+                SOURCE_ARTIFACT_PROVENANCE,
+                ByteArrayInputStream(sourceArtifactByteArray),
+                sourceArtifactByteArray.size.toLong()
+            )
+            storage.putData(
+                SOURCE_ARTIFACT_PROVENANCE,
+                ByteArrayInputStream(sourceArtifactUpdatedByteArray),
+                sourceArtifactUpdatedByteArray.size.toLong()
+            )
 
             storage.getData(SOURCE_ARTIFACT_PROVENANCE) shouldNotBeNull {
                 String(use { readBytes() }) shouldBe "source artifact updated"

diff --git a/model/src/main/kotlin/utils/FileArchiver.kt b/model/src/main/kotlin/utils/FileArchiver.kt
@@ -90,7 +90,7 @@ class FileArchiver(
 
         logger.info { "Archived directory '$directory' in $zipDuration." }
 
-        val writeDuration = measureTime { storage.putData(provenance, zipFile.inputStream()) }
+        val writeDuration = measureTime { storage.putData(provenance, zipFile.inputStream(), zipFile.length()) }
 
         logger.info { "Wrote archive of directory '$directory' to storage in $writeDuration." }
 

diff --git a/model/src/main/kotlin/utils/FileProvenanceFileStorage.kt b/model/src/main/kotlin/utils/FileProvenanceFileStorage.kt
@@ -59,7 +59,7 @@ class FileProvenanceFileStorage(
         return storage.exists(filePath)
     }
 
-    override fun putData(provenance: KnownProvenance, data: InputStream) {
+    override fun putData(provenance: KnownProvenance, data: InputStream, size: Long) {
         storage.write(getFilePath(provenance), data)
     }
 

diff --git a/model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt b/model/src/main/kotlin/utils/PostgresProvenanceFileStorage.kt
@@ -86,7 +86,7 @@ class PostgresProvenanceFileStorage(
             }.first()[table.provenance.count()].toInt()
         } == 1
 
-    override fun putData(provenance: KnownProvenance, data: InputStream) {
+    override fun putData(provenance: KnownProvenance, data: InputStream, size: Long) {
         database.transaction {
             table.deleteWhere {
                 table.provenance eq provenance.storageKey()

diff --git a/model/src/main/kotlin/utils/ProvenanceFileStorage.kt b/model/src/main/kotlin/utils/ProvenanceFileStorage.kt
@@ -32,10 +32,10 @@ interface ProvenanceFileStorage {
     fun hasData(provenance: KnownProvenance): Boolean
 
     /**
-     * Associate [provenance] with the given [data]. Replaces any existing association by [provenance]. The function
-     * implementation is responsible for closing the stream.
+     * Associate [provenance] with the given [data] of the provided [size]. Replaces any existing association by
+     * [provenance]. The function implementation is responsible for closing the stream.
      */
-    fun putData(provenance: KnownProvenance, data: InputStream)
+    fun putData(provenance: KnownProvenance, data: InputStream, size: Long)
 
     /**
      * Return the data associated by [provenance], or null if there is no such data. Note that it is the responsibility

diff --git a/scanner/src/main/kotlin/utils/FileListResolver.kt b/scanner/src/main/kotlin/utils/FileListResolver.kt
@@ -21,6 +21,7 @@ package org.ossreviewtoolkit.scanner.utils
 
 import com.fasterxml.jackson.module.kotlin.readValue
 
+import java.io.ByteArrayInputStream
 import java.io.File
 
 import org.ossreviewtoolkit.model.HashAlgorithm
@@ -51,7 +52,8 @@ internal class FileListResolver(
 }
 
 private fun ProvenanceFileStorage.putFileList(provenance: KnownProvenance, fileList: FileList) {
-    putData(provenance, fileList.toYaml().byteInputStream())
+    val byteArray = fileList.toYaml().toByteArray()
+    putData(provenance, ByteArrayInputStream(byteArray), byteArray.size.toLong())
 }
 
 private fun ProvenanceFileStorage.getFileList(provenance: KnownProvenance): FileList? {