refactor(dos): Edit job query parameters

Add declared licenses of the packages to the request body in order to save this information about the package on DOS server side (so the information can be shown in the DOS Clearance UI). Signed-off-by: Johanna Lamppu <johanna.lamppu@doubleopen.org>
oss-review-toolkit · Jul 3, 2024 · e929d4d · e929d4d
1 parent 61866be
commit e929d4d
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 21 deletions.
diff --git a/clients/dos/src/main/kotlin/DosClient.kt b/clients/dos/src/main/kotlin/DosClient.kt
@@ -96,22 +96,25 @@ class DosClient(private val service: DosService) {
     }
 
     /**
-     * Add a new scan job [zipFileKey] and [purls]. Return a [JobResponseBody] or null on error.
+     * Add a new scan job [zipFileKey] and [packages]. Return a [JobResponseBody] or null on error.
      */
-    suspend fun addScanJob(zipFileKey: String, purls: List<String>): JobResponseBody? {
-        if (zipFileKey.isEmpty() || purls.isEmpty()) {
+    suspend fun addScanJob(zipFileKey: String, packages: List<PackageInfo>): JobResponseBody? {
+        if (zipFileKey.isEmpty() || packages.isEmpty()) {
             logger.error { "The ZIP file key and Package URLs are required to add a scan job." }
             return null
         }
 
-        val requestBody = JobRequestBody(zipFileKey, purls)
+        val requestBody = JobRequestBody(zipFileKey, packages)
         val response = service.addScanJob(requestBody)
         val responseBody = response.body()
 
         return if (response.isSuccessful && responseBody != null) {
             responseBody
         } else {
-            logger.error { "Error adding a new scan job for $zipFileKey and $purls: ${response.errorBody()?.string()}" }
+            logger.error {
+                "Error adding a new scan job for $zipFileKey and ${packages.map { it.purl }}: " +
+                    "${response.errorBody()?.string()}"
+            }
 
             null
         }

diff --git a/clients/dos/src/main/kotlin/DosModel.kt b/clients/dos/src/main/kotlin/DosModel.kt
@@ -46,6 +46,18 @@ data class UploadUrlResponseBody(
     val message: String? = null
 )
 
+@Serializable
+data class PackageInfo(
+    /** The purl for the package (includes the VCS information). */
+    val purl: String,
+
+    /**
+     * The declared license for the package, if any. TODO: Use SpdxExpression type instead of String when the
+     * SpdxExpression class has been migrated to use kotlinx-serialization.
+     */
+    val declaredLicenseExpressionSPDX: String?
+)
+
 @Serializable
 data class ScanResultsRequestBody(
     /**
@@ -90,8 +102,8 @@ data class JobRequestBody(
     /** The key of the previously uploaded ZIP file to scan. */
     val zipFileKey: String,
 
-    /** The list of purls whose packages' source code is contained in the ZIP file. */
-    val purls: List<String>
+    /** The list of packages whose source code is contained in the ZIP file. */
+    val packages: List<PackageInfo>
 )
 
 @Serializable

diff --git a/plugins/scanners/dos/src/main/kotlin/DosScanner.kt b/plugins/scanners/dos/src/main/kotlin/DosScanner.kt
@@ -33,6 +33,7 @@ import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.clients.dos.DosClient
 import org.ossreviewtoolkit.clients.dos.DosService
+import org.ossreviewtoolkit.clients.dos.PackageInfo
 import org.ossreviewtoolkit.clients.dos.ScanResultsResponseBody
 import org.ossreviewtoolkit.downloader.DefaultWorkingTreeCache
 import org.ossreviewtoolkit.model.Issue
@@ -104,13 +105,13 @@ class DosScanner internal constructor(
                 return@runBlocking null
             }
 
-            val purls = context.coveredPackages.getDosPurls(provenance)
+            val packages = context.coveredPackages.getDosPackages(provenance)
 
-            logger.info { "Packages requested for scanning: ${purls.joinToString()}" }
+            logger.info { "Packages requested for scanning: ${packages.joinToString { it.purl }}" }
 
             // Ask for scan results from DOS API
             val existingScanResults = runCatching {
-                client.getScanResults(purls, config.fetchConcluded)
+                client.getScanResults(packages.map{it.purl}, config.fetchConcluded)
             }.onFailure {
                 issues += createAndLogIssue(name, it.collectMessages())
             }.onSuccess {
@@ -124,7 +125,7 @@ class DosScanner internal constructor(
                     runCatching {
                         downloader.download(provenance)
                     }.mapCatching { sourceDir ->
-                        runBackendScan(purls, sourceDir, startTime, issues)
+                        runBackendScan(packages, sourceDir, startTime, issues)
                     }.onFailure {
                         issues += createAndLogIssue(name, it.collectMessages())
                     }.getOrNull()
@@ -135,7 +136,7 @@ class DosScanner internal constructor(
                         "The job ID must not be null for 'pending' status."
                     }
 
-                    pollForCompletion(purls.first(), jobId, "Pending scan", startTime, issues)
+                    pollForCompletion(packages.first().purl, jobId, "Pending scan", startTime, issues)
                 }
 
                 "ready" -> existingScanResults
@@ -162,12 +163,12 @@ class DosScanner internal constructor(
     }
 
     internal suspend fun runBackendScan(
-        purls: List<String>,
+        packages: List<PackageInfo>,
         sourceDir: File,
         startTime: Instant,
         issues: MutableList<Issue>
     ): ScanResultsResponseBody? {
-        logger.info { "Initiating a backend scan for $purls." }
+        logger.info { "Initiating a backend scan for ${packages.map { it.purl }}." }
 
         val tmpDir = createOrtTempDir()
         val zipName = "${sourceDir.name}.zip"
@@ -189,18 +190,18 @@ class DosScanner internal constructor(
             return null
         }
 
-        val jobResponse = client.addScanJob(zipName, purls)
+        val jobResponse = client.addScanJob(zipName, packages)
         val id = jobResponse?.scannerJobId
 
         if (id == null) {
-            issues += createAndLogIssue(name, "Failed to add scan job for '$zipName' and $purls.")
+            issues += createAndLogIssue(name, "Failed to add scan job for '$zipName' and ${packages.map { it.purl }}.")
             return null
         }
 
         // In case of multiple PURLs, they all point to packages with the same provenance. So if one package scan is
         // complete, all package scans are complete, which is why it is enough to arbitrarily pool for the first
         // package here.
-        return pollForCompletion(purls.first(), id, "New scan", startTime, issues)
+        return pollForCompletion(packages.first().purl, id, "New scan", startTime, issues)
     }
 
     private suspend fun pollForCompletion(
@@ -235,15 +236,25 @@ class DosScanner internal constructor(
     }
 }
 
-private fun Collection<Package>.getDosPurls(provenance: Provenance = UnknownProvenance): List<String> {
+private fun Collection<Package>.getDosPackages(provenance: Provenance = UnknownProvenance): List<PackageInfo> {
     val extras = provenance.toPurlExtras()
 
     return when (provenance) {
         is RepositoryProvenance -> {
             // Maintain the VCS path to get the "bookmarking" right for the file tree in the package configuration UI.
-            map { it.id.toPurl(extras.qualifiers, it.vcsProcessed.path) }
+            map {
+                PackageInfo(
+                    purl = it.id.toPurl(extras.qualifiers, it.vcsProcessed.path),
+                    declaredLicenseExpressionSPDX = it.declaredLicensesProcessed.spdxExpression?.toString()
+                )
+            }
         }
 
-        else -> map { it.id.toPurl(extras) }
+        else -> map {
+            PackageInfo(
+                purl = it.id.toPurl(extras),
+                declaredLicenseExpressionSPDX = it.declaredLicensesProcessed.spdxExpression?.toString()
+            )
+        }
     }
 }
diff --git a/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt b/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt
@@ -40,6 +40,7 @@ import java.time.Instant
 import kotlinx.serialization.encodeToString
 
 import org.ossreviewtoolkit.clients.dos.JSON
+import org.ossreviewtoolkit.clients.dos.PackageInfo
 import org.ossreviewtoolkit.clients.dos.ScanResultsResponseBody
 import org.ossreviewtoolkit.model.Identifier
 import org.ossreviewtoolkit.model.Issue
@@ -163,7 +164,12 @@ class DosScannerTest : StringSpec({
         val issues = mutableListOf<Issue>()
 
         val result = scanner.runBackendScan(
-            purls = listOf(pkg.purl),
+            packages = listOf(
+                PackageInfo(
+                    purl = pkg.purl,
+                    declaredLicenseExpressionSPDX = null
+                )
+            ),
             sourceDir = tempdir(),
             startTime = Instant.now(),
             issues = issues