Save scan record async (#730)

Signed-off-by: Paolo Di Tommaso <[email protected]>

src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy

@@ -84,6 +84,7 @@ class ScanConfig {
     @Value('${wave.scan.environment}')
     List<String> environment
 
+    @Nullable
     @Value('${wave.scan.vulnerability.limit:100}')
     Integer vulnerabilityLimit
 

src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy

@@ -54,9 +54,15 @@ interface SurrealClient {
     @Post("/sql")
     Flux<Map<String, Object>> sqlAsync(@Header String authorization, @Body String body)
 
+    @Post("/sql")
+    Flux<List<Map<String, Object>>> sqlAsyncMany(@Header String authorization, @Body String body)
+
     @Post("/sql")
     Map<String, Object> sqlAsMap(@Header String authorization, @Body String body)
 
+    @Post("/sql")
+    List<Map<String, Object>> sqlAsList(@Header String authorization, @Body String body)
+
     @Post("/sql")
     String sqlAsString(@Header String authorization, @Body String body)
 

src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy

@@ -256,25 +256,36 @@ class SurrealPersistenceService implements PersistenceService {
     void saveScanRecord(WaveScanRecord scanRecord) {
         final vulnerabilities = scanRecord.vulnerabilities ?: List.<ScanVulnerability>of()
 
+        // create a multi-command surreal sql statement to insert all vulnerabilities
+        // and the scan record in a single operation
+        List<String> ids = new ArrayList<>(101)
+        String statement = ''
         // save all vulnerabilities
         for( ScanVulnerability it : vulnerabilities ) {
-            surrealDb.insertScanVulnerability(authorization, it)
+            statement += "INSERT INTO wave_scan_vuln ${JacksonHelper.toJson(it)};\n"
+            ids << "wave_scan_vuln:⟨$it.id⟩".toString()
         }
 
-        // compose the list of ids
-        final ids = vulnerabilities
-                .collect(it-> "wave_scan_vuln:⟨$it.id⟩".toString())
-
-
         // scan object
         final copy = scanRecord.clone()
         copy.vulnerabilities = List.of()
         final json = JacksonHelper.toJson(copy)
 
-        // create the scan record
-        final statement = "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)}".toString()
-        final result = surrealDb.sqlAsMap(authorization, statement)
-        log.trace "Scan update result=$result"
+        // add the wave_scan record
+        statement += "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)};\n".toString()
+        // store the statement using an async operation
+        surrealDb
+                .sqlAsyncMany(getAuthorization(), statement)
+                .subscribe({result ->
+                    log.trace "Scan record save result=$result"
+                },
+                {error->
+                    def msg = error.message
+                    if( error instanceof HttpClientResponseException ){
+                        msg += ":\n $error.response.body"
+                    }
+                    log.error("Error saving scan record => ${msg}\n", error)
+                })
     }
 
     protected String patchScanVulnerabilities(String json, List<String> ids) {

src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy

@@ -270,7 +270,7 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler<ScanE
         if( state.succeeded() ) {
             try {
                 final scanFile = job.workDir.resolve(Trivy.OUTPUT_FILE_NAME)
-                final vulnerabilities = TrivyResultProcessor.parse(scanFile, config.vulnerabilityLimit)
+                final vulnerabilities = TrivyResultProcessor.parseFile(scanFile, config.vulnerabilityLimit)
                 result = entry.success(vulnerabilities)
                 log.info("Container scan succeeded - id=${entry.scanId}; exit=${state.exitCode}; stdout=${state.stdout}")
             }

src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy

@@ -35,17 +35,25 @@ import io.seqera.wave.exception.ScanRuntimeException
 @CompileStatic
 class TrivyResultProcessor {
 
-    static List<ScanVulnerability> parse(Path scanFile) {
-        return parse(scanFile.getText())
-    }
-
-    static List<ScanVulnerability> parse(Path scanFile, int maxEntries) {
-        final result = parse(scanFile)
-        return filter(result, maxEntries)
+    /**
+     * Parse a Trivy vulnerabilities JSON file and return a list of {@link ScanVulnerability} entries
+     *
+     * @param scanFile
+     *      The {@link Path} of the Trivy JSON file to be scanned
+     * @param maxEntries
+     *      The max number of vulnerabilities that should be returned giving precedence to the
+     *      most severe vulnerabilities e.g. one critical and one medium issues are found and
+     *      1 is specified as {@code maxEntries} only the critical issues is returned.
+     * @return
+     *      The list of {@link ScanVulnerability} entries as parsed in from the JSON file.
+     */
+    static List<ScanVulnerability> parseFile(Path scanFile, Integer maxEntries=null) {
+        final result = parseJson(scanFile.getText())
+        return maxEntries>0 ? filter(result, maxEntries) : result
     }
 
     @CompileDynamic
-    static List<ScanVulnerability> parse(String scanJson) {
+    static List<ScanVulnerability> parseJson(String scanJson) {
         final slurper = new JsonSlurper()
         try{
             final jsonMap = slurper.parseText(scanJson) as Map

src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy

@@ -324,6 +324,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe
         def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null)
         when:
         persistence.saveScanRecord(scan)
+        sleep 200
         then:
         def result = persistence.loadScanRecord(SCAN_ID)
         and:
@@ -343,6 +344,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe
         and:
         // should save the same CVE into another build
         persistence.saveScanRecord(scanRecord2)
+        sleep 200
         then:
         def result2 = persistence.loadScanRecord(SCAN_ID2)
         and:
@@ -372,6 +374,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe
 
         when:
         persistence.saveScanRecord(scan)
+        sleep 200
         then:
         persistence.existsScanRecord(SCAN_ID)
     }
@@ -566,7 +569,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe
         persistence.saveScanRecord(scan2)
         persistence.saveScanRecord(scan3)
         persistence.saveScanRecord(scan4)
-
+        and:
+        sleep 200
         then:
         persistence.allScans("1234567890abcdef") == [scan3, scan2, scan1]
         and:

src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy

@@ -18,11 +18,11 @@
 
 package io.seqera.wave.service.scan
 
-
 import spock.lang.Specification
 
-import io.seqera.wave.exception.ScanRuntimeException
+import java.nio.file.Files
 
+import io.seqera.wave.exception.ScanRuntimeException
 /**
  *
  * @author Munish Chouhan <[email protected]>
@@ -91,7 +91,7 @@ class TrivyResultProcessorTest extends Specification {
         """
 
         when:
-        def result = TrivyResultProcessor.parse(trivyDockerResulJson)
+        def result = TrivyResultProcessor.parseJson(trivyDockerResulJson)
 
         then:
         def vulnerability = result[0]
@@ -107,7 +107,10 @@ class TrivyResultProcessorTest extends Specification {
 
     def "should return a sorted map of vulnerabilities"() {
         given:
-        def trivyDockerResulJson = """
+        def folder = Files.createTempDirectory('test')
+        def scan = folder.resolve('scan.json')
+        and:
+        scan.text = """
             { "Results": [
                 {
                    "Target": "sample-application",
@@ -170,19 +173,22 @@ class TrivyResultProcessorTest extends Specification {
         }""".stripIndent()
 
         when:
-        def result = TrivyResultProcessor.parse(trivyDockerResulJson)
-        result = TrivyResultProcessor.filter(result, 4)
+        def topIssues = TrivyResultProcessor.parseFile(scan, 2)
 
         then:
-        result.size() == 4
-        result[0].severity == "CRITICAL"
-        result[0].id == "CVE-2023-0005"
-        result[1].severity == "HIGH"
-        result[1].id == "CVE-2023-0003"
-        result[2].severity == "HIGH"
-        result[2].id == "CVE-2023-0004"
-        result[3].severity == "MEDIUM"
-        result[3].id == "CVE-2023-0002"
+        topIssues.size() == 2
+        topIssues[0].severity == "CRITICAL"
+        topIssues[0].id == "CVE-2023-0005"
+        topIssues[1].severity == "HIGH"
+        topIssues[1].id == "CVE-2023-0003"
+
+        when:
+        def allIssues = TrivyResultProcessor.parseFile(scan)
+        then:
+        allIssues.size() == 5
+
+        cleanup:
+        folder?.deleteDir()
     }
 
     def 'should not fail with empty list' () {
@@ -192,7 +198,7 @@ class TrivyResultProcessorTest extends Specification {
 
     def "process should throw exception if json is not correct"() {
         when:
-        TrivyResultProcessor.parse("invalid json")
+        TrivyResultProcessor.parseJson("invalid json")
         then:
         thrown ScanRuntimeException
     }