Revamp the test and run fork process infrastructure

.drone.yml

@@ -4,13 +4,15 @@ matrix:
   SBT_RUN:
     - sbt "benchmarks/compile" \
           "jsonConfig/test" \
-          "sbtBloop/scripted" \
           "frontend/integrationSetUpBloop" \
-          "benchmarks/jmh:run .*HotBloopBenchmark.* -wi 0 -i 1 -f1 -t1 -p project=with-tests -p projectName=with-tests" \
           "backend/test" \
           "frontend/test" \
           "docs/makeSite"
 
+  SBT_RUN_BENCHMARKS_AND_SCRIPTED:
+    - sbt "sbtBloop/scripted" \
+          "benchmarks/jmh:run .*HotBloopBenchmark.* -wi 0 -i 1 -f1 -t1 -p project=with-tests -p projectName=with-tests"
+
   SBT_PUBLISH:
     - sbt "set pgpPublicRing in Global := file(\"/drone/.gnupg/pubring.asc\")" \
           "set pgpSecretRing in Global := file(\"/drone/.gnupg/secring.asc\")" \
@@ -23,6 +25,9 @@ matrix:
   OS:
     - windows
     - linux
+  OPS:
+    - basic
+    - extra
 
 clone:
   git:
@@ -62,20 +67,36 @@ pipeline:
       ref: [ refs/heads/master, refs/tags/*, refs/pull/*/head ]
       matrix:
         OS: linux
+        OPS: basic
     commands:
       - export DRONE_DIR="/drone"
       - git log | head -n 20
       - ./bin/add-coursier.sh
       - . bin/detect-community-build.sh # We need to source it for the env variable to be exported
       - ${SBT_RUN}; ./bin/ci-clean-cache.sh
 
+  run_benchmarks_scripted:
+    image: scalacenter/scala-docs:1.1
+    group: build
+    when:
+      ref: [ refs/heads/master, refs/tags/*, refs/pull/*/head ]
+      matrix:
+        OS: linux
+        OPS: extra
+    commands:
+      - export DRONE_DIR="/drone"
+      - git log | head -n 20
+      - ./bin/add-coursier.sh
+      - ${SBT_RUN_BENCHMARKS_AND_SCRIPTED}; ./bin/ci-clean-cache.sh
+
   build_windows:
     group: build
     image: scalacenter/scala-docs:1.0
     when:
       ref: [ refs/heads/master, refs/tags/*, refs/pull/*/head ]
       matrix:
         OS: windows
+        OPS: basic
     secrets: [ bloop_jenkins_token ]
     commands:
       - ./bin/stream-jenkins-log.sh "bloop:$BLOOP_JENKINS_TOKEN"
@@ -91,6 +112,7 @@ pipeline:
       status: success
       matrix:
         OS: linux
+        OPS: basic
     commands:
       - git log | head -n 20
       # I have no idea why this has to be done manually... TODO: inspect why.
@@ -112,6 +134,7 @@ pipeline:
       status: success
       matrix:
         OS: linux
+        OPS: basic
     commands:
       - export DRONE_DIR="/drone"
       - ./bin/add-coursier.sh
@@ -124,6 +147,7 @@ pipeline:
       ref: [ refs/heads/master, refs/tags/*, refs/pull/*/head ]
       matrix:
         OS: linux
+        OPS: basic
     secrets: [ sftp_cache_username, sftp_cache_private_key, sftp_cache_server, sftp_cache_path ]
     rebuild: true
     mount:

build.sbt

@@ -58,6 +58,7 @@ val jsonConfig = project
           Dependencies.metaconfigDocs,
           Dependencies.metaconfigConfig,
           Dependencies.circeDerivation,
+          Dependencies.nuprocess,
           Dependencies.scalacheck % Test,
         )
       } else {

frontend/src/main/scala/bloop/Cli.scala

@@ -257,7 +257,6 @@ object Cli {
         .flatMap(start => taskState.materialize.map(s => (s, start)))
         .map { case (state, start) => logElapsed(start); state }
         .dematerialize
-        .executeWithOptions(_.enableAutoCancelableRunLoops)
         .runAsync(ExecutionContext.scheduler)
 
     def handleException(t: Throwable) = {

frontend/src/main/scala/bloop/bsp/BspServer.scala

@@ -97,7 +97,7 @@ object BspServer {
         .doOnFinish(_ => me.Task { handle.serverSocket.close() })
     }
 
-    val runTask = initServer(cmd, state).materialize.flatMap {
+    initServer(cmd, state).materialize.flatMap {
       case scala.util.Success(handle: ConnectionHandle) =>
         startServer(handle).onErrorRecover {
           case t: Throwable =>
@@ -112,7 +112,5 @@ object BspServer {
           state
         }
     }
-
-    runTask.executeWithOptions(_.enableAutoCancelableRunLoops)
   }
 }

frontend/src/main/scala/bloop/engine/tasks/Tasks.scala

@@ -4,7 +4,7 @@ import bloop.cli.ExitStatus
 import bloop.config.Config
 import bloop.engine.caches.ResultsCache
 import bloop.engine.{Dag, Leaf, Parent, State}
-import bloop.exec.ForkProcess
+import bloop.exec.Forker
 import bloop.io.AbsolutePath
 import bloop.logging.BspLogger
 import bloop.reporter.{BspReporter, LogReporter, Problem, Reporter, ReporterConfig}
@@ -271,52 +271,54 @@ object Tasks {
       isolated: Boolean,
       frameworkSpecificRawArgs: List[String],
       testFilter: String => Boolean
-  ): Task[State] = Task {
-    // TODO(jvican): This method should cache the test loader always.
+  ): Task[State] = {
     import state.logger
     import bloop.util.JavaCompat.EnrichOptional
+    def foundFrameworks(frameworks: Array[Framework]) = frameworks.map(_.name).mkString(", ")
+
+    // Test arguments coming after `--` can only be used if only one mapping is found
+    def considerFrameworkArgs(frameworks: Array[Framework]) = {
+      if (frameworkSpecificRawArgs.isEmpty) Nil
+      else {
+        frameworks match {
+          case Array(oneFramework) =>
+            val rawArgs = frameworkSpecificRawArgs.toArray
+            val cls = oneFramework.getClass.getName()
+            logger.debug(s"Test options '$rawArgs' assigned to the only found framework $cls'.")
+            List(Config.TestArgument(rawArgs, Some(Config.TestFramework(List(cls)))))
+          case _ =>
+            val ignoredArgs = frameworkSpecificRawArgs.mkString(" ")
+            logger.warn(
+              s"Framework-specific test options '${ignoredArgs}' are ignored because several frameworks were found: ${foundFrameworks(frameworks)}")
+            Nil
+        }
+      }
+    }
 
     val projectsToTest = if (isolated) List(project) else Dag.dfs(state.build.getDagFor(project))
-    projectsToTest.foreach { project =>
+    val testTasks = projectsToTest.map { project =>
       val projectName = project.name
       val projectTestArgs = project.testOptions.arguments
-      val forkProcess = ForkProcess(project.javaEnv, project.classpath)
-      val testLoader = forkProcess.toExecutionClassLoader(Some(TestInternals.filteredLoader))
+      val forker = Forker(project.javaEnv, project.classpath)
+      val testLoader = forker.newClassLoader(Some(TestInternals.filteredLoader))
       val frameworks = project.testFrameworks
         .flatMap(f => TestInternals.loadFramework(testLoader, f.names, logger))
-      def foundFrameworks = frameworks.map(_.name).mkString(", ")
-      logger.debug(s"Found frameworks: $foundFrameworks")
+      logger.debug(s"Found frameworks: ${foundFrameworks(frameworks)}")
 
-      // Test arguments coming after `--` can only be used if only one mapping is found
-      val frameworkArgs = {
-        if (frameworkSpecificRawArgs.isEmpty) Nil
-        else {
-          frameworks match {
-            case Array(oneFramework) =>
-              val rawArgs = frameworkSpecificRawArgs.toArray
-              val cls = oneFramework.getClass.getName()
-              logger.debug(s"Test options '$rawArgs' assigned to the only found framework $cls'.")
-              List(Config.TestArgument(rawArgs, Some(Config.TestFramework(List(cls)))))
-            case _ =>
-              val ignoredArgs = frameworkSpecificRawArgs.mkString(" ")
-              logger.warn(
-                s"Framework-specific test options '${ignoredArgs}' are ignored because several frameworks were found: $foundFrameworks")
-              Nil
-          }
-        }
-      }
-
-      val analysis = state.results.lastSuccessfulResult(project).analysis().toOption.getOrElse {
+      val frameworkArgs = considerFrameworkArgs(frameworks)
+      val lastCompileResult = state.results.lastSuccessfulResult(project)
+      val analysis = lastCompileResult.analysis().toOption.getOrElse {
         logger.warn(s"Test execution is triggered but no compilation detected for ${projectName}.")
         Analysis.empty
       }
 
-      val discoveredTests = {
+      val discovered = {
         val tests = discoverTests(analysis, frameworks)
         val excluded = project.testOptions.excludes.toSet
         val ungroupedTests = tests.toList.flatMap {
           case (framework, tasks) => tasks.map(t => (framework, t))
         }
+
         val (includedTests, excludedTests) = ungroupedTests.partition {
           case (_, task) =>
             val fqn = task.fullyQualifiedName()
@@ -335,13 +337,17 @@ object Tasks {
         DiscoveredTests(testLoader, includedTests.groupBy(_._1).mapValues(_.map(_._2)))
       }
 
+      val opts = state.commonOptions
       val args = project.testOptions.arguments ++ frameworkArgs
-      val env = state.commonOptions.env
-      TestInternals.executeTasks(cwd, forkProcess, discoveredTests, args, handler, logger, env)
+      TestInternals.execute(cwd, forker, discovered, args, handler, logger, opts)
     }
 
-    // Return the previous state, test execution doesn't modify it.
-    state.mergeStatus(ExitStatus.Ok)
+    // For now, test execution is only sequential.
+    Task.sequence(testTasks).map { exitCodes =>
+      val isOk = exitCodes.forall(_ == 0)
+      if (isOk) state.mergeStatus(ExitStatus.Ok)
+      else state.copy(status = ExitStatus.TestExecutionError)
+    }
   }
 
   /**
@@ -357,16 +363,16 @@ object Tasks {
           project: Project,
           cwd: AbsolutePath,
           fqn: String,
-          args: Array[String]): Task[State] = Task {
+          args: Array[String]): Task[State] = {
     val classpath = project.classpath
-    val processConfig = ForkProcess(project.javaEnv, classpath)
-    val exitCode = processConfig.runMain(cwd, fqn, args, state.logger, state.commonOptions.env)
-    val exitStatus = {
-      if (exitCode == ForkProcess.EXIT_OK) ExitStatus.Ok
-      else ExitStatus.UnexpectedError
+    val processConfig = Forker(project.javaEnv, classpath)
+    val runTask = processConfig.runMain(cwd, fqn, args, state.logger, state.commonOptions)
+    runTask.map { exitCode =>
+      state.mergeStatus {
+        if (exitCode == Forker.EXIT_OK) ExitStatus.Ok
+        else ExitStatus.UnexpectedError
+      }
     }
-
-    state.mergeStatus(exitStatus)
   }
 
   /**