Persist analysis files on the background

Previously, bloop would persist the analysis file on `shutDown`.
However, this hook has been removed upstream in
facebookarchive/nailgun#131.

As a result, we take another (more resilient) strategy to persist the
analysis file. We persist analysis file on the io pool after every
`Interpreter` execution, without blocking the return of the request.

All the logs are redirected to the nailgun server output.

frontend/src/main/scala/bloop/Bloop.scala

@@ -53,7 +53,7 @@ object Bloop extends CaseApp[CliOptions] {
     val input = reader.readLine()
     input.split(" ") match {
       case Array("exit") =>
-        timed(state.logger) { Tasks.persist(state) }
+        waitForState(Exit(ExitStatus.Ok), Tasks.persist(state).map(_ => state))
         ()
 
       case Array("projects") =>

frontend/src/main/scala/bloop/Cli.scala

@@ -10,14 +10,13 @@ import bloop.logging.{BloopLogger, Logger}
 import caseapp.core.{DefaultBaseCommand, Messages}
 import com.martiansoftware.nailgun.NGContext
 import _root_.monix.eval.Task
-import bloop.io.Timer
+import bloop.engine.tasks.Tasks
 
 import scala.util.control.NonFatal
 
 class Cli
 object Cli {
   def main(args: Array[String]): Unit = {
-    State.setUpShutdownHoook()
     val action = parse(args, CommonOptions.default)
     val exitStatus = run(action, NoPool, args)
     sys.exit(exitStatus.code)
@@ -227,6 +226,8 @@ object Cli {
     waitUntilEndOfWorld(action, cliOptions, pool, configDirectory.underlying, logger, userArgs) {
       Interpreter.execute(action, currentState).map { newState =>
         State.stateCache.updateBuild(newState.copy(status = ExitStatus.Ok))
+        // Persist successful result on the background for the new state -- it doesn't block!
+        Tasks.persist(newState).runAsync(ExecutionContext.ioScheduler)
         newState
       }
     }

frontend/src/main/scala/bloop/Server.scala

@@ -53,9 +53,6 @@ object Server {
   }
 
   private def shutDown(server: NGServer): Unit = {
-    import bloop.engine.State
-    import bloop.engine.tasks.Tasks
-    State.stateCache.allStates.foreach(s => Tasks.persist(s))
     server.shutdown( /* exitVM = */ false)
   }
 }

frontend/src/main/scala/bloop/engine/State.scala

@@ -65,16 +65,6 @@ object State {
     State(build, results, compilerCache, pool, opts, ExitStatus.Ok, logger)
   }
 
-  def setUpShutdownHoook(): Unit = {
-    Runtime
-      .getRuntime()
-      .addShutdownHook(new Thread {
-        import bloop.engine.tasks.Tasks
-        override def run(): Unit =
-          State.stateCache.allStates.foreach(s => Tasks.persist(s))
-      })
-  }
-
   /**
    * Sets up the cores for the execution context to be used for compiling and testing the project.
    *

frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala

@@ -47,5 +47,8 @@ final class ResultsCache(cache: Map[Project, PreviousResult], logger: Logger) {
 }
 
 object ResultsCache {
+  import java.util.concurrent.ConcurrentHashMap
+  // TODO: Enrich this with a guava cache that stores maximum 200 analysis file
+  private[bloop] val persisted = ConcurrentHashMap.newKeySet[PreviousResult]()
   def getEmpty(logger: Logger): ResultsCache = new ResultsCache(Map.empty, logger)
 }

frontend/src/main/scala/bloop/engine/tasks/Tasks.scala

@@ -1,14 +1,14 @@
 package bloop.engine.tasks
 
 import bloop.cli.ExitStatus
+import bloop.engine.caches.ResultsCache
 import bloop.engine.{Dag, Leaf, Parent, State}
 import bloop.exec.ForkProcess
 import bloop.io.AbsolutePath
 import bloop.reporter.{Reporter, ReporterConfig}
 import bloop.testing.{DiscoveredTests, TestInternals}
 import bloop.{CompileInputs, Compiler, Project}
 import monix.eval.Task
-
 import sbt.internal.inc.{Analysis, AnalyzingCompiler, ConcreteAnalysisContents, FileAnalysisStore}
 import sbt.internal.inc.classpath.ClasspathUtilities
 import sbt.testing.{Event, EventHandler, Framework, SuiteSelector, TaskDef}
@@ -195,35 +195,40 @@ object Tasks {
   }
 
   /**
-   * Persists on disk the state of the incremental compiler.
+   * Persists every analysis file (the state of the incremental compiler) on disk in parallel.
    *
    * @param state The current state of Bloop
-   * @return The same state, unchanged.
+   * @return The task that will persist all the results in parallel.
    */
-  def persist(state: State): State = {
-    import state.logger
+  def persist(state: State): Task[Unit] = {
+    val out = state.commonOptions.ngout
     import bloop.util.JavaCompat.EnrichOptional
-    def persistResult(project: Project, result: PreviousResult): Unit = {
+    def persist(project: Project, result: PreviousResult): Unit = {
       def toBinaryFile(analysis: CompileAnalysis, setup: MiniSetup): Unit = {
         val storeFile = project.out.resolve(s"${project.name}-analysis.bin")
+        state.commonOptions.ngout.println(s"Writing ${storeFile.syntax}.")
         FileAnalysisStore.binary(storeFile.toFile).set(ConcreteAnalysisContents(analysis, setup))
+        ResultsCache.persisted.add(result)
+        ()
       }
 
       val analysis = result.analysis().toOption
       val setup = result.setup().toOption
       (analysis, setup) match {
-        case (Some(analysis), Some(setup)) => toBinaryFile(analysis, setup)
+        case (Some(analysis), Some(setup)) =>
+          if (ResultsCache.persisted.contains(result)) ()
+          else toBinaryFile(analysis, setup)
         case (Some(analysis), None) =>
-          logger.warn(s"$project has analysis but not setup after compilation. Report upstream.")
+          out.println(s"$project has analysis but not setup after compilation. Report upstream.")
         case (None, Some(analysis)) =>
-          logger.warn(s"$project has setup but not analysis after compilation. Report upstream.")
-        case (None, None) => logger.debug(s"Project $project has no analysis file.")
+          out.println(s"$project has setup but not analysis after compilation. Report upstream.")
+        case (None, None) => out.println(s"Project $project has no analysis and setup.")
       }
 
     }
 
-    state.results.iterator.foreach(kv => persistResult(kv._1, kv._2))
-    state
+    val ts = state.results.iterator.map { case (project, result) => Task(persist(project, result)) }
+    Task.gatherUnordered(ts).map(_ => ())
   }
 
   /**

project/Dependencies.scala

@@ -1,7 +1,7 @@
 package build
 
 object Dependencies {
-  val nailgunVersion = "51ddd0d9"
+  val nailgunVersion = "615737a9"
   val zincVersion = "1.1.1+49-1c290cbb"
   val bspVersion = "03e9b72d"
   val coursierVersion = "1.0.0-RC8"