From c80bdcc48f624abbad89414b269df15d45e5b5a3 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Tue, 13 Feb 2024 14:36:37 +0100 Subject: [PATCH] refactor and make it work --- .../scala/bloop/ClientClassesObserver.scala | 65 +++++++ .../scala/bloop/CompileBackgroundTasks.scala | 4 +- backend/src/main/scala/bloop/Compiler.scala | 63 ++++--- .../scala/bloop/bsp/BloopBspServices.scala | 5 +- .../bloop/dap/BloopDebugToolsResolver.scala | 13 +- .../main/scala/bloop/dap/BloopDebuggee.scala | 106 ++++++----- .../bloop/dap/DapCancellableFuture.scala | 7 +- .../main/scala/bloop/data/ClientInfo.scala | 13 ++ .../src/main/scala/bloop/data/Project.scala | 3 - .../bloop/engine/tasks/CompileTask.scala | 69 +------ .../tasks/compilation/CompileBundle.scala | 9 +- .../tasks/compilation/CompileGraph.scala | 7 +- .../bloop/dap/DebugAdapterConnection.scala | 8 +- .../scala/bloop/dap/DebugServerSpec.scala | 170 ++++++++++-------- project/Dependencies.scala | 2 +- 15 files changed, 303 insertions(+), 241 deletions(-) create mode 100644 backend/src/main/scala/bloop/ClientClassesObserver.scala diff --git a/backend/src/main/scala/bloop/ClientClassesObserver.scala b/backend/src/main/scala/bloop/ClientClassesObserver.scala new file mode 100644 index 0000000000..e84810e5aa --- /dev/null +++ b/backend/src/main/scala/bloop/ClientClassesObserver.scala @@ -0,0 +1,65 @@ +package bloop + +import java.io.File +import java.util.concurrent.atomic.AtomicReference + +import scala.jdk.CollectionConverters._ + +import bloop.io.AbsolutePath +import bloop.task.Task + +import monix.reactive.Observable +import monix.reactive.subjects.PublishSubject +import sbt.internal.inc.PlainVirtualFileConverter +import xsbti.VirtualFileRef +import xsbti.compile.CompileAnalysis +import xsbti.compile.analysis.Stamp + +/** + * Each time a new compile analysis is produced for a given client, it is given to + * the [[ClientClassObserver]] which computes the list of classes that changed or got created. + * + * A client can subscribe to the observer to get notified of classes to update. + * It is used by DAP to hot reload classes in the debuggee process. + * + * @param clientClassesDir the class directory for the client + */ +private[bloop] class ClientClassesObserver(val classesDir: AbsolutePath) { + private val converter = PlainVirtualFileConverter.converter + private val previousAnalysis: AtomicReference[CompileAnalysis] = new AtomicReference() + private val classesSubject: PublishSubject[Seq[String]] = PublishSubject() + + def observable: Observable[Seq[String]] = classesSubject + + def nextAnalysis(analysis: CompileAnalysis): Task[Unit] = { + val prev = previousAnalysis.getAndSet(analysis) + if (prev != null && classesSubject.size > 0) { + Task { + val previousStamps = prev.readStamps.getAllProductStamps + analysis.readStamps.getAllProductStamps.asScala.iterator.collect { + case (vf, stamp) if isClassFile(vf) && isNewer(stamp, previousStamps.get(vf)) => + getFullyQualifiedClassName(vf) + }.toSeq + } + .flatMap { classesToUpdate => + Task.fromFuture(classesSubject.onNext(classesToUpdate)).map(_ => ()) + } + } else Task.unit + } + + private def isClassFile(vf: VirtualFileRef): Boolean = vf.id.endsWith(".class") + + private def isNewer(current: Stamp, previous: Stamp): Boolean = + previous == null || { + val currentHash = current.getHash + val previousHash = previous.getHash + currentHash.isPresent && + (!previousHash.isPresent || currentHash.get != previousHash.get) + } + + private def getFullyQualifiedClassName(vf: VirtualFileRef): String = { + val path = converter.toPath(vf) + val relativePath = classesDir.underlying.relativize(path) + relativePath.toString.replace(File.separator, ".").stripSuffix(".class") + } +} diff --git a/backend/src/main/scala/bloop/CompileBackgroundTasks.scala b/backend/src/main/scala/bloop/CompileBackgroundTasks.scala index 41306919ae..5b4b57c5d2 100644 --- a/backend/src/main/scala/bloop/CompileBackgroundTasks.scala +++ b/backend/src/main/scala/bloop/CompileBackgroundTasks.scala @@ -8,7 +8,7 @@ import bloop.tracing.BraveTracer abstract class CompileBackgroundTasks { def trigger( - clientClassesDir: AbsolutePath, + clientClassesObserver: ClientClassesObserver, clientReporter: Reporter, clientTracer: BraveTracer, clientLogger: Logger @@ -20,7 +20,7 @@ object CompileBackgroundTasks { val empty: CompileBackgroundTasks = { new CompileBackgroundTasks { def trigger( - clientClassesDir: AbsolutePath, + clientClassesObserver: ClientClassesObserver, clientReporter: Reporter, clientTracer: BraveTracer, clientLogger: Logger diff --git a/backend/src/main/scala/bloop/Compiler.scala b/backend/src/main/scala/bloop/Compiler.scala index 14d7430ef3..87941fdb6e 100644 --- a/backend/src/main/scala/bloop/Compiler.scala +++ b/backend/src/main/scala/bloop/Compiler.scala @@ -452,11 +452,12 @@ object Compiler { val backgroundTasks = new CompileBackgroundTasks { def trigger( - clientClassesDir: AbsolutePath, + clientClassesObserver: ClientClassesObserver, clientReporter: Reporter, clientTracer: BraveTracer, clientLogger: Logger ): Task[Unit] = Task.defer { + val clientClassesDir = clientClassesObserver.classesDir clientLogger.debug(s"Triggering background tasks for $clientClassesDir") val updateClientState = updateExternalClassesDirWithReadOnly(clientClassesDir, clientTracer, clientLogger) @@ -472,10 +473,20 @@ object Compiler { } val deleteNewClassesDir = Task(BloopPaths.delete(AbsolutePath(newClassesDir))) - val allTasks = List(deleteNewClassesDir, updateClientState, writeAnalysisIfMissing) + val publishClientAnalysis = Task { + rebaseAnalysisClassFiles( + analysis, + readOnlyClassesDir, + clientClassesDir.underlying, + sourcesWithFatal + ) + } + .flatMap(clientClassesObserver.nextAnalysis) Task - .gatherUnordered(allTasks) - .map(_ => ()) + .gatherUnordered( + List(deleteNewClassesDir, updateClientState, writeAnalysisIfMissing) + ) + .flatMap(_ => publishClientAnalysis) .onErrorHandleWith(err => { clientLogger.debug("Caught error in background tasks"); clientLogger.trace(err); Task.raiseError(err) @@ -495,14 +506,12 @@ object Compiler { ) } else { val allGeneratedProducts = allGeneratedRelativeClassFilePaths.toMap - val analysisForFutureCompilationRuns = { - rebaseAnalysisClassFiles( - analysis, - readOnlyClassesDir, - newClassesDir, - sourcesWithFatal - ) - } + val analysisForFutureCompilationRuns = rebaseAnalysisClassFiles( + analysis, + readOnlyClassesDir, + newClassesDir, + sourcesWithFatal + ) val resultForFutureCompilationRuns = { resultForDependentCompilationsInSameRun.withAnalysis( @@ -517,12 +526,12 @@ object Compiler { // Schedule the tasks to run concurrently after the compilation end val backgroundTasksExecution = new CompileBackgroundTasks { def trigger( - clientClassesDir: AbsolutePath, + clientClassesObserver: ClientClassesObserver, clientReporter: Reporter, clientTracer: BraveTracer, clientLogger: Logger ): Task[Unit] = { - val clientClassesDirPath = clientClassesDir.toString + val clientClassesDir = clientClassesObserver.classesDir val successBackgroundTasks = backgroundTasksWhenNewSuccessfulAnalysis .map(f => f(clientClassesDir, clientReporter, clientTracer)) @@ -543,7 +552,7 @@ object Compiler { val syntax = path.syntax if (syntax.startsWith(readOnlyClassesDirPath)) { val rebasedFile = AbsolutePath( - syntax.replace(readOnlyClassesDirPath, clientClassesDirPath) + syntax.replace(readOnlyClassesDirPath, clientClassesDir.toString) ) if (rebasedFile.exists) { Files.delete(rebasedFile.underlying) @@ -551,7 +560,18 @@ object Compiler { } } } - Task.gatherUnordered(List(firstTask, secondTask)).map(_ => ()) + + val publishClientAnalysis = Task { + rebaseAnalysisClassFiles( + analysis, + newClassesDir, + clientClassesDir.underlying, + sourcesWithFatal + ) + }.flatMap(clientClassesObserver.nextAnalysis) + Task + .gatherUnordered(List(firstTask, secondTask)) + .flatMap(_ => publishClientAnalysis) } allClientSyncTasks.doOnFinish(_ => Task(clientReporter.reportEndCompilation())) @@ -691,11 +711,12 @@ object Compiler { ): CompileBackgroundTasks = { new CompileBackgroundTasks { def trigger( - clientClassesDir: AbsolutePath, + clientClassesObserver: ClientClassesObserver, clientReporter: Reporter, tracer: BraveTracer, clientLogger: Logger ): Task[Unit] = { + val clientClassesDir = clientClassesObserver.classesDir val backgroundTasks = tasks.map(f => f(clientClassesDir, clientReporter, tracer)) Task.gatherUnordered(backgroundTasks).memoize.map(_ => ()) } @@ -783,8 +804,8 @@ object Compiler { */ def rebaseAnalysisClassFiles( analysis0: CompileAnalysis, - readOnlyClassesDir: Path, - newClassesDir: Path, + origin: Path, + target: Path, sourceFilesWithFatalWarnings: scala.collection.Set[File] ): Analysis = { // Cast to the only internal analysis that we support @@ -792,10 +813,10 @@ object Compiler { def rebase(file: VirtualFileRef): VirtualFileRef = { val filePath = converter.toPath(file).toAbsolutePath() - if (!filePath.startsWith(readOnlyClassesDir)) file + if (!filePath.startsWith(origin)) file else { // Hash for class file is the same because the copy duplicates metadata - val path = newClassesDir.resolve(readOnlyClassesDir.relativize(filePath)) + val path = target.resolve(origin.relativize(filePath)) converter.toVirtualFile(path) } } diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala index 6624c919a6..79af099da2 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala @@ -603,10 +603,7 @@ final class BloopBspServices( params: bsp.DebugSessionParams ): BspEndpointResponse[bsp.DebugSessionAddress] = { - def inferDebuggee( - projects: Seq[Project], - state: State - ): BspResponse[Debuggee] = { + def inferDebuggee(projects: Seq[Project], state: State): BspResponse[Debuggee] = { def convert[A: JsonValueCodec]( f: A => Either[String, Debuggee] ): Either[Response.Error, Debuggee] = { diff --git a/frontend/src/main/scala/bloop/dap/BloopDebugToolsResolver.scala b/frontend/src/main/scala/bloop/dap/BloopDebugToolsResolver.scala index 4d23d0edf4..75fa303592 100644 --- a/frontend/src/main/scala/bloop/dap/BloopDebugToolsResolver.scala +++ b/frontend/src/main/scala/bloop/dap/BloopDebugToolsResolver.scala @@ -34,13 +34,12 @@ class BloopDebugToolsResolver(logger: Logger) extends DebugToolsResolver { } } - override def resolveUnpickler(scalaVersion: ScalaVersion): Try[ClassLoader] = { - getOrTryUpdate(stepFilterCache, scalaVersion) { - val unpicklerModule = s"${BuildInfo.unpicklerName}_${scalaVersion.binaryVersion}" - val stepFilter = Artifact(BuildInfo.organization, unpicklerModule, BuildInfo.version) - val tastyCore = Artifact("org.scala-lang", "tasty-core_3", scalaVersion.value) + override def resolveDecoder(scalaVersion: ScalaVersion): Try[ClassLoader] = { + getOrTryUpdate(decoderCache, scalaVersion) { + val decoderModule = s"${BuildInfo.decoderName}_${scalaVersion.binaryVersion}" + val artifact = Artifact(BuildInfo.organization, decoderModule, BuildInfo.version) DependencyResolution - .resolveWithErrors(List(stepFilter, tastyCore), logger) + .resolveWithErrors(List(artifact), logger) .map(jars => toClassLoader(jars, true)) .toTry } @@ -66,5 +65,5 @@ class BloopDebugToolsResolver(logger: Logger) extends DebugToolsResolver { object BloopDebugToolsResolver { private val expressionCompilerCache: mutable.Map[ScalaVersion, ClassLoader] = mutable.Map.empty - private val stepFilterCache: mutable.Map[ScalaVersion, ClassLoader] = mutable.Map.empty + private val decoderCache: mutable.Map[ScalaVersion, ClassLoader] = mutable.Map.empty } diff --git a/frontend/src/main/scala/bloop/dap/BloopDebuggee.scala b/frontend/src/main/scala/bloop/dap/BloopDebuggee.scala index 6cdab68685..88e04cfc70 100644 --- a/frontend/src/main/scala/bloop/dap/BloopDebuggee.scala +++ b/frontend/src/main/scala/bloop/dap/BloopDebuggee.scala @@ -1,6 +1,9 @@ package bloop.dap +import java.io.Closeable + import scala.collection.mutable +import scala.concurrent.Future import ch.epfl.scala.bsp import ch.epfl.scala.bsp.ScalaMainClass @@ -15,20 +18,23 @@ import bloop.engine.Dag import bloop.engine.State import bloop.engine.tasks.RunMode import bloop.engine.tasks.Tasks +import bloop.io.AbsolutePath import bloop.task.Task import bloop.testing.DebugLoggingEventHandler import bloop.testing.TestInternals +import monix.execution.Ack import monix.execution.Scheduler -import io.reactivex.Observable +import monix.reactive.Observable abstract class BloopDebuggee( initialState: State, - ioScheduler: Scheduler -) extends Debuggee { - val debuggeeScalaVersion: Option[String] + classUpdates: Observable[Seq[String]] +)(implicit ioScheduler: Scheduler) + extends Debuggee { + protected def scalaVersionOpt: Option[String] // The version doesn't matter for project without Scala version (Java only) - lazy val scalaVersion = ScalaVersion(debuggeeScalaVersion.getOrElse("2.13.8")) + override val scalaVersion = ScalaVersion(scalaVersionOpt.getOrElse("2.13.8")) override def run(listener: DebuggeeListener): CancelableFuture[Unit] = { val debugSessionLogger = new DebuggeeLogger(listener, initialState.logger) @@ -37,7 +43,13 @@ abstract class BloopDebuggee( .map { status => if (!status.isOk) throw new Exception(s"debuggee failed with ${status.name}") } - DapCancellableFuture.runAsync(task, ioScheduler) + DapCancellableFuture.runAsync(task) + } + + override def observeClassUpdates(onClassUpdate: Seq[String] => Unit): Closeable = { + val subscription = + classUpdates.subscribe(onClassUpdate.andThen(_ => Future.successful(Ack.Continue))) + () => subscription.cancel() } protected def start(state: State, listener: DebuggeeListener): Task[ExitStatus] @@ -49,14 +61,13 @@ private final class MainClassDebugAdapter( val modules: Seq[Module], val libraries: Seq[Library], val unmanagedEntries: Seq[UnmanagedEntry], + val classUpdates: Observable[Seq[String]], env: JdkConfig, initialState: State, ioScheduler: Scheduler -) extends BloopDebuggee(initialState, ioScheduler) { - - override val debuggeeScalaVersion = project.scalaInstance.map(_.version) +) extends BloopDebuggee(initialState, classUpdates)(ioScheduler) { - val classesToUpdate: Observable[Seq[String]] = project.classObserver + protected def scalaVersionOpt: Option[String] = project.scalaInstance.map(_.version) val javaRuntime: Option[JavaRuntime] = JavaRuntime(env.javaHome.underlying) def name: String = s"${getClass.getSimpleName}(${project.name}, ${mainClass.className})" @@ -87,14 +98,13 @@ private final class TestSuiteDebugAdapter( val libraries: Seq[Library], val unmanagedEntries: Seq[UnmanagedEntry], val javaRuntime: Option[JavaRuntime], + val classUpdates: Observable[Seq[String]], initialState: State, ioScheduler: Scheduler -) extends BloopDebuggee(initialState, ioScheduler) { +) extends BloopDebuggee(initialState, classUpdates)(ioScheduler) { - override val debuggeeScalaVersion = projects.headOption.flatMap(_.scalaInstance.map(_.version)) - - val classesToUpdate: Observable[Seq[String]] = - projects.map(_.classObserver).fold(Observable.empty[Seq[String]])(_ mergeWith _) + protected def scalaVersionOpt: Option[String] = + projects.headOption.flatMap(_.scalaInstance.map(_.version)) override def name: String = { val projectsStr = projects.map(_.bspUri).mkString("[", ", ", "]") @@ -130,33 +140,21 @@ private final class AttachRemoteDebugAdapter( val libraries: Seq[Library], val unmanagedEntries: Seq[UnmanagedEntry], val javaRuntime: Option[JavaRuntime], + val classUpdates: Observable[Seq[String]], initialState: State, ioScheduler: Scheduler -) extends BloopDebuggee(initialState, ioScheduler) { +) extends BloopDebuggee(initialState, classUpdates)(ioScheduler) { - override val debuggeeScalaVersion = projects.headOption.flatMap(_.scalaInstance.map(_.version)) + protected def scalaVersionOpt: Option[String] = + projects.headOption.flatMap(_.scalaInstance.map(_.version)) override def name: String = s"${getClass.getSimpleName}(${initialState.build.origin})" override def start(state: State, listener: DebuggeeListener): Task[ExitStatus] = Task( ExitStatus.Ok ) - val classesToUpdate: Observable[Seq[String]] = - projects.map(_.classObserver).fold(Observable.empty[Seq[String]])(_ mergeWith _) } object BloopDebuggeeRunner { - def getEntries( - project: Project, - state: State - ): (Seq[Module], Seq[Library], Seq[UnmanagedEntry]) = { - val dag = state.build.getDagFor(project) - val modules = getModules(dag, state.client) - val libraries = getLibraries(dag) - val unmanagedEntries = - getUnmanagedEntries(project, dag, state.client, modules ++ libraries) - (modules, libraries, unmanagedEntries) - } - def forMainClass( projects: Seq[Project], mainClass: ScalaMainClass, @@ -168,7 +166,8 @@ object BloopDebuggeeRunner { case Seq(project) => project.platform match { case jvm: Platform.Jvm => - val (modules, libraries, unmanagedEntries) = getEntries(project, state) + val (modules, libraries, unmanagedEntries, classUpdates) = + getEntriesAndClassUpdates(project, state) Right( new MainClassDebugAdapter( project, @@ -176,6 +175,7 @@ object BloopDebuggeeRunner { modules, libraries, unmanagedEntries, + classUpdates, jvm.runtimeConfig.getOrElse(jvm.config), state, ioScheduler @@ -200,7 +200,8 @@ object BloopDebuggeeRunner { s"No projects specified for the test suites: [${testClasses.suites.map(_.className).sorted}]" ) case Seq(project) if project.platform.isInstanceOf[Platform.Jvm] => - val (modules, libraries, unmanagedEntries) = getEntries(project, state) + val (modules, libraries, unmanagedEntries, classUpdates) = + getEntriesAndClassUpdates(project, state) val Platform.Jvm(config, _, _, runtimeConfig, _, _) = project.platform val javaRuntime = JavaRuntime(runtimeConfig.getOrElse(config).javaHome.underlying) Right( @@ -211,6 +212,7 @@ object BloopDebuggeeRunner { libraries, unmanagedEntries, javaRuntime, + classUpdates, state, ioScheduler ) @@ -225,6 +227,7 @@ object BloopDebuggeeRunner { Seq.empty, Seq.empty, None, + Observable.empty, state, ioScheduler ) @@ -240,7 +243,8 @@ object BloopDebuggeeRunner { ): Debuggee = { projects match { case Seq(project) if project.platform.isInstanceOf[Platform.Jvm] => - val (modules, libraries, unmanagedEntries) = getEntries(project, state) + val (modules, libraries, unmanagedEntries, classUpdates) = + getEntriesAndClassUpdates(project, state) val Platform.Jvm(config, _, _, runtimeConfig, _, _) = project.platform val javaRuntime = JavaRuntime(runtimeConfig.getOrElse(config).javaHome.underlying) new AttachRemoteDebugAdapter( @@ -249,6 +253,7 @@ object BloopDebuggeeRunner { libraries, unmanagedEntries, javaRuntime, + classUpdates, state, ioScheduler ) @@ -259,15 +264,30 @@ object BloopDebuggeeRunner { Seq.empty, Seq.empty, None, + Observable.empty, state, ioScheduler ) } } - private def getLibraries(dag: Dag[Project]): Seq[Library] = { - Dag - .dfs(dag, mode = Dag.PreOrder) + private def getEntriesAndClassUpdates( + project: Project, + state: State + ): (Seq[Module], Seq[Library], Seq[UnmanagedEntry], Observable[Seq[String]]) = { + val dag = state.build.getDagFor(project) + val projects = Dag.dfs(dag, mode = Dag.PreOrder) + val modules = getModules(projects, state.client) + val libraries = getLibraries(projects) + val fullClasspath = project.fullClasspath(dag, state.client) + val unmanagedEntries = getUnmanagedEntries(fullClasspath, modules ++ libraries) + val allClassUpdates = projects.map(state.client.getClassesObserverFor(_).observable) + val mergedClassUpdates = Observable.fromIterable(allClassUpdates).merge + (modules, libraries, unmanagedEntries, mergedClassUpdates) + } + + private def getLibraries(projects: Seq[Project]): Seq[Library] = { + projects .flatMap(_.resolution) .flatMap(_.modules) .distinct @@ -284,20 +304,16 @@ object BloopDebuggeeRunner { } private def getUnmanagedEntries( - project: Project, - dag: Dag[Project], - client: ClientInfo, + fullClasspath: Seq[AbsolutePath], managedEntries: Seq[ManagedEntry] ): Seq[UnmanagedEntry] = { val managedPaths = managedEntries.map(_.absolutePath).toSet - val fullClasspath = project.fullClasspath(dag, client).map(_.underlying).toSeq fullClasspath - .filter(p => !managedPaths.contains(p)) - .map(UnmanagedEntry.apply) + .collect { case p if !managedPaths.contains(p.underlying) => UnmanagedEntry(p.underlying) } } - private def getModules(dag: Dag[Project], client: ClientInfo): Seq[Module] = { - Dag.dfs(dag, mode = Dag.PreOrder).map { project => + private def getModules(projects: Seq[Project], client: ClientInfo): Seq[Module] = { + projects.map { project => val sourceBuffer = mutable.Buffer.empty[SourceEntry] for (sourcePath <- project.sources) { if (sourcePath.isDirectory) { diff --git a/frontend/src/main/scala/bloop/dap/DapCancellableFuture.scala b/frontend/src/main/scala/bloop/dap/DapCancellableFuture.scala index fdf68ff8dc..f3b32070fb 100644 --- a/frontend/src/main/scala/bloop/dap/DapCancellableFuture.scala +++ b/frontend/src/main/scala/bloop/dap/DapCancellableFuture.scala @@ -3,6 +3,8 @@ package bloop.dap import scala.concurrent.Future import scala.concurrent.Promise +import ch.epfl.scala.debugadapter.CancelableFuture + import bloop.task.Task import monix.execution.Cancelable @@ -15,10 +17,7 @@ private class DapCancellableFuture(future: Future[Unit], cancelable: Cancelable) } object DapCancellableFuture { - def runAsync( - task: Task[Unit], - ioScheduler: Scheduler - ): ch.epfl.scala.debugadapter.CancelableFuture[Unit] = { + def runAsync(task: Task[Unit])(implicit ioScheduler: Scheduler): CancelableFuture[Unit] = { val promise = Promise[Unit]() val cancelable = task .doOnFinish { diff --git a/frontend/src/main/scala/bloop/data/ClientInfo.scala b/frontend/src/main/scala/bloop/data/ClientInfo.scala index 53de37989e..478985dc6f 100644 --- a/frontend/src/main/scala/bloop/data/ClientInfo.scala +++ b/frontend/src/main/scala/bloop/data/ClientInfo.scala @@ -15,6 +15,7 @@ import bloop.io.AbsolutePath import bloop.io.Filenames import bloop.io.Paths import bloop.util.UUIDUtil +import bloop.ClientClassesObserver sealed trait ClientInfo { @@ -40,6 +41,18 @@ sealed trait ClientInfo { */ def getUniqueClassesDirFor(project: Project, forceGeneration: Boolean): AbsolutePath + /** + * Provides the classes observer for a given project. One can subscribe to it + * to get notified when some classes change or get created. + * It is used by DAP to hot reload classes in the debuggee process. + */ + private val classesObserver = new ConcurrentHashMap[Project, ClientClassesObserver]() + def getClassesObserverFor(project: Project): ClientClassesObserver = + classesObserver.computeIfAbsent( + project, + project => new ClientClassesObserver(getUniqueClassesDirFor(project, true)) + ) + /** * Tells the caller whether this client manages its own client classes * directories or whether bloop should take care of any created resources. diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index e0f9b9e8eb..cabb75d403 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -28,7 +28,6 @@ import bloop.util.JavaRuntime import scalaz.Cord import xsbti.compile.ClasspathOptions import xsbti.compile.CompileOrder -import io.reactivex.subjects.PublishSubject final case class Project( name: String, @@ -57,8 +56,6 @@ final case class Project( origin: Origin ) { - val classObserver = PublishSubject.create[Seq[String]]() - /** The bsp uri associated with this project. */ val bspUri: Bsp.Uri = Bsp.Uri(ProjectUris.toURI(baseDirectory, name)) diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index c01f6f3e35..19e20f4096 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -36,10 +36,6 @@ import bloop.tracing.BraveTracer import monix.execution.CancelableFuture import monix.reactive.MulticastStrategy import monix.reactive.Observable -import xsbti.compile.analysis.ReadStamps -import xsbti.compile.analysis.Stamp -import xsbti.VirtualFileRef -import scala.jdk.CollectionConverters._ object CompileTask { private implicit val logContext: DebugFilter = DebugFilter.Compilation @@ -102,7 +98,6 @@ object CompileTask { compileProjectTracer.terminate() Task.now(earlyResultBundle) case Right(CompileSourcesAndInstance(sources, instance, _)) => - val externalUserClassesDir = bundle.clientClassesDir val readOnlyClassesDir = lastSuccessful.classesDir val newClassesDir = compileOut.internalNewClassesDir val classpath = bundle.dependenciesData.buildFullCompileClasspathFor( @@ -177,7 +172,7 @@ object CompileTask { val postCompilationTasks = backgroundTasks .trigger( - externalUserClassesDir, + bundle.clientClassesObserver, reporter.underlying, compileProjectTracer, logger @@ -223,52 +218,6 @@ object CompileTask { } } - def registerUpdatedClasses(res: FinalNormalCompileResult, project: Project) = { - res.result.fromCompiler match { - case s: Success => - val currentAnalysis = - Option(s.products.resultForFutureCompilationRuns.analysis().orElse(null)) - val previous = res.result.previous - val previousAnalysis = - if (previous.isEmpty) None - else Option(previous.get.previous.analysis().orElse(null)) - - (currentAnalysis, previousAnalysis) match { - case (None, _) | (_, None) => () - case (Some(curr), Some(prev)) => - val currentStamps = curr.readStamps - val previousStamps = prev.readStamps - val newClasses = getNewClasses(currentStamps, previousStamps) - project.classObserver.onNext(newClasses) - } - case _ => () - } - } - - def getNewClasses(currentStamps: ReadStamps, previousStamps: ReadStamps): Seq[String] = { - def isNewer(current: Stamp, previous: Stamp) = { - if (previous == null) true - else { - val newHash = current.getHash - val oldHash = previous.getHash - newHash.isPresent && (!oldHash.isPresent || newHash.get != oldHash.get) - } - } - - object ClassFile { - def unapply(vf: VirtualFileRef): Option[String] = { - val fqcn = vf.name - if (fqcn.toString.endsWith(".class")) Some(fqcn.stripSuffix(".class")) - else None - } - } - - val oldStamps = previousStamps.getAllProductStamps - currentStamps.getAllProductStamps.asScala.collect { - case (file @ ClassFile(fqcn), stamp) if isNewer(stamp, oldStamps.get(file)) => fqcn - }.toSeq - } - def setup(inputs: CompileDefinitions.BundleInputs): Task[CompileBundle] = { // Create a multicast observable stream to allow multiple mirrors of loggers val (observer, obs) = { @@ -297,14 +246,14 @@ object CompileTask { val o = state.commonOptions val cancel = cancelCompilation val logger = ObservedLogger(rawLogger, observer) - val dir = state.client.getUniqueClassesDirFor(inputs.project, forceGeneration = true) + val clientClassesObserver = state.client.getClassesObserverFor(inputs.project) val underlying = createReporter(ReporterInputs(inputs.project, cwd, rawLogger)) val reporter = new ObservedReporter(logger, underlying) val sourceGeneratorCache = state.sourceGeneratorCache CompileBundle.computeFrom( inputs, sourceGeneratorCache, - dir, + clientClassesObserver, reporter, last, prev, @@ -338,11 +287,6 @@ object CompileTask { val newState: State = { val stateWithResults = state.copy(results = state.results.addFinalResults(results)) if (failures.isEmpty) { - results.foreach { - case res: FinalNormalCompileResult => - registerUpdatedClasses(res, res.project) - case _ => () - } stateWithResults.copy(status = ExitStatus.Ok) } else { results.foreach { @@ -374,9 +318,12 @@ object CompileTask { runIOTasksInParallel(cleanUpTasksToRunInBackground) val runningTasksRequiredForCorrectness = Task.sequence { - results.collect { + results.flatMap { case FinalNormalCompileResult(_, result) => - Task.fromFuture(result.runningBackgroundTasks) + val tasksAtEndOfBuildCompilation = + Task.fromFuture(result.runningBackgroundTasks) + List(tasksAtEndOfBuildCompilation) + case _ => Nil } } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileBundle.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileBundle.scala index f741483000..89e695c540 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileBundle.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileBundle.scala @@ -23,6 +23,7 @@ import bloop.tracing.BraveTracer import monix.reactive.Observable import sbt.internal.inc.PlainVirtualFileConverter +import bloop.ClientClassesObserver sealed trait CompileBundle @@ -75,7 +76,7 @@ case object CancelledCompileBundle extends CompileBundle */ final case class SuccessfulCompileBundle( project: Project, - clientClassesDir: AbsolutePath, + clientClassesObserver: ClientClassesObserver, dependenciesData: CompileDependenciesData, javaSources: List[AbsolutePath], scalaSources: List[AbsolutePath], @@ -95,7 +96,7 @@ final case class SuccessfulCompileBundle( project.out, project.analysisOut, project.genericClassesDir, - clientClassesDir, + clientClassesObserver.classesDir, readOnlyClassesDir ) } @@ -152,7 +153,7 @@ object CompileBundle { def computeFrom( inputs: CompileDefinitions.BundleInputs, sourceGeneratorCache: SourceGeneratorCache, - clientExternalClassesDir: AbsolutePath, + clientClassesObserver: ClientClassesObserver, reporter: ObservedReporter, lastSuccessful: LastSuccessfulResult, lastResult: Compiler.Result, @@ -228,7 +229,7 @@ object CompileBundle { new SuccessfulCompileBundle( project, - clientExternalClassesDir, + clientClassesObserver, compileDependenciesData, javaSources, scalaSources, diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala index 1ad3840fda..f596ce335f 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala @@ -166,8 +166,7 @@ object CompileGraph { val previousProblems = Compiler.previousProblemsFromResult(bundle.latestResult, previousSuccessfulProblems) - val externalClassesDir = - client.getUniqueClassesDirFor(bundle.project, forceGeneration = true) + val clientClassesObserver = client.getClassesObserverFor(bundle.project) // Replay events asynchronously to waiting for the compilation result import scala.concurrent.duration.FiniteDuration @@ -210,7 +209,7 @@ object CompileGraph { reporter.processEndCompilation( previousSuccessfulProblems, a.code, - Some(externalClassesDir), + Some(clientClassesObserver.classesDir), Some(bundle.out.analysisOut) ) } @@ -263,7 +262,7 @@ object CompileGraph { case s: Compiler.Result.Success => // Wait on new classes to be populated for correctness val runningBackgroundTasks = s.backgroundTasks - .trigger(externalClassesDir, reporter, bundle.tracer, logger) + .trigger(clientClassesObserver, reporter, bundle.tracer, logger) .runAsync(ExecutionContext.ioScheduler) Task.now(results.copy(runningBackgroundTasks = runningBackgroundTasks)) case _: Compiler.Result.Cancelled => diff --git a/frontend/src/test/scala/bloop/dap/DebugAdapterConnection.scala b/frontend/src/test/scala/bloop/dap/DebugAdapterConnection.scala index eb73e22719..0aa7154506 100644 --- a/frontend/src/test/scala/bloop/dap/DebugAdapterConnection.scala +++ b/frontend/src/test/scala/bloop/dap/DebugAdapterConnection.scala @@ -14,13 +14,7 @@ import bloop.task.Task import com.microsoft.java.debug.core.protocol.Events import com.microsoft.java.debug.core.protocol.Requests._ -import com.microsoft.java.debug.core.protocol.Responses.ContinueResponseBody -import com.microsoft.java.debug.core.protocol.Responses.EvaluateResponseBody -import com.microsoft.java.debug.core.protocol.Responses.RedefineClassesResponse -import com.microsoft.java.debug.core.protocol.Responses.ScopesResponseBody -import com.microsoft.java.debug.core.protocol.Responses.SetBreakpointsResponseBody -import com.microsoft.java.debug.core.protocol.Responses.StackTraceResponseBody -import com.microsoft.java.debug.core.protocol.Responses.VariablesResponseBody +import com.microsoft.java.debug.core.protocol.Responses._ import com.microsoft.java.debug.core.protocol.Types.Capabilities import monix.execution.Cancelable import monix.execution.Scheduler diff --git a/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala b/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala index 1b2ca82aa2..06f18d71e2 100644 --- a/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala +++ b/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala @@ -1,11 +1,11 @@ package bloop.dap +import java.io.Closeable import java.net.ConnectException import java.net.SocketException import java.net.SocketTimeoutException import java.util.NoSuchElementException import java.util.concurrent.TimeUnit.MILLISECONDS -import java.util.concurrent.TimeUnit.SECONDS import scala.collection.JavaConverters._ import scala.collection.mutable @@ -14,15 +14,19 @@ import scala.concurrent.Promise import scala.concurrent.TimeoutException import scala.concurrent.duration.Duration import scala.concurrent.duration.FiniteDuration +import scala.concurrent.duration._ import ch.epfl.scala.bsp import ch.epfl.scala.bsp.ScalaMainClass import ch.epfl.scala.debugadapter._ +import bloop.Cli import bloop.ScalaInstance +import bloop.cli.CommonOptions import bloop.cli.ExitStatus import bloop.data.Platform import bloop.data.Project +import bloop.engine.NoPool import bloop.engine.State import bloop.engine.tasks.RunMode import bloop.engine.tasks.Tasks @@ -30,6 +34,7 @@ import bloop.internal.build import bloop.internal.build.BuildTestInfo import bloop.io.AbsolutePath import bloop.io.Environment.lineSeparator +import bloop.logging.BspClientLogger import bloop.logging.Logger import bloop.logging.LoggerAction import bloop.logging.LoggerAction.LogInfoMessage @@ -48,85 +53,12 @@ import coursierapi.Dependency import coursierapi.Fetch import monix.execution.Ack import monix.reactive.Observer -import io.reactivex.Observable object DebugServerSpec extends DebugBspBaseSuite { private val ServerNotListening = new IllegalStateException("Server is not accepting connections") private val Success: ExitStatus = ExitStatus.Ok private val resolver = new BloopDebugToolsResolver(NoopLogger) - testTask("Performs Hot Code Replace", FiniteDuration(16, SECONDS)) { - val source2 = - """|/example/Main.scala - |package example - |class A { - | def m() = { - | println("B") - | } - |} - |""".stripMargin - TestUtil.withinWorkspace { workspace => - val source = - """|/Main.scala - |import example.A - |object Main { - | def main(args: Array[String]): Unit = { - | println("A") - | new A().m() - | } - |} - |""".stripMargin - val Asource = - """|/example/A.scala - |package example - |class A { - | def m() = { - | println("A") - | } - |} - |""".stripMargin - - val logger = new RecordingLogger(ansiCodesSupported = false) - val projectDep = TestProject(workspace, "a", List(Asource)) - val mainProject = TestProject(workspace, "r", List(source), List(projectDep)) - - loadBspStateWithTask(workspace, List(mainProject, projectDep), logger) { state => - val testState = state.compile(mainProject).toTestState - val buildProject = testState.getProjectFor(mainProject) - def srcFor(srcName: String) = - buildProject.sources.map(_.resolve(s"$srcName")).find(_.exists).get - val `A.scala` = srcFor("../../a/src/example/A.scala") - val breakpoints = breakpointsArgs(srcFor("Main.scala"), 5) - - val runner = mainRunner(mainProject, state) - - startDebugServer(runner) { server => - for { - client <- server.startConnection - _ <- client.initialize() - _ <- client.launch(noDebug = false) - _ <- client.initialized - bps <- client.setBreakpoints(breakpoints) - _ = assert(bps.breakpoints.forall(_.verified)) - _ <- client.configurationDone() - stopped <- client.stopped - _ = writeFile(`A.scala`, source2) - _ = state.compile(mainProject) - _ <- client.redefineClasses() - _ <- client.continue(stopped.threadId) - _ <- client.exited - _ <- client.terminated - _ <- Task.fromFuture(client.closedPromise.future) - output <- client.takeCurrentOutput - } yield { - assert(client.socket.isClosed) - assertNoDiff(output, "A\nB") - } - } - } - } - } - testTask("cancelling server closes server connection", FiniteDuration(10, SECONDS)) { startDebugServer(Task.now(Success)) { server => for { @@ -1037,6 +969,87 @@ object DebugServerSpec extends DebugBspBaseSuite { } } + testTask("hot code replace", 30.seconds) { + val mainSource = + """|/Main.scala + |object Main { + | def main(args: Array[String]): Unit = { + | val a = new example.A + | a.m() + | } + |} + |""".stripMargin + val originalSource = + """|/example/A.scala + |package example + |class A { + | def m() = { + | println("A") + | } + |} + |""".stripMargin + val modifiedSource = + """|/example/A.scala + |package example + |class A { + | def m() = { + | println("B") + | } + |} + |""".stripMargin + val logger = new RecordingLogger(ansiCodesSupported = false) + TestUtil.withinWorkspace { workspace => + val dependency = TestProject(workspace, "a", List(originalSource)) + val mainProject = TestProject(workspace, "main", List(mainSource), List(dependency)) + val configDir = TestProject.populateWorkspace(workspace, List(mainProject, dependency)) + + def cliCompile(project: TestProject) = { + val compileArgs = Array("compile", project.config.name, "--config-dir", configDir.syntax) + val compileAction = Cli.parse(compileArgs, CommonOptions.default) + Task.eval(Cli.run(compileAction, NoPool)).executeAsync + } + + def bspCommand() = createBspCommand(configDir) + val state = TestUtil.loadTestProject(configDir.underlying, logger) + openBspConnection(state, bspCommand, configDir, new BspClientLogger(logger)) + .withinSession { state => + val testState = state.compile(mainProject).toTestState + val `A.scala` = testState + .getProjectFor(dependency) + .sources + .map(_.resolve("example/A.scala")) + .find(_.exists) + .get + + val runner = mainRunner(mainProject, state) + startDebugServer(runner) { server => + for { + client <- server.startConnection + _ <- client.initialize() + _ <- client.launch(noDebug = false) + _ <- client.initialized + response <- client.setBreakpoints(breakpointsArgs(`A.scala`, 4)) + _ = assert(response.breakpoints.forall(_.verified)) + _ <- client.configurationDone() + stopped <- client.stopped + _ = writeFile(`A.scala`, modifiedSource) + _ <- cliCompile(mainProject) // another client trigger a compilation + _ = state.compile(mainProject) // noop + _ <- client.redefineClasses() + _ <- client.continue(stopped.threadId) + _ <- client.exited + _ <- client.terminated + _ <- Task.fromFuture(client.closedPromise.future) + output <- client.takeCurrentOutput + } yield { + assert(client.socket.isClosed) + assertNoDiff(output, "B") + } + } + } + } + } + private def startRemoteProcess(buildProject: Project, testState: TestState): Task[Int] = { val attachPort = Promise[Int]() @@ -1169,13 +1182,14 @@ object DebugServerSpec extends DebugBspBaseSuite { override def modules: Seq[Module] = Seq.empty override def libraries: Seq[Library] = Seq.empty override def unmanagedEntries: Seq[UnmanagedEntry] = Seq.empty - override val classesToUpdate = Observable.empty[Seq[String]] override def javaRuntime: Option[JavaRuntime] = None def name: String = "MockRunner" def run(listener: DebuggeeListener): CancelableFuture[Unit] = { - DapCancellableFuture.runAsync(task.map(_ => ()), defaultScheduler) + DapCancellableFuture.runAsync(task.map(_ => ()))(defaultScheduler) } def scalaVersion: ScalaVersion = ScalaVersion("2.12.17") + + override def observeClassUpdates(onClassUpdate: Seq[String] => Unit): Closeable = () => () } startDebugServer( @@ -1188,7 +1202,7 @@ object DebugServerSpec extends DebugBspBaseSuite { def startDebugServer( debuggee: Debuggee, - gracePeriod: Duration = Duration(5, SECONDS) + gracePeriod: Duration = 5.seconds )(f: TestServer => Task[Unit]): Task[Unit] = { val logger = new RecordingLogger(ansiCodesSupported = false) val dapLogger = new DebugServerLogger(logger) @@ -1215,7 +1229,7 @@ object DebugServerSpec extends DebugBspBaseSuite { override def close(): Unit = { cancel() val allClientsClosed = clients.map(c => Task.fromFuture(c.closedPromise.future)) - TestUtil.await(10, SECONDS)(Task.sequence(allClientsClosed)); () + TestUtil.await(10.seconds)(Task.sequence(allClientsClosed)); () } def startConnection: Task[DebugAdapterConnection] = Task { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 529618d071..ea331d8a85 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -45,7 +45,7 @@ object Dependencies { val asmVersion = "9.6" val snailgunVersion = "0.4.0" val ztExecVersion = "1.12" - val debugAdapterVersion = "4.0.0" + val debugAdapterVersion = "4.0.1" val bloopConfigVersion = "1.5.5" val semanticdbVersion = "4.7.8" val zinc = "org.scala-sbt" %% "zinc" % zincVersion