feat(build-plan): output invocations after running build scripts

src/cargo/core/compiler/build_config.rs

@@ -23,7 +23,7 @@ pub struct BuildConfig {
     /// Force Cargo to do a full rebuild and treat each target as changed.
     pub force_rebuild: bool,
     /// Output a build plan to stdout instead of actually compiling.
-    pub build_plan: bool,
+    pub build_plan: BuildPlanConfig,
     /// An optional wrapper, if any, used to wrap rustc invocations
     pub rustc_wrapper: Option<ProcessBuilder>,
     pub rustfix_diagnostic_server: RefCell<Option<RustfixDiagnosticServer>>,
@@ -97,7 +97,7 @@ impl BuildConfig {
             mode,
             message_format: MessageFormat::Human,
             force_rebuild: false,
-            build_plan: false,
+            build_plan: BuildPlanConfig(None),
             rustc_wrapper: None,
             rustfix_diagnostic_server: RefCell::new(None),
             cache_messages: config.cli_unstable().cache_messages,
@@ -127,6 +127,73 @@ pub enum MessageFormat {
     Short,
 }
 
+/// Configuration structure for the `--build-plan` option, mainly a wrapper
+/// for the `BuildPlanStage` selected (`None` if we're not generating a build
+/// plan, then compile all units as usual).
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct BuildPlanConfig(pub Option<BuildPlanStage>);
+
+/// Stage in the build process at which to generate the build plan. At later
+/// stages more units will be compiled and less will be outputted to the build
+/// plan. In one extreme, `Init` is the earliest possible stage where all units
+/// are routed to the build plan; the `None` of this `Option` in
+/// `BuildPlanConfig` can be thought of as the other extreme at the end, where
+/// all units are already compiled so the build plan is empty (not even
+/// generated).
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum BuildPlanStage {
+    /// Initial stage before anything is compiled (default option), all units
+    /// will be outputted as `Invocation`s in the build plan instead of calling
+    /// `rustc` for them (actually compiling). Ideally (but this is not
+    /// guaranteed), generating a build plan at the `Init` stage shouldn't
+    /// change anything in the working environment (e.g., it shouldn't compile,
+    /// update fingerprints, remove an old `rlib` file).
+    Init,
+    /// In contrast to `Init`, this stage signals to output the build plan after
+    /// *all* custom build scripts (and their dependencies) have been compiled
+    /// and executed.
+    ///
+    /// At this stage in the build process, the remaining units (which will be
+    /// outputted to the build plan instead of being compiled) are the ones
+    /// actually linked inside the final binary/library target, whereas the
+    /// compiled build scripts (and their dependencies) were used only for
+    /// their side effects to modify the `rustc` invocations (which will be
+    /// reflected in the build plan, something that doesn't happen at the
+    /// `Init` stage when the build script haven't been executed yet).
+    PostBuildScripts,
+}
+
+impl BuildPlanConfig {
+    /// Signal if the `--build-plan` option was requested (independent of the
+    /// particular stage selected).
+    pub fn requested(&self) -> bool {
+        self.0.is_some()
+    }
+
+    /// Signal if the `PostBuildScripts` stage was selected for the build plan
+    /// (implies `requested`).
+    pub fn post_build_scripts(&self) -> bool {
+        self.0 == Some(BuildPlanStage::PostBuildScripts)
+    }
+
+    /// Determine (based on the build plan stage selected, if any) whether the
+    /// current unit should be compiled, or included in the build plan instead
+    /// (returning `false`, we use a `bool` because there's no other
+    /// alternative of what can happen to a unit). The result is usually stored
+    /// in `Unit::to_be_compiled`. Based on these rules:
+    /// 1. If we didn't request a build plan, compile *any* unit.
+    /// 2. If we requested it at the `Init` stage, compile *nothing*.
+    /// 3. If we requested it at the `PostBuildScripts` stage, compile *only* a
+    ///    `unit_used_for_build_script` (usually determined by `UnitFor::build`).
+    pub fn should_compile_unit(&self, unit_used_for_build_script: bool) -> bool {
+        match self.0 {
+            None => true,
+            Some(BuildPlanStage::Init) => false,
+            Some(BuildPlanStage::PostBuildScripts) => unit_used_for_build_script,
+        }
+    }
+}
+
 /// The general "mode" for what to do.
 /// This is used for two purposes. The commands themselves pass this in to
 /// `compile_ws` to tell it the general execution strategy. This influences

src/cargo/core/compiler/build_plan.rs

@@ -115,7 +115,12 @@ impl BuildPlan {
         let deps = cx
             .dep_targets(unit)
             .iter()
-            .map(|dep| self.invocation_map[&dep.buildkey()])
+            .filter_map(|&dep| {
+                self.invocation_map.get(&dep.buildkey()).copied()
+                // In case `BuildPlanStage::PostBuildScripts` was selected some
+                // dependencies (custom build scripts and related) will be
+                // compiled and won't be present in the `invocation_map`.
+            })
             .collect();
         let invocation = Invocation::new(unit, deps);
         self.plan.invocations.push(invocation);

src/cargo/core/compiler/context/mod.rs

@@ -108,7 +108,6 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
     ) -> CargoResult<Compilation<'cfg>> {
         let mut queue = JobQueue::new(self.bcx);
         let mut plan = BuildPlan::new();
-        let build_plan = self.bcx.build_config.build_plan;
         self.prepare_units(export_dir, units)?;
         self.prepare()?;
         custom_build::build_map(&mut self, units)?;
@@ -127,7 +126,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
         // Now that we've figured out everything that we're going to do, do it!
         queue.execute(&mut self, &mut plan)?;
 
-        if build_plan {
+        if self.bcx.build_config.build_plan.requested() {
             plan.set_inputs(self.build_plan_inputs()?);
             plan.output_plan();
         }
@@ -437,7 +436,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
         let mut keys = self
             .unit_dependencies
             .keys()
-            .filter(|unit| !unit.mode.is_run_custom_build())
+            .filter(|unit| !unit.mode.is_run_custom_build() && unit.to_be_compiled)
             .collect::<Vec<_>>();
         // Sort for consistent error messages.
         keys.sort_unstable();

src/cargo/core/compiler/context/unit_dependencies.rs

@@ -382,6 +382,7 @@ fn dep_build_script<'a>(
                 bcx.profiles.get_profile_run_custom_build(&unit.profile),
                 unit.kind,
                 CompileMode::RunCustomBuild,
+                bcx.build_config.build_plan.should_compile_unit(true),
             );
 
             (unit, UnitFor::new_build())
@@ -421,8 +422,12 @@ fn new_unit<'a>(
         mode,
         bcx.build_config.release,
     );
-
-    bcx.units.intern(pkg, target, profile, kind, mode)
+    let to_be_compiled = bcx
+        .build_config
+        .build_plan
+        .should_compile_unit(unit_for.is_build());
+    bcx.units
+        .intern(pkg, target, profile, kind, mode, to_be_compiled)
 }
 
 /// Fill in missing dependencies for units of the `RunCustomBuild`

src/cargo/core/compiler/custom_build.rs

@@ -126,7 +126,7 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes
     let script_dir = cx.files().build_script_dir(build_script_unit);
     let script_out_dir = cx.files().build_script_out_dir(unit);
     let script_run_dir = cx.files().build_script_run_dir(unit);
-    let build_plan = bcx.build_config.build_plan;
+    let unit_is_to_be_compiled = unit.to_be_compiled;
     let invocation_name = unit.buildkey();
 
     if let Some(deps) = unit.pkg.manifest().metabuild() {
@@ -278,7 +278,7 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes
         // along to this custom build command. We're also careful to augment our
         // dynamic library search path in case the build script depended on any
         // native dynamic libraries.
-        if !build_plan {
+        if unit_is_to_be_compiled {
             let build_state = build_state.outputs.lock().unwrap();
             for (name, id) in lib_deps {
                 let key = (id, kind);
@@ -300,9 +300,11 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes
             if let Some(build_scripts) = build_scripts {
                 super::add_plugin_deps(&mut cmd, &build_state, &build_scripts, &host_target_root)?;
             }
+            // Release `build_state.outputs` lock in this inner scope, otherwise
+            // it will deadlock with the `build_state.insert` call below.
         }
 
-        if build_plan {
+        if !unit_is_to_be_compiled {
             state.build_plan(invocation_name, cmd.clone(), Arc::new(Vec::new()));
             return Ok(());
         }
@@ -373,7 +375,7 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes
         Ok(())
     });
 
-    let mut job = if cx.bcx.build_config.build_plan {
+    let mut job = if !unit.to_be_compiled {
         Job::new(Work::noop(), Freshness::Dirty)
     } else {
         fingerprint::prepare_target(cx, unit, false)?

src/cargo/core/compiler/job_queue.rs

@@ -422,7 +422,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> {
                 "{} [{}] target(s) in {}",
                 build_type, opt_type, time_elapsed
             );
-            if !cx.bcx.build_config.build_plan {
+            if !cx.bcx.build_config.build_plan.requested() {
                 cx.bcx.config.shell().status("Finished", message)?;
             }
             Ok(())
@@ -513,7 +513,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> {
             my_tx.send(Message::Finish(id, Artifact::All, res)).unwrap();
         };
 
-        if !cx.bcx.build_config.build_plan {
+        if !cx.bcx.build_config.build_plan.requested() {
             // Print out some nice progress information.
             self.note_working_on(cx.bcx.config, unit, fresh)?;
         }

src/cargo/core/compiler/mod.rs

@@ -24,7 +24,9 @@ use log::debug;
 use same_file::is_same_file;
 use serde::Serialize;
 
-pub use self::build_config::{BuildConfig, CompileMode, MessageFormat};
+pub use self::build_config::{
+    BuildConfig, BuildPlanConfig, BuildPlanStage, CompileMode, MessageFormat,
+};
 pub use self::build_context::{BuildContext, FileFlavor, TargetConfig, TargetInfo};
 use self::build_plan::BuildPlan;
 pub use self::compilation::{Compilation, Doctest};
@@ -112,7 +114,6 @@ fn compile<'a, 'cfg: 'a>(
     force_rebuild: bool,
 ) -> CargoResult<()> {
     let bcx = cx.bcx;
-    let build_plan = bcx.build_config.build_plan;
     if !cx.compiled.insert(*unit) {
         return Ok(());
     }
@@ -128,7 +129,7 @@ fn compile<'a, 'cfg: 'a>(
     } else if unit.mode.is_doc_test() {
         // We run these targets later, so this is just a no-op for now.
         Job::new(Work::noop(), Freshness::Fresh)
-    } else if build_plan {
+    } else if !unit.to_be_compiled {
         Job::new(rustc(cx, unit, &exec.clone())?, Freshness::Dirty)
     } else {
         let force = exec.force_rebuild(unit) || force_rebuild;
@@ -167,7 +168,7 @@ fn compile<'a, 'cfg: 'a>(
     for unit in cx.dep_targets(unit).iter() {
         compile(cx, jobs, plan, unit, exec, false)?;
     }
-    if build_plan {
+    if !unit.to_be_compiled {
         plan.add(cx, unit)?;
     }
 
@@ -183,7 +184,8 @@ fn rustc<'a, 'cfg>(
     if cx.is_primary_package(unit) {
         rustc.env("CARGO_PRIMARY_PACKAGE", "1");
     }
-    let build_plan = cx.bcx.build_config.build_plan;
+    let unit_is_to_be_compiled = unit.to_be_compiled;
+    let post_build_scripts = cx.bcx.build_config.build_plan.post_build_scripts();
 
     let name = unit.pkg.name().to_string();
     let buildkey = unit.buildkey();
@@ -243,7 +245,11 @@ fn rustc<'a, 'cfg>(
         // previous build scripts, we include them in the rustc invocation.
         if let Some(build_deps) = build_deps {
             let build_state = build_state.outputs.lock().unwrap();
-            if !build_plan {
+            if unit_is_to_be_compiled || post_build_scripts {
+                // Even if we're not compiling this unit, if a build plan was
+                // requested *after* the custom build scripts are executed this
+                // information generated by them will be available (and is
+                // exactly what we want to include in the build plan).
                 add_native_deps(
                     &mut rustc,
                     &build_state,
@@ -257,14 +263,16 @@ fn rustc<'a, 'cfg>(
             add_custom_env(&mut rustc, &build_state, current_id, kind)?;
         }
 
-        for output in outputs.iter() {
-            // If there is both an rmeta and rlib, rustc will prefer to use the
-            // rlib, even if it is older. Therefore, we must delete the rlib to
-            // force using the new rmeta.
-            if output.path.extension() == Some(OsStr::new("rmeta")) {
-                let dst = root.join(&output.path).with_extension("rlib");
-                if dst.exists() {
-                    paths::remove_file(&dst)?;
+        if unit_is_to_be_compiled {
+            for output in outputs.iter() {
+                // If there is both an rmeta and rlib, rustc will prefer to use the
+                // rlib, even if it is older. Therefore, we must delete the rlib to
+                // force using the new rmeta.
+                if output.path.extension() == Some(OsStr::new("rmeta")) {
+                    let dst = root.join(&output.path).with_extension("rlib");
+                    if dst.exists() {
+                        paths::remove_file(&dst)?;
+                    }
                 }
             }
         }
@@ -284,7 +292,7 @@ fn rustc<'a, 'cfg>(
 
         state.running(&rustc);
         let timestamp = paths::set_invocation_time(&fingerprint_dir)?;
-        if build_plan {
+        if !unit_is_to_be_compiled {
             state.build_plan(buildkey, rustc.clone(), outputs.clone());
         } else {
             exec.exec(

src/cargo/core/compiler/unit.rs

@@ -48,6 +48,19 @@ pub struct UnitInner<'a> {
     pub kind: Kind,
     /// The "mode" this unit is being compiled for. See [`CompileMode`] for more details.
     pub mode: CompileMode,
+    /// Used only for the `--build-plan` option, this flag indicates whether this unit is intended
+    /// to be passed to a `rustc` call (roughly speaking, to be "compiled"), the usual case, or if
+    /// instead such invocation will be registered in a build plan (`false`).
+    pub to_be_compiled: bool,
+    // This flag adds a new dimension to allow for the (otherwise) same unit to exist as both, a
+    // compiled build script dependency and a "normal" dependency that will be outputted to the
+    // build plan when requesting the `post-build-scripts` stage, which departs from the basic
+    // all/none scenario of routing all units either to `rustc` or to the build plan.
+    // Without this flag our internal bookkeeping  structures like `Context::unit_dependencies`
+    // and `Context::compiled` would confuse the two and ignore one of them. However, we do *not*
+    // want to add this attribute to the metadata or fingerprint (hence it exists as standalone
+    // here and not inside other structures like `CompileMode` or `Profile`), since even if they
+    // may have different purposes they should be able to be reused.
 }
 
 impl UnitInner<'_> {
@@ -94,13 +107,18 @@ impl<'a> Deref for Unit<'a> {
 
 impl<'a> fmt::Debug for Unit<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("Unit")
+        let mut debug_struct = f.debug_struct("Unit");
+        debug_struct
             .field("pkg", &self.pkg)
             .field("target", &self.target)
             .field("profile", &self.profile)
             .field("kind", &self.kind)
-            .field("mode", &self.mode)
-            .finish()
+            .field("mode", &self.mode);
+        if !self.to_be_compiled {
+            // Only print in the (rare) case of use of using `--build-plan`.
+            debug_struct.field("to_be_compiled", &false);
+        }
+        debug_struct.finish()
     }
 }
 
@@ -139,13 +157,15 @@ impl<'a> UnitInterner<'a> {
         profile: Profile,
         kind: Kind,
         mode: CompileMode,
+        to_be_compiled: bool,
     ) -> Unit<'a> {
         let inner = self.intern_inner(&UnitInner {
             pkg,
             target,
             profile,
             kind,
             mode,
+            to_be_compiled,
         });
         Unit { inner }
     }

src/cargo/ops/cargo_clean.rs

@@ -93,7 +93,7 @@ pub fn clean(ws: &Workspace<'_>, opts: &CleanOptions<'_>) -> CargoResult<()> {
                                 opts.release,
                             )
                         };
-                        units.push(bcx.units.intern(pkg, target, profile, *kind, *mode));
+                        units.push(bcx.units.intern(pkg, target, profile, *kind, *mode, true));
                     }
                 }
             }

src/cargo/ops/cargo_compile.rs

@@ -653,7 +653,9 @@ fn generate_targets<'a>(
             target_mode,
             bcx.build_config.release,
         );
-        bcx.units.intern(pkg, target, profile, kind, target_mode)
+        let to_be_compiled = bcx.build_config.build_plan.should_compile_unit(false);
+        bcx.units
+            .intern(pkg, target, profile, kind, target_mode, to_be_compiled)
     };
 
     // Create a list of proposed targets.