Refactor opt-dist to simplify local building

.github/workflows/ci.yml

@@ -393,7 +393,7 @@ jobs:
           - name: dist-x86_64-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-2019-8core-32gb
           - name: dist-i686-msvc

Cargo.lock

@@ -858,14 +858,38 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "darling"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
+dependencies = [
+ "darling_core 0.14.4",
+ "darling_macro 0.14.4",
+]
+
 [[package]]
 name = "darling"
 version = "0.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
 dependencies = [
- "darling_core",
- "darling_macro",
+ "darling_core 0.20.3",
+ "darling_macro 0.20.3",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -882,13 +906,24 @@ dependencies = [
  "syn 2.0.29",
 ]
 
+[[package]]
+name = "darling_macro"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
+dependencies = [
+ "darling_core 0.14.4",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "darling_macro"
 version = "0.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
- "darling_core",
+ "darling_core 0.20.3",
  "quote",
  "syn 2.0.29",
 ]
@@ -919,6 +954,37 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "derive_builder"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
+dependencies = [
+ "derive_builder_macro",
+]
+
+[[package]]
+name = "derive_builder_core"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
+dependencies = [
+ "darling 0.14.4",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
+dependencies = [
+ "derive_builder_core",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "derive_more"
 version = "0.99.17"
@@ -938,7 +1004,7 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d"
 dependencies = [
- "darling",
+ "darling 0.20.3",
  "proc-macro2",
  "quote",
  "syn 2.0.29",
@@ -2584,6 +2650,8 @@ dependencies = [
  "anyhow",
  "build_helper",
  "camino",
+ "clap",
+ "derive_builder",
  "env_logger 0.10.0",
  "fs_extra",
  "glob",

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \
       --set rust.lto=thin
 
 ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
-    ./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \
+    ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \
     --host $HOSTS --target $HOSTS \
     --include-default-paths \
     build-manifest bootstrap

src/ci/github-actions/ci.yml

@@ -624,7 +624,7 @@ jobs:
                 --target=x86_64-pc-windows-msvc
                 --enable-full-tools
                 --enable-profiler
-              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
+              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             <<: *job-windows-8c
 

src/tools/opt-dist/Cargo.toml

@@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 glob = "0.3"
 tempfile = "3.5"
+derive_builder = "0.12"
+clap = { version = "4", features = ["derive"] }

src/tools/opt-dist/src/environment.rs

@@ -0,0 +1,108 @@
+use camino::Utf8PathBuf;
+use derive_builder::Builder;
+
+#[derive(Builder)]
+pub struct Environment {
+    host_triple: String,
+    python_binary: String,
+    /// The rustc checkout, where the compiler source is located.
+    checkout_dir: Utf8PathBuf,
+    /// The main directory where the build occurs. Stage0 rustc and cargo have to be available in
+    /// this directory before `opt-dist` is started.
+    build_dir: Utf8PathBuf,
+    /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
+    /// will be stored.
+    artifact_dir: Utf8PathBuf,
+    /// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
+    host_llvm_dir: Utf8PathBuf,
+    /// List of test paths that should be skipped when testing the optimized artifacts.
+    skipped_tests: Vec<String>,
+    /// Directory containing a pre-built rustc-perf checkout.
+    #[builder(default)]
+    prebuilt_rustc_perf: Option<Utf8PathBuf>,
+    use_bolt: bool,
+    shared_llvm: bool,
+}
+
+impl Environment {
+    pub fn host_triple(&self) -> &str {
+        &self.host_triple
+    }
+
+    pub fn python_binary(&self) -> &str {
+        &self.python_binary
+    }
+
+    pub fn checkout_path(&self) -> Utf8PathBuf {
+        self.checkout_dir.clone()
+    }
+
+    pub fn build_root(&self) -> Utf8PathBuf {
+        self.build_dir.clone()
+    }
+
+    pub fn build_artifacts(&self) -> Utf8PathBuf {
+        self.build_root().join("build").join(&self.host_triple)
+    }
+
+    pub fn artifact_dir(&self) -> Utf8PathBuf {
+        self.artifact_dir.clone()
+    }
+
+    pub fn cargo_stage_0(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage0")
+            .join("bin")
+            .join(format!("cargo{}", executable_extension()))
+    }
+
+    pub fn rustc_stage_0(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage0")
+            .join("bin")
+            .join(format!("rustc{}", executable_extension()))
+    }
+
+    pub fn rustc_stage_2(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage2")
+            .join("bin")
+            .join(format!("rustc{}", executable_extension()))
+    }
+
+    pub fn prebuilt_rustc_perf(&self) -> Option<Utf8PathBuf> {
+        self.prebuilt_rustc_perf.clone()
+    }
+
+    /// Path to the built rustc-perf benchmark suite.
+    pub fn rustc_perf_dir(&self) -> Utf8PathBuf {
+        self.artifact_dir.join("rustc-perf")
+    }
+
+    pub fn host_llvm_dir(&self) -> Utf8PathBuf {
+        self.host_llvm_dir.clone()
+    }
+
+    pub fn use_bolt(&self) -> bool {
+        self.use_bolt
+    }
+
+    pub fn supports_shared_llvm(&self) -> bool {
+        self.shared_llvm
+    }
+
+    pub fn skipped_tests(&self) -> &[String] {
+        &self.skipped_tests
+    }
+}
+
+/// What is the extension of binary executables on this platform?
+#[cfg(target_family = "unix")]
+pub fn executable_extension() -> &'static str {
+    ""
+}
+
+#[cfg(target_family = "windows")]
+pub fn executable_extension() -> &'static str {
+    ".exe"
+}