Rollup merge of rust-lang#52716 - tromey:rustup-lldb, r=Mark-Simulacrum

Add lldb to the build

This optionally adds lldb (and clang, which it needs) to the build.

Because rust uses LLVM 7, and because clang 7 is not yet released, a
recent git master version of clang is used.

The lldb that is used includes the Rust plugin.

lldb is only built when asked for, or when doing a nightly build on
macOS.  Only macOS is done for now due to difficulties with the Python
dependency.

.gitmodules

@@ -56,3 +56,11 @@
 [submodule "src/libbacktrace"]
 	path = src/libbacktrace
 	url = https://github.com/rust-lang-nursery/libbacktrace
+[submodule "src/lldb"]
+	path = src/lldb
+	url = https://github.com/rust-lang-nursery/lldb/
+	branch = rust-release-70
+[submodule "src/clang"]
+	path = src/clang
+	url = https://github.com/rust-lang-nursery/clang/
+	branch = master

config.toml.example

@@ -354,6 +354,10 @@
 # sysroot.
 #llvm-tools = false
 
+# Indicates whether LLDB will be made available in the sysroot.
+# This is only built if LLVM is also being built.
+#lldb = false
+
 # Whether to deny warnings in crates
 #deny-warnings = true
 

src/Cargo.lock

@@ -1011,7 +1011,7 @@ name = "installer"
 version = "0.0.0"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",

src/bootstrap/builder.rs

@@ -461,6 +461,7 @@ impl<'a> Builder<'a> {
                 dist::Rustfmt,
                 dist::Clippy,
                 dist::LlvmTools,
+                dist::Lldb,
                 dist::Extended,
                 dist::HashSign
             ),

src/bootstrap/config.rs

@@ -87,6 +87,7 @@ pub struct Config {
     pub llvm_link_jobs: Option<u32>,
 
     pub lld_enabled: bool,
+    pub lldb_enabled: bool,
     pub llvm_tools_enabled: bool,
 
     // rust codegen options
@@ -310,6 +311,7 @@ struct Rust {
     codegen_backends_dir: Option<String>,
     wasm_syscall: Option<bool>,
     lld: Option<bool>,
+    lldb: Option<bool>,
     llvm_tools: Option<bool>,
     deny_warnings: Option<bool>,
     backtrace_on_ice: Option<bool>,
@@ -538,6 +540,7 @@ impl Config {
             }
             set(&mut config.wasm_syscall, rust.wasm_syscall);
             set(&mut config.lld_enabled, rust.lld);
+            set(&mut config.lldb_enabled, rust.lldb);
             set(&mut config.llvm_tools_enabled, rust.llvm_tools);
             config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false);
             config.rustc_default_linker = rust.default_linker.clone();

src/bootstrap/configure.py

@@ -335,6 +335,7 @@ def set(key, value):
     elif option.name == 'full-tools':
         set('rust.codegen-backends', ['llvm', 'emscripten'])
         set('rust.lld', True)
+        set('rust.lldb', True)
         set('rust.llvm-tools', True)
         set('build.extended', True)
     elif option.name == 'option-checking':

src/bootstrap/dist.rs

@@ -47,6 +47,8 @@ pub fn pkgname(builder: &Builder, component: &str) -> String {
         format!("{}-{}", component, builder.rustfmt_package_vers())
     } else if component == "llvm-tools" {
         format!("{}-{}", component, builder.llvm_tools_package_vers())
+    } else if component == "lldb" {
+        format!("{}-{}", component, builder.lldb_package_vers())
     } else {
         assert!(component.starts_with("rust"));
         format!("{}-{}", component, builder.rust_package_vers())
@@ -1396,6 +1398,7 @@ impl Step for Extended {
         let rls_installer = builder.ensure(Rls { stage, target });
         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
         let clippy_installer = builder.ensure(Clippy { stage, target });
+        let lldb_installer = builder.ensure(Lldb { target });
         let mingw_installer = builder.ensure(Mingw { host: target });
         let analysis_installer = builder.ensure(Analysis {
             compiler: builder.compiler(stage, self.host),
@@ -1435,6 +1438,7 @@ impl Step for Extended {
         tarballs.extend(clippy_installer.clone());
         tarballs.extend(rustfmt_installer.clone());
         tarballs.extend(llvm_tools_installer.clone());
+        tarballs.extend(lldb_installer.clone());
         tarballs.push(analysis_installer);
         tarballs.push(std_installer);
         if builder.config.docs {
@@ -1869,6 +1873,7 @@ impl Step for HashSign {
         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
         cmd.arg(builder.llvm_tools_package_vers());
+        cmd.arg(builder.lldb_package_vers());
         cmd.arg(addr);
 
         builder.create_dir(&distdir(builder));
@@ -1963,3 +1968,121 @@ impl Step for LlvmTools {
         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
     }
 }
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct Lldb {
+    pub target: Interned<String>,
+}
+
+impl Step for Lldb {
+    type Output = Option<PathBuf>;
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("src/lldb")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(Lldb {
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder) -> Option<PathBuf> {
+        let target = self.target;
+
+        if builder.config.dry_run {
+            return None;
+        }
+
+        let bindir = builder
+            .llvm_out(target)
+            .join("bin");
+        let lldb_exe = bindir.join(exe("lldb", &target));
+        if !lldb_exe.exists() {
+            return None;
+        }
+
+        builder.info(&format!("Dist Lldb ({})", target));
+        let src = builder.src.join("src/lldb");
+        let name = pkgname(builder, "lldb");
+
+        let tmp = tmpdir(builder);
+        let image = tmp.join("lldb-image");
+        drop(fs::remove_dir_all(&image));
+
+        // Prepare the image directory
+        let dst = image.join("bin");
+        t!(fs::create_dir_all(&dst));
+        for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
+            let exe = bindir.join(exe(program, &target));
+            builder.install(&exe, &dst, 0o755);
+        }
+
+        // The libraries.
+        let libdir = builder.llvm_out(target).join("lib");
+        let dst = image.join("lib");
+        t!(fs::create_dir_all(&dst));
+        for entry in t!(fs::read_dir(&libdir)) {
+            // let entry = t!(entry);
+            let entry = entry.unwrap();
+            if let Ok(name) = entry.file_name().into_string() {
+                if name.starts_with("liblldb.") && !name.ends_with(".a") {
+                    if t!(entry.file_type()).is_symlink() {
+                        builder.copy_to_folder(&entry.path(), &dst);
+                    } else {
+                       builder.install(&entry.path(), &dst, 0o755);
+                    }
+                }
+            }
+        }
+
+        // The lldb scripts might be installed in lib/python$version
+        // or in lib64/python$version.  If lib64 exists, use it;
+        // otherwise lib.
+        let libdir = builder.llvm_out(target).join("lib64");
+        let (libdir, libdir_name) = if libdir.exists() {
+            (libdir, "lib64")
+        } else {
+            (builder.llvm_out(target).join("lib"), "lib")
+        };
+        for entry in t!(fs::read_dir(&libdir)) {
+            let entry = t!(entry);
+            if let Ok(name) = entry.file_name().into_string() {
+                if name.starts_with("python") {
+                    let dst = image.join(libdir_name)
+                        .join(entry.file_name());
+                    t!(fs::create_dir_all(&dst));
+                    builder.cp_r(&entry.path(), &dst);
+                    break;
+                }
+            }
+        }
+
+        // Prepare the overlay
+        let overlay = tmp.join("lldb-overlay");
+        drop(fs::remove_dir_all(&overlay));
+        builder.create_dir(&overlay);
+        builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
+        builder.create(&overlay.join("version"), &builder.lldb_vers());
+
+        // Generate the installer tarball
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+            .arg("--product-name=Rust")
+            .arg("--rel-manifest-dir=rustlib")
+            .arg("--success-message=lldb-installed.")
+            .arg("--image-dir").arg(&image)
+            .arg("--work-dir").arg(&tmpdir(builder))
+            .arg("--output-dir").arg(&distdir(builder))
+            .arg("--non-installed-overlay").arg(&overlay)
+            .arg(format!("--package-name={}-{}", name, target))
+            .arg("--legacy-manifest-dirs=rustlib,cargo")
+            .arg("--component-name=lldb-preview");
+
+
+        builder.run(&mut cmd);
+        Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
+    }
+}

src/bootstrap/lib.rs

@@ -151,6 +151,11 @@ use std::process::{self, Command};
 use std::slice;
 use std::str;
 
+#[cfg(unix)]
+use std::os::unix::fs::symlink as symlink_file;
+#[cfg(windows)]
+use std::os::windows::fs::symlink_file;
+
 use build_helper::{run_silent, run_suppressed, try_run_silent, try_run_suppressed, output, mtime};
 use filetime::FileTime;
 
@@ -1005,6 +1010,14 @@ impl Build {
         self.rust_version()
     }
 
+    fn lldb_package_vers(&self) -> String {
+        self.package_vers(&self.rust_version())
+    }
+
+    fn lldb_vers(&self) -> String {
+        self.rust_version()
+    }
+
     /// Returns the `version` string associated with this compiler for Rust
     /// itself.
     ///
@@ -1123,20 +1136,24 @@ impl Build {
     pub fn copy(&self, src: &Path, dst: &Path) {
         if self.config.dry_run { return; }
         let _ = fs::remove_file(&dst);
-        // Attempt to "easy copy" by creating a hard link (symlinks don't work on
-        // windows), but if that fails just fall back to a slow `copy` operation.
-        if let Ok(()) = fs::hard_link(src, dst) {
-            return
-        }
-        if let Err(e) = fs::copy(src, dst) {
-            panic!("failed to copy `{}` to `{}`: {}", src.display(),
-                dst.display(), e)
+        let metadata = t!(src.symlink_metadata());
+        if metadata.file_type().is_symlink() {
+            let link = t!(fs::read_link(src));
+            t!(symlink_file(link, dst));
+        } else if let Ok(()) = fs::hard_link(src, dst) {
+            // Attempt to "easy copy" by creating a hard link
+            // (symlinks don't work on windows), but if that fails
+            // just fall back to a slow `copy` operation.
+        } else {
+            if let Err(e) = fs::copy(src, dst) {
+                panic!("failed to copy `{}` to `{}`: {}", src.display(),
+                       dst.display(), e)
+            }
+            t!(fs::set_permissions(dst, metadata.permissions()));
+            let atime = FileTime::from_last_access_time(&metadata);
+            let mtime = FileTime::from_last_modification_time(&metadata);
+            t!(filetime::set_file_times(dst, atime, mtime));
         }
-        let metadata = t!(src.metadata());
-        t!(fs::set_permissions(dst, metadata.permissions()));
-        let atime = FileTime::from_last_access_time(&metadata);
-        let mtime = FileTime::from_last_modification_time(&metadata);
-        t!(filetime::set_file_times(dst, atime, mtime));
     }
 
     /// Search-and-replaces within a file. (Not maximally efficiently: allocates a

src/bootstrap/native.rs

@@ -149,7 +149,6 @@ impl Step for Llvm {
            .define("WITH_POLLY", "OFF")
            .define("LLVM_ENABLE_TERMINFO", "OFF")
            .define("LLVM_ENABLE_LIBEDIT", "OFF")
-           .define("LLVM_ENABLE_LIBXML2", "OFF")
            .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string())
            .define("LLVM_TARGET_ARCH", target.split('-').next().unwrap())
            .define("LLVM_DEFAULT_TARGET_TRIPLE", target);
@@ -163,19 +162,28 @@ impl Step for Llvm {
         cfg.define("LLVM_OCAML_INSTALL_PATH",
             env::var_os("LLVM_OCAML_INSTALL_PATH").unwrap_or_else(|| "usr/lib/ocaml".into()));
 
+        // Build clang and lldb if asked, or if doing a macOS nightly
+        // build.  LLVM_ENABLE_PROJECTS allows them to be checked out
+        // parallel to src/llvm, and makes conditionally building them
+        // simpler.
+        let want_lldb = builder.config.lldb_enabled ||
+            (target.contains("apple-darwin") &&
+             builder.config.channel == "nightly");
+
         // This setting makes the LLVM tools link to the dynamic LLVM library,
         // which saves both memory during parallel links and overall disk space
         // for the tools.  We don't distribute any of those tools, so this is
         // just a local concern.  However, it doesn't work well everywhere.
         //
         // If we are shipping llvm tools then we statically link them LLVM
         if (target.contains("linux-gnu") || target.contains("apple-darwin")) &&
-            !builder.config.llvm_tools_enabled {
+            !builder.config.llvm_tools_enabled &&
+            !want_lldb {
                 cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
         }
 
         // For distribution we want the LLVM tools to be *statically* linked to libstdc++
-        if builder.config.llvm_tools_enabled {
+        if builder.config.llvm_tools_enabled || want_lldb {
             if !target.contains("windows") {
                 if target.contains("apple") {
                     cfg.define("CMAKE_EXE_LINKER_FLAGS", "-static-libstdc++");
@@ -196,6 +204,12 @@ impl Step for Llvm {
             cfg.define("LLVM_BUILD_32_BITS", "ON");
         }
 
+        if want_lldb {
+            cfg.define("LLVM_ENABLE_PROJECTS", "clang;lldb");
+        } else {
+            cfg.define("LLVM_ENABLE_LIBXML2", "OFF");
+        }
+
         if let Some(num_linkers) = builder.config.llvm_link_jobs {
             if num_linkers > 0 {
                 cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string());

src/tools/tidy/src/lib.rs

@@ -56,8 +56,10 @@ pub mod libcoretest;
 
 fn filter_dirs(path: &Path) -> bool {
     let skip = [
+        "src/clang",
         "src/dlmalloc",
         "src/jemalloc",
+        "src/lldb",
         "src/llvm",
         "src/llvm-emscripten",
         "src/libbacktrace",