Rollup of 10 pull requests

src/bootstrap/builder/tests.rs

@@ -54,6 +54,11 @@ fn dist_baseline() {
         &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },]
     );
     assert_eq!(first(builder.cache.all::<dist::Src>()), &[dist::Src]);
+    // Make sure rustdoc is only built once.
+    assert_eq!(
+        first(builder.cache.all::<tool::Rustdoc>()),
+        &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },]
+    );
 }
 
 #[test]
@@ -414,3 +419,77 @@ fn test_exclude() {
     // Ensure other tests are not affected.
     assert!(builder.cache.contains::<test::RustdocUi>());
 }
+
+#[test]
+fn doc_default() {
+    let mut config = configure(&[], &[]);
+    config.compiler_docs = true;
+    config.cmd = Subcommand::Doc { paths: Vec::new(), open: false };
+    let build = Build::new(config);
+    let mut builder = Builder::new(&build);
+    builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]);
+    let a = INTERNER.intern_str("A");
+
+    // error_index_generator uses stage 1 to share rustdoc artifacts with the
+    // rustdoc tool.
+    assert_eq!(
+        first(builder.cache.all::<doc::ErrorIndex>()),
+        &[doc::ErrorIndex { compiler: Compiler { host: a, stage: 1 }, target: a },]
+    );
+    assert_eq!(
+        first(builder.cache.all::<tool::ErrorIndex>()),
+        &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }]
+    );
+    // This is actually stage 1, but Rustdoc::run swaps out the compiler with
+    // stage minus 1 if --stage is not 0. Very confusing!
+    assert_eq!(
+        first(builder.cache.all::<tool::Rustdoc>()),
+        &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },]
+    );
+}
+
+#[test]
+fn test_docs() {
+    // Behavior of `x.py test` doing various documentation tests.
+    let mut config = configure(&[], &[]);
+    config.cmd = Subcommand::Test {
+        paths: vec![],
+        test_args: vec![],
+        rustc_args: vec![],
+        fail_fast: true,
+        doc_tests: DocTests::Yes,
+        bless: false,
+        compare_mode: None,
+        rustfix_coverage: false,
+        pass: None,
+    };
+    let build = Build::new(config);
+    let mut builder = Builder::new(&build);
+    builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]);
+    let a = INTERNER.intern_str("A");
+
+    // error_index_generator uses stage 1 to share rustdoc artifacts with the
+    // rustdoc tool.
+    assert_eq!(
+        first(builder.cache.all::<doc::ErrorIndex>()),
+        &[doc::ErrorIndex { compiler: Compiler { host: a, stage: 1 }, target: a },]
+    );
+    assert_eq!(
+        first(builder.cache.all::<tool::ErrorIndex>()),
+        &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }]
+    );
+    // Unfortunately rustdoc is built twice. Once from stage1 for compiletest
+    // (and other things), and once from stage0 for std crates. Ideally it
+    // would only be built once. If someone wants to fix this, it might be
+    // worth investigating if it would be possible to test std from stage1.
+    // Note that the stages here are +1 than what they actually are because
+    // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is
+    // not 0.
+    assert_eq!(
+        first(builder.cache.all::<tool::Rustdoc>()),
+        &[
+            tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } },
+            tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },
+        ]
+    );
+}

src/bootstrap/doc.rs

@@ -518,8 +518,7 @@ impl Step for Rustc {
         let out = builder.compiler_doc_out(target);
         t!(fs::create_dir_all(&out));
 
-        // Get the correct compiler for this stage.
-        let compiler = builder.compiler_for(stage, builder.config.build, target);
+        let compiler = builder.compiler(stage, builder.config.build);
 
         if !builder.config.compiler_docs {
             builder.info("\tskipping - compiler/librustdoc docs disabled");
@@ -599,8 +598,7 @@ impl Step for Rustdoc {
         let out = builder.compiler_doc_out(target);
         t!(fs::create_dir_all(&out));
 
-        // Get the correct compiler for this stage.
-        let compiler = builder.compiler_for(stage, builder.config.build, target);
+        let compiler = builder.compiler(stage, builder.config.build);
 
         if !builder.config.compiler_docs {
             builder.info("\tskipping - compiler/librustdoc docs disabled");
@@ -639,9 +637,10 @@ impl Step for Rustdoc {
     }
 }
 
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct ErrorIndex {
-    target: Interned<String>,
+    pub compiler: Compiler,
+    pub target: Interned<String>,
 }
 
 impl Step for ErrorIndex {
@@ -655,26 +654,26 @@ impl Step for ErrorIndex {
     }
 
     fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(ErrorIndex { target: run.target });
+        let target = run.target;
+        // error_index_generator depends on librustdoc. Use the compiler that
+        // is normally used to build rustdoc for other documentation so that
+        // it shares the same artifacts.
+        let compiler =
+            run.builder.compiler_for(run.builder.top_stage, run.builder.config.build, target);
+        run.builder.ensure(ErrorIndex { compiler, target });
     }
 
     /// Generates the HTML rendered error-index by running the
     /// `error_index_generator` tool.
     fn run(self, builder: &Builder<'_>) {
-        let target = self.target;
-
-        builder.info(&format!("Documenting error index ({})", target));
-        let out = builder.doc_out(target);
+        builder.info(&format!("Documenting error index ({})", self.target));
+        let out = builder.doc_out(self.target);
         t!(fs::create_dir_all(&out));
-        let compiler = builder.compiler(2, builder.config.build);
-        let mut index = tool::ErrorIndex::command(builder, compiler);
+        let mut index = tool::ErrorIndex::command(builder, self.compiler);
         index.arg("html");
         index.arg(out.join("error-index.html"));
         index.arg(crate::channel::CFG_RELEASE_NUM);
 
-        // FIXME: shouldn't have to pass this env var
-        index.env("CFG_BUILD", &builder.config.build);
-
         builder.run(&mut index);
     }
 }

src/bootstrap/native.rs

@@ -175,6 +175,17 @@ impl Step for Llvm {
             cfg.define("LLVM_ENABLE_ZLIB", "OFF");
         }
 
+        // Are we compiling for iOS/tvOS?
+        if target.contains("apple-ios") || target.contains("apple-tvos") {
+            // These two defines prevent CMake from automatically trying to add a MacOSX sysroot, which leads to a compiler error.
+            cfg.define("CMAKE_OSX_SYSROOT", "/");
+            cfg.define("CMAKE_OSX_DEPLOYMENT_TARGET", "");
+            // Prevent cmake from adding -bundle to CFLAGS automatically, which leads to a compiler error because "-bitcode_bundle" also gets added.
+            cfg.define("LLVM_ENABLE_PLUGINS", "OFF");
+            // Zlib fails to link properly, leading to a compiler error.
+            cfg.define("LLVM_ENABLE_ZLIB", "OFF");
+        }
+
         if builder.config.llvm_thin_lto {
             cfg.define("LLVM_ENABLE_LTO", "Thin");
             if !target.contains("apple") {
@@ -412,6 +423,14 @@ fn configure_cmake(
     if let Some(ref s) = builder.config.llvm_cflags {
         cflags.push_str(&format!(" {}", s));
     }
+    // Some compiler features used by LLVM (such as thread locals) will not work on a min version below iOS 10.
+    if target.contains("apple-ios") {
+        if target.contains("86-") {
+            cflags.push_str(" -miphonesimulator-version-min=10.0");
+        } else {
+            cflags.push_str(" -miphoneos-version-min=10.0");
+        }
+    }
     cfg.define("CMAKE_C_FLAGS", cflags);
     let mut cxxflags = builder.cflags(target, GitRepo::Llvm).join(" ");
     if builder.config.llvm_static_stdcpp && !target.contains("msvc") && !target.contains("netbsd") {

src/bootstrap/test.rs

@@ -1454,8 +1454,11 @@ impl Step for ErrorIndex {
     }
 
     fn make_run(run: RunConfig<'_>) {
-        run.builder
-            .ensure(ErrorIndex { compiler: run.builder.compiler(run.builder.top_stage, run.host) });
+        // error_index_generator depends on librustdoc. Use the compiler that
+        // is normally used to build rustdoc for other tests (like compiletest
+        // tests in src/test/rustdoc) so that it shares the same artifacts.
+        let compiler = run.builder.compiler_for(run.builder.top_stage, run.host, run.host);
+        run.builder.ensure(ErrorIndex { compiler });
     }
 
     /// Runs the error index generator tool to execute the tests located in the error
@@ -1467,22 +1470,23 @@ impl Step for ErrorIndex {
     fn run(self, builder: &Builder<'_>) {
         let compiler = self.compiler;
 
-        builder.ensure(compile::Std { compiler, target: compiler.host });
-
         let dir = testdir(builder, compiler.host);
         t!(fs::create_dir_all(&dir));
         let output = dir.join("error-index.md");
 
-        let mut tool = tool::ErrorIndex::command(
-            builder,
-            builder.compiler(compiler.stage, builder.config.build),
-        );
-        tool.arg("markdown").arg(&output).env("CFG_BUILD", &builder.config.build);
+        let mut tool = tool::ErrorIndex::command(builder, compiler);
+        tool.arg("markdown").arg(&output);
 
-        builder.info(&format!("Testing error-index stage{}", compiler.stage));
+        // Use the rustdoc that was built by self.compiler. This copy of
+        // rustdoc is shared with other tests (like compiletest tests in
+        // src/test/rustdoc). This helps avoid building rustdoc multiple
+        // times.
+        let rustdoc_compiler = builder.compiler(builder.top_stage, builder.config.build);
+        builder.info(&format!("Testing error-index stage{}", rustdoc_compiler.stage));
         let _time = util::timeit(&builder);
         builder.run_quiet(&mut tool);
-        markdown_test(builder, compiler, &output);
+        builder.ensure(compile::Std { compiler: rustdoc_compiler, target: rustdoc_compiler.host });
+        markdown_test(builder, rustdoc_compiler, &output);
     }
 }
 
@@ -1797,9 +1801,13 @@ impl Step for CrateRustdoc {
 
     fn run(self, builder: &Builder<'_>) {
         let test_kind = self.test_kind;
+        let target = self.host;
 
-        let compiler = builder.compiler(builder.top_stage, self.host);
-        let target = compiler.host;
+        // Use the previous stage compiler to reuse the artifacts that are
+        // created when running compiletest for src/test/rustdoc. If this used
+        // `compiler`, then it would cause rustdoc to be built *again*, which
+        // isn't really necessary.
+        let compiler = builder.compiler_for(builder.top_stage, target, target);
         builder.ensure(compile::Rustc { compiler, target });
 
         let mut cargo = tool::prepare_tool_cargo(
@@ -1825,6 +1833,32 @@ impl Step for CrateRustdoc {
             cargo.arg("'-Ctarget-feature=-crt-static'");
         }
 
+        // This is needed for running doctests on librustdoc. This is a bit of
+        // an unfortunate interaction with how bootstrap works and how cargo
+        // sets up the dylib path, and the fact that the doctest (in
+        // html/markdown.rs) links to rustc-private libs. For stage1, the
+        // compiler host dylibs (in stage1/lib) are not the same as the target
+        // dylibs (in stage1/lib/rustlib/...). This is different from a normal
+        // rust distribution where they are the same.
+        //
+        // On the cargo side, normal tests use `target_process` which handles
+        // setting up the dylib for a *target* (stage1/lib/rustlib/... in this
+        // case). However, for doctests it uses `rustdoc_process` which only
+        // sets up the dylib path for the *host* (stage1/lib), which is the
+        // wrong directory.
+        //
+        // It should be considered to just stop running doctests on
+        // librustdoc. There is only one test, and it doesn't look too
+        // important. There might be other ways to avoid this, but it seems
+        // pretty convoluted.
+        //
+        // See also https://github.com/rust-lang/rust/issues/13983 where the
+        // host vs target dylibs for rustdoc are consistently tricky to deal
+        // with.
+        let mut dylib_path = dylib_path();
+        dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target)));
+        cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
+
         if !builder.config.verbose_tests {
             cargo.arg("--quiet");
         }

src/bootstrap/tool.rs

@@ -366,7 +366,7 @@ bootstrap_tool!(
     ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
 );
 
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]
 pub struct ErrorIndex {
     pub compiler: Compiler,
 }
@@ -392,9 +392,9 @@ impl Step for ErrorIndex {
     fn make_run(run: RunConfig<'_>) {
         // Compile the error-index in the same stage as rustdoc to avoid
         // recompiling rustdoc twice if we can.
-        let stage = if run.builder.top_stage >= 2 { run.builder.top_stage } else { 0 };
-        run.builder
-            .ensure(ErrorIndex { compiler: run.builder.compiler(stage, run.builder.config.build) });
+        let host = run.builder.config.build;
+        let compiler = run.builder.compiler_for(run.builder.top_stage, host, host);
+        run.builder.ensure(ErrorIndex { compiler });
     }
 
     fn run(self, builder: &Builder<'_>) -> PathBuf {
@@ -449,7 +449,7 @@ impl Step for RemoteTestServer {
     }
 }
 
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]
 pub struct Rustdoc {
     /// This should only ever be 0 or 2.
     /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here.

src/ci/docker/wasm32/Dockerfile

@@ -25,7 +25,18 @@ RUN ln `which python3` /usr/bin/python
 
 ENV PATH=$PATH:/emsdk-portable
 ENV PATH=$PATH:/emsdk-portable/upstream/emscripten/
-ENV PATH=$PATH:/emsdk-portable/node/12.9.1_64bit/bin/
+
+# Rust's build system requires NodeJS to be in the path, but the directory in
+# which emsdk stores it contains the version number. This caused breakages in
+# the past when emsdk bumped the node version causing the path to point to a
+# missing directory.
+#
+# To avoid the problem this symlinks the latest NodeJs version available to
+# "latest", and adds that to the path.
+RUN ln -s /emsdk-portable/node/$(ls /emsdk-portable/node | sort -V | tail -n 1) \
+          /emsdk-portable/node/latest
+ENV PATH=$PATH:/emsdk-portable/node/latest/bin/
+
 ENV BINARYEN_ROOT=/emsdk-portable/upstream/
 ENV EMSDK=/emsdk-portable
 ENV EM_CONFIG=/emsdk-portable/.emscripten

src/libcore/alloc/global.rs

@@ -127,9 +127,12 @@ pub unsafe trait GlobalAlloc {
     #[stable(feature = "global_alloc", since = "1.28.0")]
     unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
         let size = layout.size();
-        let ptr = self.alloc(layout);
+        // SAFETY: the safety contract for `alloc` must be upheld by the caller.
+        let ptr = unsafe { self.alloc(layout) };
         if !ptr.is_null() {
-            ptr::write_bytes(ptr, 0, size);
+            // SAFETY: as allocation succeeded, the region from `ptr`
+            // of size `size` is guaranteed to be valid for writes.
+            unsafe { ptr::write_bytes(ptr, 0, size) };
         }
         ptr
     }
@@ -187,11 +190,18 @@ pub unsafe trait GlobalAlloc {
     /// [`handle_alloc_error`]: ../../alloc/alloc/fn.handle_alloc_error.html
     #[stable(feature = "global_alloc", since = "1.28.0")]
     unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
-        let new_layout = Layout::from_size_align_unchecked(new_size, layout.align());
-        let new_ptr = self.alloc(new_layout);
+        // SAFETY: the caller must ensure that the `new_size` does not overflow.
+        // `layout.align()` comes from a `Layout` and is thus guaranteed to be valid.
+        let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
+        // SAFETY: the caller must ensure that `new_layout` is greater than zero.
+        let new_ptr = unsafe { self.alloc(new_layout) };
         if !new_ptr.is_null() {
-            ptr::copy_nonoverlapping(ptr, new_ptr, cmp::min(layout.size(), new_size));
-            self.dealloc(ptr, layout);
+            // SAFETY: the previously allocated block cannot overlap the newly allocated block.
+            // The safety contract for `dealloc` must be upheld by the caller.
+            unsafe {
+                ptr::copy_nonoverlapping(ptr, new_ptr, cmp::min(layout.size(), new_size));
+                self.dealloc(ptr, layout);
+            }
         }
         new_ptr
     }

src/libcore/alloc/layout.rs

@@ -90,7 +90,8 @@ impl Layout {
     #[rustc_const_stable(feature = "alloc_layout", since = "1.28.0")]
     #[inline]
     pub const unsafe fn from_size_align_unchecked(size: usize, align: usize) -> Self {
-        Layout { size_: size, align_: NonZeroUsize::new_unchecked(align) }
+        // SAFETY: the caller must ensure that `align` is greater than zero.
+        Layout { size_: size, align_: unsafe { NonZeroUsize::new_unchecked(align) } }
     }
 
     /// The minimum size in bytes for a memory block of this layout.