From f846aaf81f000b19d6f0ac5bde55f70bab47ec70 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 1 Mar 2017 15:34:54 -0800 Subject: [PATCH] rustbuild: Build documentation for `proc_macro` This commit fixes #38749 by building documentation for the `proc_macro` crate by default for configured hosts. Unfortunately did not turn out to be a trivial fix. Currently rustbuild generates documentation into multiple locations: one for std, one for test, and one for rustc. The initial fix for this issue simply actually executed `cargo doc -p proc_macro` which was otherwise completely elided before. Unfortunately rustbuild was the left to merge two documentation trees together. One for the standard library and one for the rustc tree (which only had docs for the `proc_macro` crate). Rustdoc itself knows how to merge documentation files (specifically around search indexes, etc) but rustbuild was unaware of this, so an initial fix ended up destroying the sidebar and the search bar from the libstd docs. To solve this issue the method of documentation has been tweaked slightly in rustbuild. The build system will not use symlinks (or directory junctions on Windows) to generate all documentation into the same location initially. This'll rely on rustdoc's logic to weave together all the output and ensure that it ends up all consistent. Closes #38749 --- src/bootstrap/doc.rs | 67 ++++++++++++++++--- src/bootstrap/lib.rs | 7 ++ src/bootstrap/step.rs | 2 +- src/bootstrap/util.rs | 139 +++++++++++++++++++++++++++++++++++++++ src/ci/docker/run.sh | 1 + src/libproc_macro/lib.rs | 2 +- 6 files changed, 207 insertions(+), 11 deletions(-) diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index d19e5b1b88456..5a5cfe0c682d4 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -19,10 +19,12 @@ use std::fs::{self, File}; use std::io::prelude::*; +use std::io; +use std::path::Path; use std::process::Command; use {Build, Compiler, Mode}; -use util::cp_r; +use util::{cp_r, symlink_dir}; use build_helper::up_to_date; /// Invoke `rustbook` as compiled in `stage` for `target` for the doc book @@ -141,7 +143,22 @@ pub fn std(build: &Build, stage: u32, target: &str) { .join(target).join("doc"); let rustdoc = build.rustdoc(&compiler); - build.clear_if_dirty(&out_dir, &rustdoc); + // Here what we're doing is creating a *symlink* (directory junction on + // Windows) to the final output location. This is not done as an + // optimization but rather for correctness. We've got three trees of + // documentation, one for std, one for test, and one for rustc. It's then + // our job to merge them all together. + // + // Unfortunately rustbuild doesn't know nearly as well how to merge doc + // trees as rustdoc does itself, so instead of actually having three + // separate trees we just have rustdoc output to the same location across + // all of them. + // + // This way rustdoc generates output directly into the output, and rustdoc + // will also directly handle merging. + let my_out = build.crate_doc_out(target); + build.clear_if_dirty(&my_out, &rustdoc); + t!(symlink_dir_force(&my_out, &out_dir)); let mut cargo = build.cargo(&compiler, Mode::Libstd, target, "doc"); cargo.arg("--manifest-path") @@ -166,7 +183,7 @@ pub fn std(build: &Build, stage: u32, target: &str) { build.run(&mut cargo); - cp_r(&out_dir, &out) + cp_r(&my_out, &out); } /// Compile all libtest documentation. @@ -187,13 +204,16 @@ pub fn test(build: &Build, stage: u32, target: &str) { .join(target).join("doc"); let rustdoc = build.rustdoc(&compiler); - build.clear_if_dirty(&out_dir, &rustdoc); + // See docs in std above for why we symlink + let my_out = build.crate_doc_out(target); + build.clear_if_dirty(&my_out, &rustdoc); + t!(symlink_dir_force(&my_out, &out_dir)); let mut cargo = build.cargo(&compiler, Mode::Libtest, target, "doc"); cargo.arg("--manifest-path") .arg(build.src.join("src/libtest/Cargo.toml")); build.run(&mut cargo); - cp_r(&out_dir, &out) + cp_r(&my_out, &out); } /// Generate all compiler documentation. @@ -213,15 +233,28 @@ pub fn rustc(build: &Build, stage: u32, target: &str) { let out_dir = build.stage_out(&compiler, Mode::Librustc) .join(target).join("doc"); let rustdoc = build.rustdoc(&compiler); - if !up_to_date(&rustdoc, &out_dir.join("rustc/index.html")) && out_dir.exists() { - t!(fs::remove_dir_all(&out_dir)); - } + + // See docs in std above for why we symlink + let my_out = build.crate_doc_out(target); + build.clear_if_dirty(&my_out, &rustdoc); + t!(symlink_dir_force(&my_out, &out_dir)); + let mut cargo = build.cargo(&compiler, Mode::Librustc, target, "doc"); cargo.arg("--manifest-path") .arg(build.src.join("src/rustc/Cargo.toml")) .arg("--features").arg(build.rustc_features()); + + // Like with libstd above if compiler docs aren't enabled then we're not + // documenting internal dependencies, so we have a whitelist. + if !build.config.compiler_docs { + cargo.arg("--no-deps"); + for krate in &["proc_macro"] { + cargo.arg("-p").arg(krate); + } + } + build.run(&mut cargo); - cp_r(&out_dir, &out) + cp_r(&my_out, &out); } /// Generates the HTML rendered error-index by running the @@ -240,3 +273,19 @@ pub fn error_index(build: &Build, target: &str) { build.run(&mut index); } + +fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> { + if let Ok(m) = fs::symlink_metadata(dst) { + if m.file_type().is_dir() { + try!(fs::remove_dir_all(dst)); + } else { + // handle directory junctions on windows by falling back to + // `remove_dir`. + try!(fs::remove_file(dst).or_else(|_| { + fs::remove_dir(dst) + })); + } + } + + symlink_dir(src, dst) +} diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 071d0b0b09009..97fc5886478c8 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -707,6 +707,13 @@ impl Build { self.out.join(target).join("doc") } + /// Output directory for all crate documentation for a target (temporary) + /// + /// The artifacts here are then copied into `doc_out` above. + fn crate_doc_out(&self, target: &str) -> PathBuf { + self.out.join(target).join("crate-docs") + } + /// Returns true if no custom `llvm-config` is set for the specified target. /// /// If no custom `llvm-config` was specified then Rust's llvm will be used. diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index a5c0d11d21985..c24a555280ba9 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -640,7 +640,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.doc(&krate.doc_step, path) .dep(|s| s.name("librustc-link")) .host(true) - .default(default && build.config.compiler_docs) + .default(default && build.config.docs) .run(move |s| doc::rustc(build, s.stage, s.target)); } diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index fc63655d79b6c..dab20f44bc361 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -16,6 +16,7 @@ use std::env; use std::ffi::OsString; use std::fs; +use std::io; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Instant; @@ -183,3 +184,141 @@ impl Drop for TimeIt { time.subsec_nanos() / 1_000_000); } } + +/// Symlinks two directories, using junctions on Windows and normal symlinks on +/// Unix. +pub fn symlink_dir(src: &Path, dest: &Path) -> io::Result<()> { + let _ = fs::remove_dir(dest); + return symlink_dir_inner(src, dest); + + #[cfg(not(windows))] + fn symlink_dir_inner(src: &Path, dest: &Path) -> io::Result<()> { + use std::os::unix::fs; + fs::symlink(src, dest) + } + + // Creating a directory junction on windows involves dealing with reparse + // points and the DeviceIoControl function, and this code is a skeleton of + // what can be found here: + // + // http://www.flexhex.com/docs/articles/hard-links.phtml + // + // Copied from std + #[cfg(windows)] + #[allow(bad_style)] + fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> { + use std::ptr; + use std::ffi::OsStr; + use std::os::windows::ffi::OsStrExt; + + const MAXIMUM_REPARSE_DATA_BUFFER_SIZE: usize = 16 * 1024; + const GENERIC_WRITE: DWORD = 0x40000000; + const OPEN_EXISTING: DWORD = 3; + const FILE_FLAG_OPEN_REPARSE_POINT: DWORD = 0x00200000; + const FILE_FLAG_BACKUP_SEMANTICS: DWORD = 0x02000000; + const FSCTL_SET_REPARSE_POINT: DWORD = 0x900a4; + const IO_REPARSE_TAG_MOUNT_POINT: DWORD = 0xa0000003; + const FILE_SHARE_DELETE: DWORD = 0x4; + const FILE_SHARE_READ: DWORD = 0x1; + const FILE_SHARE_WRITE: DWORD = 0x2; + + type BOOL = i32; + type DWORD = u32; + type HANDLE = *mut u8; + type LPCWSTR = *const u16; + type LPDWORD = *mut DWORD; + type LPOVERLAPPED = *mut u8; + type LPSECURITY_ATTRIBUTES = *mut u8; + type LPVOID = *mut u8; + type WCHAR = u16; + type WORD = u16; + + #[repr(C)] + struct REPARSE_MOUNTPOINT_DATA_BUFFER { + ReparseTag: DWORD, + ReparseDataLength: DWORD, + Reserved: WORD, + ReparseTargetLength: WORD, + ReparseTargetMaximumLength: WORD, + Reserved1: WORD, + ReparseTarget: WCHAR, + } + + extern "system" { + fn CreateFileW(lpFileName: LPCWSTR, + dwDesiredAccess: DWORD, + dwShareMode: DWORD, + lpSecurityAttributes: LPSECURITY_ATTRIBUTES, + dwCreationDisposition: DWORD, + dwFlagsAndAttributes: DWORD, + hTemplateFile: HANDLE) + -> HANDLE; + fn DeviceIoControl(hDevice: HANDLE, + dwIoControlCode: DWORD, + lpInBuffer: LPVOID, + nInBufferSize: DWORD, + lpOutBuffer: LPVOID, + nOutBufferSize: DWORD, + lpBytesReturned: LPDWORD, + lpOverlapped: LPOVERLAPPED) -> BOOL; + } + + fn to_u16s>(s: S) -> io::Result> { + Ok(s.as_ref().encode_wide().chain(Some(0)).collect()) + } + + // We're using low-level APIs to create the junction, and these are more + // picky about paths. For example, forward slashes cannot be used as a + // path separator, so we should try to canonicalize the path first. + let target = try!(fs::canonicalize(target)); + + try!(fs::create_dir(junction)); + + let path = try!(to_u16s(junction)); + + unsafe { + let h = CreateFileW(path.as_ptr(), + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + 0 as *mut _, + OPEN_EXISTING, + FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, + ptr::null_mut()); + + let mut data = [0u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; + let mut db = data.as_mut_ptr() + as *mut REPARSE_MOUNTPOINT_DATA_BUFFER; + let buf = &mut (*db).ReparseTarget as *mut _; + let mut i = 0; + // FIXME: this conversion is very hacky + let v = br"\??\"; + let v = v.iter().map(|x| *x as u16); + for c in v.chain(target.as_os_str().encode_wide().skip(4)) { + *buf.offset(i) = c; + i += 1; + } + *buf.offset(i) = 0; + i += 1; + (*db).ReparseTag = IO_REPARSE_TAG_MOUNT_POINT; + (*db).ReparseTargetMaximumLength = (i * 2) as WORD; + (*db).ReparseTargetLength = ((i - 1) * 2) as WORD; + (*db).ReparseDataLength = + (*db).ReparseTargetLength as DWORD + 12; + + let mut ret = 0; + let res = DeviceIoControl(h as *mut _, + FSCTL_SET_REPARSE_POINT, + data.as_ptr() as *mut _, + (*db).ReparseDataLength + 8, + ptr::null_mut(), 0, + &mut ret, + ptr::null_mut()); + + if res == 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } + } + } +} diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 1e61f2169106c..fb9edf33921f8 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -54,6 +54,7 @@ exec docker \ --env DEPLOY_ALT=$DEPLOY_ALT \ --env LOCAL_USER_ID=`id -u` \ --volume "$HOME/.cargo:/cargo" \ + --privileged \ --rm \ rust-ci \ /checkout/src/ci/run.sh diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs index 8d7fe655c23b2..0d2a467b57702 100644 --- a/src/libproc_macro/lib.rs +++ b/src/libproc_macro/lib.rs @@ -21,7 +21,7 @@ //! This functionality is intended to be expanded over time as more surface //! area for macro authors is stabilized. //! -//! See [the book](../../book/procedural-macros.html) for more. +//! See [the book](../book/procedural-macros.html) for more. #![crate_name = "proc_macro"] #![stable(feature = "proc_macro_lib", since = "1.15.0")]