Skip to content

Commit

Permalink
Auto merge of #96687 - jyn514:download-rustc, r=Mark-Simulacrum
Browse files Browse the repository at this point in the history 
Move download-rustc from python to rustbuild

- Remove download-rustc handling from bootstrap.py
- Allow a custom `pattern` in `builder.unpack()`
- Only download rustc once another part of bootstrap depends on it.

  This is somewhat necessary since the download functions rely on having a full
  `Builder`, which isn't available until after config parsing finishes.

Helps with #94829.
  • Loading branch information
@bors
bors committed May 29, 2022
2 parents 0acc4a3 + 00bb4df commit 303d916
Show file tree
Hide file tree
Showing 8 changed files with 418 additions and 352 deletions.
154 changes: 38 additions & 116 deletions src/bootstrap/bootstrap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,9 +444,8 @@ def __init__(self):
self.verbose = False
self.git_version = None
self.nix_deps_dir = None
self.rustc_commit = None

def download_toolchain(self, stage0=True, rustc_channel=None):
def download_toolchain(self):
"""Fetch the build system for Rust, written in Rust
This method will build a cache directory, then it will fetch the
Expand All @@ -456,45 +455,35 @@ def download_toolchain(self, stage0=True, rustc_channel=None):
Each downloaded tarball is extracted, after that, the script
will move all the content to the right place.
"""
if rustc_channel is None:
rustc_channel = self.stage0_compiler.version
bin_root = self.bin_root(stage0)
rustc_channel = self.stage0_compiler.version
bin_root = self.bin_root()

key = self.stage0_compiler.date
if not stage0:
key += str(self.rustc_commit)
if self.rustc(stage0).startswith(bin_root) and \
(not os.path.exists(self.rustc(stage0)) or
self.program_out_of_date(self.rustc_stamp(stage0), key)):
if self.rustc().startswith(bin_root) and \
(not os.path.exists(self.rustc()) or
self.program_out_of_date(self.rustc_stamp(), key)):
if os.path.exists(bin_root):
shutil.rmtree(bin_root)
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
filename = "rust-std-{}-{}{}".format(
rustc_channel, self.build, tarball_suffix)
pattern = "rust-std-{}".format(self.build)
self._download_component_helper(filename, pattern, tarball_suffix, stage0)
self._download_component_helper(filename, pattern, tarball_suffix)
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
# download-rustc doesn't need its own cargo, it can just use beta's.
if stage0:
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "cargo", tarball_suffix)
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
else:
filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
self._download_component_helper(
filename, "rustc-dev", tarball_suffix, stage0
)
self._download_component_helper(filename, "rustc", tarball_suffix)
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "cargo", tarball_suffix)
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))

self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
lib_dir = "{}/lib".format(bin_root)
for lib in os.listdir(lib_dir):
if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
with output(self.rustc_stamp(stage0)) as rust_stamp:
with output(self.rustc_stamp()) as rust_stamp:
rust_stamp.write(key)

if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
Expand All @@ -518,24 +507,17 @@ def download_toolchain(self, stage0=True, rustc_channel=None):
rustfmt_stamp.write(self.stage0_rustfmt.channel())

def _download_component_helper(
self, filename, pattern, tarball_suffix, stage0=True, key=None
self, filename, pattern, tarball_suffix, key=None
):
if key is None:
if stage0:
key = self.stage0_compiler.date
else:
key = self.rustc_commit
key = self.stage0_compiler.date
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, key)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)

if stage0:
base = self._download_url
url = "dist/{}".format(key)
else:
base = "https://ci-artifacts.rust-lang.org"
url = "rustc-builds/{}".format(self.rustc_commit)
base = self._download_url
url = "dist/{}".format(key)
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get(
Expand All @@ -544,9 +526,9 @@ def _download_component_helper(
tarball,
self.checksums_sha256,
verbose=self.verbose,
do_verify=stage0,
do_verify=True,
)
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)

def fix_bin_or_dylib(self, fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
Expand Down Expand Up @@ -644,62 +626,15 @@ def fix_bin_or_dylib(self, fname):
print("warning: failed to call patchelf:", reason)
return

# If `download-rustc` is set, download the most recent commit with CI artifacts
def maybe_download_ci_toolchain(self):
# If `download-rustc` is not set, default to rebuilding.
download_rustc = self.get_toml("download-rustc", section="rust")
if download_rustc is None or download_rustc == "false":
return None
assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc

# Handle running from a directory other than the top level
rev_parse = ["git", "rev-parse", "--show-toplevel"]
top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
compiler = "{}/compiler/".format(top_level)
library = "{}/library/".format(top_level)

# Look for a version to compare to based on the current commit.
# Only commits merged by bors will have CI artifacts.
merge_base = [
"git", "rev-list", "--author=bors@rust-lang.org", "-n1",
"--first-parent", "HEAD"
]
commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
if not commit:
print("error: could not find commit hash for downloading rustc")
print("help: maybe your repository history is too shallow?")
print("help: consider disabling `download-rustc`")
print("help: or fetch enough history to include one upstream commit")
exit(1)

# Warn if there were changes to the compiler or standard library since the ancestor commit.
status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
if status != 0:
if download_rustc == "if-unchanged":
if self.verbose:
print("warning: saw changes to compiler/ or library/ since {}; " \
"ignoring `download-rustc`".format(commit))
return None
print("warning: `download-rustc` is enabled, but there are changes to " \
"compiler/ or library/")

if self.verbose:
print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
self.rustc_commit = commit
# FIXME: support downloading artifacts from the beta channel
self.download_toolchain(False, "nightly")

def rustc_stamp(self, stage0):
def rustc_stamp(self):
"""Return the path for .rustc-stamp at the given stage
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
True
>>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
>>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
True
"""
return os.path.join(self.bin_root(stage0), '.rustc-stamp')
return os.path.join(self.bin_root(), '.rustc-stamp')

def rustfmt_stamp(self):
"""Return the path for .rustfmt-stamp
Expand All @@ -709,7 +644,7 @@ def rustfmt_stamp(self):
>>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
True
"""
return os.path.join(self.bin_root(True), '.rustfmt-stamp')
return os.path.join(self.bin_root(), '.rustfmt-stamp')

def program_out_of_date(self, stamp_path, key):
"""Check if the given program stamp is out of date"""
Expand All @@ -718,26 +653,21 @@ def program_out_of_date(self, stamp_path, key):
with open(stamp_path, 'r') as stamp:
return key != stamp.read()

def bin_root(self, stage0):
def bin_root(self):
"""Return the binary root directory for the given stage
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.bin_root(True) == os.path.join("build", "stage0")
True
>>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
>>> rb.bin_root() == os.path.join("build", "stage0")
True
When the 'build' property is given should be a nested directory:
>>> rb.build = "devel"
>>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
>>> rb.bin_root() == os.path.join("build", "devel", "stage0")
True
"""
if stage0:
subdir = "stage0"
else:
subdir = "ci-rustc"
subdir = "stage0"
return os.path.join(self.build_dir, self.build, subdir)

def get_toml(self, key, section=None):
Expand Down Expand Up @@ -785,37 +715,33 @@ def cargo(self):
"""Return config path for cargo"""
return self.program_config('cargo')

def rustc(self, stage0):
def rustc(self):
"""Return config path for rustc"""
return self.program_config('rustc', stage0)
return self.program_config('rustc')

def rustfmt(self):
"""Return config path for rustfmt"""
if self.stage0_rustfmt is None:
return None
return self.program_config('rustfmt')

def program_config(self, program, stage0=True):
def program_config(self, program):
"""Return config path for the given program at the given stage
>>> rb = RustBuild()
>>> rb.config_toml = 'rustc = "rustc"\\n'
>>> rb.program_config('rustc')
'rustc'
>>> rb.config_toml = ''
>>> cargo_path = rb.program_config('cargo', True)
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
... "bin", "cargo")
True
>>> cargo_path = rb.program_config('cargo', False)
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
>>> cargo_path = rb.program_config('cargo')
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
... "bin", "cargo")
True
"""
config = self.get_toml(program)
if config:
return os.path.expanduser(config)
return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
return os.path.join(self.bin_root(), "bin", "{}{}".format(
program, self.exe_suffix()))

@staticmethod
Expand Down Expand Up @@ -871,14 +797,14 @@ def build_bootstrap(self):
if "CARGO_BUILD_TARGET" in env:
del env["CARGO_BUILD_TARGET"]
env["CARGO_TARGET_DIR"] = build_dir
env["RUSTC"] = self.rustc(True)
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
env["RUSTC"] = self.rustc()
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LD_LIBRARY_PATH"]) \
if "LD_LIBRARY_PATH" in env else ""
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \
if "DYLD_LIBRARY_PATH" in env else ""
env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LIBRARY_PATH"]) \
if "LIBRARY_PATH" in env else ""

Expand All @@ -900,7 +826,7 @@ def build_bootstrap(self):
if self.get_toml("deny-warnings", "rust") != "false":
env["RUSTFLAGS"] += " -Dwarnings"

env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
env["PATH"] = os.path.join(self.bin_root(), "bin") + \
os.pathsep + env["PATH"]
if not os.path.isfile(self.cargo()):
raise Exception("no cargo executable found at `{}`".format(
Expand Down Expand Up @@ -1171,8 +1097,6 @@ def bootstrap(help_triggered):

# Fetch/build the bootstrap
build.download_toolchain()
# Download the master compiler if `download-rustc` is set
build.maybe_download_ci_toolchain()
sys.stdout.flush()
build.ensure_vendored()
build.build_bootstrap()
Expand All @@ -1184,8 +1108,6 @@ def bootstrap(help_triggered):
env = os.environ.copy()
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
env["BOOTSTRAP_PYTHON"] = sys.executable
if build.rustc_commit is not None:
env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
run(args, env=env, verbose=build.verbose, is_bootstrap=True)


Expand Down
Loading

0 comments on commit 303d916

Please sign in to comment.