Auto merge of rust-lang#117771 - tmandry:fuchsia-gha, r=Mark-Simulacrum

Build Fuchsia in CI Fittingly, when I first put this up it was failing due to discovering an ICE in clippy (looks like fixed in rust-lang/rust-clippy#11760), probably more fallout from recent type system changes. Other recent regressions this would have caught include - rust-lang#117455 and rust-lang#117493 - rust-lang#117602 Originally we discussed basing this on cargotest, but they ended up not sharing anything. Fuchsia has its own tool to manage checkouts and its own build system. What it requires is a fully "install"ed toolchain with a host and fuchsia target. We share logic from the dist-various-2 builder to build the fuchsia target. Right now this runs clippy and skips linking a bunch of targets, since most issues we catch are in the frontend. In theory we could probably get the build CPU time down quite a bit with this approach, but right now some linked targets are creeping into the dependencies anyway and we don't have a good way of preventing that yet. The approach is basically to get a checkout at a pinned commit and then run a [script](https://fuchsia-review.git.corp.google.com/c/fuchsia/+/943833/6/scripts/rust/build_fuchsia_from_rust_ci.sh) at a predetermined location. I would like to update that pin every few weeks. Partial checkouts are used to minimize clone time, but we don't filter out prebuilt packages. r? `@Mark-Simulacrum` Based on discussion in [this Zulip thread](https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/Putting.20Fuchsia.20in.20crater).
rust-lang-ci · Dec 9, 2023 · c722d51 · c722d51
2 parents 08587a5 + a1b9a59
commit c722d51
Show file tree

Hide file tree

Showing 9 changed files with 144 additions and 11 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -288,6 +288,9 @@ jobs:
           - name: x86_64-gnu-aux
             os: ubuntu-20.04-4core-16gb
             env: {}
+          - name: x86_64-gnu-integration
+            os: ubuntu-20.04-16core-64gb
+            env: {}
           - name: x86_64-gnu-debug
             os: ubuntu-20.04-8core-32gb
             env: {}

diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
@@ -1722,7 +1722,7 @@ impl Step for Assemble {
             let dst_exe = exe("rust-lld", target_compiler.host);
             builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe));
             let self_contained_lld_dir = libdir_bin.join("gcc-ld");
-            t!(fs::create_dir(&self_contained_lld_dir));
+            t!(fs::create_dir_all(&self_contained_lld_dir));
             let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper {
                 compiler: build_compiler,
                 target: target_compiler.host,

diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile
@@ -76,8 +76,8 @@ RUN env \
     rm -rf /build/*
 
 WORKDIR /tmp
-COPY host-x86_64/dist-various-2/shared.sh /tmp/
-COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/
+COPY scripts/shared.sh /tmp/
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
 RUN /tmp/build-fuchsia-toolchain.sh
 COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/
 RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386  pc

diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile
@@ -0,0 +1,68 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  build-essential \
+  g++ \
+  make \
+  ninja-build \
+  file \
+  curl \
+  ca-certificates \
+  python3 \
+  git \
+  cmake \
+  libssl-dev \
+  sudo \
+  xz-utils \
+  pkg-config \
+  unzip \
+  && rm -rf /var/lib/apt/lists/*
+
+# Duplicated in dist-various-2 Dockerfile.
+# FIXME: Move to canonical triple
+ENV \
+    AR_x86_64_fuchsia=x86_64-unknown-fuchsia-ar \
+    CC_x86_64_fuchsia=x86_64-unknown-fuchsia-clang \
+    CFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \
+    CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib"
+
+WORKDIR /tmp
+COPY scripts/shared.sh /tmp/
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
+RUN /tmp/build-fuchsia-toolchain.sh
+
+ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar
+ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \
+  -C panic=abort \
+  -C force-unwind-tables=yes \
+  -C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \
+  -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \
+  -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib
+
+ENV TARGETS=x86_64-fuchsia
+ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV RUST_INSTALL_DIR /checkout/obj/install
+RUN mkdir -p $RUST_INSTALL_DIR/etc
+
+ENV RUST_CONFIGURE_ARGS \
+  --prefix=$RUST_INSTALL_DIR \
+  --sysconfdir=etc \
+  --enable-lld \
+  --llvm-libunwind=in-tree \
+  --enable-extended \
+  --disable-docs \
+  --set target.x86_64-fuchsia.cc=/usr/local/bin/clang \
+  --set target.x86_64-fuchsia.cxx=/usr/local/bin/clang++ \
+  --set target.x86_64-fuchsia.ar=/usr/local/bin/llvm-ar \
+  --set target.x86_64-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \
+  --set target.x86_64-fuchsia.linker=/usr/local/bin/ld.lld
+ENV SCRIPT \
+    python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \
+    bash ../src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh b/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Downloads and builds the Fuchsia operating system using a toolchain installed
+# in $RUST_INSTALL_DIR.
+
+set -euf -o pipefail
+
+INTEGRATION_SHA=66793c4894bf6204579bbee3b79956335f31c768
+PICK_REFS=()
+
+checkout=fuchsia
+jiri=.jiri_root/bin/jiri
+
+set -x
+
+# This script will:
+# - create a directory named "fuchsia" if it does not exist
+# - download "jiri" to "fuchsia/.jiri_root/bin"
+curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \
+    | base64 --decode \
+    | bash -s $checkout
+
+cd $checkout
+
+$jiri init \
+    -partial=true \
+    -analytics-opt=false \
+    .
+
+$jiri import \
+    -name=integration \
+    -revision=$INTEGRATION_SHA \
+    -overwrite=true \
+    flower \
+    "https://fuchsia.googlesource.com/integration"
+
+if [ -d ".git" ]; then
+    # Wipe out any local changes if we're reusing a checkout.
+    git checkout --force JIRI_HEAD
+fi
+
+$jiri update -autoupdate=false
+
+echo integration commit = $(git -C integration rev-parse HEAD)
+
+for git_ref in "${PICK_REFS[@]}"; do
+    git fetch https://fuchsia.googlesource.com/fuchsia $git_ref
+    git cherry-pick --no-commit FETCH_HEAD
+done
+
+bash scripts/rust/build_fuchsia_from_rust_ci.sh
diff --git a/...dist-various-2/build-fuchsia-toolchain.sh → ...docker/scripts/build-fuchsia-toolchain.sh b/...dist-various-2/build-fuchsia-toolchain.sh → ...docker/scripts/build-fuchsia-toolchain.sh
@@ -4,13 +4,13 @@ set -ex
 source shared.sh
 
 FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64
-FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC
-FUCHSIA_SDK_SHA256=e318f1ac652b0db43aff32708fa70337521b5ac595e5a0905c2ff33bf1eed179
+FUCHSIA_SDK_ID=MrhQwtmP8CpZre-i_PNOREcThbUcrX3bA-45d6WQr-cC
+FUCHSIA_SDK_SHA256=32b850c2d98ff02a59adefa2fcf34e44471385b51cad7ddb03ee3977a590afe7
 FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk
 CLANG_DOWNLOAD_URL=\
 https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64
-CLANG_DOWNLOAD_ID=vU0vNjSihOV4Q6taQYCpy03JXGiCyVwxen3rFMNMIgsC
-CLANG_DOWNLOAD_SHA256=bd4d2f3634a284e57843ab5a4180a9cb4dc95c6882c95c317a7deb14c34c220b
+CLANG_DOWNLOAD_ID=Tpc85d1ZwSlZ6UKl2d96GRUBGNA5JKholOKe24sRDr0C
+CLANG_DOWNLOAD_SHA256=4e973ce5dd59c12959e942a5d9df7a19150118d03924a86894e29edb8b110ebd
 
 install_clang() {
   mkdir -p clang_download

diff --git a/...cker/host-x86_64/dist-various-2/shared.sh → src/ci/docker/scripts/shared.sh b/...cker/host-x86_64/dist-various-2/shared.sh → src/ci/docker/scripts/shared.sh
@@ -1,5 +1,11 @@
-#!/usr/bin/env bash
-hide_output() {
+#!/bin/false
+# shellcheck shell=bash
+
+# This file is intended to be sourced with `. shared.sh` or
+# `source shared.sh`, hence the invalid shebang and not being
+# marked as an executable file in git.
+
+function hide_output {
   { set +x; } 2>/dev/null
   on_err="
 echo ERROR: An error was encountered with the build.
@@ -15,7 +21,8 @@ exit 1
   set -x
 }
 
-# Copied from ../../shared.sh
+# See https://unix.stackexchange.com/questions/82598
+# Duplicated in src/ci/shared.sh
 function retry {
   echo "Attempting with retry:" "$@"
   local n=1

diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
@@ -470,6 +470,9 @@ jobs:
           - name: x86_64-gnu-aux
             <<: *job-linux-4c
 
+          - name: x86_64-gnu-integration
+            <<: *job-linux-16c
+
           - name: x86_64-gnu-debug
             <<: *job-linux-8c
 
@@ -730,6 +733,7 @@ jobs:
               CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-16c
 
+
   master:
     name: master
     runs-on: ubuntu-latest

diff --git a/src/ci/shared.sh b/src/ci/shared.sh
@@ -8,7 +8,7 @@
 export MIRRORS_BASE="https://ci-mirrors.rust-lang.org/rustc"
 
 # See https://unix.stackexchange.com/questions/82598
-# Duplicated in docker/dist-various-2/shared.sh
+# Duplicated in docker/scripts/shared.sh
 function retry {
   echo "Attempting with retry:" "$@"
   local n=1