From d4048407880212bb820f85d2e06e5fc262cce897 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 27 Nov 2020 20:48:53 +0100 Subject: [PATCH 01/26] Merge commit '5988bbd24aa87732bfa1d111ba00bcdaa22c481a' into sync_cg_clif-2020-11-27 --- .gitattributes | 1 + Cargo.lock | 104 +++++++++--------- Cargo.toml | 4 +- Readme.md | 6 +- build.sh | 25 ++++- build_sysroot/Cargo.lock | 20 ++-- build_sysroot/build_sysroot.sh | 13 ++- build_sysroot/prepare_sysroot_src.sh | 12 +- example/std_example.rs | 1 + ...022-core-Disable-not-compiling-tests.patch | 8 +- prepare.sh | 1 + rust-toolchain | 2 +- scripts/cargo.sh | 8 +- scripts/config.sh | 19 ++-- scripts/rustup.sh | 38 +++++-- scripts/test_bootstrap.sh | 6 +- scripts/tests.sh | 30 ++--- src/abi/mod.rs | 6 +- src/archive.rs | 6 +- src/atomic_shim.rs | 3 +- src/base.rs | 31 +++++- src/bin/cg_clif.rs | 18 +-- src/common.rs | 4 +- src/constant.rs | 9 +- src/debuginfo/emit.rs | 7 +- src/debuginfo/unwind.rs | 4 +- src/discriminant.rs | 31 +++++- src/driver/aot.rs | 6 +- src/driver/jit.rs | 16 ++- src/driver/mod.rs | 26 ++--- src/intrinsics/mod.rs | 80 +++++++++++--- src/intrinsics/simd.rs | 65 +++++++++-- src/trap.rs | 12 ++ 33 files changed, 400 insertions(+), 222 deletions(-) diff --git a/.gitattributes b/.gitattributes index 6313b56c57848..0ceb3fe646cca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ * text=auto eol=lf +*.rs diff=rust diff --git a/Cargo.lock b/Cargo.lock index 2889fac77f6a4..67ed41e765231 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "anyhow" -version = "1.0.33" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1fd36ffbb1fb7c834eac128ea8d0e310c5aeb635548f9d58861e1308d46e71c" +checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" [[package]] name = "ar" @@ -31,9 +31,9 @@ checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" [[package]] name = "cc" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" +checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40" [[package]] name = "cfg-if" @@ -41,18 +41,24 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "cranelift-bforest" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" dependencies = [ "byteorder", "cranelift-bforest", @@ -69,8 +75,8 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -78,18 +84,18 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" [[package]] name = "cranelift-entity" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" [[package]] name = "cranelift-frontend" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" dependencies = [ "cranelift-codegen", "log", @@ -99,8 +105,8 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" dependencies = [ "anyhow", "cranelift-codegen", @@ -111,8 +117,8 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -121,8 +127,8 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" dependencies = [ "anyhow", "cranelift-codegen", @@ -134,8 +140,8 @@ dependencies = [ [[package]] name = "cranelift-simplejit" -version = "0.67.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#44cbdecea03c360ea82e6482f0cf6c614effef21" +version = "0.68.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -151,18 +157,18 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] name = "errno" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eab5ee3df98a279d9b316b1af6ac95422127b1290317e6d18c1743c99418b01" +checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe" dependencies = [ "errno-dragonfly", "libc", @@ -187,9 +193,9 @@ checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" [[package]] name = "gimli" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" dependencies = [ "indexmap", ] @@ -212,17 +218,17 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" +checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" [[package]] name = "libloading" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3557c9384f7f757f6d139cd3a4c62ef4e850696c16bf27924a5538c8a09717a1" +checksum = "1090080fe06ec2648d0da3881d9453d97e71a45f00eb179af7fdd7e3f686fdb0" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "winapi", ] @@ -232,7 +238,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -246,9 +252,9 @@ dependencies = [ [[package]] name = "object" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37fd5004feb2ce328a52b0b3d01dbf4ffff72583493900ed15f22d4111c51693" +checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" dependencies = [ "crc32fast", "indexmap", @@ -274,9 +280,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "7.0.3" +version = "8.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a349ca83373cfa5d6dbb66fd76e58b2cca08da71a5f6400de0a0a6a9bceeaf" +checksum = "1fdf7d9dbd43f3d81d94a49c1c3df73cc2b3827995147e6cf7f89d4ec5483e73" dependencies = [ "bitflags", "cc", @@ -361,9 +367,9 @@ checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] name = "syn" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e03e57e4fcbfe7749842d53e24ccb9aa12b7252dbe5e91d2acad31834c8b8fdd" +checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" dependencies = [ "proc-macro2", "quote", @@ -372,24 +378,24 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe2635952a442a01fd4cb53d98858b5e4bb461b02c0d111f22f31772e3e7a8b2" +checksum = "4ee5a98e506fb7231a304c3a1bd7c132a55016cf65001e0282480665870dfcb9" [[package]] name = "thiserror" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 1c8e350d24297..cbff06749d3e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,8 @@ cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", bran cranelift-simplejit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } target-lexicon = "0.11.0" -gimli = { version = "0.22.0", default-features = false, features = ["write"]} -object = { version = "0.21.1", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] } +gimli = { version = "0.23.0", default-features = false, features = ["write"]} +object = { version = "0.22.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] } ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" } indexmap = "1.0.2" diff --git a/Readme.md b/Readme.md index f8a5e13ed54c1..de54bf67f4a19 100644 --- a/Readme.md +++ b/Readme.md @@ -51,7 +51,7 @@ This should build and run your project with rustc_codegen_cranelift instead of t > You should prefer using the Cargo method. ```bash -$ $cg_clif_dir/build/cg_clif my_crate.rs +$ $cg_clif_dir/build/bin/cg_clif my_crate.rs ``` ### Jit mode @@ -68,7 +68,7 @@ $ $cg_clif_dir/build/cargo.sh jit or ```bash -$ $cg_clif_dir/build/cg_clif --jit my_crate.rs +$ $cg_clif_dir/build/bin/cg_clif --jit my_crate.rs ``` ### Shell @@ -77,7 +77,7 @@ These are a few functions that allow you to easily run rust code from the shell ```bash function jit_naked() { - echo "$@" | $cg_clif_dir/build/cg_clif - --jit + echo "$@" | $cg_clif_dir/build/bin/cg_clif - --jit } function jit() { diff --git a/build.sh b/build.sh index f9a87e68a046a..26041b59cca18 100755 --- a/build.sh +++ b/build.sh @@ -26,22 +26,35 @@ while [[ $# != 0 ]]; do done # Build cg_clif +unset CARGO_TARGET_DIR export RUSTFLAGS="-Zrun_dsymutil=no" +unamestr=$(uname) +if [[ "$unamestr" == 'Linux' ]]; then + export RUSTFLAGS='-Clink-arg=-Wl,-rpath=$ORIGIN/../lib '$RUSTFLAGS +elif [[ "$unamestr" == 'Darwin' ]]; then + export RUSTFLAGS='-Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS + dylib_ext='dylib' +else + echo "Unsupported os" + exit 1 +fi if [[ "$CHANNEL" == "release" ]]; then cargo build --release else cargo build fi -rm -rf $target_dir -mkdir $target_dir -cp -a target/$CHANNEL/cg_clif{,_build_sysroot} target/$CHANNEL/*rustc_codegen_cranelift* $target_dir/ -cp -a rust-toolchain scripts/config.sh scripts/cargo.sh $target_dir +rm -rf "$target_dir" +mkdir "$target_dir" +mkdir "$target_dir"/bin "$target_dir"/lib +ln target/$CHANNEL/cg_clif{,_build_sysroot} "$target_dir"/bin +ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib +ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir" if [[ "$build_sysroot" == "1" ]]; then echo "[BUILD] sysroot" export CG_CLIF_INCR_CACHE_DISABLED=1 dir=$(pwd) - cd $target_dir - time $dir/build_sysroot/build_sysroot.sh + cd "$target_dir" + time "$dir/build_sysroot/build_sysroot.sh" fi diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index 03ba5b53d2e40..a2b8f449f00ff 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "addr2line" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +checksum = "7c0929d69e78dd9bf5408269919fcbcaeb2e35e5d43e5815517cdc6a8e11a423" dependencies = [ "compiler_builtins", "gimli", @@ -47,9 +47,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "cc" -version = "1.0.61" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" +checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" [[package]] name = "cfg-if" @@ -76,9 +76,9 @@ version = "0.0.0" [[package]] name = "dlmalloc" -version = "0.1.4" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35055b1021724f4eb5262eb49130eebff23fc59fc5a14160e05faad8eeb36673" +checksum = "332570860c2edf2d57914987bf9e24835425f75825086b6ba7d1e6a3e4f1f254" dependencies = [ "compiler_builtins", "libc", @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", @@ -163,9 +163,9 @@ dependencies = [ [[package]] name = "object" -version = "0.20.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" +checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index eba15c0dd4308..d7a72df2eb283 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -10,10 +10,10 @@ dir=$(pwd) # Use rustc with cg_clif as hotpluggable backend instead of the custom cg_clif driver so that # build scripts are still compiled using cg_llvm. -export RUSTC=$dir"/cg_clif_build_sysroot" +export RUSTC=$dir"/bin/cg_clif_build_sysroot" export RUSTFLAGS=$RUSTFLAGS" --clif" -cd $(dirname "$0") +cd "$(dirname "$0")" # Cleanup for previous run # v Clean target dir except for build scripts and incremental cache @@ -28,12 +28,13 @@ if [[ "$1" != "--debug" ]]; then sysroot_channel='release' # FIXME Enable incremental again once rust-lang/rust#74946 is fixed # FIXME Enable -Zmir-opt-level=2 again once it doesn't ice anymore - CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS" cargo build --target $TARGET_TRIPLE --release + CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS" cargo build --target "$TARGET_TRIPLE" --release else sysroot_channel='debug' - cargo build --target $TARGET_TRIPLE + cargo build --target "$TARGET_TRIPLE" fi # Copy files to sysroot -mkdir -p $dir/sysroot/lib/rustlib/$TARGET_TRIPLE/lib/ -cp -a target/$TARGET_TRIPLE/$sysroot_channel/deps/* $dir/sysroot/lib/rustlib/$TARGET_TRIPLE/lib/ +mkdir -p "$dir/lib/rustlib/$TARGET_TRIPLE/lib/" +ln "target/$TARGET_TRIPLE/$sysroot_channel/deps/"* "$dir/lib/rustlib/$TARGET_TRIPLE/lib/" +rm "$dir/lib/rustlib/$TARGET_TRIPLE/lib/"*.{rmeta,d} diff --git a/build_sysroot/prepare_sysroot_src.sh b/build_sysroot/prepare_sysroot_src.sh index d0fb09ce745d4..40fbaf646a2f6 100755 --- a/build_sysroot/prepare_sysroot_src.sh +++ b/build_sysroot/prepare_sysroot_src.sh @@ -1,18 +1,18 @@ #!/bin/bash set -e -cd $(dirname "$0") +cd "$(dirname "$0")" -SRC_DIR=$(dirname $(rustup which rustc))"/../lib/rustlib/src/rust/" +SRC_DIR="$(dirname "$(rustup which rustc)")/../lib/rustlib/src/rust/" DST_DIR="sysroot_src" -if [ ! -e $SRC_DIR ]; then +if [ ! -e "$SRC_DIR" ]; then echo "Please install rust-src component" exit 1 fi rm -rf $DST_DIR mkdir -p $DST_DIR/library -cp -a $SRC_DIR/library $DST_DIR/ +cp -a "$SRC_DIR/library" $DST_DIR/ pushd $DST_DIR echo "[GIT] init" @@ -22,8 +22,8 @@ git add . echo "[GIT] commit" git commit -m "Initial commit" -q for file in $(ls ../../patches/ | grep -v patcha); do -echo "[GIT] apply" $file -git apply ../../patches/$file +echo "[GIT] apply" "$file" +git apply ../../patches/"$file" git add -A git commit --no-gpg-sign -m "Patch $file" done diff --git a/example/std_example.rs b/example/std_example.rs index cb512a4aa335e..b38e25328a4ee 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -53,6 +53,7 @@ fn main() { assert_eq!(0b0000000000000000000000000010000010000000000000000000000000000000_0000000000100000000000000000000000001000000000000100000000000000u128.leading_zeros(), 26); assert_eq!(0b0000000000000000000000000010000000000000000000000000000000000000_0000000000000000000000000000000000001000000000000000000010000000u128.trailing_zeros(), 7); + assert_eq!(core::intrinsics::saturating_sub(0, -170141183460469231731687303715884105728i128), 170141183460469231731687303715884105727i128); let _d = 0i128.checked_div(2i128); let _d = 0u128.checked_div(2u128); diff --git a/patches/0022-core-Disable-not-compiling-tests.patch b/patches/0022-core-Disable-not-compiling-tests.patch index ee8548783de65..8cfffe580a1f0 100644 --- a/patches/0022-core-Disable-not-compiling-tests.patch +++ b/patches/0022-core-Disable-not-compiling-tests.patch @@ -52,8 +52,8 @@ index 0475aeb..9558198 100644 fn test_rotate() { assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A); @@ -112,6 +113,7 @@ mod tests { - assert_eq!(B.rotate_left(64), B); - assert_eq!(C.rotate_left(64), C); + assert_eq!(B.rotate_left(128), B); + assert_eq!(C.rotate_left(128), C); } + */ @@ -72,8 +72,8 @@ index 04ed14f..a6e372e 100644 fn test_rotate() { assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A); @@ -76,6 +77,7 @@ mod tests { - assert_eq!(B.rotate_left(64), B); - assert_eq!(C.rotate_left(64), C); + assert_eq!(B.rotate_left(128), B); + assert_eq!(C.rotate_left(128), C); } + */ diff --git a/prepare.sh b/prepare.sh index 87f96f5dcf41f..08e7cb1802999 100755 --- a/prepare.sh +++ b/prepare.sh @@ -24,6 +24,7 @@ git checkout -- . git checkout 804a7a21b9e673a482797aa289a18ed480e4d813 # build with cg_llvm for perf comparison +unset CARGO_TARGET_DIR cargo build mv target/debug/main raytracer_cg_llvm popd diff --git a/rust-toolchain b/rust-toolchain index 0ca96be9ae731..ed1e64f45db08 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-10-31 +nightly-2020-11-27 diff --git a/scripts/cargo.sh b/scripts/cargo.sh index 947b4a28798f2..dcd40acc02a53 100755 --- a/scripts/cargo.sh +++ b/scripts/cargo.sh @@ -1,16 +1,16 @@ #!/bin/bash dir=$(dirname "$0") -source $dir/config.sh +source "$dir/config.sh" # read nightly compiler from rust-toolchain file -TOOLCHAIN=$(cat $dir/rust-toolchain) +TOOLCHAIN=$(cat "$dir/rust-toolchain") cmd=$1 shift || true if [[ "$cmd" = "jit" ]]; then -cargo +${TOOLCHAIN} rustc "$@" -- --jit +cargo "+${TOOLCHAIN}" rustc "$@" -- --jit else -cargo +${TOOLCHAIN} $cmd "$@" +cargo "+${TOOLCHAIN}" "$cmd" "$@" fi diff --git a/scripts/config.sh b/scripts/config.sh index 6120a550a27a1..dea037e2bc002 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -1,7 +1,8 @@ -#!/usr/bin/env bash +# Note to people running shellcheck: this file should only be sourced, not executed directly. + set -e -unamestr=`uname` +unamestr=$(uname) if [[ "$unamestr" == 'Linux' ]]; then dylib_ext='so' elif [[ "$unamestr" == 'Darwin' ]]; then @@ -40,19 +41,19 @@ echo export RUSTC_WRAPPER= fi -dir=$(cd $(dirname "$BASH_SOURCE"); pwd) +dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd) -export RUSTC=$dir"/cg_clif" -export RUSTFLAGS=$linker -export RUSTDOCFLAGS=$linker' -Ztrim-diagnostic-paths=no -Cpanic=abort -Zpanic-abort-tests '\ -'-Zcodegen-backend='$dir'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir'/sysroot' +export RUSTC=$dir"/bin/cg_clif" +export RUSTFLAGS=$linker" "$RUSTFLAGS +export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\ +'-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir # FIXME remove once the atomic shim is gone -if [[ `uname` == 'Darwin' ]]; then +if [[ $(uname) == 'Darwin' ]]; then export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" fi -export LD_LIBRARY_PATH="$dir:$(rustc --print sysroot)/lib:$dir/target/out:$dir/sysroot/lib/rustlib/"$TARGET_TRIPLE"/lib" +export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib" export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH export CG_CLIF_DISPLAY_CG_TIME=1 diff --git a/scripts/rustup.sh b/scripts/rustup.sh index 541b3c6563bab..430f5c469b4b2 100755 --- a/scripts/rustup.sh +++ b/scripts/rustup.sh @@ -7,13 +7,13 @@ case $1 in TOOLCHAIN=$(date +%Y-%m-%d) echo "=> Installing new nightly" - rustup toolchain install --profile minimal nightly-${TOOLCHAIN} # Sanity check to see if the nightly exists - echo nightly-${TOOLCHAIN} > rust-toolchain + rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists + echo "nightly-${TOOLCHAIN}" > rust-toolchain rustup component add rustfmt || true echo "=> Uninstalling all old nighlies" - for nightly in $(rustup toolchain list | grep nightly | grep -v $TOOLCHAIN | grep -v nightly-x86_64); do - rustup toolchain uninstall $nightly + for nightly in $(rustup toolchain list | grep nightly | grep -v "$TOOLCHAIN" | grep -v nightly-x86_64); do + rustup toolchain uninstall "$nightly" done ./clean_all.sh @@ -27,14 +27,30 @@ case $1 in git commit -m "Rustup to $(rustc -V)" ;; "push") - cg_clif=$(pwd) - pushd ../rust - branch=update_cg_clif-$(date +%Y-%m-%d) - git checkout -b $branch - git subtree pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master - git push -u my $branch - popd + cg_clif=$(pwd) + pushd ../rust + git pull origin master + branch=sync_cg_clif-$(date +%Y-%m-%d) + git checkout -b "$branch" + git subtree pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master + git push -u my "$branch" + + # immediately merge the merge commit into cg_clif to prevent merge conflicts when syncing + # from rust-lang/rust later + git subtree push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust + popd + git merge sync_from_rust ;; + "pull") + cg_clif=$(pwd) + pushd ../rust + git pull origin master + rust_vers="$(git rev-parse HEAD)" + git subtree push --prefix=compiler/rustc_codegen_cranelift/ "$cg_clif" sync_from_rust + popd + git merge sync_from_rust -m "Sync from rust $rust_vers" + git branch -d sync_from_rust + ;; *) echo "Unknown command '$1'" echo "Usage: ./rustup.sh prepare|commit" diff --git a/scripts/test_bootstrap.sh b/scripts/test_bootstrap.sh index 7f43f81a6cdcd..db69541b226cf 100755 --- a/scripts/test_bootstrap.sh +++ b/scripts/test_bootstrap.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -cd $(dirname "$0")/../ +cd "$(dirname "$0")/../" ./build.sh source build/config.sh @@ -11,7 +11,7 @@ git clone https://github.com/rust-lang/rust.git || true pushd rust git fetch git checkout -- . -git checkout $(rustc -V | cut -d' ' -f3 | tr -d '(') +git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')" git apply - < config.toml <( support_vararg: bool, ) -> (String, Signature) { assert!(!inst.substs.needs_infer()); - let fn_sig = tcx.normalize_erasing_late_bound_regions( - ParamEnv::reveal_all(), - fn_sig_for_fn_abi(tcx, inst), - ); + let fn_sig = tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_sig_for_fn_abi(tcx, inst)); if fn_sig.c_variadic && !support_vararg { tcx.sess.span_fatal( tcx.def_span(inst.def_id()), diff --git a/src/archive.rs b/src/archive.rs index daf9fa6158f1c..96579054389ab 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -8,7 +8,7 @@ use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder}; use rustc_codegen_ssa::METADATA_FILENAME; use rustc_session::Session; -use object::{Object, SymbolKind}; +use object::{Object, ObjectSymbol, SymbolKind}; #[derive(Debug)] enum ArchiveEntry { @@ -184,7 +184,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { entry_name.as_bytes().to_vec(), object .symbols() - .filter_map(|(_index, symbol)| { + .filter_map(|symbol| { if symbol.is_undefined() || symbol.is_local() || symbol.kind() != SymbolKind::Data @@ -193,7 +193,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { { None } else { - symbol.name().map(|name| name.as_bytes().to_vec()) + symbol.name().map(|name| name.as_bytes().to_vec()).ok() } }) .collect::>(), diff --git a/src/atomic_shim.rs b/src/atomic_shim.rs index 2f0157c257b98..674e6d907510a 100644 --- a/src/atomic_shim.rs +++ b/src/atomic_shim.rs @@ -7,8 +7,7 @@ use crate::prelude::*; #[cfg(all(feature = "jit", unix))] #[no_mangle] -static mut __cg_clif_global_atomic_mutex: libc::pthread_mutex_t = - libc::PTHREAD_MUTEX_INITIALIZER; +static mut __cg_clif_global_atomic_mutex: libc::pthread_mutex_t = libc::PTHREAD_MUTEX_INITIALIZER; pub(crate) fn init_global_lock( module: &mut impl Module, diff --git a/src/base.rs b/src/base.rs index a4df371c88aa6..72073896a723b 100644 --- a/src/base.rs +++ b/src/base.rs @@ -12,6 +12,10 @@ pub(crate) fn codegen_fn<'tcx>( ) { let tcx = cx.tcx; + let _inst_guard = + crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); + debug_assert!(!instance.substs.needs_infer()); + let mir = tcx.instance_mir(instance.def); // Declare function @@ -499,7 +503,8 @@ fn codegen_stmt<'tcx>( UnOp::Neg => match layout.ty.kind() { ty::Int(IntTy::I128) => { // FIXME remove this case once ineg.i128 works - let zero = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size)); + let zero = + CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size)); crate::num::codegen_int_binop(fx, BinOp::Sub, zero, operand) } ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout), @@ -509,7 +514,11 @@ fn codegen_stmt<'tcx>( }; lval.write_cvalue(fx, res); } - Rvalue::Cast(CastKind::Pointer(PointerCast::ReifyFnPointer), ref operand, to_ty) => { + Rvalue::Cast( + CastKind::Pointer(PointerCast::ReifyFnPointer), + ref operand, + to_ty, + ) => { let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx)); let to_layout = fx.layout_of(fx.monomorphize(to_ty)); match *from_ty.kind() { @@ -530,9 +539,21 @@ fn codegen_stmt<'tcx>( _ => bug!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty), } } - Rvalue::Cast(CastKind::Pointer(PointerCast::UnsafeFnPointer), ref operand, to_ty) - | Rvalue::Cast(CastKind::Pointer(PointerCast::MutToConstPointer), ref operand, to_ty) - | Rvalue::Cast(CastKind::Pointer(PointerCast::ArrayToPointer), ref operand, to_ty) => { + Rvalue::Cast( + CastKind::Pointer(PointerCast::UnsafeFnPointer), + ref operand, + to_ty, + ) + | Rvalue::Cast( + CastKind::Pointer(PointerCast::MutToConstPointer), + ref operand, + to_ty, + ) + | Rvalue::Cast( + CastKind::Pointer(PointerCast::ArrayToPointer), + ref operand, + to_ty, + ) => { let to_layout = fx.layout_of(fx.monomorphize(to_ty)); let operand = codegen_operand(fx, operand); lval.write_cvalue(fx, operand.cast_pointer_to(to_layout)); diff --git a/src/bin/cg_clif.rs b/src/bin/cg_clif.rs index cd01acc9a8329..f4d23ebcf4e4d 100644 --- a/src/bin/cg_clif.rs +++ b/src/bin/cg_clif.rs @@ -26,15 +26,15 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks { config.opts.cg.panic = Some(PanicStrategy::Abort); config.opts.debugging_opts.panic_abort_tests = true; - config.opts.maybe_sysroot = Some( - config.opts.maybe_sysroot.clone().unwrap_or_else( - || std::env::current_exe() - .unwrap() - .parent() - .unwrap() - .join("sysroot"), - ), - ); + config.opts.maybe_sysroot = Some(config.opts.maybe_sysroot.clone().unwrap_or_else(|| { + std::env::current_exe() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .to_owned() + })); } } diff --git a/src/common.rs b/src/common.rs index d7d6c3e16773b..1485d4451b815 100644 --- a/src/common.rs +++ b/src/common.rs @@ -233,7 +233,7 @@ pub(crate) fn type_min_max_value( let min_msb = bcx.ins().iconst(types::I64, (min >> 64) as u64 as i64); let min = bcx.ins().iconcat(min_lsb, min_msb); - let max = i128::MIN as u128; + let max = i128::MAX as u128; let max_lsb = bcx.ins().iconst(types::I64, max as u64 as i64); let max_msb = bcx.ins().iconst(types::I64, (max >> 64) as u64 as i64); let max = bcx.ins().iconcat(max_lsb, max_msb); @@ -364,7 +364,7 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> { self.instance.subst_mir_and_normalize_erasing_regions( self.tcx, ty::ParamEnv::reveal_all(), - value + value, ) } diff --git a/src/constant.rs b/src/constant.rs index 351bb6ecd2383..544b020b71190 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -163,10 +163,7 @@ pub(crate) fn codegen_const_value<'tcx>( assert!(!layout.is_unsized(), "sized const value"); if layout.is_zst() { - return CValue::by_ref( - crate::Pointer::dangling(layout.align.pref), - layout, - ); + return CValue::by_ref(crate::Pointer::dangling(layout.align.pref), layout); } match const_val { @@ -186,9 +183,7 @@ pub(crate) fn codegen_const_value<'tcx>( } match x { - Scalar::Int(int) => { - CValue::const_val(fx, layout, int) - } + Scalar::Int(int) => CValue::const_val(fx, layout, int), Scalar::Ptr(ptr) => { let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id); let base_addr = match alloc_kind { diff --git a/src/debuginfo/emit.rs b/src/debuginfo/emit.rs index f6f795e45615c..c21835b1fc3aa 100644 --- a/src/debuginfo/emit.rs +++ b/src/debuginfo/emit.rs @@ -76,7 +76,7 @@ impl WriterRelocate { #[cfg(feature = "jit")] pub(super) fn relocate_for_jit( mut self, - jit_product: &cranelift_simplejit::SimpleJITProduct, + jit_module: &cranelift_simplejit::SimpleJITModule, ) -> Vec { use std::convert::TryInto; @@ -84,8 +84,9 @@ impl WriterRelocate { match reloc.name { super::DebugRelocName::Section(_) => unreachable!(), super::DebugRelocName::Symbol(sym) => { - let addr = jit_product - .lookup_func(cranelift_module::FuncId::from_u32(sym.try_into().unwrap())); + let addr = jit_module.get_finalized_function( + cranelift_module::FuncId::from_u32(sym.try_into().unwrap()), + ); let val = (addr as u64 as i64 + reloc.addend) as u64; self.writer .write_udata_at(reloc.offset as usize, val, reloc.size) diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index 68138404c2436..e0f62b64e6bbb 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -80,7 +80,7 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(feature = "jit")] pub(crate) unsafe fn register_jit( self, - jit_product: &cranelift_simplejit::SimpleJITProduct, + jit_module: &cranelift_simplejit::SimpleJITModule, ) -> Option { let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian( self.tcx, @@ -91,7 +91,7 @@ impl<'tcx> UnwindContext<'tcx> { return None; } - let mut eh_frame = eh_frame.0.relocate_for_jit(jit_product); + let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module); // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. eh_frame.extend(&[0, 0, 0, 0]); diff --git a/src/discriminant.rs b/src/discriminant.rs index 1e8e86add1a59..ad635016a9136 100644 --- a/src/discriminant.rs +++ b/src/discriminant.rs @@ -30,8 +30,16 @@ pub(crate) fn codegen_set_discriminant<'tcx>( .ty .discriminant_for_variant(fx.tcx, variant_index) .unwrap() - .val - .into(); + .val; + let to = if ptr.layout().abi.is_signed() { + ty::ScalarInt::try_from_int( + ptr.layout().size.sign_extend(to) as i128, + ptr.layout().size, + ) + .unwrap() + } else { + ty::ScalarInt::try_from_uint(to, ptr.layout().size).unwrap() + }; let discr = CValue::const_val(fx, ptr.layout(), to); ptr.write_cvalue(fx, discr); } @@ -49,8 +57,12 @@ pub(crate) fn codegen_set_discriminant<'tcx>( if variant_index != dataful_variant { let niche = place.place_field(fx, mir::Field::new(tag_field)); let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); - let niche_value = u128::from(niche_value).wrapping_add(niche_start); - let niche_llval = CValue::const_val(fx, niche.layout(), niche_value.into()); + let niche_value = ty::ScalarInt::try_from_uint( + u128::from(niche_value).wrapping_add(niche_start), + niche.layout().size, + ) + .unwrap(); + let niche_llval = CValue::const_val(fx, niche.layout(), niche_value); niche.write_cvalue(fx, niche_llval); } } @@ -78,7 +90,16 @@ pub(crate) fn codegen_get_discriminant<'tcx>( .ty .discriminant_for_variant(fx.tcx, *index) .map_or(u128::from(index.as_u32()), |discr| discr.val); - return CValue::const_val(fx, dest_layout, discr_val.into()); + let discr_val = if dest_layout.abi.is_signed() { + ty::ScalarInt::try_from_int( + dest_layout.size.sign_extend(discr_val) as i128, + dest_layout.size, + ) + .unwrap() + } else { + ty::ScalarInt::try_from_uint(discr_val, dest_layout.size).unwrap() + }; + return CValue::const_val(fx, dest_layout, discr_val); } Variants::Multiple { tag, diff --git a/src/driver/aot.rs b/src/driver/aot.rs index c0245aa1e0213..491d6cbbf7969 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -145,7 +145,11 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege } let mut cx = crate::CodegenCx::new(tcx, module, tcx.sess.opts.debuginfo != DebugInfo::None); - super::codegen_mono_items(&mut cx, mono_items); + super::predefine_mono_items(&mut cx, &mono_items); + for (mono_item, (linkage, visibility)) in mono_items { + let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + super::codegen_mono_item(&mut cx, mono_item, linkage); + } let (mut module, global_asm, debug, mut unwind_context) = tcx.sess.time("finalize CodegenCx", || cx.finalize()); crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context, false); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 3f47df7d844b3..5a844841c2ce5 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -70,7 +70,11 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let (mut jit_module, global_asm, _debug, mut unwind_context) = super::time(tcx, "codegen mono items", || { - super::codegen_mono_items(&mut cx, mono_items); + super::predefine_mono_items(&mut cx, &mono_items); + for (mono_item, (linkage, visibility)) in mono_items { + let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + super::codegen_mono_item(&mut cx, mono_item, linkage); + } tcx.sess.time("finalize CodegenCx", || cx.finalize()) }); if !global_asm.is_empty() { @@ -81,11 +85,11 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { tcx.sess.abort_if_errors(); - let jit_product = jit_module.finish(); + jit_module.finalize_definitions(); - let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_product) }; + let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; - let finalized_main: *const u8 = jit_product.lookup_func(main_func_id); + let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); println!("Rustc codegen cranelift will JIT run the executable, because --jit was passed"); @@ -140,11 +144,11 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { let mut imported_symbols = Vec::new(); for path in dylib_paths { - use object::Object; + use object::{Object, ObjectSymbol}; let lib = libloading::Library::new(&path).unwrap(); let obj = std::fs::read(path).unwrap(); let obj = object::File::parse(&obj).unwrap(); - imported_symbols.extend(obj.dynamic_symbols().filter_map(|(_idx, symbol)| { + imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| { let name = symbol.name().unwrap().to_string(); if name.is_empty() || !symbol.is_global() || symbol.is_undefined() { return None; diff --git a/src/driver/mod.rs b/src/driver/mod.rs index a11dc57ee6453..7b8cc2ddd48d6 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -1,4 +1,4 @@ -//! Drivers are responsible for calling [`codegen_mono_items`] and performing any further actions +//! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions //! like JIT executing or writing object files. use std::any::Any; @@ -40,12 +40,12 @@ pub(crate) fn codegen_crate( aot::run_aot(tcx, metadata, need_metadata_module) } -fn codegen_mono_items<'tcx>( +fn predefine_mono_items<'tcx>( cx: &mut crate::CodegenCx<'tcx, impl Module>, - mono_items: Vec<(MonoItem<'tcx>, (RLinkage, Visibility))>, + mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))], ) { cx.tcx.sess.time("predefine functions", || { - for &(mono_item, (linkage, visibility)) in &mono_items { + for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { let (name, sig) = get_function_name_and_sig( @@ -61,11 +61,6 @@ fn codegen_mono_items<'tcx>( } } }); - - for (mono_item, (linkage, visibility)) in mono_items { - let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - codegen_mono_item(cx, mono_item, linkage); - } } fn codegen_mono_item<'tcx, M: Module>( @@ -73,20 +68,15 @@ fn codegen_mono_item<'tcx, M: Module>( mono_item: MonoItem<'tcx>, linkage: Linkage, ) { - let tcx = cx.tcx; match mono_item { MonoItem::Fn(inst) => { - let _inst_guard = - crate::PrintOnPanic(|| format!("{:?} {}", inst, tcx.symbol_name(inst).name)); - debug_assert!(!inst.substs.needs_infer()); - tcx.sess + cx.tcx + .sess .time("codegen fn", || crate::base::codegen_fn(cx, inst, linkage)); } - MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id); - } + MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx.constants_cx, def_id), MonoItem::GlobalAsm(hir_id) => { - let item = tcx.hir().expect_item(hir_id); + let item = cx.tcx.hir().expect_item(hir_id); if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { cx.global_asm.push_str(&*asm.as_str()); cx.global_asm.push_str("\n\n"); diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index ab16fabd348a5..3563aa250a9b8 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -263,6 +263,48 @@ fn simd_pair_for_each_lane<'tcx, M: Module>( } } +fn simd_reduce<'tcx, M: Module>( + fx: &mut FunctionCx<'_, 'tcx, M>, + val: CValue<'tcx>, + ret: CPlace<'tcx>, + f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value, +) { + let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + assert_eq!(lane_layout, ret.layout()); + + let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); + for lane_idx in 1..lane_count { + let lane = val + .value_field(fx, mir::Field::new(lane_idx.into())) + .load_scalar(fx); + res_val = f(fx, lane_layout, res_val, lane); + } + let res = CValue::by_val(res_val, lane_layout); + ret.write_cvalue(fx, res); +} + +fn simd_reduce_bool<'tcx, M: Module>( + fx: &mut FunctionCx<'_, 'tcx, M>, + val: CValue<'tcx>, + ret: CPlace<'tcx>, + f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value, +) { + let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + assert!(ret.layout().ty.is_bool()); + + let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); + let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean + for lane_idx in 1..lane_count { + let lane = val + .value_field(fx, mir::Field::new(lane_idx.into())) + .load_scalar(fx); + let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean + res_val = f(fx, res_val, lane); + } + let res = CValue::by_val(res_val, ret.layout()); + ret.write_cvalue(fx, res); +} + fn bool_to_zero_or_max_uint<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, layout: TyAndLayout<'tcx>, @@ -287,7 +329,7 @@ fn bool_to_zero_or_max_uint<'tcx>( } macro simd_cmp { - ($fx:expr, $cc:ident($x:ident, $y:ident) -> $ret:ident) => { + ($fx:expr, $cc:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => { let vector_ty = clif_vector_type($fx.tcx, $x.layout()); if let Some(vector_ty) = vector_ty { @@ -308,6 +350,7 @@ macro simd_cmp { |fx, lane_layout, res_lane_layout, x_lane, y_lane| { let res_lane = match lane_layout.ty.kind() { ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane), + ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane), _ => unreachable!("{:?}", lane_layout.ty), }; bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane) @@ -315,7 +358,7 @@ macro simd_cmp { ); } }, - ($fx:expr, $cc_u:ident|$cc_s:ident($x:ident, $y:ident) -> $ret:ident) => { + ($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) => { // FIXME use vector icmp when possible simd_pair_for_each_lane( $fx, @@ -326,6 +369,7 @@ macro simd_cmp { let res_lane = match lane_layout.ty.kind() { ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane), ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane), + ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane), _ => unreachable!("{:?}", lane_layout.ty), }; bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane) @@ -497,12 +541,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; copy | copy_nonoverlapping, (v src, v dst, v count) { let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); - let elem_size = fx - .bcx - .ins() - .iconst(fx.pointer_type, elem_size as i64); assert_eq!(args.len(), 3); - let byte_amount = fx.bcx.ins().imul(count, elem_size); + let byte_amount = if elem_size != 1 { + fx.bcx.ins().imul_imm(count, elem_size as i64) + } else { + count + }; if intrinsic.contains("nonoverlapping") { // FIXME emit_small_memcpy @@ -515,12 +559,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( // NOTE: the volatile variants have src and dst swapped volatile_copy_memory | volatile_copy_nonoverlapping_memory, (v dst, v src, v count) { let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); - let elem_size = fx - .bcx - .ins() - .iconst(fx.pointer_type, elem_size as i64); assert_eq!(args.len(), 3); - let byte_amount = fx.bcx.ins().imul(count, elem_size); + let byte_amount = if elem_size != 1 { + fx.bcx.ins().imul_imm(count, elem_size as i64) + } else { + count + }; // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} if intrinsic.contains("nonoverlapping") { @@ -676,7 +720,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( offset | arith_offset, (c base, v offset) { let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty; let pointee_size = fx.layout_of(pointee_ty).size.bytes(); - let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64); + let ptr_diff = if pointee_size != 1 { + fx.bcx.ins().imul_imm(offset, pointee_size as i64) + } else { + offset + }; let base_val = base.load_scalar(fx); let res = fx.bcx.ins().iadd(base_val, ptr_diff); ret.write_cvalue(fx, CValue::by_val(res, base.layout())); @@ -688,7 +736,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( write_bytes | volatile_set_memory, (c dst, v val, v count) { let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty; let pointee_size = fx.layout_of(pointee_ty).size.bytes(); - let count = fx.bcx.ins().imul_imm(count, pointee_size as i64); + let count = if pointee_size != 1 { + fx.bcx.ins().imul_imm(count, pointee_size as i64) + } else { + count + }; let dst_ptr = dst.load_scalar(fx); // FIXME make the memset actually volatile when switching to emit_small_memset // FIXME use emit_small_memset diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index 2e31c4669e25b..2b32e866e5ef6 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -35,30 +35,33 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }); }; - // FIXME support float comparisons simd_eq, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - simd_cmp!(fx, Equal(x, y) -> ret); + simd_cmp!(fx, Equal|Equal(x, y) -> ret); }; simd_ne, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - simd_cmp!(fx, NotEqual(x, y) -> ret); + simd_cmp!(fx, NotEqual|NotEqual(x, y) -> ret); }; simd_lt, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - simd_cmp!(fx, UnsignedLessThan|SignedLessThan(x, y) -> ret); + simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret); }; simd_le, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual(x, y) -> ret); + simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret); }; simd_gt, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan(x, y) -> ret); + simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret); }; simd_ge, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - simd_cmp!(fx, UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual(x, y) -> ret); + simd_cmp!( + fx, + UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual + (x, y) -> ret + ); }; // simd_shuffle32(x: T, y: T, idx: [u32; 32]) -> U @@ -107,9 +110,9 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( for (out_idx, in_idx) in indexes.into_iter().enumerate() { let in_lane = if in_idx < lane_count { - x.value_field(fx, mir::Field::new(in_idx.try_into().unwrap())) + x.value_field(fx, mir::Field::new(in_idx.into())) } else { - y.value_field(fx, mir::Field::new((in_idx - lane_count).try_into().unwrap())) + y.value_field(fx, mir::Field::new((in_idx - lane_count).into())) }; let out_lane = ret.place_field(fx, mir::Field::new(out_idx)); out_lane.write_cvalue(fx, in_lane); @@ -143,10 +146,17 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) { idx_const } else { - fx.tcx.sess.span_fatal( + fx.tcx.sess.span_warn( span, "Index argument for `simd_extract` is not a constant", ); + let res = crate::trap::trap_unimplemented_ret_value( + fx, + ret.layout(), + "Index argument for `simd_extract` is not a constant", + ); + ret.write_cvalue(fx, res); + return; }; let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); @@ -207,7 +217,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( assert_eq!(lane_count, ret_lane_count); for lane in 0..lane_count { - let lane = mir::Field::new(lane.try_into().unwrap()); + let lane = mir::Field::new(lane.into()); let a_lane = a.value_field(fx, lane).load_scalar(fx); let b_lane = b.value_field(fx, lane).load_scalar(fx); let c_lane = c.value_field(fx, lane).load_scalar(fx); @@ -228,11 +238,42 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( simd_flt_binop!(fx, fmax(x, y) -> ret); }; + simd_reduce_add_ordered | simd_reduce_add_unordered, (c v) { + validate_simd_type!(fx, intrinsic, span, v.layout().ty); + simd_reduce(fx, v, ret, |fx, lane_layout, a, b| { + if lane_layout.ty.is_floating_point() { + fx.bcx.ins().fadd(a, b) + } else { + fx.bcx.ins().iadd(a, b) + } + }); + }; + + simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v) { + validate_simd_type!(fx, intrinsic, span, v.layout().ty); + simd_reduce(fx, v, ret, |fx, lane_layout, a, b| { + if lane_layout.ty.is_floating_point() { + fx.bcx.ins().fmul(a, b) + } else { + fx.bcx.ins().imul(a, b) + } + }); + }; + + simd_reduce_all, (c v) { + validate_simd_type!(fx, intrinsic, span, v.layout().ty); + simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().band(a, b)); + }; + + simd_reduce_any, (c v) { + validate_simd_type!(fx, intrinsic, span, v.layout().ty); + simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().bor(a, b)); + }; + // simd_fabs // simd_saturating_add // simd_bitmask // simd_select - // simd_reduce_add_{,un}ordered // simd_rem } } diff --git a/src/trap.rs b/src/trap.rs index 690d96764a8f5..67495c7414840 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -67,3 +67,15 @@ pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, impl Module>, msg: let true_ = fx.bcx.ins().iconst(types::I32, 1); fx.bcx.ins().trapnz(true_, TrapCode::User(!0)); } + +/// Like `trap_unimplemented` but returns a fake value of the specified type. +/// +/// Trap code: user65535 +pub(crate) fn trap_unimplemented_ret_value<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + dest_layout: TyAndLayout<'tcx>, + msg: impl AsRef, +) -> CValue<'tcx> { + trap_unimplemented(fx, msg); + CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout) +} From 7760894d3fcd2a1048e8f61d3e64124a94a726d0 Mon Sep 17 00:00:00 2001 From: oli Date: Sun, 29 Nov 2020 14:56:19 +0000 Subject: [PATCH 02/26] Allow cranelift to handle atomic pointers --- src/intrinsics/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index ab16fabd348a5..e5482187a731d 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -146,12 +146,12 @@ macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) { match $ty.kind() { - ty::Uint(_) | ty::Int(_) => {} + ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} _ => { $fx.tcx.sess.span_err( $span, &format!( - "`{}` intrinsic: expected basic integer type, found `{:?}`", + "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`", $intrinsic, $ty ), ); From a8cda1aee15733a9994a8a6c98c6676c18a5caf7 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 3 Dec 2020 11:52:11 +0100 Subject: [PATCH 03/26] Rustup to rustc 1.50.0-nightly (f4db9ffb2 2020-12-02) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index ed1e64f45db08..547f1e0d8f0c3 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-11-27 +nightly-2020-12-03 From 64e7ff25f65c29c80b7e1659a67562e68d1c8fac Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 3 Dec 2020 12:12:46 +0100 Subject: [PATCH 04/26] Update Cranelift --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67ed41e765231..1e873d3a01cd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,7 +50,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" dependencies = [ "cranelift-entity", ] @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" dependencies = [ "byteorder", "cranelift-bforest", @@ -76,7 +76,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -85,17 +85,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" [[package]] name = "cranelift-entity" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" [[package]] name = "cranelift-frontend" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" dependencies = [ "cranelift-codegen", "log", @@ -106,7 +106,7 @@ dependencies = [ [[package]] name = "cranelift-module" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" dependencies = [ "anyhow", "cranelift-codegen", @@ -118,7 +118,7 @@ dependencies = [ [[package]] name = "cranelift-native" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -128,7 +128,7 @@ dependencies = [ [[package]] name = "cranelift-object" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" dependencies = [ "anyhow", "cranelift-codegen", @@ -141,7 +141,7 @@ dependencies = [ [[package]] name = "cranelift-simplejit" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" dependencies = [ "cranelift-codegen", "cranelift-entity", From d95d03ae8ad10f253dce81a62a9ac372835b9bb4 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 3 Dec 2020 12:59:36 +0100 Subject: [PATCH 05/26] Support #[repr(simd)] on array wrappers Complement to rust-lang/rust#78863 --- src/intrinsics/llvm.rs | 4 ++-- src/intrinsics/mod.rs | 42 +++++++++++++----------------------------- src/intrinsics/simd.rs | 25 +++++++++++++------------ 3 files changed, 28 insertions(+), 43 deletions(-) diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index 171445f2d71b6..d58e4d4995842 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -23,8 +23,8 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) { - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, a.layout()); - let lane_ty = fx.clif_type(lane_layout.ty).unwrap(); + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); + let lane_ty = fx.clif_type(lane_ty).unwrap(); assert!(lane_count <= 32); let mut res = fx.bcx.ins().iconst(types::I32, 0); diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 3563aa250a9b8..4cfd4569760ee 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -171,27 +171,6 @@ macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) { } } -fn lane_type_and_count<'tcx>( - tcx: TyCtxt<'tcx>, - layout: TyAndLayout<'tcx>, -) -> (TyAndLayout<'tcx>, u16) { - assert!(layout.ty.is_simd()); - let lane_count = match layout.fields { - rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(), - _ => unreachable!("lane_type_and_count({:?})", layout), - }; - let lane_layout = layout - .field( - &ty::layout::LayoutCx { - tcx, - param_env: ParamEnv::reveal_all(), - }, - 0, - ) - .unwrap(); - (lane_layout, lane_count) -} - pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option { let (element, count) = match &layout.abi { Abi::Vector { element, count } => (element.clone(), *count), @@ -218,8 +197,10 @@ fn simd_for_each_lane<'tcx, M: Module>( ) { let layout = val.layout(); - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); assert_eq!(lane_count, ret_lane_count); for lane_idx in 0..lane_count { @@ -248,8 +229,10 @@ fn simd_pair_for_each_lane<'tcx, M: Module>( assert_eq!(x.layout(), y.layout()); let layout = x.layout(); - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); assert_eq!(lane_count, ret_lane_count); for lane in 0..lane_count { @@ -269,13 +252,14 @@ fn simd_reduce<'tcx, M: Module>( ret: CPlace<'tcx>, f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value, ) { - let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); assert_eq!(lane_layout, ret.layout()); let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); for lane_idx in 1..lane_count { let lane = val - .value_field(fx, mir::Field::new(lane_idx.into())) + .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())) .load_scalar(fx); res_val = f(fx, lane_layout, res_val, lane); } @@ -289,14 +273,14 @@ fn simd_reduce_bool<'tcx, M: Module>( ret: CPlace<'tcx>, f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value, ) { - let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout()); + let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx); assert!(ret.layout().ty.is_bool()); let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx); let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean for lane_idx in 1..lane_count { let lane = val - .value_field(fx, mir::Field::new(lane_idx.into())) + .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())) .load_scalar(fx); let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean res_val = f(fx, res_val, lane); diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index 2b32e866e5ef6..e0eb5c59590ff 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -73,11 +73,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( assert_eq!(x.layout(), y.layout()); let layout = x.layout(); - let (lane_type, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); - assert_eq!(lane_type, ret_lane_type); - assert_eq!(n, ret_lane_count); + assert_eq!(lane_ty, ret_lane_ty); + assert_eq!(u64::from(n), ret_lane_count); let total_len = lane_count * 2; @@ -105,14 +105,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; for &idx in &indexes { - assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len); + assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len); } for (out_idx, in_idx) in indexes.into_iter().enumerate() { - let in_lane = if in_idx < lane_count { + let in_lane = if u64::from(in_idx) < lane_count { x.value_field(fx, mir::Field::new(in_idx.into())) } else { - y.value_field(fx, mir::Field::new((in_idx - lane_count).into())) + y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap())) }; let out_lane = ret.place_field(fx, mir::Field::new(out_idx)); out_lane.write_cvalue(fx, in_lane); @@ -131,7 +131,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); - let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, base.layout()); + let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count)); } @@ -160,7 +160,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); - let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, v.layout()); + let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count)); } @@ -212,12 +212,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( assert_eq!(a.layout(), c.layout()); let layout = a.layout(); - let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); - let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); assert_eq!(lane_count, ret_lane_count); + let ret_lane_layout = fx.layout_of(ret_lane_ty); for lane in 0..lane_count { - let lane = mir::Field::new(lane.into()); + let lane = mir::Field::new(lane.try_into().unwrap()); let a_lane = a.value_field(fx, lane).load_scalar(fx); let b_lane = b.value_field(fx, lane).load_scalar(fx); let c_lane = c.value_field(fx, lane).load_scalar(fx); From 3f28a49177e662b9e16ec9b0fb3e4c4fdf810ad1 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 4 Dec 2020 10:19:32 +0100 Subject: [PATCH 06/26] Update Cranelift This includes bytecodealliance/wasmtime#2403 which enables hotswapping with SimpleJIT --- Cargo.lock | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e873d3a01cd2..4b19a7b827ee3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,7 +50,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" dependencies = [ "cranelift-entity", ] @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" dependencies = [ "byteorder", "cranelift-bforest", @@ -76,7 +76,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -85,17 +85,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" [[package]] name = "cranelift-entity" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" [[package]] name = "cranelift-frontend" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" dependencies = [ "cranelift-codegen", "log", @@ -106,7 +106,7 @@ dependencies = [ [[package]] name = "cranelift-module" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" dependencies = [ "anyhow", "cranelift-codegen", @@ -118,7 +118,7 @@ dependencies = [ [[package]] name = "cranelift-native" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -128,7 +128,7 @@ dependencies = [ [[package]] name = "cranelift-object" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" dependencies = [ "anyhow", "cranelift-codegen", @@ -141,8 +141,9 @@ dependencies = [ [[package]] name = "cranelift-simplejit" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#b93381e12684c948119fd4406650c804d6396d9d" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" dependencies = [ + "anyhow", "cranelift-codegen", "cranelift-entity", "cranelift-module", From 5f21ff20b3ab8cfce37a3dfc4641bf0d8e1b1c74 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 11 Oct 2020 13:59:31 +0200 Subject: [PATCH 07/26] Inline codegen_mono_item --- src/driver/aot.rs | 22 ++++++++++++++++++++-- src/driver/jit.rs | 19 +++++++++++++++++-- src/driver/mod.rs | 24 ------------------------ 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 491d6cbbf7969..c443483be884e 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -8,7 +8,7 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::middle::cstore::EncodedMetadata; -use rustc_middle::mir::mono::CodegenUnit; +use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{DebugInfo, OutputType}; @@ -148,7 +148,25 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege super::predefine_mono_items(&mut cx, &mono_items); for (mono_item, (linkage, visibility)) in mono_items { let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - super::codegen_mono_item(&mut cx, mono_item, linkage); + match mono_item { + MonoItem::Fn(inst) => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id) + } + MonoItem::GlobalAsm(hir_id) => { + let item = cx.tcx.hir().expect_item(hir_id); + if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { + cx.global_asm.push_str(&*asm.as_str()); + cx.global_asm.push_str("\n\n"); + } else { + bug!("Expected GlobalAsm found {:?}", item); + } + } + } } let (mut module, global_asm, debug, mut unwind_context) = tcx.sess.time("finalize CodegenCx", || cx.finalize()); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 5a844841c2ce5..4d40debc956de 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -5,6 +5,7 @@ use std::ffi::CString; use std::os::raw::{c_char, c_int}; use rustc_codegen_ssa::CrateInfo; +use rustc_middle::mir::mono::MonoItem; use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule}; @@ -73,12 +74,26 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { super::predefine_mono_items(&mut cx, &mono_items); for (mono_item, (linkage, visibility)) in mono_items { let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - super::codegen_mono_item(&mut cx, mono_item, linkage); + match mono_item { + MonoItem::Fn(inst) => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id) + } + MonoItem::GlobalAsm(hir_id) => { + let item = cx.tcx.hir().expect_item(hir_id); + tcx.sess + .span_fatal(item.span, "Global asm is not supported in JIT mode"); + } + } } tcx.sess.time("finalize CodegenCx", || cx.finalize()) }); if !global_asm.is_empty() { - tcx.sess.fatal("Global asm is not supported in JIT mode"); + tcx.sess.fatal("Inline asm is not supported in JIT mode"); } crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true); crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 7b8cc2ddd48d6..0b0ea8a852c65 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -63,30 +63,6 @@ fn predefine_mono_items<'tcx>( }); } -fn codegen_mono_item<'tcx, M: Module>( - cx: &mut crate::CodegenCx<'tcx, M>, - mono_item: MonoItem<'tcx>, - linkage: Linkage, -) { - match mono_item { - MonoItem::Fn(inst) => { - cx.tcx - .sess - .time("codegen fn", || crate::base::codegen_fn(cx, inst, linkage)); - } - MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx.constants_cx, def_id), - MonoItem::GlobalAsm(hir_id) => { - let item = cx.tcx.hir().expect_item(hir_id); - if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { - cx.global_asm.push_str(&*asm.as_str()); - cx.global_asm.push_str("\n\n"); - } else { - bug!("Expected GlobalAsm found {:?}", item); - } - } - } -} - fn time(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R { if std::env::var("CG_CLIF_DISPLAY_CG_TIME") .as_ref() From 994b91ac73016e2671138115ecd7fbbb31f13d28 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 9 Dec 2020 19:55:11 +0100 Subject: [PATCH 08/26] Optimize branches when the target is statically known to a jump This can happen in generic code --- src/base.rs | 24 ++++++++++++++++++------ src/optimize/peephole.rs | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/base.rs b/src/base.rs index 72073896a723b..3af4897b4f0b0 100644 --- a/src/base.rs +++ b/src/base.rs @@ -307,7 +307,9 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { } => { let discr = codegen_operand(fx, discr).load_scalar(fx); - if switch_ty.kind() == fx.tcx.types.bool.kind() { + let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind() + || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0); + if use_bool_opt { assert_eq!(targets.iter().count(), 1); let (then_value, then_block) = targets.iter().next().unwrap(); let then_block = fx.get_block(then_block); @@ -325,12 +327,22 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr); let discr = crate::optimize::peephole::make_branchable_value(&mut fx.bcx, discr); - if test_zero { - fx.bcx.ins().brz(discr, then_block, &[]); - fx.bcx.ins().jump(else_block, &[]); + if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken( + &fx.bcx, discr, test_zero, + ) { + if taken { + fx.bcx.ins().jump(then_block, &[]); + } else { + fx.bcx.ins().jump(else_block, &[]); + } } else { - fx.bcx.ins().brnz(discr, then_block, &[]); - fx.bcx.ins().jump(else_block, &[]); + if test_zero { + fx.bcx.ins().brz(discr, then_block, &[]); + fx.bcx.ins().jump(else_block, &[]); + } else { + fx.bcx.ins().brnz(discr, then_block, &[]); + fx.bcx.ins().jump(else_block, &[]); + } } } else { let mut switch = ::cranelift_frontend::Switch::new(); diff --git a/src/optimize/peephole.rs b/src/optimize/peephole.rs index f8e0f3af3d0ad..a575ed8dc35f8 100644 --- a/src/optimize/peephole.rs +++ b/src/optimize/peephole.rs @@ -73,7 +73,7 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) - })() .unwrap_or_else(|| { match bcx.func.dfg.value_type(arg) { - types::I8 | types::I32 => { + types::I8 | types::I16 => { // WORKAROUND for brz.i8 and brnz.i8 not yet being implemented bcx.ins().uextend(types::I32, arg) } @@ -81,3 +81,40 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) - } }) } + +/// Returns whether the branch is statically known to be taken or `None` if it isn't statically known. +pub(crate) fn maybe_known_branch_taken( + bcx: &FunctionBuilder<'_>, + arg: Value, + test_zero: bool, +) -> Option { + let arg_inst = if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) { + arg_inst + } else { + return None; + }; + + match bcx.func.dfg[arg_inst] { + InstructionData::UnaryBool { + opcode: Opcode::Bconst, + imm, + } => { + if test_zero { + Some(!imm) + } else { + Some(imm) + } + } + InstructionData::UnaryImm { + opcode: Opcode::Iconst, + imm, + } => { + if test_zero { + Some(imm.bits() == 0) + } else { + Some(imm.bits() != 0) + } + } + _ => None, + } +} From 42b267d22137218588d75708ad99a5d9acefa9ab Mon Sep 17 00:00:00 2001 From: Jack Huey Date: Fri, 11 Dec 2020 15:02:46 -0500 Subject: [PATCH 09/26] Move binder for dyn to each list item --- src/value_and_place.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/value_and_place.rs b/src/value_and_place.rs index cb40d4ed9a6df..5bcb11fd515a0 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -480,17 +480,19 @@ impl<'tcx> CPlace<'tcx> { // fn(&T) -> for<'l> fn(&'l T) is allowed } (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => { - let from_traits = fx - .tcx - .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from_traits); - let to_traits = fx - .tcx - .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_traits); - assert_eq!( - from_traits, to_traits, - "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}", - from_traits, to_traits, fx, - ); + for (from, to) in from_traits.iter().zip(to_traits) { + let from = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from); + let to = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to); + assert_eq!( + from, to, + "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}", + from_traits, to_traits, fx, + ); + } // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed } _ => { From f8e22bfb1bff658466cf97e99026b91c2213df5d Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 12 Dec 2020 10:37:10 +0100 Subject: [PATCH 10/26] Update Cranelift --- Cargo.lock | 56 ++++++++++++++++++++--------------------- Cargo.toml | 6 ++--- src/debuginfo/emit.rs | 2 +- src/debuginfo/unwind.rs | 2 +- src/driver/jit.rs | 6 ++--- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b19a7b827ee3..0382835269d1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,7 +50,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-entity", ] @@ -58,7 +58,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "byteorder", "cranelift-bforest", @@ -76,7 +76,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -85,17 +85,17 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" [[package]] name = "cranelift-entity" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" [[package]] name = "cranelift-frontend" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "cranelift-codegen", "log", @@ -104,56 +104,56 @@ dependencies = [ ] [[package]] -name = "cranelift-module" +name = "cranelift-jit" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "anyhow", "cranelift-codegen", "cranelift-entity", + "cranelift-module", + "cranelift-native", + "errno", + "libc", "log", - "thiserror", + "region", + "target-lexicon", + "winapi", ] [[package]] -name = "cranelift-native" +name = "cranelift-module" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ + "anyhow", "cranelift-codegen", - "raw-cpuid", - "target-lexicon", + "cranelift-entity", + "log", + "thiserror", ] [[package]] -name = "cranelift-object" +name = "cranelift-native" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ - "anyhow", "cranelift-codegen", - "cranelift-module", - "log", - "object", + "raw-cpuid", "target-lexicon", ] [[package]] -name = "cranelift-simplejit" +name = "cranelift-object" version = "0.68.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#bfd10512c10ef08d1969ab961e88dbbb41a3b201" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb" dependencies = [ "anyhow", "cranelift-codegen", - "cranelift-entity", "cranelift-module", - "cranelift-native", - "errno", - "libc", "log", - "region", + "object", "target-lexicon", - "winapi", ] [[package]] @@ -326,9 +326,9 @@ dependencies = [ "ar", "cranelift-codegen", "cranelift-frontend", + "cranelift-jit", "cranelift-module", "cranelift-object", - "cranelift-simplejit", "gimli", "indexmap", "libloading", diff --git a/Cargo.toml b/Cargo.toml index cbff06749d3e9..8e1933bb14e7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ crate-type = ["dylib"] cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] } cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -cranelift-simplejit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } +cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } target-lexicon = "0.11.0" gimli = { version = "0.23.0", default-features = false, features = ["write"]} @@ -27,7 +27,7 @@ libloading = { version = "0.6.0", optional = true } #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } #cranelift-module = { path = "../wasmtime/cranelift/module" } -#cranelift-simplejit = { path = "../wasmtime/cranelift/simplejit" } +#cranelift-jit = { path = "../wasmtime/cranelift/jit" } #cranelift-object = { path = "../wasmtime/cranelift/object" } #[patch.crates-io] @@ -35,7 +35,7 @@ libloading = { version = "0.6.0", optional = true } [features] default = ["jit", "inline_asm"] -jit = ["cranelift-simplejit", "libloading"] +jit = ["cranelift-jit", "libloading"] inline_asm = [] [profile.dev] diff --git a/src/debuginfo/emit.rs b/src/debuginfo/emit.rs index c21835b1fc3aa..7a0d8907c5005 100644 --- a/src/debuginfo/emit.rs +++ b/src/debuginfo/emit.rs @@ -76,7 +76,7 @@ impl WriterRelocate { #[cfg(feature = "jit")] pub(super) fn relocate_for_jit( mut self, - jit_module: &cranelift_simplejit::SimpleJITModule, + jit_module: &cranelift_jit::JITModule, ) -> Vec { use std::convert::TryInto; diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index e0f62b64e6bbb..dc630fa5fd421 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -80,7 +80,7 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(feature = "jit")] pub(crate) unsafe fn register_jit( self, - jit_module: &cranelift_simplejit::SimpleJITModule, + jit_module: &cranelift_jit::JITModule, ) -> Option { let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian( self.tcx, diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 4d40debc956de..05bcade353562 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -7,7 +7,7 @@ use std::os::raw::{c_char, c_int}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; -use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule}; +use cranelift_jit::{JITBuilder, JITModule}; use crate::prelude::*; @@ -36,12 +36,12 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let imported_symbols = load_imported_symbols_for_jit(tcx); - let mut jit_builder = SimpleJITBuilder::with_isa( + let mut jit_builder = JITBuilder::with_isa( crate::build_isa(tcx.sess, false), cranelift_module::default_libcall_names(), ); jit_builder.symbols(imported_symbols); - let mut jit_module = SimpleJITModule::new(jit_builder); + let mut jit_module = JITModule::new(jit_builder); assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); let sig = Signature { From 3f47f938ba5303be9b6fe8c13aee6dce4aaa4b0b Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 12 Dec 2020 10:38:46 +0100 Subject: [PATCH 11/26] Enable Cranelift optimizations when optimizing LICM in Cranelift has been fixed recently --- src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ba9ee0d450ee6..7b751102f8b42 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -283,8 +283,6 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box { @@ -297,7 +295,7 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box { sess.warn("Optimizing for size is not supported. Just ignoring the request"); } - }*/ + } let flags = settings::Flags::new(flags_builder); From cd21269ae0b3c9f516b0322b522255da03afcc76 Mon Sep 17 00:00:00 2001 From: Jesse Date: Sun, 13 Dec 2020 10:58:47 +0100 Subject: [PATCH 12/26] Fix Cranelift link in readme (#1118) --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index de54bf67f4a19..b0ea95692f3f7 100644 --- a/Readme.md +++ b/Readme.md @@ -2,7 +2,7 @@ > ⚠⚠⚠ Certain kinds of FFI don't work yet. ⚠⚠⚠ -The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift). +The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/main/cranelift). This has the potential to improve compilation times in debug mode. If your project doesn't use any of the things listed under "Not yet supported", it should work fine. If not please open an issue. From 44b331047dd5ab70d2e1e8671aeb8ee799a3b38d Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 14 Dec 2020 12:52:41 +0100 Subject: [PATCH 13/26] Also emit vcode when emitting clif ir while using new style backends --- src/base.rs | 10 +++++ src/pretty_clif.rs | 96 ++++++++++++++++++++++++++-------------------- 2 files changed, 64 insertions(+), 42 deletions(-) diff --git a/src/base.rs b/src/base.rs index 3af4897b4f0b0..5ba992d93724f 100644 --- a/src/base.rs +++ b/src/base.rs @@ -118,6 +118,8 @@ pub(crate) fn codegen_fn<'tcx>( context.eliminate_unreachable_code(cx.module.isa()).unwrap(); context.dce(cx.module.isa()).unwrap(); + context.want_disasm = crate::pretty_clif::should_write_ir(tcx); + // Define function let module = &mut cx.module; tcx.sess.time("define function", || { @@ -140,6 +142,14 @@ pub(crate) fn codegen_fn<'tcx>( &clif_comments, ); + if let Some(mach_compile_result) = &context.mach_compile_result { + if let Some(disasm) = &mach_compile_result.disasm { + crate::pretty_clif::write_ir_file(tcx, &format!("{}.vcode", tcx.symbol_name(instance).name), |file| { + file.write_all(disasm.as_bytes()) + }) + } + } + // Define debuginfo for function let isa = cx.module.isa(); let debug_context = &mut cx.debug_context; diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index a9f060e51d8f8..1d17157a8761d 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -53,6 +53,7 @@ //! ``` use std::fmt; +use std::io::Write; use cranelift_codegen::{ entity::SecondaryMap, @@ -200,32 +201,24 @@ impl FunctionCx<'_, '_, M> { } } -pub(crate) fn write_clif_file<'tcx>( - tcx: TyCtxt<'tcx>, - postfix: &str, - isa: Option<&dyn cranelift_codegen::isa::TargetIsa>, - instance: Instance<'tcx>, - context: &cranelift_codegen::Context, - mut clif_comments: &CommentWriter, -) { - use std::io::Write; - - if !cfg!(debug_assertions) - && !tcx +pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { + cfg!(debug_assertions) + || tcx .sess .opts .output_types .contains_key(&OutputType::LlvmAssembly) - { +} + +pub(crate) fn write_ir_file<'tcx>( + tcx: TyCtxt<'tcx>, + name: &str, + write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, +) { + if !should_write_ir(tcx) { return; } - let value_ranges = isa.map(|isa| { - context - .build_value_labels_ranges(isa) - .expect("value location ranges") - }); - let clif_output_dir = tcx.output_filenames(LOCAL_CRATE).with_extension("clif"); match std::fs::create_dir(&clif_output_dir) { @@ -234,39 +227,58 @@ pub(crate) fn write_clif_file<'tcx>( res @ Err(_) => res.unwrap(), } - let clif_file_name = clif_output_dir.join(format!( + let clif_file_name = clif_output_dir.join(name); + + let res: std::io::Result<()> = try { + let mut file = std::fs::File::create(clif_file_name)?; + write(&mut file)?; + }; + if let Err(err) = res { + tcx.sess.warn(&format!("error writing ir file: {}", err)); + } +} + +pub(crate) fn write_clif_file<'tcx>( + tcx: TyCtxt<'tcx>, + postfix: &str, + isa: Option<&dyn cranelift_codegen::isa::TargetIsa>, + instance: Instance<'tcx>, + context: &cranelift_codegen::Context, + mut clif_comments: &CommentWriter, +) { + write_ir_file(tcx, &format!( "{}.{}.clif", tcx.symbol_name(instance).name, postfix - )); + ), |file| { + let value_ranges = isa.map(|isa| { + context + .build_value_labels_ranges(isa) + .expect("value location ranges") + }); - let mut clif = String::new(); - cranelift_codegen::write::decorate_function( - &mut clif_comments, - &mut clif, - &context.func, - &DisplayFunctionAnnotations { - isa: Some(&*crate::build_isa( - tcx.sess, true, /* PIC doesn't matter here */ - )), - value_ranges: value_ranges.as_ref(), - }, - ) - .unwrap(); + let mut clif = String::new(); + cranelift_codegen::write::decorate_function( + &mut clif_comments, + &mut clif, + &context.func, + &DisplayFunctionAnnotations { + isa: Some(&*crate::build_isa( + tcx.sess, true, /* PIC doesn't matter here */ + )), + value_ranges: value_ranges.as_ref(), + }, + ) + .unwrap(); - let res: std::io::Result<()> = try { - let mut file = std::fs::File::create(clif_file_name)?; - let target_triple = crate::target_triple(tcx.sess); writeln!(file, "test compile")?; writeln!(file, "set is_pic")?; writeln!(file, "set enable_simd")?; - writeln!(file, "target {} haswell", target_triple)?; + writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; writeln!(file)?; file.write_all(clif.as_bytes())?; - }; - if let Err(err) = res { - tcx.sess.warn(&format!("err writing clif file: {}", err)); - } + Ok(()) + }); } impl fmt::Debug for FunctionCx<'_, '_, M> { From 5f6c32af159c427fedb6fd25dfd9b6933affe068 Mon Sep 17 00:00:00 2001 From: David Wood Date: Sun, 8 Nov 2020 14:01:23 +0000 Subject: [PATCH 14/26] cg_clif: fix build with split dwarf This commit makes minor changes to the cranelift backend so that it can build given changes in cg_ssa for Split DWARF. Signed-off-by: David Wood --- src/driver/aot.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 491d6cbbf7969..78d6ff0cb001c 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -75,6 +75,7 @@ fn emit_module( name, kind, object: Some(tmp_file), + dwarf_object: None, bytecode: None, }, work_product, @@ -111,6 +112,7 @@ fn reuse_workproduct_for_cgu( name: cgu.name().to_string(), kind: ModuleKind::Regular, object, + dwarf_object: None, bytecode: None, } } @@ -290,6 +292,7 @@ pub(super) fn run_aot( name: metadata_cgu_name, kind: ModuleKind::Metadata, object: Some(tmp_file), + dwarf_object: None, bytecode: None, }) } else { From 8e234d5d4b5985fe80af50472e637d77c472c6cc Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 17 Dec 2020 18:35:44 +0100 Subject: [PATCH 15/26] Rustup to rustc 1.50.0-nightly (b32e6e6ac 2020-12-16) --- build_sysroot/Cargo.lock | 8 ++++---- build_sysroot/Cargo.toml | 3 ++- rust-toolchain | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index a2b8f449f00ff..990557694ead4 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -47,9 +47,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "cc" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" [[package]] name = "cfg-if" @@ -141,9 +141,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.80" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" dependencies = [ "rustc-std-workspace-core", ] diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml index e562dedb5324b..3dbd28c286a24 100644 --- a/build_sysroot/Cargo.toml +++ b/build_sysroot/Cargo.toml @@ -5,13 +5,14 @@ version = "0.0.0" [dependencies] core = { path = "./sysroot_src/library/core" } -compiler_builtins = "0.1" alloc = { path = "./sysroot_src/library/alloc" } std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] } test = { path = "./sysroot_src/library/test" } alloc_system = { path = "./alloc_system" } +compiler_builtins = { version = "=0.1.36", default-features = false } + [patch.crates-io] rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" } rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" } diff --git a/rust-toolchain b/rust-toolchain index 547f1e0d8f0c3..eeee23d6a8ba2 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-12-03 +nightly-2020-12-17 From 32f92aa34f87afa705c553d2455345a6107c3309 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 18 Dec 2020 11:11:48 +0100 Subject: [PATCH 16/26] Rustup to rustc 1.50.0-nightly (eb4fc71dc 2020-12-17) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index eeee23d6a8ba2..fca771e13e144 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-12-17 +nightly-2020-12-18 From 54ba238a6d4d1c2106848ffeb44b6932e95d27fa Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 18 Dec 2020 11:12:52 +0100 Subject: [PATCH 17/26] Remove implementation of intrinsics that are now lowered See rust-lang/rust#80040 --- src/intrinsics/mod.rs | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 2910af8c2b75f..be5b247bb9f0b 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -444,9 +444,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( "abort" => { trap_abort(fx, "Called intrinsic::abort."); } - "unreachable" => { - trap_unreachable(fx, "[corruption] Called intrinsic::unreachable."); - } "transmute" => { crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span); } @@ -559,12 +556,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); } }; - discriminant_value, (c ptr) { - let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); - let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout); - let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout()); - ret.write_cvalue(fx, discr); - }; size_of_val, (c ptr) { let layout = fx.layout_of(T); let size = if layout.is_unsized() { @@ -625,22 +616,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ); ret.write_cvalue(fx, res); }; - _ if intrinsic.starts_with("wrapping_"), (c x, c y) { - assert_eq!(x.layout().ty, y.layout().ty); - let bin_op = match intrinsic { - "wrapping_add" => BinOp::Add, - "wrapping_sub" => BinOp::Sub, - "wrapping_mul" => BinOp::Mul, - _ => unreachable!("intrinsic {}", intrinsic), - }; - let res = crate::num::codegen_int_binop( - fx, - bin_op, - x, - y, - ); - ret.write_cvalue(fx, res); - }; _ if intrinsic.starts_with("saturating_"), (c lhs, c rhs) { assert_eq!(lhs.layout().ty, rhs.layout().ty); let bin_op = match intrinsic { @@ -900,7 +875,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( dest.write_cvalue(fx, val); }; - size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () { + pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () { let const_val = fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap(); let val = crate::constant::codegen_const_value( From 6c1fc324b2ebec435c8f2815f25698a6c5ad5314 Mon Sep 17 00:00:00 2001 From: Jack Huey Date: Fri, 18 Dec 2020 13:24:55 -0500 Subject: [PATCH 18/26] Make BoundRegion have a kind of BoungRegionKind --- src/abi/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index aee274ab4a823..76e1987459f87 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -64,7 +64,7 @@ pub(crate) fn fn_sig_for_fn_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx ty::Generator(_, substs, _) => { let sig = substs.as_generator().poly_sig(); - let env_region = ty::ReLateBound(ty::INNERMOST, ty::BrEnv); + let env_region = ty::ReLateBound(ty::INNERMOST, ty::BoundRegion { kind: ty::BrEnv }); let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty); let pin_did = tcx.require_lang_item(rustc_hir::LangItem::Pin, None); From 510616fc0705c94c389710ad5fd47bc97eb2bfa1 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 23 Dec 2020 12:47:20 +0100 Subject: [PATCH 19/26] Rustup to rustc 1.50.0-nightly (bb1fbbf84 2020-12-22) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index fca771e13e144..d6ad24bcf26dd 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-12-18 +nightly-2020-12-23 From c556e4dd354056ef3c4a9fbcd1ab67510e935833 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 17 Dec 2020 19:31:38 +0100 Subject: [PATCH 20/26] Use PIC in JIT mode too --- src/backend.rs | 4 ++-- src/debuginfo/unwind.rs | 4 ++-- src/driver/aot.rs | 9 +++++++-- src/driver/jit.rs | 4 ++-- src/lib.rs | 12 ++++-------- src/pretty_clif.rs | 4 +--- 6 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/backend.rs b/src/backend.rs index 9e32259716f51..0ce34c904bdcc 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -162,7 +162,7 @@ impl AddConstructor for ObjectProduct { } pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec { - let triple = crate::build_isa(sess, true).triple().clone(); + let triple = crate::build_isa(sess).triple().clone(); let binary_format = match triple.binary_format { target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, @@ -193,7 +193,7 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule { let mut builder = ObjectBuilder::new( - crate::build_isa(sess, true), + crate::build_isa(sess), name + ".o", cranelift_module::default_libcall_names(), ) diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index dc630fa5fd421..49de927cdba05 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -15,11 +15,11 @@ pub(crate) struct UnwindContext<'tcx> { } impl<'tcx> UnwindContext<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self { + pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { let mut frame_table = FrameTable::default(); let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { - if isa.flags().is_pic() { + if pic_eh_frame { cie.fde_address_encoding = gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0); } diff --git a/src/driver/aot.rs b/src/driver/aot.rs index cc79a4234eb98..16f9bfc99189f 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -146,7 +146,12 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege } } - let mut cx = crate::CodegenCx::new(tcx, module, tcx.sess.opts.debuginfo != DebugInfo::None); + let mut cx = crate::CodegenCx::new( + tcx, + module, + tcx.sess.opts.debuginfo != DebugInfo::None, + true, + ); super::predefine_mono_items(&mut cx, &mono_items); for (mono_item, (linkage, visibility)) in mono_items { let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); @@ -254,7 +259,7 @@ pub(super) fn run_aot( tcx.sess.abort_if_errors(); let mut allocator_module = new_module(tcx, "allocator_shim".to_string()); - let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa()); + let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 05bcade353562..4be317f83917e 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -37,7 +37,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let imported_symbols = load_imported_symbols_for_jit(tcx); let mut jit_builder = JITBuilder::with_isa( - crate::build_isa(tcx.sess, false), + crate::build_isa(tcx.sess), cranelift_module::default_libcall_names(), ); jit_builder.symbols(imported_symbols); @@ -67,7 +67,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { .into_iter() .collect::>(); - let mut cx = crate::CodegenCx::new(tcx, jit_module, false); + let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); let (mut jit_module, global_asm, _debug, mut unwind_context) = super::time(tcx, "codegen mono items", || { diff --git a/src/lib.rs b/src/lib.rs index 7b751102f8b42..ee20e449bb9c6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -141,8 +141,8 @@ struct CodegenCx<'tcx, M: Module> { } impl<'tcx, M: Module> CodegenCx<'tcx, M> { - fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool) -> Self { - let unwind_context = UnwindContext::new(tcx, module.isa()); + fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool, pic_eh_frame: bool) -> Self { + let unwind_context = UnwindContext::new(tcx, module.isa(), pic_eh_frame); let debug_context = if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { @@ -250,17 +250,13 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple { sess.target.llvm_target.parse().unwrap() } -fn build_isa(sess: &Session, enable_pic: bool) -> Box { +fn build_isa(sess: &Session) -> Box { use target_lexicon::BinaryFormat; let target_triple = crate::target_triple(sess); let mut flags_builder = settings::builder(); - if enable_pic { - flags_builder.enable("is_pic").unwrap(); - } else { - flags_builder.set("is_pic", "false").unwrap(); - } + flags_builder.enable("is_pic").unwrap(); flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided flags_builder .set( diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index 1d17157a8761d..1326bac74b1da 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -263,9 +263,7 @@ pub(crate) fn write_clif_file<'tcx>( &mut clif, &context.func, &DisplayFunctionAnnotations { - isa: Some(&*crate::build_isa( - tcx.sess, true, /* PIC doesn't matter here */ - )), + isa: Some(&*crate::build_isa(tcx.sess)), value_ranges: value_ranges.as_ref(), }, ) From 20ffea6b8a9f614eed854adbba75626effc1f111 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 25 Dec 2020 11:31:33 +0100 Subject: [PATCH 21/26] Change the way JIT mode is selected --- Readme.md | 4 +-- scripts/cargo.sh | 2 +- scripts/filter_profile.rs | 2 +- scripts/tests.sh | 4 +-- src/bin/cg_clif.rs | 19 ++-------- src/bin/cg_clif_build_sysroot.rs | 4 +-- src/driver/jit.rs | 2 +- src/driver/mod.rs | 32 +++++++++-------- src/lib.rs | 61 ++++++++++++++++++++++++++++---- 9 files changed, 81 insertions(+), 49 deletions(-) diff --git a/Readme.md b/Readme.md index b0ea95692f3f7..283eec91e219b 100644 --- a/Readme.md +++ b/Readme.md @@ -68,7 +68,7 @@ $ $cg_clif_dir/build/cargo.sh jit or ```bash -$ $cg_clif_dir/build/bin/cg_clif --jit my_crate.rs +$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs ``` ### Shell @@ -77,7 +77,7 @@ These are a few functions that allow you to easily run rust code from the shell ```bash function jit_naked() { - echo "$@" | $cg_clif_dir/build/bin/cg_clif - --jit + echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic } function jit() { diff --git a/scripts/cargo.sh b/scripts/cargo.sh index dcd40acc02a53..934b2c6d7a1c3 100755 --- a/scripts/cargo.sh +++ b/scripts/cargo.sh @@ -10,7 +10,7 @@ cmd=$1 shift || true if [[ "$cmd" = "jit" ]]; then -cargo "+${TOOLCHAIN}" rustc "$@" -- --jit +cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit -Cprefer-dynamic else cargo "+${TOOLCHAIN}" "$cmd" "$@" fi diff --git a/scripts/filter_profile.rs b/scripts/filter_profile.rs index 3327c10089d9b..15388926ec9ec 100755 --- a/scripts/filter_profile.rs +++ b/scripts/filter_profile.rs @@ -4,7 +4,7 @@ pushd $(dirname "$0")/../ source build/config.sh popd -PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS --jit $0 +PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS -Cllvm-args=mode=jit -Cprefer-dynamic $0 #*/ //! This program filters away uninteresting samples and trims uninteresting frames for stackcollapse diff --git a/scripts/tests.sh b/scripts/tests.sh index 114b6f30a4a91..60ed7aac8aa51 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -15,7 +15,7 @@ function no_sysroot_tests() { if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] mini_core_hello_world" - CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" + CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" else echo "[JIT] mini_core_hello_world (skipped)" fi @@ -37,7 +37,7 @@ function base_sysroot_tests() { if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] std_example" - $MY_RUSTC --jit example/std_example.rs --target "$HOST_TRIPLE" + $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE" else echo "[JIT] std_example (skipped)" fi diff --git a/src/bin/cg_clif.rs b/src/bin/cg_clif.rs index f4d23ebcf4e4d..58e45b4e9b972 100644 --- a/src/bin/cg_clif.rs +++ b/src/bin/cg_clif.rs @@ -44,9 +44,7 @@ fn main() { let mut callbacks = CraneliftPassesCallbacks::default(); rustc_driver::install_ice_hook(); let exit_code = rustc_driver::catch_with_exit_code(|| { - let mut use_jit = false; - - let mut args = std::env::args_os() + let args = std::env::args_os() .enumerate() .map(|(i, arg)| { arg.into_string().unwrap_or_else(|arg| { @@ -56,23 +54,10 @@ fn main() { ) }) }) - .filter(|arg| { - if arg == "--jit" { - use_jit = true; - false - } else { - true - } - }) .collect::>(); - if use_jit { - args.push("-Cprefer-dynamic".to_string()); - } let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks); run_compiler.set_make_codegen_backend(Some(Box::new(move |_| { - Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { - config: rustc_codegen_cranelift::BackendConfig { use_jit }, - }) + Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None }) }))); run_compiler.run() }); diff --git a/src/bin/cg_clif_build_sysroot.rs b/src/bin/cg_clif_build_sysroot.rs index 165d33dcfb509..8ee4cd46c94e0 100644 --- a/src/bin/cg_clif_build_sysroot.rs +++ b/src/bin/cg_clif_build_sysroot.rs @@ -92,9 +92,7 @@ fn main() { let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks); if use_clif { run_compiler.set_make_codegen_backend(Some(Box::new(move |_| { - Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { - config: rustc_codegen_cranelift::BackendConfig { use_jit: false }, - }) + Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None }) }))); } run_compiler.run() diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 4be317f83917e..3a942a1549132 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -106,7 +106,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - println!("Rustc codegen cranelift will JIT run the executable, because --jit was passed"); + println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"); let f: extern "C" fn(c_int, *const *const c_char) -> c_int = unsafe { ::std::mem::transmute(finalized_main) }; diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 0b0ea8a852c65..49bb79c101a3f 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -7,6 +7,7 @@ use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; +use crate::CodegenMode; mod aot; #[cfg(feature = "jit")] @@ -20,24 +21,25 @@ pub(crate) fn codegen_crate( ) -> Box { tcx.sess.abort_if_errors(); - if config.use_jit { - let is_executable = tcx - .sess - .crate_types() - .contains(&rustc_session::config::CrateType::Executable); - if !is_executable { - tcx.sess.fatal("can't jit non-executable crate"); - } + match config.codegen_mode { + CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module), + CodegenMode::Jit => { + let is_executable = tcx + .sess + .crate_types() + .contains(&rustc_session::config::CrateType::Executable); + if !is_executable { + tcx.sess.fatal("can't jit non-executable crate"); + } - #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx); + #[cfg(feature = "jit")] + let _: ! = jit::run_jit(tcx); - #[cfg(not(feature = "jit"))] - tcx.sess - .fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + #[cfg(not(feature = "jit"))] + tcx.sess + .fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } } - - aot::run_aot(tcx, metadata, need_metadata_module) } fn predefine_mono_items<'tcx>( diff --git a/src/lib.rs b/src/lib.rs index ee20e449bb9c6..91e1efb71b920 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,8 @@ associated_type_bounds, never_type, try_blocks, - hash_drain_filter + hash_drain_filter, + str_split_once )] #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] @@ -34,6 +35,7 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; +use std::str::FromStr; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; @@ -172,12 +174,53 @@ impl<'tcx, M: Module> CodegenCx<'tcx, M> { } #[derive(Copy, Clone, Debug)] +pub enum CodegenMode { + Aot, + Jit, +} + +impl Default for CodegenMode { + fn default() -> Self { + CodegenMode::Aot + } +} + +impl FromStr for CodegenMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "aot" => Ok(CodegenMode::Aot), + "jit" => Ok(CodegenMode::Jit), + _ => Err(format!("Unknown codegen mode `{}`", s)), + } + } +} + +#[derive(Copy, Clone, Debug, Default)] pub struct BackendConfig { - pub use_jit: bool, + pub codegen_mode: CodegenMode, +} + +impl BackendConfig { + fn from_opts(opts: &[String]) -> Result { + let mut config = BackendConfig::default(); + for opt in opts { + if let Some((name, value)) = opt.split_once('=') { + match name { + "mode" => config.codegen_mode = value.parse()?, + _ => return Err(format!("Unknown option `{}`", name)), + } + } else { + return Err(format!("Invalid option `{}`", opt)); + } + } + Ok(config) + } } pub struct CraneliftCodegenBackend { - pub config: BackendConfig, + pub config: Option, } impl CodegenBackend for CraneliftCodegenBackend { @@ -204,7 +247,13 @@ impl CodegenBackend for CraneliftCodegenBackend { metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box { - let res = driver::codegen_crate(tcx, metadata, need_metadata_module, self.config); + let config = if let Some(config) = self.config { + config + } else { + BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) + .unwrap_or_else(|err| tcx.sess.fatal(&err)) + }; + let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config); rustc_symbol_mangling::test::report_symbol_names(tcx); @@ -305,7 +354,5 @@ fn build_isa(sess: &Session) -> Box { /// This is the entrypoint for a hot plugged rustc_codegen_cranelift #[no_mangle] pub fn __rustc_codegen_backend() -> Box { - Box::new(CraneliftCodegenBackend { - config: BackendConfig { use_jit: false }, - }) + Box::new(CraneliftCodegenBackend { config: None }) } From 0b9b2532d262bf590b769a7cd2f32602b8925183 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 25 Dec 2020 11:49:50 +0100 Subject: [PATCH 22/26] Move finalize CodegenCx timer out of codegen mono items timer --- src/driver/jit.rs | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 3a942a1549132..cb386ea83b981 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -69,32 +69,36 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); - let (mut jit_module, global_asm, _debug, mut unwind_context) = - super::time(tcx, "codegen mono items", || { - super::predefine_mono_items(&mut cx, &mono_items); - for (mono_item, (linkage, visibility)) in mono_items { - let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - match mono_item { - MonoItem::Fn(inst) => { - cx.tcx.sess.time("codegen fn", || { - crate::base::codegen_fn(&mut cx, inst, linkage) - }); - } - MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id) - } - MonoItem::GlobalAsm(hir_id) => { - let item = cx.tcx.hir().expect_item(hir_id); - tcx.sess - .span_fatal(item.span, "Global asm is not supported in JIT mode"); - } + super::time(tcx, "codegen mono items", || { + super::predefine_mono_items(&mut cx, &mono_items); + for (mono_item, (linkage, visibility)) in mono_items { + let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + match mono_item { + MonoItem::Fn(inst) => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id) + } + MonoItem::GlobalAsm(hir_id) => { + let item = cx.tcx.hir().expect_item(hir_id); + tcx.sess + .span_fatal(item.span, "Global asm is not supported in JIT mode"); } } - tcx.sess.time("finalize CodegenCx", || cx.finalize()) - }); + } + }); + + let (mut jit_module, global_asm, _debug, mut unwind_context) = + tcx.sess.time("finalize CodegenCx", || cx.finalize()); + jit_module.finalize_definitions(); + if !global_asm.is_empty() { tcx.sess.fatal("Inline asm is not supported in JIT mode"); } + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true); crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); From 3f6a3b5ebece5f6d3b1010b68a5fd86756593098 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 25 Dec 2020 11:41:48 +0100 Subject: [PATCH 23/26] Implement lazy compilation in JIT mode Lazy compilation has the potential to significantly improve the startup time of a program. While functions have to be codegened when called, it is expected that a significant amount of all code is only required when an error occurs or only when the program is used in certain ways. The basic approach is to first codegen a shim for each function. This shim calls the `__cg_clif_jit` function of cg_clif with a pointer to the `Instance` corresponding to the function for which it is a shim. `__cg_clif_jit` function then codegens this function and uses the hot code swapping support of SimpleJIT to redirect future calls to the function to the real version. Finally it calls the newly codegened function. --- example/std_example.rs | 2 + scripts/tests.sh | 6 ++ src/constant.rs | 3 +- src/driver/jit.rs | 129 ++++++++++++++++++++++++++++++++++++++--- src/driver/mod.rs | 4 +- src/lib.rs | 2 + src/vtable.rs | 3 +- 7 files changed, 138 insertions(+), 11 deletions(-) diff --git a/example/std_example.rs b/example/std_example.rs index b38e25328a4ee..015bbdfed4648 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -15,6 +15,8 @@ fn main() { let stderr = ::std::io::stderr(); let mut stderr = stderr.lock(); + // FIXME support lazy jit when multi threading + #[cfg(not(lazy_jit))] std::thread::spawn(move || { println!("Hello from another thread!"); }); diff --git a/scripts/tests.sh b/scripts/tests.sh index 60ed7aac8aa51..a61774f479ec7 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -16,6 +16,9 @@ function no_sysroot_tests() { if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] mini_core_hello_world" CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" + + echo "[JIT-lazy] mini_core_hello_world" + CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE" else echo "[JIT] mini_core_hello_world (skipped)" fi @@ -38,6 +41,9 @@ function base_sysroot_tests() { if [[ "$JIT_SUPPORTED" = "1" ]]; then echo "[JIT] std_example" $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE" + + echo "[JIT-lazy] std_example" + $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/std_example.rs --cfg lazy_jit --target "$HOST_TRIPLE" else echo "[JIT] std_example (skipped)" fi diff --git a/src/constant.rs b/src/constant.rs index 544b020b71190..bfa2076d3d969 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -447,7 +447,8 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64); } - module.define_data(data_id, &data_ctx).unwrap(); + // FIXME don't duplicate definitions in lazy jit mode + let _ = module.define_data(data_id, &data_ctx); cx.done.insert(data_id); } diff --git a/src/driver/jit.rs b/src/driver/jit.rs index cb386ea83b981..9a42c675cc144 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -1,6 +1,7 @@ //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object //! files. +use std::cell::RefCell; use std::ffi::CString; use std::os::raw::{c_char, c_int}; @@ -10,8 +11,13 @@ use rustc_middle::mir::mono::MonoItem; use cranelift_jit::{JITBuilder, JITModule}; use crate::prelude::*; +use crate::{CodegenCx, CodegenMode}; -pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { +thread_local! { + pub static CURRENT_MODULE: RefCell> = RefCell::new(None); +} + +pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! { if !tcx.sess.opts.output_types.should_codegen() { tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); } @@ -40,6 +46,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { crate::build_isa(tcx.sess), cranelift_module::default_libcall_names(), ); + jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy)); jit_builder.symbols(imported_symbols); let mut jit_module = JITModule::new(jit_builder); assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); @@ -74,13 +81,17 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { for (mono_item, (linkage, visibility)) in mono_items { let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); match mono_item { - MonoItem::Fn(inst) => { - cx.tcx.sess.time("codegen fn", || { - crate::base::codegen_fn(&mut cx, inst, linkage) - }); - } + MonoItem::Fn(inst) => match codegen_mode { + CodegenMode::Aot => unreachable!(), + CodegenMode::Jit => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + CodegenMode::JitLazy => codegen_shim(&mut cx, inst), + }, MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id) + crate::constant::codegen_static(&mut cx.constants_cx, def_id); } MonoItem::GlobalAsm(hir_id) => { let item = cx.tcx.hir().expect_item(hir_id); @@ -126,11 +137,50 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { // useful as some dynamic linkers use it as a marker to jump over. argv.push(std::ptr::null()); + CURRENT_MODULE + .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none())); + let ret = f(args.len() as c_int, argv.as_ptr()); std::process::exit(ret); } +#[no_mangle] +extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 { + rustc_middle::ty::tls::with(|tcx| { + // lift is used to ensure the correct lifetime for instance. + let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); + + CURRENT_MODULE.with(|jit_module| { + let mut jit_module = jit_module.borrow_mut(); + let jit_module = jit_module.as_mut().unwrap(); + let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); + + let (name, sig) = crate::abi::get_function_name_and_sig( + tcx, + cx.module.isa().triple(), + instance, + true, + ); + let func_id = cx + .module + .declare_function(&name, Linkage::Export, &sig) + .unwrap(); + cx.module.prepare_for_function_redefine(func_id).unwrap(); + + tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, instance, Linkage::Export) + }); + + let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize(); + assert!(global_asm.is_empty()); + jit_module.finalize_definitions(); + std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); + jit_module.get_finalized_function(func_id) + }) + }) +} + fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { use rustc_middle::middle::dependency_format::Linkage; @@ -190,3 +240,68 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } + +pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) { + let tcx = cx.tcx; + + let pointer_type = cx.module.target_config().pointer_type(); + + let (name, sig) = + crate::abi::get_function_name_and_sig(tcx, cx.module.isa().triple(), inst, true); + let func_id = cx + .module + .declare_function(&name, Linkage::Export, &sig) + .unwrap(); + + let instance_ptr = Box::into_raw(Box::new(inst)); + + let jit_fn = cx + .module + .declare_function( + "__clif_jit_fn", + Linkage::Import, + &Signature { + call_conv: cx.module.target_config().default_call_conv, + params: vec![AbiParam::new(pointer_type)], + returns: vec![AbiParam::new(pointer_type)], + }, + ) + .unwrap(); + + let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); + let mut builder_ctx = FunctionBuilderContext::new(); + let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); + + let jit_fn = cx + .module + .declare_func_in_func(jit_fn, trampoline_builder.func); + let sig_ref = trampoline_builder.func.import_signature(sig); + + let entry_block = trampoline_builder.create_block(); + trampoline_builder.append_block_params_for_function_params(entry_block); + let fn_args = trampoline_builder + .func + .dfg + .block_params(entry_block) + .to_vec(); + + trampoline_builder.switch_to_block(entry_block); + let instance_ptr = trampoline_builder + .ins() + .iconst(pointer_type, instance_ptr as u64 as i64); + let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]); + let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; + let call_inst = trampoline_builder + .ins() + .call_indirect(sig_ref, jitted_fn, &fn_args); + let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); + trampoline_builder.ins().return_(&ret_vals); + + cx.module + .define_function( + func_id, + &mut Context::for_function(trampoline), + &mut cranelift_codegen::binemit::NullTrapSink {}, + ) + .unwrap(); +} diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 49bb79c101a3f..9f4ea9a386551 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -23,7 +23,7 @@ pub(crate) fn codegen_crate( match config.codegen_mode { CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module), - CodegenMode::Jit => { + CodegenMode::Jit | CodegenMode::JitLazy => { let is_executable = tcx .sess .crate_types() @@ -33,7 +33,7 @@ pub(crate) fn codegen_crate( } #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx); + let _: ! = jit::run_jit(tcx, config.codegen_mode); #[cfg(not(feature = "jit"))] tcx.sess diff --git a/src/lib.rs b/src/lib.rs index 91e1efb71b920..6e4f3bf2898d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -177,6 +177,7 @@ impl<'tcx, M: Module> CodegenCx<'tcx, M> { pub enum CodegenMode { Aot, Jit, + JitLazy, } impl Default for CodegenMode { @@ -192,6 +193,7 @@ impl FromStr for CodegenMode { match s { "aot" => Ok(CodegenMode::Aot), "jit" => Ok(CodegenMode::Jit), + "jit-lazy" => Ok(CodegenMode::JitLazy), _ => Err(format!("Unknown codegen mode `{}`", s)), } } diff --git a/src/vtable.rs b/src/vtable.rs index 238abc0d8bdfa..8f15586a9dc06 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -158,7 +158,8 @@ fn build_vtable<'tcx>( ) .unwrap(); - fx.cx.module.define_data(data_id, &data_ctx).unwrap(); + // FIXME don't duplicate definitions in lazy jit mode + let _ = fx.cx.module.define_data(data_id, &data_ctx); data_id } From 35f4a25c6b216c27c5d87a171620d3f2c5c0a18b Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 25 Dec 2020 12:13:11 +0100 Subject: [PATCH 24/26] Add documentation --- Readme.md | 8 ++++++++ scripts/cargo.sh | 2 ++ 2 files changed, 10 insertions(+) diff --git a/Readme.md b/Readme.md index 283eec91e219b..22d9e00923f00 100644 --- a/Readme.md +++ b/Readme.md @@ -71,6 +71,14 @@ or $ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs ``` +There is also an experimental lazy jit mode. In this mode functions are only compiled once they are +first called. It currently does not work with multi-threaded programs. When a not yet compiled +function is called from another thread than the main thread, you will get an ICE. + +```bash +$ $cg_clif_dir/build/cargo.sh lazy-jit +``` + ### Shell These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit. diff --git a/scripts/cargo.sh b/scripts/cargo.sh index 934b2c6d7a1c3..a3d6d303057b8 100755 --- a/scripts/cargo.sh +++ b/scripts/cargo.sh @@ -11,6 +11,8 @@ shift || true if [[ "$cmd" = "jit" ]]; then cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit -Cprefer-dynamic +elif [[ "$cmd" = "lazy-jit" ]]; then +cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit-lazy -Cprefer-dynamic else cargo "+${TOOLCHAIN}" "$cmd" "$@" fi From 5b32489e7224eea5fc5ebc388732efa10c510fb2 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 25 Dec 2020 12:47:03 +0100 Subject: [PATCH 25/26] Set rust-analyzer importMergeBehaviour option to last --- .vscode/settings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 04ab5085c196c..7618251acd5c2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"], + "rust-analyzer.assist.importMergeBehaviour": "last", "rust-analyzer.cargo.loadOutDirsFromCheck": true, "rust-analyzer.linkedProjects": [ "./Cargo.toml", From dbee13661efa269cb4cd57bb4c6b99a19732b484 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 25 Dec 2020 12:48:19 +0100 Subject: [PATCH 26/26] Rustfmt --- src/base.rs | 8 +++--- src/constant.rs | 5 +++- src/debuginfo/emit.rs | 5 +--- src/pretty_clif.rs | 58 +++++++++++++++++++++---------------------- 4 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/base.rs b/src/base.rs index 5ba992d93724f..34c9561d67622 100644 --- a/src/base.rs +++ b/src/base.rs @@ -144,9 +144,11 @@ pub(crate) fn codegen_fn<'tcx>( if let Some(mach_compile_result) = &context.mach_compile_result { if let Some(disasm) = &mach_compile_result.disasm { - crate::pretty_clif::write_ir_file(tcx, &format!("{}.vcode", tcx.symbol_name(instance).name), |file| { - file.write_all(disasm.as_bytes()) - }) + crate::pretty_clif::write_ir_file( + tcx, + &format!("{}.vcode", tcx.symbol_name(instance).name), + |file| file.write_all(disasm.as_bytes()), + ) } } diff --git a/src/constant.rs b/src/constant.rs index bfa2076d3d969..beff84fb2e217 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -100,7 +100,10 @@ fn codegen_static_ref<'tcx>( let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); assert!(!layout.is_unsized(), "unsized statics aren't supported"); assert!( - matches!(fx.bcx.func.global_values[local_data_id], GlobalValueData::Symbol { tls: false, ..}), + matches!( + fx.bcx.func.global_values[local_data_id], + GlobalValueData::Symbol { tls: false, .. } + ), "tls static referenced without Rvalue::ThreadLocalRef" ); CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout) diff --git a/src/debuginfo/emit.rs b/src/debuginfo/emit.rs index 7a0d8907c5005..6160f9b78d8b3 100644 --- a/src/debuginfo/emit.rs +++ b/src/debuginfo/emit.rs @@ -74,10 +74,7 @@ impl WriterRelocate { /// Perform the collected relocations to be usable for JIT usage. #[cfg(feature = "jit")] - pub(super) fn relocate_for_jit( - mut self, - jit_module: &cranelift_jit::JITModule, - ) -> Vec { + pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec { use std::convert::TryInto; for reloc in self.relocs.drain(..) { diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index 1326bac74b1da..22c94fec82fc1 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -246,37 +246,37 @@ pub(crate) fn write_clif_file<'tcx>( context: &cranelift_codegen::Context, mut clif_comments: &CommentWriter, ) { - write_ir_file(tcx, &format!( - "{}.{}.clif", - tcx.symbol_name(instance).name, - postfix - ), |file| { - let value_ranges = isa.map(|isa| { - context - .build_value_labels_ranges(isa) - .expect("value location ranges") - }); + write_ir_file( + tcx, + &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), + |file| { + let value_ranges = isa.map(|isa| { + context + .build_value_labels_ranges(isa) + .expect("value location ranges") + }); - let mut clif = String::new(); - cranelift_codegen::write::decorate_function( - &mut clif_comments, - &mut clif, - &context.func, - &DisplayFunctionAnnotations { - isa: Some(&*crate::build_isa(tcx.sess)), - value_ranges: value_ranges.as_ref(), - }, - ) - .unwrap(); + let mut clif = String::new(); + cranelift_codegen::write::decorate_function( + &mut clif_comments, + &mut clif, + &context.func, + &DisplayFunctionAnnotations { + isa: Some(&*crate::build_isa(tcx.sess)), + value_ranges: value_ranges.as_ref(), + }, + ) + .unwrap(); - writeln!(file, "test compile")?; - writeln!(file, "set is_pic")?; - writeln!(file, "set enable_simd")?; - writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; - writeln!(file)?; - file.write_all(clif.as_bytes())?; - Ok(()) - }); + writeln!(file, "test compile")?; + writeln!(file, "set is_pic")?; + writeln!(file, "set enable_simd")?; + writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; + writeln!(file)?; + file.write_all(clif.as_bytes())?; + Ok(()) + }, + ); } impl fmt::Debug for FunctionCx<'_, '_, M> {