riscv: Support zabha target feature

taiki-e · Sep 1, 2024 · 694364a · 694364a
1 parent e7002b9
commit 694364a
Show file tree

Hide file tree

Showing 6 changed files with 80 additions and 14 deletions.
diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt
@@ -46,6 +46,7 @@ espup
 exynos
 getauxval
 getpid
+Halfword
 HWCAP
 hwprobe
 ifunc
@@ -175,4 +176,5 @@ xmmword
 xsave
 xsub
 zaamo
+zabha
 Zhaoxin
diff --git a/build.rs b/build.rs
@@ -47,7 +47,7 @@ fn main() {
 
     if version.minor >= 80 {
         println!(
-            r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
+            r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","zabha","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"#
         );
 
         // Custom cfgs set by build script. Not public API.
@@ -58,7 +58,7 @@ fn main() {
         // TODO: handle multi-line target_feature_fallback
         // grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/'
         println!(
-            r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo"))"#
+            r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo","zabha"))"#
         );
     }
 
@@ -311,9 +311,18 @@ fn main() {
             }
         }
         "riscv32" | "riscv64" => {
-            // As of rustc 1.80, target_feature "zaamo" is not available on rustc side:
+            // As of rustc 1.80, target_feature "zaamo"/"zabha" is not available on rustc side:
             // https://github.com/rust-lang/rust/blob/1.80.0/compiler/rustc_target/src/target_features.rs#L273
-            target_feature_fallback("zaamo", false); // amo*.{w,d}
+            // zabha implies zaamo in GCC, but do not in LLVM (but enabling it without zaamo is not allowed).
+            // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0-rc3/llvm/lib/TargetParser/RISCVISAInfo.cpp#L776-L778
+            // https://github.com/gcc-mirror/gcc/blob/08693e29ec186fd7941d0b73d4d466388971fe2f/gcc/config/riscv/arch-canonicalize#L45
+            if version.llvm >= 19 {
+                // amo*.{b,h}
+                // available since 19 https://github.com/llvm/llvm-project/commit/89f87c387627150d342722b79c78cea2311cddf7 / https://github.com/llvm/llvm-project/commit/6b7444964a8d028989beee554a1f5c61d16a1cac
+                target_feature_fallback("zabha", false);
+            }
+            // amo*.{w,d}
+            target_feature_fallback("zaamo", false);
         }
         "powerpc64" => {
             // For Miri and ThreadSanitizer.

diff --git a/src/imp/interrupt/mod.rs b/src/imp/interrupt/mod.rs
@@ -604,7 +604,8 @@ macro_rules! atomic_int {
             }
         }
     };
-    // 32-bit(RV32)/{32,64}-bit(RV64) RMW with Zaamo extension
+    // RISC-V 32-bit(RV32)/{32,64}-bit(RV64) RMW with Zaamo extension
+    // RISC-V 8-bit/16-bit RMW with Zabha extension
     (cas, $atomic_type:ident, $int_type:ident) => {
         impl $atomic_type {
             #[inline]
@@ -711,8 +712,17 @@ macro_rules! atomic_int {
             }
         }
     };
-    // {8,16}-bit RMW with Zaamo extension
+    // RISC-V 8-bit/16-bit RMW with Zaamo extension
     (cas[sub_word], $atomic_type:ident, $int_type:ident) => {
+        #[cfg(all(
+            any(target_arch = "riscv32", target_arch = "riscv64"),
+            any(target_feature = "zabha", portable_atomic_target_feature = "zabha"),
+        ))]
+        atomic_int!(cas, $atomic_type, $int_type);
+        #[cfg(not(all(
+            any(target_arch = "riscv32", target_arch = "riscv64"),
+            any(target_feature = "zabha", portable_atomic_target_feature = "zabha"),
+        )))]
         impl $atomic_type {
             #[inline]
             pub(crate) fn swap(&self, val: $int_type, _order: Ordering) -> $int_type {

diff --git a/src/imp/riscv.rs b/src/imp/riscv.rs
@@ -12,10 +12,12 @@
 //   https://riscv.org/wp-content/uploads/2019/12/riscv-spec-20191213.pdf
 // - RISC-V Atomics ABI Specification
 //   https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20240829-13bfa9f54634cb60d86b9b333e109f077805b4b3/riscv-atomic.adoc
-// - "Mappings from C/C++ primitives to RISC-V primitives." table in RISC-V Instruction Set Manual:
+// - "Mappings from C/C++ primitives to RISC-V primitives." table in RISC-V Instruction Set Manual
 //   https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/mm-eplan.adoc#code-porting-and-mapping-guidelines
-// - ""Zaamo" Extension for Atomic Memory Operations" in RISC-V Instruction Set Manual:
+// - ""Zaamo" Extension for Atomic Memory Operations" in RISC-V Instruction Set Manual
 //   https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/a-st-ext.adoc#zaamo-extension-for-atomic-memory-operations
+// - ""Zabha" Extension for Byte and Halfword Atomic Memory Operations" in RISC-V Instruction Set Manual
+//   https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/zabha.adoc
 // - atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit
 //
 // Generated asm:
@@ -32,6 +34,7 @@ use core::{cell::UnsafeCell, sync::atomic::Ordering};
     target_feature = "zaamo",
     portable_atomic_target_feature = "zaamo",
 ))]
+#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
 #[cfg(target_arch = "riscv32")]
 macro_rules! w {
     () => {
@@ -44,6 +47,7 @@ macro_rules! w {
     target_feature = "zaamo",
     portable_atomic_target_feature = "zaamo",
 ))]
+#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
 #[cfg(target_arch = "riscv64")]
 macro_rules! w {
     () => {
@@ -86,7 +90,7 @@ macro_rules! atomic_rmw_amo {
                 // The caller of this macro must guarantee the validity of the pointer.
                 asm!(
                     ".option push",
-                    // https://github.com/riscv-non-isa/riscv-asm-manual/blob/HEAD/riscv-asm.md#arch
+                    // https://github.com/riscv-non-isa/riscv-asm-manual/blob/ad0de8c004e29c9a7ac33cfd054f4d4f9392f2fb/src/asm-manual.adoc#arch
                     ".option arch, +a",
                     concat!("amo", stringify!($op), ".", $asm_suffix, $asm_order, " {out}, {val}, 0({dst})"),
                     ".option pop",
@@ -108,6 +112,7 @@ macro_rules! atomic_rmw_amo {
     target_feature = "zaamo",
     portable_atomic_target_feature = "zaamo",
 ))]
+#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
 #[inline]
 fn sllw(val: u32, shift: u32) -> u32 {
     // SAFETY: Calling sll{,w} is safe.
@@ -130,6 +135,7 @@ fn sllw(val: u32, shift: u32) -> u32 {
     target_feature = "zaamo",
     portable_atomic_target_feature = "zaamo",
 ))]
+#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
 macro_rules! srlw {
     ($val:expr, $shift:expr) => {
         // SAFETY: Calling srl{,w} is safe.
@@ -376,6 +382,7 @@ macro_rules! atomic {
     target_feature = "zaamo",
     portable_atomic_target_feature = "zaamo",
 ))]
+#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
 trait ZeroExtend: Copy {
     /// Zero-extends `self` to `u32` if it is smaller than 32-bit.
     fn zero_extend(self) -> u32;
@@ -388,6 +395,7 @@ macro_rules! zero_extend {
             target_feature = "zaamo",
             portable_atomic_target_feature = "zaamo",
         ))]
+        #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
         impl ZeroExtend for $uint {
             #[inline]
             fn zero_extend(self) -> u32 {
@@ -400,6 +408,7 @@ macro_rules! zero_extend {
             target_feature = "zaamo",
             portable_atomic_target_feature = "zaamo",
         ))]
+        #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
         impl ZeroExtend for $int {
             #[allow(clippy::cast_sign_loss)]
             #[inline]
@@ -413,14 +422,18 @@ zero_extend!(i8, u8);
 zero_extend!(i16, u16);
 
 macro_rules! atomic_sub_word {
-    ($atomic_type:ident, $value_type:ty, $unsigned_type:ty, $asm_suffix:tt) => {
+    ($atomic_type:ident, $value_type:ty, $asm_suffix:tt, $max:tt, $min:tt) => {
+        #[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))]
+        atomic!($atomic_type, $value_type, $asm_suffix, $max, $min);
+        #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
         atomic_load_store!($atomic_type, $value_type, $asm_suffix);
         #[cfg(any(
             test,
             portable_atomic_force_amo,
             target_feature = "zaamo",
             portable_atomic_target_feature = "zaamo",
         ))]
+        #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
         impl $atomic_type {
             #[inline]
             pub(crate) fn fetch_and(&self, val: $value_type, order: Ordering) -> $value_type {
@@ -465,10 +478,10 @@ macro_rules! atomic_sub_word {
     };
 }
 
-atomic_sub_word!(AtomicI8, i8, u8, "b");
-atomic_sub_word!(AtomicU8, u8, u8, "b");
-atomic_sub_word!(AtomicI16, i16, u16, "h");
-atomic_sub_word!(AtomicU16, u16, u16, "h");
+atomic_sub_word!(AtomicI8, i8, "b", max, min);
+atomic_sub_word!(AtomicU8, u8, "b", maxu, minu);
+atomic_sub_word!(AtomicI16, i16, "h", max, min);
+atomic_sub_word!(AtomicU16, u16, "h", maxu, minu);
 atomic!(AtomicI32, i32, "w", max, min);
 atomic!(AtomicU32, u32, "w", maxu, minu);
 #[cfg(target_arch = "riscv64")]
@@ -660,6 +673,9 @@ mod tests {
                 )]
                 mod [<test_atomic_ $int_type _amo>] {
                     use super::*;
+                    #[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))]
+                    test_atomic_int_amo!([<Atomic $int_type:camel>], $int_type);
+                    #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))]
                     test_atomic_int_amo_sub_word!([<Atomic $int_type:camel>], $int_type);
                 }
             }

diff --git a/tools/build.sh b/tools/build.sh
@@ -465,6 +465,12 @@ build() {
                                         CARGO_TARGET_DIR="${target_dir}/assume-single-core-zaamo" \
                                             RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core -C target-feature=+zaamo" \
                                             x_cargo "${args[@]}" --exclude-features "critical-section" "$@"
+                                        # Support for Zabha extension requires LLVM 19+.
+                                        if [[ "${llvm_version}" -ge 19 ]]; then
+                                            CARGO_TARGET_DIR="${target_dir}/assume-single-core-zabha" \
+                                                RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core -C target-feature=+zaamo,+zabha" \
+                                                x_cargo "${args[@]}" --exclude-features "critical-section" "$@"
+                                        fi
                                     fi
                                     ;;
                             esac

diff --git a/tools/no-std.sh b/tools/no-std.sh
@@ -85,6 +85,8 @@ rustc_target_list=$(rustc ${pre_args[@]+"${pre_args[@]}"} --print target-list)
 rustc_version=$(rustc ${pre_args[@]+"${pre_args[@]}"} -vV | grep -E '^release:' | cut -d' ' -f2)
 rustc_minor_version="${rustc_version#*.}"
 rustc_minor_version="${rustc_minor_version%%.*}"
+llvm_version=$(rustc ${pre_args[@]+"${pre_args[@]}"} -vV | { grep -E '^LLVM version:' || true; } | cut -d' ' -f3)
+llvm_version="${llvm_version%%.*}"
 target_dir=$(pwd)/target
 nightly=''
 if [[ "${rustc_version}" =~ nightly|dev ]]; then
@@ -99,6 +101,9 @@ export PORTABLE_ATOMIC_DENY_WARNINGS=1
 run() {
     local target="$1"
     shift
+    target_lower="${target//-/_}"
+    target_lower="${target_lower//./_}"
+    target_upper=$(tr '[:lower:]' '[:upper:]' <<<"${target_lower}")
     local args=(${pre_args[@]+"${pre_args[@]}"})
     local target_rustflags="${RUSTFLAGS:-}"
     if ! grep -Eq "^${target}$" <<<"${rustc_target_list}" || [[ -f "target-specs/${target}.json" ]]; then
@@ -215,6 +220,24 @@ run() {
                     CARGO_TARGET_DIR="${target_dir}/no-std-test-zaamo" \
                         RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo" \
                         x_cargo "${args[@]}" --release "$@"
+                    local arch
+                    case "${target}" in
+                        riscv32*) arch=riscv32 ;;
+                        riscv64*) arch=riscv64 ;;
+                        *) bail "${target}" ;;
+                    esac
+                    # Support for Zabha extension requires LLVM 19+ and QEMU 9.1+.
+                    # https://github.com/qemu/qemu/commit/be4a8db7f304347395b081ae5848bad2f507d0c4
+                    qemu_version=$(qemu-system-"${arch}" --version | sed -En '1 s/QEMU emulator version [^ ]+ \(v([^ )]+)\)/\1/p')
+                    if [[ "${llvm_version}" -ge 19 ]] && [[ "${qemu_version}" =~ ^(9\.[^0]|[1-9][0-9]+\.) ]]; then
+                        export "CARGO_TARGET_${target_upper}_RUNNER"="qemu-system-${arch} -M virt -cpu max -display none -semihosting -kernel"
+                        CARGO_TARGET_DIR="${target_dir}/no-std-test-zabha" \
+                            RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo,+zabha" \
+                            x_cargo "${args[@]}" "$@"
+                        CARGO_TARGET_DIR="${target_dir}/no-std-test-zabha" \
+                            RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo,+zabha" \
+                            x_cargo "${args[@]}" --release "$@"
+                    fi
                 fi
                 ;;
         esac