diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt index b5bf56f3..86777d73 100644 --- a/.github/.cspell/project-dictionary.txt +++ b/.github/.cspell/project-dictionary.txt @@ -46,6 +46,7 @@ espup exynos getauxval getpid +Halfword HWCAP hwprobe ifunc @@ -175,4 +176,5 @@ xmmword xsave xsub zaamo +zabha Zhaoxin diff --git a/build.rs b/build.rs index 7c870085..e17a562f 100644 --- a/build.rs +++ b/build.rs @@ -47,7 +47,7 @@ fn main() { if version.minor >= 80 { println!( - r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"# + r#"cargo:rustc-check-cfg=cfg(target_feature,values("zaamo","zabha","quadword-atomics","fast-serialization","load-store-on-cond","distinct-ops","miscellaneous-extensions-3"))"# ); // Custom cfgs set by build script. Not public API. @@ -58,7 +58,7 @@ fn main() { // TODO: handle multi-line target_feature_fallback // grep -F 'target_feature_fallback("' build.rs | grep -Ev '^ *//' | sed -E 's/^.*target_feature_fallback\(//; s/",.*$/"/' | LC_ALL=C sort -u | tr '\n' ',' | sed -E 's/,$/\n/' println!( - r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo"))"# + r#"cargo:rustc-check-cfg=cfg(portable_atomic_target_feature,values("cmpxchg16b","distinct-ops","fast-serialization","load-store-on-cond","lse","lse128","lse2","mclass","miscellaneous-extensions-3","quadword-atomics","rcpc3","v6","zaamo","zabha"))"# ); } @@ -311,9 +311,18 @@ fn main() { } } "riscv32" | "riscv64" => { - // As of rustc 1.80, target_feature "zaamo" is not available on rustc side: + // As of rustc 1.80, target_feature "zaamo"/"zabha" is not available on rustc side: // https://github.com/rust-lang/rust/blob/1.80.0/compiler/rustc_target/src/target_features.rs#L273 - target_feature_fallback("zaamo", false); // amo*.{w,d} + // zabha implies zaamo in GCC, but do not in LLVM (but enabling it without zaamo is not allowed). + // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0-rc3/llvm/lib/TargetParser/RISCVISAInfo.cpp#L776-L778 + // https://github.com/gcc-mirror/gcc/blob/08693e29ec186fd7941d0b73d4d466388971fe2f/gcc/config/riscv/arch-canonicalize#L45 + if version.llvm >= 19 { + // amo*.{b,h} + // available since 19 https://github.com/llvm/llvm-project/commit/89f87c387627150d342722b79c78cea2311cddf7 / https://github.com/llvm/llvm-project/commit/6b7444964a8d028989beee554a1f5c61d16a1cac + target_feature_fallback("zabha", false); + } + // amo*.{w,d} + target_feature_fallback("zaamo", false); } "powerpc64" => { // For Miri and ThreadSanitizer. diff --git a/src/imp/interrupt/mod.rs b/src/imp/interrupt/mod.rs index f13c28ac..1fa778eb 100644 --- a/src/imp/interrupt/mod.rs +++ b/src/imp/interrupt/mod.rs @@ -604,7 +604,8 @@ macro_rules! atomic_int { } } }; - // 32-bit(RV32)/{32,64}-bit(RV64) RMW with Zaamo extension + // RISC-V 32-bit(RV32)/{32,64}-bit(RV64) RMW with Zaamo extension + // RISC-V 8-bit/16-bit RMW with Zabha extension (cas, $atomic_type:ident, $int_type:ident) => { impl $atomic_type { #[inline] @@ -711,8 +712,17 @@ macro_rules! atomic_int { } } }; - // {8,16}-bit RMW with Zaamo extension + // RISC-V 8-bit/16-bit RMW with Zaamo extension (cas[sub_word], $atomic_type:ident, $int_type:ident) => { + #[cfg(all( + any(target_arch = "riscv32", target_arch = "riscv64"), + any(target_feature = "zabha", portable_atomic_target_feature = "zabha"), + ))] + atomic_int!(cas, $atomic_type, $int_type); + #[cfg(not(all( + any(target_arch = "riscv32", target_arch = "riscv64"), + any(target_feature = "zabha", portable_atomic_target_feature = "zabha"), + )))] impl $atomic_type { #[inline] pub(crate) fn swap(&self, val: $int_type, _order: Ordering) -> $int_type { diff --git a/src/imp/riscv.rs b/src/imp/riscv.rs index a4c3b7d0..0ee9bddf 100644 --- a/src/imp/riscv.rs +++ b/src/imp/riscv.rs @@ -12,10 +12,12 @@ // https://riscv.org/wp-content/uploads/2019/12/riscv-spec-20191213.pdf // - RISC-V Atomics ABI Specification // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/draft-20240829-13bfa9f54634cb60d86b9b333e109f077805b4b3/riscv-atomic.adoc -// - "Mappings from C/C++ primitives to RISC-V primitives." table in RISC-V Instruction Set Manual: +// - "Mappings from C/C++ primitives to RISC-V primitives." table in RISC-V Instruction Set Manual // https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/mm-eplan.adoc#code-porting-and-mapping-guidelines -// - ""Zaamo" Extension for Atomic Memory Operations" in RISC-V Instruction Set Manual: +// - ""Zaamo" Extension for Atomic Memory Operations" in RISC-V Instruction Set Manual // https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/a-st-ext.adoc#zaamo-extension-for-atomic-memory-operations +// - ""Zabha" Extension for Byte and Halfword Atomic Memory Operations" in RISC-V Instruction Set Manual +// https://github.com/riscv/riscv-isa-manual/blob/riscv-isa-release-8b9dc50-2024-08-30/src/zabha.adoc // - atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit // // Generated asm: @@ -32,6 +34,7 @@ use core::{cell::UnsafeCell, sync::atomic::Ordering}; target_feature = "zaamo", portable_atomic_target_feature = "zaamo", ))] +#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] #[cfg(target_arch = "riscv32")] macro_rules! w { () => { @@ -44,6 +47,7 @@ macro_rules! w { target_feature = "zaamo", portable_atomic_target_feature = "zaamo", ))] +#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] #[cfg(target_arch = "riscv64")] macro_rules! w { () => { @@ -86,7 +90,7 @@ macro_rules! atomic_rmw_amo { // The caller of this macro must guarantee the validity of the pointer. asm!( ".option push", - // https://github.com/riscv-non-isa/riscv-asm-manual/blob/HEAD/riscv-asm.md#arch + // https://github.com/riscv-non-isa/riscv-asm-manual/blob/ad0de8c004e29c9a7ac33cfd054f4d4f9392f2fb/src/asm-manual.adoc#arch ".option arch, +a", concat!("amo", stringify!($op), ".", $asm_suffix, $asm_order, " {out}, {val}, 0({dst})"), ".option pop", @@ -108,6 +112,7 @@ macro_rules! atomic_rmw_amo { target_feature = "zaamo", portable_atomic_target_feature = "zaamo", ))] +#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] #[inline] fn sllw(val: u32, shift: u32) -> u32 { // SAFETY: Calling sll{,w} is safe. @@ -130,6 +135,7 @@ fn sllw(val: u32, shift: u32) -> u32 { target_feature = "zaamo", portable_atomic_target_feature = "zaamo", ))] +#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] macro_rules! srlw { ($val:expr, $shift:expr) => { // SAFETY: Calling srl{,w} is safe. @@ -376,6 +382,7 @@ macro_rules! atomic { target_feature = "zaamo", portable_atomic_target_feature = "zaamo", ))] +#[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] trait ZeroExtend: Copy { /// Zero-extends `self` to `u32` if it is smaller than 32-bit. fn zero_extend(self) -> u32; @@ -388,6 +395,7 @@ macro_rules! zero_extend { target_feature = "zaamo", portable_atomic_target_feature = "zaamo", ))] + #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] impl ZeroExtend for $uint { #[inline] fn zero_extend(self) -> u32 { @@ -400,6 +408,7 @@ macro_rules! zero_extend { target_feature = "zaamo", portable_atomic_target_feature = "zaamo", ))] + #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] impl ZeroExtend for $int { #[allow(clippy::cast_sign_loss)] #[inline] @@ -413,7 +422,10 @@ zero_extend!(i8, u8); zero_extend!(i16, u16); macro_rules! atomic_sub_word { - ($atomic_type:ident, $value_type:ty, $unsigned_type:ty, $asm_suffix:tt) => { + ($atomic_type:ident, $value_type:ty, $asm_suffix:tt, $max:tt, $min:tt) => { + #[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))] + atomic!($atomic_type, $value_type, $asm_suffix, $max, $min); + #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] atomic_load_store!($atomic_type, $value_type, $asm_suffix); #[cfg(any( test, @@ -421,6 +433,7 @@ macro_rules! atomic_sub_word { target_feature = "zaamo", portable_atomic_target_feature = "zaamo", ))] + #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] impl $atomic_type { #[inline] pub(crate) fn fetch_and(&self, val: $value_type, order: Ordering) -> $value_type { @@ -465,10 +478,10 @@ macro_rules! atomic_sub_word { }; } -atomic_sub_word!(AtomicI8, i8, u8, "b"); -atomic_sub_word!(AtomicU8, u8, u8, "b"); -atomic_sub_word!(AtomicI16, i16, u16, "h"); -atomic_sub_word!(AtomicU16, u16, u16, "h"); +atomic_sub_word!(AtomicI8, i8, "b", max, min); +atomic_sub_word!(AtomicU8, u8, "b", maxu, minu); +atomic_sub_word!(AtomicI16, i16, "h", max, min); +atomic_sub_word!(AtomicU16, u16, "h", maxu, minu); atomic!(AtomicI32, i32, "w", max, min); atomic!(AtomicU32, u32, "w", maxu, minu); #[cfg(target_arch = "riscv64")] @@ -660,6 +673,9 @@ mod tests { )] mod [] { use super::*; + #[cfg(any(target_feature = "zabha", portable_atomic_target_feature = "zabha"))] + test_atomic_int_amo!([], $int_type); + #[cfg(not(any(target_feature = "zabha", portable_atomic_target_feature = "zabha")))] test_atomic_int_amo_sub_word!([], $int_type); } } diff --git a/tools/build.sh b/tools/build.sh index b57c32fb..afd1c68a 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -465,6 +465,12 @@ build() { CARGO_TARGET_DIR="${target_dir}/assume-single-core-zaamo" \ RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core -C target-feature=+zaamo" \ x_cargo "${args[@]}" --exclude-features "critical-section" "$@" + # Support for Zabha extension requires LLVM 19+. + if [[ "${llvm_version}" -ge 19 ]]; then + CARGO_TARGET_DIR="${target_dir}/assume-single-core-zabha" \ + RUSTFLAGS="${target_rustflags} --cfg portable_atomic_unsafe_assume_single_core -C target-feature=+zaamo,+zabha" \ + x_cargo "${args[@]}" --exclude-features "critical-section" "$@" + fi fi ;; esac diff --git a/tools/no-std.sh b/tools/no-std.sh index 2ef9e657..f443d6a2 100755 --- a/tools/no-std.sh +++ b/tools/no-std.sh @@ -85,6 +85,8 @@ rustc_target_list=$(rustc ${pre_args[@]+"${pre_args[@]}"} --print target-list) rustc_version=$(rustc ${pre_args[@]+"${pre_args[@]}"} -vV | grep -E '^release:' | cut -d' ' -f2) rustc_minor_version="${rustc_version#*.}" rustc_minor_version="${rustc_minor_version%%.*}" +llvm_version=$(rustc ${pre_args[@]+"${pre_args[@]}"} -vV | { grep -E '^LLVM version:' || true; } | cut -d' ' -f3) +llvm_version="${llvm_version%%.*}" target_dir=$(pwd)/target nightly='' if [[ "${rustc_version}" =~ nightly|dev ]]; then @@ -99,6 +101,9 @@ export PORTABLE_ATOMIC_DENY_WARNINGS=1 run() { local target="$1" shift + target_lower="${target//-/_}" + target_lower="${target_lower//./_}" + target_upper=$(tr '[:lower:]' '[:upper:]' <<<"${target_lower}") local args=(${pre_args[@]+"${pre_args[@]}"}) local target_rustflags="${RUSTFLAGS:-}" if ! grep -Eq "^${target}$" <<<"${rustc_target_list}" || [[ -f "target-specs/${target}.json" ]]; then @@ -215,6 +220,24 @@ run() { CARGO_TARGET_DIR="${target_dir}/no-std-test-zaamo" \ RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo" \ x_cargo "${args[@]}" --release "$@" + local arch + case "${target}" in + riscv32*) arch=riscv32 ;; + riscv64*) arch=riscv64 ;; + *) bail "${target}" ;; + esac + # Support for Zabha extension requires LLVM 19+ and QEMU 9.1+. + # https://github.com/qemu/qemu/commit/be4a8db7f304347395b081ae5848bad2f507d0c4 + qemu_version=$(qemu-system-"${arch}" --version | sed -En '1 s/QEMU emulator version [^ ]+ \(v([^ )]+)\)/\1/p') + if [[ "${llvm_version}" -ge 19 ]] && [[ "${qemu_version}" =~ ^(9\.[^0]|[1-9][0-9]+\.) ]]; then + export "CARGO_TARGET_${target_upper}_RUNNER"="qemu-system-${arch} -M virt -cpu max -display none -semihosting -kernel" + CARGO_TARGET_DIR="${target_dir}/no-std-test-zabha" \ + RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo,+zabha" \ + x_cargo "${args[@]}" "$@" + CARGO_TARGET_DIR="${target_dir}/no-std-test-zabha" \ + RUSTFLAGS="${target_rustflags} -C target-feature=+zaamo,+zabha" \ + x_cargo "${args[@]}" --release "$@" + fi fi ;; esac