Change the order of arguments in simd_masked_store

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -1059,7 +1059,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         }
 
         sym::simd_scatter => {
-            intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
+            intrinsic_args!(fx, args => (mask, ptr, val); intrinsic);
 
             let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
             let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -1596,81 +1596,87 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_masked_store {
-        // simd_masked_store(values: <N x T>, pointer: *mut T,
-        //                   mask: <N x i{M}>) -> ()
+        // simd_masked_store(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> ()
         // * N: number of elements in the input vectors
         // * T: type of the element to load
         // * M: any integer width is supported, will be truncated to i1
         // Stores contiguous elements to memory behind `pointer`, but only for
         // those lanes whose `mask` bit is enabled.
         // The memory addresses corresponding to the “off” lanes are not accessed.
 
-        // The first argument is a passthrough vector providing values for disabled lanes
-
         // The element type of the "mask" argument must be a signed integer type of any width
-        let (mask_len, mask_elem) = require_simd!(arg_tys[2], SimdThird);
+        let mask_ty = in_ty;
+        let (mask_len, mask_elem) = (in_len, in_elem);
+
+        // The second argument must be a pointer matching the element type
+        let pointer_ty = arg_tys[1];
+
+        // The last argument specifies the values to store to memory
+        let values_ty = arg_tys[2];
+        let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
 
         // Of the same length:
         require!(
-            in_len == mask_len,
+            values_len == mask_len,
             InvalidMonomorphization::ThirdArgumentLength {
                 span,
                 name,
-                in_len,
-                in_ty,
-                arg_ty: arg_tys[2],
-                out_len: mask_len
+                in_len: mask_len,
+                in_ty: mask_ty,
+                arg_ty: values_ty,
+                out_len: values_len
             }
         );
 
         // The second argument must be a mutable pointer type matching the element type
         require!(
             matches!(
-                arg_tys[1].kind(),
-                ty::RawPtr(p) if p.ty == in_elem && p.ty.kind() == in_elem.kind() && p.mutbl.is_mut()
+                pointer_ty.kind(),
+                ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() && p.mutbl.is_mut()
             ),
             InvalidMonomorphization::ExpectedElementType {
                 span,
                 name,
-                expected_element: in_elem,
-                second_arg: arg_tys[1],
-                in_elem,
-                in_ty,
+                expected_element: values_elem,
+                second_arg: pointer_ty,
+                in_elem: values_elem,
+                in_ty: values_ty,
                 mutability: ExpectedPointerMutability::Mut,
             }
         );
 
-        // Mask needs to be an integer type
-        match mask_elem.kind() {
-            ty::Int(_) => (),
-            _ => {
-                return_error!(InvalidMonomorphization::ThirdArgElementType {
-                    span,
-                    name,
-                    expected_element: mask_elem,
-                    third_arg: arg_tys[2]
-                });
+        let expected_int_bits = (mask_len.max(8) - 1).next_power_of_two();
+        let expected_bytes = mask_len / 8 + ((mask_len % 8 > 0) as u64);
+
+        require!(
+            matches!(mask_elem.kind(), ty::Int(_)),
+            InvalidMonomorphization::InvalidBitmask {
+                span,
+                name,
+                mask_ty,
+                expected_int_bits,
+                expected_bytes
             }
-        }
+        );
 
         // Alignment of T, must be a constant integer value:
         let alignment_ty = bx.type_i32();
-        let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
+        let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
 
         // Truncate the mask vector to a vector of i1s:
         let (mask, mask_ty) = {
             let i1 = bx.type_i1();
             let i1xn = bx.type_vector(i1, in_len);
-            (bx.trunc(args[2].immediate(), i1xn), i1xn)
+            (bx.trunc(args[0].immediate(), i1xn), i1xn)
         };
 
         let ret_t = bx.type_void();
 
         let llvm_pointer = bx.type_ptr();
 
         // Type of the vector of elements:
-        let llvm_elem_vec_ty = llvm_vector_ty(bx, in_elem, in_len);
-        let llvm_elem_vec_str = llvm_vector_str(bx, in_elem, in_len);
+        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
+        let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
 
         let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0");
         let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t);
@@ -1680,7 +1686,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             None,
             None,
             f,
-            &[args[0].immediate(), args[1].immediate(), alignment, mask],
+            &[args[2].immediate(), args[1].immediate(), alignment, mask],
             None,
         );
         return Ok(v);

tests/ui/simd/masked-load-store-build-fail.rs

@@ -3,7 +3,7 @@
 
 extern "platform-intrinsic" {
     fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
-    fn simd_masked_store<T, P, M>(values: T, pointer: P, mask: M) -> ();
+    fn simd_masked_store<M, P, T>(mask: M, pointer: P, values: T) -> ();
 }
 
 #[derive(Copy, Clone)]
@@ -44,31 +44,31 @@ fn main() {
         //~^^^^^ ERROR invalid bitmask `Simd<u8, 4>`, expected `u8` or `[u8; 1]`
 
         simd_masked_store(
-            Simd([5u32; 4]),
+            Simd([-1i8; 4]),
             arr.as_ptr(),
-            Simd([-1i8; 4])
+            Simd([5u32; 4])
         );
         //~^^^^^ ERROR expected element type `u32` of second argument `*const u8` to be a pointer to the element type `u32` of the first argument `Simd<u32, 4>`, found `u32` != `*mut u32`
 
         simd_masked_store(
-            Simd([5u8; 4]),
+            Simd([-1i8; 4]),
             arr.as_ptr(),
-            Simd([-1i8; 4])
+            Simd([5u8; 4])
         );
         //~^^^^^ ERROR expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*mut u8`
 
         simd_masked_store(
-            Simd([5u8; 2]),
+            Simd([-1i8; 4]),
             arr.as_mut_ptr(),
-            Simd([-1i8; 4])
+            Simd([5u8; 2])
         );
-        //~^^^^^ ERROR expected third argument with length 2 (same as input type `Simd<u8, 2>`), found `Simd<i8, 4>` with length 4
+        //~^^^^^ ERROR expected third argument with length 4 (same as input type `Simd<i8, 4>`), found `Simd<u8, 2>` with length 2
 
         simd_masked_store(
-            Simd([5u8; 4]),
+            Simd([1u32; 4]),
             arr.as_mut_ptr(),
-            Simd([1u32; 4])
+            Simd([5u8; 4])
         );
-        //~^^^^^ ERROR expected element type `u32` of third argument `Simd<u32, 4>` to be a signed integer type
+        //~^^^^^ ERROR invalid bitmask `Simd<u32, 4>`, expected `u8` or `[u8; 1]
     }
 }

tests/ui/simd/masked-load-store-build-fail.stderr

@@ -42,39 +42,39 @@ error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expecte
   --> $DIR/masked-load-store-build-fail.rs:46:9
    |
 LL | /         simd_masked_store(
-LL | |             Simd([5u32; 4]),
+LL | |             Simd([-1i8; 4]),
 LL | |             arr.as_ptr(),
-LL | |             Simd([-1i8; 4])
+LL | |             Simd([5u32; 4])
 LL | |         );
    | |_________^
 
 error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u8` of second argument `*const u8` to be a pointer to the element type `u8` of the first argument `Simd<u8, 4>`, found `u8` != `*mut u8`
   --> $DIR/masked-load-store-build-fail.rs:53:9
    |
 LL | /         simd_masked_store(
-LL | |             Simd([5u8; 4]),
+LL | |             Simd([-1i8; 4]),
 LL | |             arr.as_ptr(),
-LL | |             Simd([-1i8; 4])
+LL | |             Simd([5u8; 4])
 LL | |         );
    | |_________^
 
-error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected third argument with length 2 (same as input type `Simd<u8, 2>`), found `Simd<i8, 4>` with length 4
+error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected third argument with length 4 (same as input type `Simd<i8, 4>`), found `Simd<u8, 2>` with length 2
   --> $DIR/masked-load-store-build-fail.rs:60:9
    |
 LL | /         simd_masked_store(
-LL | |             Simd([5u8; 2]),
+LL | |             Simd([-1i8; 4]),
 LL | |             arr.as_mut_ptr(),
-LL | |             Simd([-1i8; 4])
+LL | |             Simd([5u8; 2])
 LL | |         );
    | |_________^
 
-error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: expected element type `u32` of third argument `Simd<u32, 4>` to be a signed integer type
+error[E0511]: invalid monomorphization of `simd_masked_store` intrinsic: invalid bitmask `Simd<u32, 4>`, expected `u8` or `[u8; 1]`
   --> $DIR/masked-load-store-build-fail.rs:67:9
    |
 LL | /         simd_masked_store(
-LL | |             Simd([5u8; 4]),
+LL | |             Simd([1u32; 4]),
 LL | |             arr.as_mut_ptr(),
-LL | |             Simd([1u32; 4])
+LL | |             Simd([5u8; 4])
 LL | |         );
    | |_________^
 

tests/ui/simd/masked-load-store-check-fail.rs

@@ -3,7 +3,7 @@
 
 extern "platform-intrinsic" {
     fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
-    fn simd_masked_store<T, P, M>(values: T, pointer: P, mask: M) -> ();
+    fn simd_masked_store<M, P, T>(mask: M, pointer: P, values: T) -> ();
 }
 
 #[derive(Copy, Clone)]

tests/ui/simd/masked-load-store.rs

@@ -3,7 +3,7 @@
 
 extern "platform-intrinsic" {
     fn simd_masked_load<M, P, T>(mask: M, pointer: P, values: T) -> T;
-    fn simd_masked_store<T, P, M>(values: T, pointer: P, mask: M) -> ();
+    fn simd_masked_store<M, P, T>(mask: M, pointer: P, values: T) -> ();
 }
 
 #[derive(Copy, Clone)]
@@ -25,9 +25,9 @@ fn main() {
 
         let mut output = [u8::MAX; 5];
 
-        simd_masked_store(a, output.as_mut_ptr(), Simd::<i8, 4>([-1, -1, -1, 0]));
+        simd_masked_store(Simd::<i8, 4>([-1, -1, -1, 0]), output.as_mut_ptr(), a);
         assert_eq!(&output, &[0, 1, 2, u8::MAX, u8::MAX]);
-        simd_masked_store(b, output[1..].as_mut_ptr(), Simd::<i8, 4>([0, -1, -1, 0]));
+        simd_masked_store(Simd::<i8, 4>([0, -1, -1, 0]), output[1..].as_mut_ptr(), b);
         assert_eq!(&output, &[0, 1, 9, 6, u8::MAX]);
     }
 }