Vector bitcast support (AArch64 & Interpreter)

cranelift/codegen/meta/src/shared/instructions.rs

@@ -3280,6 +3280,9 @@ pub(crate) fn define(
         The input and output types must be storable to memory and of the same
         size. A bitcast is equivalent to storing one type and loading the other
         type from the same address.
+
+        For vector types, the lane types must also be the same size (see
+        `raw_bitcast` for changing the lane size).
         "#,
             &formats.unary,
         )

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -1738,6 +1738,13 @@
 (decl writable_zero_reg () WritableReg)
 (extern constructor writable_zero_reg writable_zero_reg)
 
+;; Helper for emitting `MInst.Mov` instructions.
+(decl mov (Reg Type) Reg)
+(rule (mov src ty)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.Mov (operand_size ty) dst src))))
+        dst))
+
 ;; Helper for emitting `MInst.MovZ` instructions.
 (decl movz (MoveWideConst OperandSize) Reg)
 (rule (movz imm size)
@@ -2093,6 +2100,17 @@
             (_ Unit (emit (MInst.FpuRound op dst rn))))
         dst))
 
+;; Helper for emitting `MInst.FpuMove64` and `MInst.FpuMove128` instructions.
+(decl fpu_move (Type Reg) Reg)
+(rule (fpu_move _ src)
+      (let ((dst WritableReg (temp_writable_reg $I8X16))
+            (_ Unit (emit (MInst.FpuMove128 dst src))))
+        dst))
+(rule (fpu_move (fits_in_64 _) src)
+      (let ((dst WritableReg (temp_writable_reg $F64))
+            (_ Unit (emit (MInst.FpuMove64 dst src))))
+        dst))
+
 ;; Helper for emitting `MInst.MovToFpu` instructions.
 (decl mov_to_fpu (Reg ScalarSize) Reg)
 (rule (mov_to_fpu x size)

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -2209,17 +2209,26 @@
 
 ;;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(rule (lower (has_type $I32 (bitcast src @ (value_type $F32))))
-      (mov_from_vec src 0 (ScalarSize.Size32)))
+; SIMD&FP <=> SIMD&FP
+(rule (lower (has_type (ty_float_or_vec out_ty) (bitcast x @ (value_type (ty_float_or_vec _)))))
+      (fpu_move out_ty x))
 
-(rule (lower (has_type $F32 (bitcast src @ (value_type $I32))))
-      (mov_to_fpu src (ScalarSize.Size32)))
+; GPR => SIMD&FP
+(rule (lower (has_type (ty_float_or_vec _) (bitcast x @ (value_type in_ty))))
+      (if (ty_int_bool_ref_scalar_64 in_ty))
+      (mov_to_fpu x (scalar_size in_ty)))
 
-(rule (lower (has_type $I64 (bitcast src @ (value_type $F64))))
-      (mov_from_vec src 0 (ScalarSize.Size64)))
+; SIMD&FP => GPR
+(rule (lower (has_type out_ty (bitcast x @ (value_type (fits_in_64 (ty_float_or_vec _))))))
+      (if (ty_int_bool_ref_scalar_64 out_ty))
+      (mov_from_vec x 0 (scalar_size out_ty)))
 
-(rule (lower (has_type $F64 (bitcast src @ (value_type $I64))))
-      (mov_to_fpu src (ScalarSize.Size64)))
+; GPR <=> GPR
+(rule (lower (has_type out_ty (bitcast x @ (value_type in_ty))))
+      (if (ty_int_bool_ref_scalar_64 out_ty))
+      (if (ty_int_bool_ref_scalar_64 in_ty))
+      x)
+(rule (lower (has_type $I128 (bitcast x @ (value_type $I128)))) x)
 
 ;;; Rules for `raw_bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -268,12 +268,7 @@ pub(crate) fn lower_insn_to_regs(
 
         Opcode::IsNull | Opcode::IsInvalid => implemented_in_isle(ctx),
 
-        Opcode::Copy => {
-            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
-            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            let ty = ctx.input_ty(insn, 0);
-            ctx.emit(Inst::gen_move(rd, rn, ty));
-        }
+        Opcode::Copy => implemented_in_isle(ctx),
 
         Opcode::Breduce | Opcode::Ireduce => implemented_in_isle(ctx),
 

cranelift/codegen/src/verifier/mod.rs

@@ -1096,15 +1096,26 @@ impl<'a> Verifier<'a> {
         let typ = self.func.dfg.ctrl_typevar(inst);
         let value_type = self.func.dfg.value_type(arg);
 
-        if typ.lane_bits() < value_type.lane_bits() {
+        if typ.lane_bits() != value_type.lane_bits() {
             errors.fatal((
                 inst,
                 format!(
-                    "The bitcast argument {} doesn't fit in a type of {} bits",
+                    "The bitcast argument {} has a lane type of {} bits, which doesn't match an expected type of {} bits",
                     arg,
+                    value_type.lane_bits(),
                     typ.lane_bits()
                 ),
             ))
+        } else if typ.bits() != value_type.bits() {
+            errors.fatal((
+                inst,
+                format!(
+                    "The bitcast argument {} has a type of {} bits, which doesn't match an expected type of {} bits",
+                    arg,
+                    value_type.bits(),
+                    typ.bits()
+                ),
+            ))
         } else {
             Ok(())
         }

cranelift/filetests/filetests/runtests/bitcast-ref64.clif

@@ -0,0 +1,26 @@
+test run
+target aarch64
+; the interpreter, x86_64, and s390x do not support bitcasting to/from
+; references
+
+function %bitcast_ir64(i64) -> b1 {
+block0(v0: i64):
+  v1 = bitcast.r64 v0
+  v2 = is_null v1
+  return v2
+}
+; run: %bitcast_ir64(0) == true
+; run: %bitcast_ir64(18446744073709551615) == false
+; run: %bitcast_ir64(-1) == false
+; run: %bitcast_ir64(127) == false
+
+function %bitcast_ri64(i64) -> i64 {
+block0(v0: i64):
+  v1 = bitcast.r64 v0
+  v2 = bitcast.i64 v1
+  return v2
+}
+; run: %bitcast_ri64(0) == 0
+; run: %bitcast_ri64(18446744073709551615) == 18446744073709551615
+; run: %bitcast_ri64(-1) == -1
+; run: %bitcast_ri64(127) == 127

cranelift/filetests/filetests/runtests/bitcast-same-type.clif

@@ -0,0 +1,69 @@
+test interpret
+test run
+target aarch64
+; x86_64 and s390x do not support bitcasting to the same type as the input.
+
+function %bitcast_i8(i8) -> i8 {
+block0(v0: i8):
+  v1 = bitcast.i8 v0
+  return v1
+}
+; run: %bitcast_i8(0) == 0
+; run: %bitcast_i8(42) == 42
+; run: %bitcast_i8(255) == 255
+
+function %bitcast_i16(i16) -> i16 {
+block0(v0: i16):
+  v1 = bitcast.i16 v0
+  return v1
+}
+; run: %bitcast_i16(0) == 0
+; run: %bitcast_i16(42) == 42
+; run: %bitcast_i16(65535) == 65535
+
+function %bitcast_i32(i32) -> i32 {
+block0(v0: i32):
+  v1 = bitcast.i32 v0
+  return v1
+}
+; run: %bitcast_i32(0) == 0
+; run: %bitcast_i32(42) == 42
+; run: %bitcast_i32(4294967295) == 4294967295
+
+function %bitcast_i64(i64) -> i64 {
+block0(v0: i64):
+  v1 = bitcast.i64 v0
+  return v1
+}
+; run: %bitcast_i64(0) == 0
+; run: %bitcast_i64(42) == 42
+; run: %bitcast_i64(18446744073709551615) == 18446744073709551615
+
+function %bitcast_i128(i128) -> i128 {
+block0(v0: i128):
+  v1 = bitcast.i128 v0
+  return v1
+}
+; run: %bitcast_i128(0) == 0
+; run: %bitcast_i128(42) == 42
+; run: %bitcast_i128(200000000000000000000) == 200000000000000000000
+
+function %bitcast_f32(f32) -> f32 {
+block0(v0: f32):
+  v1 = bitcast.f32 v0
+  return v1
+}
+; run: %bitcast_f32(0x0.0) == 0x0.0
+; run: %bitcast_f32(0x1.0) == 0x1.0
+; run: %bitcast_f32(-0x1.0) == -0x1.0
+; run: %bitcast_f32(NaN) == NaN
+
+function %bitcast_f64(f64) -> f64 {
+block0(v0: f64):
+  v1 = bitcast.f64 v0
+  return v1
+}
+; run: %bitcast_f64(0x0.0) == 0x0.0
+; run: %bitcast_f64(0x1.0) == 0x1.0
+; run: %bitcast_f64(-0x1.0) == -0x1.0
+; run: %bitcast_f64(NaN) == NaN

cranelift/filetests/filetests/runtests/bitcast.clif

@@ -0,0 +1,45 @@
+test interpret
+test run
+target aarch64
+target x86_64
+target s390x
+
+function %bitcast_if32(i32) -> f32 {
+block0(v0: i32):
+  v1 = bitcast.f32 v0
+  return v1
+}
+; run: %bitcast_if32(0) == 0x0.0
+; run: %bitcast_if32(4294967295) == -NaN:0x3fffff
+; run: %bitcast_if32(-1) == -NaN:0x3fffff
+; run: %bitcast_if32(127) == 0x0.0000fep-126
+
+function %bitcast_fi32(f32) -> i32 {
+block0(v0: f32):
+  v1 = bitcast.i32 v0
+  return v1
+}
+; run: %bitcast_fi32(0x0.0) == 0
+; run: %bitcast_fi32(-NaN:0x3fffff) == 4294967295
+; run: %bitcast_fi32(-NaN:0x3fffff) == -1
+; run: %bitcast_fi32(0x0.0000fep-126) == 127
+
+function %bitcast_if64(i64) -> f64 {
+block0(v0: i64):
+  v1 = bitcast.f64 v0
+  return v1
+}
+; run: %bitcast_if64(0) == 0x0.0
+; run: %bitcast_if64(18446744073709551615) == -NaN:0x7ffffffffffff
+; run: %bitcast_if64(-1) == -NaN:0x7ffffffffffff
+; run: %bitcast_if64(127) == 0x0.000000000007fp-1022
+
+function %bitcast_fi64(f64) -> i64 {
+block0(v0: f64):
+  v1 = bitcast.i64 v0
+  return v1
+}
+; run: %bitcast_fi64(0x0.0) == 0
+; run: %bitcast_fi64(-NaN:0x7ffffffffffff) == 18446744073709551615
+; run: %bitcast_fi64(-NaN:0x7ffffffffffff) == -1
+; run: %bitcast_fi64(0x0.000000000007fp-1022) == 127

cranelift/filetests/filetests/runtests/copy.clif

@@ -0,0 +1,121 @@
+test interpret
+test run
+target aarch64
+target s390x
+; x86_64 regards this as an unused opcode.
+
+function %copy_i8(i8) -> i8 {
+block0(v0: i8):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_i8(0) == 0
+; run: %copy_i8(255) == 255
+; run: %copy_i8(-1) == -1
+; run: %copy_i8(127) == 127
+
+function %copy_i16(i16) -> i16 {
+block0(v0: i16):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_i16(0) == 0
+; run: %copy_i16(65535) == 65535
+; run: %copy_i16(-1) == -1
+; run: %copy_i16(127) == 127
+
+function %copy_i32(i32) -> i32 {
+block0(v0: i32):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_i32(0) == 0
+; run: %copy_i32(4294967295) == 4294967295
+; run: %copy_i32(-1) == -1
+; run: %copy_i32(127) == 127
+
+function %copy_i64(i64) -> i64 {
+block0(v0: i64):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_i64(0) == 0
+; run: %copy_i64(18446744073709551615) == 18446744073709551615
+; run: %copy_i64(-1) == -1
+; run: %copy_i64(127) == 127
+
+function %copy_b1(b1) -> b1 {
+block0(v0: b1):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_b1(false) == false
+; run: %copy_b1(true) == true
+
+function %copy_b8(b8) -> b8 {
+block0(v0: b8):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_b8(false) == false
+; run: %copy_b8(true) == true
+
+function %copy_b16(b16) -> b16 {
+block0(v0: b16):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_b16(false) == false
+; run: %copy_b16(true) == true
+
+function %copy_b32(b32) -> b32 {
+block0(v0: b32):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_b32(false) == false
+; run: %copy_b32(true) == true
+
+function %copy_b64(b64) -> b64 {
+block0(v0: b64):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_b64(false) == false
+; run: %copy_b64(true) == true
+
+function %copy_f32(f32) -> f32 {
+block0(v0: f32):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_f32(0x1.0) == 0x1.0
+; run: %copy_f32(0x1.0p10) == 0x1.0p10
+
+; run: %copy_f32(0x0.0) == 0x0.0
+; run: %copy_f32(-0x0.0) == -0x0.0
+; run: %copy_f32(+Inf) == +Inf
+; run: %copy_f32(-Inf) == -Inf
+; run: %copy_f32(0x1.000002p-23) == 0x1.000002p-23
+; run: %copy_f32(0x1.fffffep127) == 0x1.fffffep127
+; run: %copy_f32(0x1.000000p-126) == 0x1.000000p-126
+; run: %copy_f32(0x0.800002p-126) ==  0x0.800002p-126
+; run: %copy_f32(-0x0.800000p-126) == -0x0.800000p-126
+
+function %copy_f64(f64) -> f64 {
+block0(v0: f64):
+  v1 = copy v0
+  return v1
+}
+; run: %copy_f64(0x2.0) == 0x2.0
+; run: %copy_f64(0x1.0p11) == 0x1.0p11
+
+; run: %copy_f64(0x0.0) == 0x0.0
+; run: %copy_f64(-0x0.0) == -0x0.0
+; run: %copy_f64(+Inf) == +Inf
+; run: %copy_f64(-Inf) == -Inf
+; run: %copy_f64(0x1.0000000000002p-52) == 0x1.0000000000002p-52
+; run: %copy_f64(0x1.fffffffffffffp1023) == 0x1.fffffffffffffp1023
+; run: %copy_f64(0x1.0000000000000p-1022) == 0x1.0000000000000p-1022
+; run: %copy_f64(0x0.8000000000002p-1022) == 0x0.8000000000002p-1022
+; run: %copy_f64(-0x0.8000000000000p-1022) == -0x0.8000000000000p-1022