diff --git a/cranelift/codegen/meta/src/cdsl/instructions.rs b/cranelift/codegen/meta/src/cdsl/instructions.rs index 6d4b6738dfc2..88a15c603882 100644 --- a/cranelift/codegen/meta/src/cdsl/instructions.rs +++ b/cranelift/codegen/meta/src/cdsl/instructions.rs @@ -62,7 +62,7 @@ impl InstructionGroup { self.instructions .iter() .find(|inst| inst.name == name) - .unwrap_or_else(|| panic!("unexisting instruction with name {}", name)) + .unwrap_or_else(|| panic!("instruction with name '{}' does not exist", name)) } } @@ -598,7 +598,7 @@ fn verify_format(inst_name: &str, operands_in: &[Operand], format: &InstructionF assert_eq!( num_values, format.num_value_operands, - "inst {} doesnt' have as many value input operand as its format {} declares; you may need \ + "inst {} doesn't have as many value input operands as its format {} declares; you may need \ to use a different format.", inst_name, format.name ); diff --git a/cranelift/codegen/meta/src/isa/x86/encodings.rs b/cranelift/codegen/meta/src/isa/x86/encodings.rs index 303b1bfaebb5..8f4a77d814a4 100644 --- a/cranelift/codegen/meta/src/isa/x86/encodings.rs +++ b/cranelift/codegen/meta/src/isa/x86/encodings.rs @@ -1676,6 +1676,8 @@ fn define_simd( let uload16x4_complex = shared.by_name("uload16x4_complex"); let uload32x2 = shared.by_name("uload32x2"); let uload32x2_complex = shared.by_name("uload32x2_complex"); + let snarrow = shared.by_name("snarrow"); + let unarrow = shared.by_name("unarrow"); let ushr_imm = shared.by_name("ushr_imm"); let usub_sat = shared.by_name("usub_sat"); let vconst = shared.by_name("vconst"); @@ -1686,7 +1688,6 @@ fn define_simd( let x86_fmin = x86.by_name("x86_fmin"); let x86_movlhps = x86.by_name("x86_movlhps"); let x86_movsd = x86.by_name("x86_movsd"); - let x86_packss = x86.by_name("x86_packss"); let x86_pblendw = x86.by_name("x86_pblendw"); let x86_pextr = x86.by_name("x86_pextr"); let x86_pinsr = x86.by_name("x86_pinsr"); @@ -1901,8 +1902,15 @@ fn define_simd( ); } for (ty, opcodes) in &[(I16, &PACKSSWB), (I32, &PACKSSDW)] { - let x86_packss = x86_packss.bind(vector(*ty, sse_vector_size)); - e.enc_both_inferred(x86_packss, rec_fa.opcodes(*opcodes)); + let snarrow = snarrow.bind(vector(*ty, sse_vector_size)); + e.enc_both_inferred(snarrow, rec_fa.opcodes(*opcodes)); + } + for (ty, opcodes, isap) in &[ + (I16, &PACKUSWB[..], None), + (I32, &PACKUSDW[..], Some(use_sse41_simd)), + ] { + let unarrow = unarrow.bind(vector(*ty, sse_vector_size)); + e.enc_both_inferred_maybe_isap(unarrow, rec_fa.opcodes(*opcodes), *isap); } // SIMD bitcast all 128-bit vectors to each other (for legalizing splat.x16x8). diff --git a/cranelift/codegen/meta/src/isa/x86/instructions.rs b/cranelift/codegen/meta/src/isa/x86/instructions.rs index 4afbc8874702..0e48784f2337 100644 --- a/cranelift/codegen/meta/src/isa/x86/instructions.rs +++ b/cranelift/codegen/meta/src/isa/x86/instructions.rs @@ -454,35 +454,6 @@ pub(crate) fn define( .operands_out(vec![a]), ); - let I16xN = &TypeVar::new( - "I16xN", - "A SIMD vector type containing integers 16-bits wide and up", - TypeSetBuilder::new() - .ints(16..32) - .simd_lanes(4..8) - .includes_scalars(false) - .build(), - ); - - let x = &Operand::new("x", I16xN); - let y = &Operand::new("y", I16xN); - let a = &Operand::new("a", &I16xN.split_lanes()); - - ig.push( - Inst::new( - "x86_packss", - r#" - Convert packed signed integers the lanes of ``x`` and ``y`` into half-width integers, using - signed saturation to handle overflows. For example, with notional i16x2 vectors, where - ``x = [x1, x0]`` and ``y = [y1, y0]``, this operation would result in - ``a = [y1', y0', x1', x0']`` (using the Intel manual's right-to-left lane ordering). - "#, - &formats.binary, - ) - .operands_in(vec![x, y]) - .operands_out(vec![a]), - ); - let x = &Operand::new("x", FxN); let y = &Operand::new("y", FxN); let a = &Operand::new("a", FxN); diff --git a/cranelift/codegen/meta/src/isa/x86/legalize.rs b/cranelift/codegen/meta/src/isa/x86/legalize.rs index 51453322e9ba..30c6789dff71 100644 --- a/cranelift/codegen/meta/src/isa/x86/legalize.rs +++ b/cranelift/codegen/meta/src/isa/x86/legalize.rs @@ -405,6 +405,7 @@ fn define_simd( let uadd_sat = insts.by_name("uadd_sat"); let umax = insts.by_name("umax"); let umin = insts.by_name("umin"); + let snarrow = insts.by_name("snarrow"); let ushr_imm = insts.by_name("ushr_imm"); let ushr = insts.by_name("ushr"); let vconst = insts.by_name("vconst"); @@ -412,7 +413,6 @@ fn define_simd( let vany_true = insts.by_name("vany_true"); let vselect = insts.by_name("vselect"); - let x86_packss = x86_instructions.by_name("x86_packss"); let x86_pmaxs = x86_instructions.by_name("x86_pmaxs"); let x86_pmaxu = x86_instructions.by_name("x86_pmaxu"); let x86_pmins = x86_instructions.by_name("x86_pmins"); @@ -575,7 +575,7 @@ fn define_simd( def!(g = raw_bitcast_i16x8_again(f)), def!(h = x86_psra(g, b)), // Re-pack the vector. - def!(z = x86_packss(e, h)), + def!(z = snarrow(e, h)), ], ); } diff --git a/cranelift/codegen/meta/src/isa/x86/opcodes.rs b/cranelift/codegen/meta/src/isa/x86/opcodes.rs index f7f7480f9b7d..c357488ddd4e 100644 --- a/cranelift/codegen/meta/src/isa/x86/opcodes.rs +++ b/cranelift/codegen/meta/src/isa/x86/opcodes.rs @@ -314,7 +314,7 @@ pub static PABSD: [u8; 4] = [0x66, 0x0f, 0x38, 0x1e]; /// xmm1 (SSSE3). pub static PABSW: [u8; 4] = [0x66, 0x0f, 0x38, 0x1d]; -/// Converts 8 packed signed word integers from xmm1 and from xxm2/m128 into 16 packed signed byte +/// Converts 8 packed signed word integers from xmm1 and from xmm2/m128 into 16 packed signed byte /// integers in xmm1 using signed saturation (SSE2). pub static PACKSSWB: [u8; 3] = [0x66, 0x0f, 0x63]; @@ -322,6 +322,14 @@ pub static PACKSSWB: [u8; 3] = [0x66, 0x0f, 0x63]; /// word integers in xmm1 using signed saturation (SSE2). pub static PACKSSDW: [u8; 3] = [0x66, 0x0f, 0x6b]; +/// Converts 8 packed signed word integers from xmm1 and from xmm2/m128 into 16 packed unsigned byte +/// integers in xmm1 using unsigned saturation (SSE2). +pub static PACKUSWB: [u8; 3] = [0x66, 0x0f, 0x67]; + +/// Converts 4 packed signed doubleword integers from xmm1 and from xmm2/m128 into 8 unpacked signed +/// word integers in xmm1 using unsigned saturation (SSE4.1). +pub static PACKUSDW: [u8; 4] = [0x66, 0x0f, 0x38, 0x2b]; + /// Add packed byte integers from xmm2/m128 and xmm1 (SSE2). pub static PADDB: [u8; 3] = [0x66, 0x0f, 0xfc]; diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs index fb91ae0ae9ca..c78787ce823f 100644 --- a/cranelift/codegen/meta/src/shared/instructions.rs +++ b/cranelift/codegen/meta/src/shared/instructions.rs @@ -3883,6 +3883,57 @@ pub(crate) fn define( .constraints(vec![WiderOrEq(Int.clone(), IntTo.clone())]), ); + let I16xN = &TypeVar::new( + "I16xN", + "A SIMD vector type containing integers 16-bits wide and up", + TypeSetBuilder::new() + .ints(16..32) + .simd_lanes(4..8) + .includes_scalars(false) + .build(), + ); + + let x = &Operand::new("x", I16xN); + let y = &Operand::new("y", I16xN); + let a = &Operand::new("a", &I16xN.split_lanes()); + + ig.push( + Inst::new( + "snarrow", + r#" + Combine `x` and `y` into a vector with twice the lanes but half the integer width while + saturating overflowing values to the signed maximum and minimum. + + The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4` + and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value + returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "unarrow", + r#" + Combine `x` and `y` into a vector with twice the lanes but half the integer width while + saturating overflowing values to the unsigned maximum and minimum. + + Note that all input lanes are considered signed: any negative lanes will overflow and be + replaced with the unsigned minimum, `0x00`. + + The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4` + and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value + returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + let IntTo = &TypeVar::new( "IntTo", "A larger integer type with the same number of lanes", diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 82eb35f13fb8..340d83ea292f 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -2060,7 +2060,6 @@ pub(crate) fn lower_insn_to_regs>( | Opcode::X86Pminu | Opcode::X86Pmullq | Opcode::X86Pmuludq - | Opcode::X86Packss | Opcode::X86Punpckh | Opcode::X86Punpckl | Opcode::X86Vcvtudq2ps @@ -2069,8 +2068,9 @@ pub(crate) fn lower_insn_to_regs>( panic!("x86-specific opcode in supposedly arch-neutral IR!"); } - Opcode::Iabs => unimplemented!(), Opcode::AvgRound => unimplemented!(), + Opcode::Iabs => unimplemented!(), + Opcode::Snarrow | Opcode::Unarrow => unimplemented!(), Opcode::TlsValue => unimplemented!(), } diff --git a/cranelift/filetests/filetests/isa/x86/simd-bitwise-legalize.clif b/cranelift/filetests/filetests/isa/x86/simd-bitwise-legalize.clif index 7193aa2b541d..ad459563efb0 100644 --- a/cranelift/filetests/filetests/isa/x86/simd-bitwise-legalize.clif +++ b/cranelift/filetests/filetests/isa/x86/simd-bitwise-legalize.clif @@ -34,7 +34,7 @@ block0: ; nextln: v9 = raw_bitcast.i16x8 v8 ; nextln: v10 = x86_psra v9, v4 - ; nextln: v2 = x86_packss v7, v10 + ; nextln: v2 = snarrow v7, v10 return v2 } diff --git a/cranelift/filetests/filetests/isa/x86/simd-lane-access-binemit.clif b/cranelift/filetests/filetests/isa/x86/simd-lane-access-binemit.clif index 24bc8cfa2409..abee26fa4b85 100644 --- a/cranelift/filetests/filetests/isa/x86/simd-lane-access-binemit.clif +++ b/cranelift/filetests/filetests/isa/x86/simd-lane-access-binemit.clif @@ -118,8 +118,9 @@ block0(v0: i32x4 [%xmm7], v1: i32x4 [%xmm6]): return } -function %packss_i16x8(i16x8, i16x8) { +function %narrowing_i16x8(i16x8, i16x8) { block0(v0: i16x8 [%xmm7], v1: i16x8 [%xmm8]): -[-, %xmm7] v2 = x86_packss v0, v1 ; bin: 66 41 0f 63 f8 +[-, %xmm7] v2 = snarrow v0, v1 ; bin: 66 41 0f 63 f8 +[-, %xmm7] v3 = unarrow v0, v1 ; bin: 66 41 0f 67 f8 return } diff --git a/cranelift/filetests/filetests/isa/x86/simd-lane-access-run.clif b/cranelift/filetests/filetests/isa/x86/simd-lane-access-run.clif index 00ebae26f657..0d58472a36fd 100644 --- a/cranelift/filetests/filetests/isa/x86/simd-lane-access-run.clif +++ b/cranelift/filetests/filetests/isa/x86/simd-lane-access-run.clif @@ -206,15 +206,16 @@ block0: } ; run -function %pack() -> b1 { -block0: - v0 = vconst.i32x4 [0 1 -1 0x0001ffff] - v1 = vconst.i32x4 [4 5 -6 0xffffffff] - v2 = x86_packss v0, v1 +function %snarrow(i32x4, i32x4) -> i16x8 { +block0(v0: i32x4, v1: i32x4): + v2 = snarrow v0, v1 + return v2 +} +; run: %snarrow([0 1 -1 0x0001ffff], [4 5 -6 0xffffffff]) == [0 1 -1 0x7fff 4 5 -6 0xffff] - v3 = vconst.i16x8 [0 1 -1 0x7fff 4 5 -6 0xffff] - v4 = icmp eq v2, v3 - v5 = vall_true v4 - return v5 +function %unarrow(i32x4, i32x4) -> i16x8 { +block0(v0: i32x4, v1: i32x4): + v2 = unarrow v0, v1 + return v2 } -; run +; run: %unarrow([0 1 -1 0x0001ffff], [4 5 -6 0xffffffff]) == [0 1 0 0xffff 4 5 0 0] diff --git a/cranelift/wasm/src/code_translator.rs b/cranelift/wasm/src/code_translator.rs index 023961035b6e..c62d29e60198 100644 --- a/cranelift/wasm/src/code_translator.rs +++ b/cranelift/wasm/src/code_translator.rs @@ -1559,11 +1559,23 @@ pub fn translate_operator( let a = pop1_with_bitcast(state, F32X4, builder); state.push1(builder.ins().fcvt_to_sint_sat(I32X4, a)) } + Operator::I8x16NarrowI16x8S => { + let (a, b) = pop2_with_bitcast(state, I16X8, builder); + state.push1(builder.ins().snarrow(a, b)) + } + Operator::I16x8NarrowI32x4S => { + let (a, b) = pop2_with_bitcast(state, I32X4, builder); + state.push1(builder.ins().snarrow(a, b)) + } + Operator::I8x16NarrowI16x8U => { + let (a, b) = pop2_with_bitcast(state, I16X8, builder); + state.push1(builder.ins().unarrow(a, b)) + } + Operator::I16x8NarrowI32x4U => { + let (a, b) = pop2_with_bitcast(state, I32X4, builder); + state.push1(builder.ins().unarrow(a, b)) + } Operator::I32x4TruncSatF32x4U - | Operator::I8x16NarrowI16x8S { .. } - | Operator::I8x16NarrowI16x8U { .. } - | Operator::I16x8NarrowI32x4S { .. } - | Operator::I16x8NarrowI32x4U { .. } | Operator::I16x8WidenLowI8x16S { .. } | Operator::I16x8WidenHighI8x16S { .. } | Operator::I16x8WidenLowI8x16U { .. }