Skip to content

Commit 1f60df4

Browse files
elliotttelliottt
committed
Lower fcvt_to_uint_sat in ISLE
1 parent 47ce5c9 commit 1f60df4

4 files changed

Lines changed: 112 additions & 150 deletions

File tree

‎cranelift/codegen/src/isa/x64/inst.isle‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1671,6 +1671,10 @@
16711671
(rule (x64_movapd src)
16721672
(xmm_unary_rm_r (SseOpcode.Movapd) src))
16731673

1674+
(decl x64_movaps (XmmMem) Xmm)
1675+
(rule (x64_movaps src)
1676+
(xmm_unary_rm_r (SseOpcode.Movaps) src))
1677+
16741678
(decl x64_pmovsxbw (XmmMem) Xmm)
16751679
(rule (x64_pmovsxbw from)
16761680
(xmm_unary_rm_r (SseOpcode.Pmovsxbw) from))

‎cranelift/codegen/src/isa/x64/lower.isle‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3105,3 +3105,95 @@
31053105
;; Keeps negative overflow lanes as is.
31063106
(x64_pxor tmp dst)))
31073107

3108+
;; The algorithm for converting floats to unsigned ints is a little tricky. The
3109+
;; complication arises because we are converting from a signed 64-bit int with a positive
3110+
;; integer range from 1..INT_MAX (0x1..0x7FFFFFFF) to an unsigned integer with an extended
3111+
;; range from (INT_MAX+1)..UINT_MAX. It's this range from (INT_MAX+1)..UINT_MAX
3112+
;; (0x80000000..0xFFFFFFFF) that needs to be accounted for as a special case since our
3113+
;; conversion instruction (cvttps2dq) only converts as high as INT_MAX (0x7FFFFFFF), but
3114+
;; which conveniently setting underflows and overflows (smaller than MIN_INT or larger than
3115+
;; MAX_INT) to be INT_MAX+1 (0x80000000). Nothing that the range (INT_MAX+1)..UINT_MAX includes
3116+
;; precisely INT_MAX values we can correctly account for and convert every value in this range
3117+
;; if we simply subtract INT_MAX+1 before doing the cvttps2dq conversion. After the subtraction
3118+
;; every value originally (INT_MAX+1)..UINT_MAX is now the range (0..INT_MAX).
3119+
;; After the conversion we add INT_MAX+1 back to this converted value, noting again that
3120+
;; values we are trying to account for were already set to INT_MAX+1 during the original conversion.
3121+
;; We simply have to create a mask and make sure we are adding together only the lanes that need
3122+
;; to be accounted for. Digesting it all the steps then are:
3123+
;;
3124+
;; Step 1 - Account for NaN and negative floats by setting these src values to zero.
3125+
;; Step 2 - Make a copy (tmp1) of the src value since we need to convert twice for
3126+
;; reasons described above.
3127+
;; Step 3 - Convert the original src values. This will convert properly all floats up to INT_MAX
3128+
;; Step 4 - Subtract INT_MAX from the copy set (tmp1). Note, all zero and negative values are those
3129+
;; values that were originally in the range (0..INT_MAX). This will come in handy during
3130+
;; step 7 when we zero negative lanes.
3131+
;; Step 5 - Create a bit mask for tmp1 that will correspond to all lanes originally less than
3132+
;; UINT_MAX that are now less than INT_MAX thanks to the subtraction.
3133+
;; Step 6 - Convert the second set of values (tmp1)
3134+
;; Step 7 - Prep the converted second set by zeroing out negative lanes (these have already been
3135+
;; converted correctly with the first set) and by setting overflow lanes to 0x7FFFFFFF
3136+
;; as this will allow us to properly saturate overflow lanes when adding to 0x80000000
3137+
;; Step 8 - Add the orginal converted src and the converted tmp1 where float values originally less
3138+
;; than and equal to INT_MAX will be unchanged, float values originally between INT_MAX+1 and
3139+
;; UINT_MAX will add together (INT_MAX) + (SRC - INT_MAX), and float values originally
3140+
;; greater than UINT_MAX will be saturated to UINT_MAX (0xFFFFFFFF) after adding (0x8000000 + 0x7FFFFFFF).
3141+
;;
3142+
;;
3143+
;; The table below illustrates the result after each step where it matters for the converted set.
3144+
;; Note the original value range (original src set) is the final dst in Step 8:
3145+
;;
3146+
;; Original src set:
3147+
;; | Original Value Range | Step 1 | Step 3 | Step 8 |
3148+
;; | -FLT_MIN..FLT_MAX | 0.0..FLT_MAX | 0..INT_MAX(w/overflow) | 0..UINT_MAX(w/saturation) |
3149+
;;
3150+
;; Copied src set (tmp1):
3151+
;; | Step 2 | Step 4 |
3152+
;; | 0.0..FLT_MAX | (0.0-(INT_MAX+1))..(FLT_MAX-(INT_MAX+1)) |
3153+
;;
3154+
;; | Step 6 | Step 7 |
3155+
;; | (0-(INT_MAX+1))..(UINT_MAX-(INT_MAX+1))(w/overflow) | ((INT_MAX+1)-(INT_MAX+1))..(INT_MAX+1) |
3156+
(rule (lower (has_type $I32X4 (fcvt_to_uint_sat val @ (value_type $F32X4))))
3157+
(let ((src Xmm val)
3158+
3159+
;; Converting to unsigned int so if float src is negative or NaN
3160+
;; will first set to zero.
3161+
(tmp2 Xmm (x64_pxor src src)) ;; TODO: unnecessary dependency on src
3162+
(dst Xmm (x64_maxps src tmp2))
3163+
3164+
;; Set tmp2 to INT_MAX+1. It is important to note here that after it looks
3165+
;; like we are only converting INT_MAX (0x7FFFFFFF) but in fact because
3166+
;; single precision IEEE-754 floats can only accurately represent contingous
3167+
;; integers up to 2^23 and outside of this range it rounds to the closest
3168+
;; integer that it can represent. In the case of INT_MAX, this value gets
3169+
;; represented as 0x4f000000 which is the integer value (INT_MAX+1).
3170+
(tmp2 Xmm (x64_pcmpeqd tmp2 tmp2))
3171+
(tmp2 Xmm (x64_psrld tmp2 (RegMemImm.Imm 1)))
3172+
(tmp2 Xmm (x64_cvtdq2ps tmp2))
3173+
3174+
;; Make a copy of these lanes and then do the first conversion.
3175+
;; Overflow lanes greater than the maximum allowed signed value will
3176+
;; set to 0x80000000. Negative and NaN lanes will be 0x0
3177+
(tmp1 Xmm dst)
3178+
(dst Xmm (x64_cvttps2dq $F32X4 dst))
3179+
3180+
;; Set lanes to src - max_signed_int
3181+
(tmp1 Xmm (x64_subps tmp1 tmp2))
3182+
3183+
;; Create mask for all positive lanes to saturate (i.e. greater than
3184+
;; or equal to the maxmimum allowable unsigned int).
3185+
(tmp2 Xmm (x64_cmpps tmp2 tmp1 (FcmpImm.LessThanOrEqual)))
3186+
3187+
;; Convert those set of lanes that have the max_signed_int factored out.
3188+
(tmp1 Xmm (x64_cvttps2dq $F32X4 tmp1))
3189+
3190+
;; Prepare converted lanes by zeroing negative lanes and prepping lanes
3191+
;; that have positive overflow (based on the mask) by setting these lanes
3192+
;; to 0x7FFFFFFF
3193+
(tmp1 Xmm (x64_pxor tmp1 tmp2))
3194+
(tmp2 Xmm (x64_pxor tmp2 tmp2)) ;; make another zero
3195+
(tmp1 Xmm (x64_pmaxsd tmp1 tmp2)))
3196+
3197+
;; Add this second set of converted lanes to the original to properly handle
3198+
;; values greater than max signed int.
3199+
(x64_paddd tmp1 dst)))

‎cranelift/codegen/src/isa/x64/lower.rs‎

Lines changed: 3 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -559,146 +559,12 @@ fn lower_insn_to_regs(
559559
| Opcode::FcvtLowFromSint
560560
| Opcode::FcvtFromUint
561561
| Opcode::FcvtToUint
562-
| Opcode::FcvtToSint => {
562+
| Opcode::FcvtToSint
563+
| Opcode::FcvtToUintSat
564+
| Opcode::FcvtToSintSat => {
563565
implemented_in_isle(ctx);
564566
}
565567

566-
Opcode::FcvtToUintSat | Opcode::FcvtToSintSat => {
567-
let src = put_input_in_reg(ctx, inputs[0]);
568-
let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
569-
570-
let input_ty = ctx.input_ty(insn, 0);
571-
if !input_ty.is_vector() {
572-
implemented_in_isle(ctx);
573-
} else {
574-
if op == Opcode::FcvtToSintSat {
575-
implemented_in_isle(ctx);
576-
} else if op == Opcode::FcvtToUintSat {
577-
// The algorithm for converting floats to unsigned ints is a little tricky. The
578-
// complication arises because we are converting from a signed 64-bit int with a positive
579-
// integer range from 1..INT_MAX (0x1..0x7FFFFFFF) to an unsigned integer with an extended
580-
// range from (INT_MAX+1)..UINT_MAX. It's this range from (INT_MAX+1)..UINT_MAX
581-
// (0x80000000..0xFFFFFFFF) that needs to be accounted for as a special case since our
582-
// conversion instruction (cvttps2dq) only converts as high as INT_MAX (0x7FFFFFFF), but
583-
// which conveniently setting underflows and overflows (smaller than MIN_INT or larger than
584-
// MAX_INT) to be INT_MAX+1 (0x80000000). Nothing that the range (INT_MAX+1)..UINT_MAX includes
585-
// precisely INT_MAX values we can correctly account for and convert every value in this range
586-
// if we simply subtract INT_MAX+1 before doing the cvttps2dq conversion. After the subtraction
587-
// every value originally (INT_MAX+1)..UINT_MAX is now the range (0..INT_MAX).
588-
// After the conversion we add INT_MAX+1 back to this converted value, noting again that
589-
// values we are trying to account for were already set to INT_MAX+1 during the original conversion.
590-
// We simply have to create a mask and make sure we are adding together only the lanes that need
591-
// to be accounted for. Digesting it all the steps then are:
592-
//
593-
// Step 1 - Account for NaN and negative floats by setting these src values to zero.
594-
// Step 2 - Make a copy (tmp1) of the src value since we need to convert twice for
595-
// reasons described above.
596-
// Step 3 - Convert the original src values. This will convert properly all floats up to INT_MAX
597-
// Step 4 - Subtract INT_MAX from the copy set (tmp1). Note, all zero and negative values are those
598-
// values that were originally in the range (0..INT_MAX). This will come in handy during
599-
// step 7 when we zero negative lanes.
600-
// Step 5 - Create a bit mask for tmp1 that will correspond to all lanes originally less than
601-
// UINT_MAX that are now less than INT_MAX thanks to the subtraction.
602-
// Step 6 - Convert the second set of values (tmp1)
603-
// Step 7 - Prep the converted second set by zeroing out negative lanes (these have already been
604-
// converted correctly with the first set) and by setting overflow lanes to 0x7FFFFFFF
605-
// as this will allow us to properly saturate overflow lanes when adding to 0x80000000
606-
// Step 8 - Add the orginal converted src and the converted tmp1 where float values originally less
607-
// than and equal to INT_MAX will be unchanged, float values originally between INT_MAX+1 and
608-
// UINT_MAX will add together (INT_MAX) + (SRC - INT_MAX), and float values originally
609-
// greater than UINT_MAX will be saturated to UINT_MAX (0xFFFFFFFF) after adding (0x8000000 + 0x7FFFFFFF).
610-
//
611-
//
612-
// The table below illustrates the result after each step where it matters for the converted set.
613-
// Note the original value range (original src set) is the final dst in Step 8:
614-
//
615-
// Original src set:
616-
// | Original Value Range | Step 1 | Step 3 | Step 8 |
617-
// | -FLT_MIN..FLT_MAX | 0.0..FLT_MAX | 0..INT_MAX(w/overflow) | 0..UINT_MAX(w/saturation) |
618-
//
619-
// Copied src set (tmp1):
620-
// | Step 2 | Step 4 |
621-
// | 0.0..FLT_MAX | (0.0-(INT_MAX+1))..(FLT_MAX-(INT_MAX+1)) |
622-
//
623-
// | Step 6 | Step 7 |
624-
// | (0-(INT_MAX+1))..(UINT_MAX-(INT_MAX+1))(w/overflow) | ((INT_MAX+1)-(INT_MAX+1))..(INT_MAX+1) |
625-
626-
// Create temporaries
627-
assert_eq!(types::F32X4, ctx.input_ty(insn, 0));
628-
let tmp1 = ctx.alloc_tmp(types::I32X4).only_reg().unwrap();
629-
let tmp2 = ctx.alloc_tmp(types::I32X4).only_reg().unwrap();
630-
631-
// Converting to unsigned int so if float src is negative or NaN
632-
// will first set to zero.
633-
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp2));
634-
ctx.emit(Inst::gen_move(dst, src, input_ty));
635-
ctx.emit(Inst::xmm_rm_r(SseOpcode::Maxps, RegMem::from(tmp2), dst));
636-
637-
// Set tmp2 to INT_MAX+1. It is important to note here that after it looks
638-
// like we are only converting INT_MAX (0x7FFFFFFF) but in fact because
639-
// single precision IEEE-754 floats can only accurately represent contingous
640-
// integers up to 2^23 and outside of this range it rounds to the closest
641-
// integer that it can represent. In the case of INT_MAX, this value gets
642-
// represented as 0x4f000000 which is the integer value (INT_MAX+1).
643-
644-
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pcmpeqd, RegMem::from(tmp2), tmp2));
645-
ctx.emit(Inst::xmm_rmi_reg(SseOpcode::Psrld, RegMemImm::imm(1), tmp2));
646-
ctx.emit(Inst::xmm_unary_rm_r(
647-
SseOpcode::Cvtdq2ps,
648-
RegMem::from(tmp2),
649-
tmp2,
650-
));
651-
652-
// Make a copy of these lanes and then do the first conversion.
653-
// Overflow lanes greater than the maximum allowed signed value will
654-
// set to 0x80000000. Negative and NaN lanes will be 0x0
655-
ctx.emit(Inst::xmm_mov(SseOpcode::Movaps, RegMem::from(dst), tmp1));
656-
ctx.emit(Inst::xmm_unary_rm_r(
657-
SseOpcode::Cvttps2dq,
658-
RegMem::from(dst),
659-
dst,
660-
));
661-
662-
// Set lanes to src - max_signed_int
663-
ctx.emit(Inst::xmm_rm_r(SseOpcode::Subps, RegMem::from(tmp2), tmp1));
664-
665-
// Create mask for all positive lanes to saturate (i.e. greater than
666-
// or equal to the maxmimum allowable unsigned int).
667-
let cond = FcmpImm::from(FloatCC::LessThanOrEqual);
668-
ctx.emit(Inst::xmm_rm_r_imm(
669-
SseOpcode::Cmpps,
670-
RegMem::from(tmp1),
671-
tmp2,
672-
cond.encode(),
673-
OperandSize::Size32,
674-
));
675-
676-
// Convert those set of lanes that have the max_signed_int factored out.
677-
ctx.emit(Inst::xmm_unary_rm_r(
678-
SseOpcode::Cvttps2dq,
679-
RegMem::from(tmp1),
680-
tmp1,
681-
));
682-
683-
// Prepare converted lanes by zeroing negative lanes and prepping lanes
684-
// that have positive overflow (based on the mask) by setting these lanes
685-
// to 0x7FFFFFFF
686-
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp1));
687-
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp2), tmp2));
688-
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pmaxsd, RegMem::from(tmp2), tmp1));
689-
690-
// Add this second set of converted lanes to the original to properly handle
691-
// values greater than max signed int.
692-
ctx.emit(Inst::xmm_rm_r(SseOpcode::Paddd, RegMem::from(tmp1), dst));
693-
} else {
694-
// Since this branch is also guarded by a check for vector types
695-
// neither Opcode::FcvtToUint nor Opcode::FcvtToSint can reach here
696-
// due to vector varients not existing. The first two branches will
697-
// cover all reachable cases.
698-
unreachable!();
699-
}
700-
}
701-
}
702568
Opcode::IaddPairwise => {
703569
if let (Some(swiden_low), Some(swiden_high)) = (
704570
matches_input(ctx, inputs[0], Opcode::SwidenLow),

‎cranelift/filetests/filetests/isa/x64/fcvt.clif‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -433,20 +433,20 @@ block0(v0: f32x4):
433433
; pushq %rbp
434434
; movq %rsp, %rbp
435435
; block0:
436-
; pxor %xmm5, %xmm5, %xmm5
437-
; maxps %xmm0, %xmm5, %xmm0
438-
; pcmpeqd %xmm5, %xmm5, %xmm5
439-
; psrld %xmm5, $1, %xmm5
440-
; cvtdq2ps %xmm5, %xmm5
441-
; movdqa %xmm0, %xmm11
436+
; movdqa %xmm0, %xmm6
437+
; pxor %xmm6, %xmm0, %xmm6
438+
; maxps %xmm0, %xmm6, %xmm0
439+
; pcmpeqd %xmm6, %xmm6, %xmm6
440+
; psrld %xmm6, $1, %xmm6
441+
; cvtdq2ps %xmm6, %xmm1
442+
; cvttps2dq %xmm0, %xmm13
443+
; subps %xmm0, %xmm1, %xmm0
444+
; cmpps $2, %xmm1, %xmm0, %xmm1
442445
; cvttps2dq %xmm0, %xmm0
443-
; subps %xmm11, %xmm5, %xmm11
444-
; cmpps $2, %xmm5, %xmm11, %xmm5
445-
; cvttps2dq %xmm11, %xmm11
446-
; pxor %xmm11, %xmm5, %xmm11
447-
; pxor %xmm5, %xmm5, %xmm5
448-
; pmaxsd %xmm11, %xmm5, %xmm11
449-
; paddd %xmm0, %xmm11, %xmm0
446+
; pxor %xmm0, %xmm1, %xmm0
447+
; pxor %xmm1, %xmm1, %xmm1
448+
; pmaxsd %xmm0, %xmm1, %xmm0
449+
; paddd %xmm0, %xmm13, %xmm0
450450
; movq %rbp, %rsp
451451
; popq %rbp
452452
; ret

0 commit comments

Comments
 (0)