diff --git a/src/mono/mono/metadata/metadata-internals.h b/src/mono/mono/metadata/metadata-internals.h index 67c6d1350473d1..befc5b83d4e837 100644 --- a/src/mono/mono/metadata/metadata-internals.h +++ b/src/mono/mono/metadata/metadata-internals.h @@ -1296,4 +1296,7 @@ mono_method_signature_has_ext_callconv (MonoMethodSignature *sig, MonoExtCallCon return (sig->ext_callconv & flags) != 0; } +guint32 +mono_metadata_decode_value_simd (const guint8 *ptr, const guint8 **new_ptr); + #endif /* __MONO_METADATA_INTERNALS_H__ */ diff --git a/src/mono/mono/metadata/metadata.c b/src/mono/mono/metadata/metadata.c index d005bfdb6b6b68..d97dc406ef6f8d 100644 --- a/src/mono/mono/metadata/metadata.c +++ b/src/mono/mono/metadata/metadata.c @@ -1502,6 +1502,135 @@ mono_metadata_decode_row_col_raw (const MonoTableInfo *t, int idx, guint col) return 0; } +// This will inline into mono_metadata_decode_value_simd on targets that can't use +// the simd version of the algorithm. +MONO_ALWAYS_INLINE static guint32 +decode_value_scalar (const guint8 *ptr, const guint8 **new_ptr) +{ + guint32 result; + guint8 b = *ptr; + + if ((b & 0x80) == 0){ + result = b; + ++ptr; + } else if ((b & 0x40) == 0){ + result = ((b & 0x3f) << 8 | ptr [1]); + ptr += 2; + } else if (b != 0xff) { + result = ((b & 0x1f) << 24) | + (ptr [1] << 16) | + (ptr [2] << 8) | + ptr [3]; + ptr += 4; + } + else { + result = (ptr [1] << 24) | (ptr [2] << 16) | (ptr [3] << 8) | ptr [4]; + ptr += 5; + } + + if (new_ptr) + *new_ptr = ptr; + + return result; +} + +// This is meant to be wrapped by our existing decode_value functions, and inline into them. +MONO_ALWAYS_INLINE guint32 +mono_metadata_decode_value_simd (const guint8 *ptr, const guint8 **new_ptr) +{ + // FIXME: Determine whether it's safe to perform this optimization on non-wasm targets. +#if defined(__clang__) && (G_BYTE_ORDER == G_LITTLE_ENDIAN) && defined(HOST_WASM) + guint32 result; + + typedef guint8 v64_u1 __attribute__ ((vector_size (8))); + typedef guint32 v64_u4 __attribute__ ((vector_size (8))); + + // this will generate vectorized code on x64 and wasm as long as SIMD is enabled at build time. + // if simd isn't enabled, it generates fairly adequate scalar code. + // *(bytes *)ptr and *(guint32 *)ptr by themselves don't force an i32 load of + // ptr in either x64 or wasm clang, so this is the only way to prefetch all the bytes + // without doing this, decode_value will do 5 conditional single-byte memory loads, + // and each individual load is potentially bounds-checked. we produce one wide load + // we could overrun the source buffer by up to 7 bytes, but doing that on wasm is + // safe unless we're decoding from the absolute end of memory. + // we pad all buffers by 16 bytes in mono_wasm_load_bytes_into_heap, so we're fine + // clang happily upsizes these 8-byte vectors into 16-byte ones for wasm, and uses 8-byte-wide + // insns on x64 as appropriate. armv8 looks fine too, albeit a little weird + union { + v64_u1 b; + v64_u4 i; + } v; + // ideally we would load 5 bytes, but it won't use a vector load then + // memcpy instead of a regular pointer dereference, to say 'this is unaligned' + memcpy(&v, ptr, sizeof(v)); + // mask and shift two bits so we can have a 4-element jump table in wasm + guint8 flags = (v.b[0] & (0x80u | 0x40u)) >> 6; + switch (flags) { + case 0b00u: + case 0b01u: + // if (b & 0x80) == 0 + result = v.b[0] & 0x7Fu; + ++ptr; + break; + case 0b10u: + // (b * 0x80) != 0, and (b & 0x40) == 0 + // v.b = { ptr[1], ptr[0], ptr[0], ptr[0] } + v.b = __builtin_shufflevector( + v.b, v.b, + 1, 0, -1, -1, -1, -1, -1, -1 + ); + // result = v.b[0..3] where v.b[1..2] = 0 and v.b[0] &= 0x3F + result = v.i[0] & 0x3FFFu; + ptr += 2; + break; + case 0b11u: + // i don't know why the default case is necessary here, but without it the jump table has 5 entries. + default: + // (b * 0x80) != 0, and (b & 0x40) != 0 + // on wasm the 'v.b[0]' load generates an '& 255', even if we cache it in a + // guint8 local. this is https://github.com/llvm/llvm-project/issues/87398 + if (v.b[0] == 0xFFu) { + // v.b = { ptr[4], ptr[3], ptr[2], ptr[1] } + // on x64 this generates kind of gross code, i.e. + // pshufd, pshufhw, pshufd, pshuflw, packuswb + // but on wasm it's fine + v.b = __builtin_shufflevector( + v.b, v.b, + 4, 3, 2, 1, -1, -1, -1, -1 + ); + // result = v.b[0..3]; + result = v.i[0]; + ptr += 5; + } else { + // v.b = { ptr[3], ptr[2], ptr[1], ptr[0] } +#ifdef USE_BSWAP + // generates much smaller x64 assembly, but terrible wasm + // interestingly, clang for arm automatically does this to the shuffle below + result = __builtin_bswap32(v.i[0]) & 0x1FFFFFFFu; +#else + // on x64 unlike the above this generates a single pshuflw + pack + v.b = __builtin_shufflevector( + v.b, v.b, + 3, 2, 1, 0, -1, -1, -1, -1 + ); + // result = v.b[0..3] where v.b[0] &= 0x1F + result = v.i[0] & 0x1FFFFFFFu; +#endif + ptr += 4; + } + break; + } + + if (new_ptr) + *new_ptr = ptr; + + return result; +#else + return decode_value_scalar (ptr, new_ptr); +#endif +} + + /** * mono_metadata_decode_blob_size: * \param ptr pointer to a blob object @@ -1514,25 +1643,7 @@ mono_metadata_decode_row_col_raw (const MonoTableInfo *t, int idx, guint col) guint32 mono_metadata_decode_blob_size (const char *xptr, const char **rptr) { - const unsigned char *ptr = (const unsigned char *)xptr; - guint32 size; - - if ((*ptr & 0x80) == 0){ - size = ptr [0] & 0x7f; - ptr++; - } else if ((*ptr & 0x40) == 0){ - size = ((ptr [0] & 0x3f) << 8) + ptr [1]; - ptr += 2; - } else { - size = ((ptr [0] & 0x1f) << 24) + - (ptr [1] << 16) + - (ptr [2] << 8) + - ptr [3]; - ptr += 4; - } - if (rptr) - *rptr = (char*)ptr; - return size; + return mono_metadata_decode_value_simd ((const guint8 *)xptr, (const guint8 **)rptr); } /** @@ -1548,27 +1659,7 @@ mono_metadata_decode_blob_size (const char *xptr, const char **rptr) guint32 mono_metadata_decode_value (const char *_ptr, const char **rptr) { - const unsigned char *ptr = (const unsigned char *) _ptr; - unsigned char b = *ptr; - guint32 len; - - if ((b & 0x80) == 0){ - len = b; - ++ptr; - } else if ((b & 0x40) == 0){ - len = ((b & 0x3f) << 8 | ptr [1]); - ptr += 2; - } else { - len = ((b & 0x1f) << 24) | - (ptr [1] << 16) | - (ptr [2] << 8) | - ptr [3]; - ptr += 4; - } - if (rptr) - *rptr = (char*)ptr; - - return len; + return mono_metadata_decode_value_simd ((const guint8 *)_ptr, (const guint8 **)rptr); } /** diff --git a/src/mono/mono/mini/aot-runtime.c b/src/mono/mono/mini/aot-runtime.c index bdd39e400edca9..0f6b89fbf97e3c 100644 --- a/src/mono/mono/mini/aot-runtime.c +++ b/src/mono/mono/mini/aot-runtime.c @@ -345,31 +345,7 @@ load_image (MonoAotModule *amodule, int index, MonoError *error) static gint32 decode_value (guint8 *ptr, guint8 **rptr) { - guint8 b = *ptr; - gint32 len; - - if ((b & 0x80) == 0){ - len = b; - ++ptr; - } else if ((b & 0x40) == 0){ - len = ((b & 0x3f) << 8 | ptr [1]); - ptr += 2; - } else if (b != 0xff) { - len = ((b & 0x1f) << 24) | - (ptr [1] << 16) | - (ptr [2] << 8) | - ptr [3]; - ptr += 4; - } - else { - len = (ptr [1] << 24) | (ptr [2] << 16) | (ptr [3] << 8) | ptr [4]; - ptr += 5; - } - if (rptr) - *rptr = ptr; - - //printf ("DECODE: %d.\n", len); - return len; + return mono_metadata_decode_value_simd ((const guint8 *)ptr, (const guint8 **)rptr); } static guint32 @@ -2480,7 +2456,7 @@ load_container_amodule (MonoAssemblyLoadContext *alc) mono_loader_lock (); // There might be several threads that passed the first check - // Adding another check to ensure single load of a container assembly due to race condition + // Adding another check to ensure single load of a container assembly due to race condition if (!container_amodule) { ERROR_DECL (error); diff --git a/src/mono/mono/mini/debug-mini.c b/src/mono/mono/mini/debug-mini.c index f307d08bf7f511..e9590954ad27a4 100644 --- a/src/mono/mono/mini/debug-mini.c +++ b/src/mono/mono/mini/debug-mini.c @@ -390,31 +390,7 @@ encode_value (gint32 value, guint8 *buf, guint8 **endbuf) static gint32 decode_value (guint8 *ptr, guint8 **rptr) { - guint8 b = *ptr; - gint32 len; - - if ((b & 0x80) == 0){ - len = b; - ++ptr; - } else if ((b & 0x40) == 0){ - len = ((b & 0x3f) << 8 | ptr [1]); - ptr += 2; - } else if (b != 0xff) { - len = ((b & 0x1f) << 24) | - (ptr [1] << 16) | - (ptr [2] << 8) | - ptr [3]; - ptr += 4; - } - else { - len = (ptr [1] << 24) | (ptr [2] << 16) | (ptr [3] << 8) | ptr [4]; - ptr += 5; - } - if (rptr) - *rptr = ptr; - - //printf ("DECODE: %d.\n", len); - return len; + return mono_metadata_decode_value_simd ((const guint8 *)ptr, (const guint8 **)rptr); } static void