diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,9 @@ # Revision history for Z-Data +## 0.8.7.0 -- 2021-06-12 + +* Add `use-avx2` and `use-avx512` flags(default to False) to enable avx2 or avx512 base64 codec, UTF8 validator, etc. + ## 0.8.6.1 -- 2021-06-09 * Change SIMD base64 codec to https://github.com/lemire/fastbase64. diff --git a/Z-Data.cabal b/Z-Data.cabal --- a/Z-Data.cabal +++ b/Z-Data.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: Z-Data -version: 0.8.6.1 +version: 0.8.7.0 synopsis: Array, vector and text description: This package provides array, slice and text operations license: BSD-3-Clause @@ -110,15 +110,21 @@ default: False manual: True -flag no-avx +flag use-avx2 description: - Do not use AVX instructions(utf8 validation, base64 codec, etc), this is added for - making some cached code runnable on VMs where AVX is not available. + Use AVX2 instructions(utf8 validation, base64 codec, etc). - default: True + default: False manual: True +flag use-avx512 + description: + Use AVX512 instructions(utf8 validation, base64 codec, etc). + default: False + manual: True + + library exposed-modules: Z.Data.Array @@ -201,10 +207,16 @@ utf8rewind.h chromiumbase64.h - if !flag(no-avx) + if flag(use-avx2) includes: fastavxbase64.h + if flag(use-avx512) + includes: + simdasciicheck_avx512.h + simdutf8check_avx512.h + fastavx512bwbase64.h + c-sources: cbits/bytes.c cbits/dtoa.c @@ -221,10 +233,14 @@ third_party/utf8rewind/source/unicodedatabase.c third_party/utf8rewind/source/utf8rewind.c - if !flag(no-avx) + if flag(use-avx2) c-sources: third_party/fastbase64/src/fastavxbase64.c + if flag(use-avx512) + c-sources: + third_party/fastbase64/src/fastavx512bwbase64.c + cxx-sources: cbits/regex.cc third_party/re2/re2/bitstate.cc @@ -253,10 +269,16 @@ default-language: Haskell2010 build-tool-depends: hsc2hs:hsc2hs -any - cc-options: -std=c11 -march=native -Wno-sign-compare - if flag(no-avx) - cc-options: -DNO_AVX + cc-options: -std=c11 -Wno-sign-compare + if flag(use-avx512) + cc-options: -march=cannonlake -DZ_USE_AVX512 + else + if flag(use-avx2) + cc-options: -mavx2 -DZ_USE_AVX2 + else + cc-options: -msse4.2 + cxx-options: -std=c++11 if os(windows) @@ -379,6 +401,3 @@ else cpp-options: -DINTEGER_GMP build-depends: integer-gmp >=0.2 && <1.2 - - if flag(no-avx) - cc-options: -DNO_AVX diff --git a/cbits/bytes.c b/cbits/bytes.c --- a/cbits/bytes.c +++ b/cbits/bytes.c @@ -36,7 +36,9 @@ #if defined(__x86_64__) #include #endif -#if defined(__AVX2__) && !defined(NO_AVX) +#if defined(Z_USE_AVX512) +#include +#elif defined(Z_USE_AVX2) #include #endif @@ -291,7 +293,9 @@ // Base64 codec void hs_base64_encode(char* output, HsInt output_off, const char* input, HsInt off, HsInt len){ -#if defined(__AVX2__) && !defined(NO_AVX) +#if defined(Z_USE_AVX512) + fast_avx512bw_base64_encode(output+output_off, input+off, len); +#elif defined(Z_USE_AVX2) fast_avx2_base64_encode(output+output_off, input+off, len); #else chromium_base64_encode(output+output_off, input+off, len); @@ -299,7 +303,9 @@ } HsInt hs_base64_decode(char* output, const char* input, HsInt off, HsInt len){ -#if defined(__AVX2__) && !defined(NO_AVX) +#if defined(Z_USE_AVX512) + size_t r = fast_avx512bw_base64_decode(output, input+off, len); +#elif defined(Z_USE_AVX2) size_t r = fast_avx2_base64_decode(output, input+off, len); #else size_t r = chromium_base64_decode(output, input+off, len); @@ -346,7 +352,7 @@ // this function should be called with large enough len, otherwise underflow will happen. size_t hs_count_simd(unsigned char *str, size_t len, unsigned char w) { -#if defined(__AVX2__) && !defined(NO_AVX) +#if defined(Z_USE_AVX2) __m256i pat = _mm256_set1_epi8(w); size_t prefix = 0, res = 0; size_t i = 0; @@ -451,7 +457,7 @@ } HsInt hs_count_ba(unsigned char *str, HsInt off, HsInt len, unsigned char w) { -#if (defined(__AVX2__) && !defined(NO_AVX)) || defined(__SSE2__) +#if defined(__SSE4_2__) || defined(Z_USE_AVX2) || defined(Z_USE_AVX512) if (len >= 1024) return (HsInt)hs_count_simd(str+off, len, w); else { #endif @@ -460,7 +466,7 @@ for (res = 0; len-- != 0; ++str) res += *str == w; return res; -#if (defined(__AVX2__) && !defined(NO_AVX)) || defined(__SSE2__) +#if defined(__SSE4_2__) || defined(Z_USE_AVX2) || defined(Z_USE_AVX512) } #endif } diff --git a/cbits/text.c b/cbits/text.c --- a/cbits/text.c +++ b/cbits/text.c @@ -30,14 +30,19 @@ #include #include -#ifdef __SSE2__ +#if defined(Z_USE_AVX512) +#include +#include +#else #include #include #endif HsInt ascii_validate(const char* p, HsInt off, HsInt len){ const char* q = p + off; -#if defined(__AVX2__) && !defined(NO_AVX) +#if defined(Z_USE_AVX512) + return (HsInt)validate_ascii_fast_avx512(q, (size_t)len); +#elif defined(Z_USE_AVX2) return (HsInt)validate_ascii_fast_avx(q, (size_t)len); #elif defined(__SSE2__) return (HsInt)validate_ascii_fast(q, (size_t)len); @@ -48,7 +53,9 @@ // for some reason unknown, on windows we have to supply a seperated version of ascii_validate // otherwise we got segfault if we import the same FFI with different type (Addr# vs ByteArray#) HsInt ascii_validate_addr(const char* p, HsInt len){ -#if defined(__AVX2__) && !defined(NO_AVX) +#if defined(Z_USE_AVX512) + return (HsInt)validate_ascii_fast_avx512(p, (size_t)len); +#elif defined(Z_USE_AVX2) return (HsInt)validate_ascii_fast_avx(p, (size_t)len); #elif defined(__SSE2__) return (HsInt)validate_ascii_fast(p, (size_t)len); @@ -59,7 +66,9 @@ HsInt utf8_validate(const char* p, HsInt off, HsInt len){ const char* q = p + off; -#if defined(__AVX2__) && !defined(NO_AVX) +#if defined(Z_USE_AVX512) + return (HsInt)validate_utf8_fast_avx512(q, (size_t)len); +#elif defined(Z_USE_AVX2) return (HsInt)validate_utf8_fast_avx(q, (size_t)len); #elif defined(__SSE2__) return (HsInt)validate_utf8_fast(q, (size_t)len); @@ -70,7 +79,9 @@ // for some reason unknown, on windows we have to supply a seperated version of utf8_validate // otherwise we got segfault if we import the same FFI with different type (Addr# vs ByteArray#) HsInt utf8_validate_addr(const char* p, HsInt len){ -#if defined(__AVX2__) && !defined(NO_AVX) +#if defined(Z_USE_AVX512) + return (HsInt)validate_utf8_fast_avx512(p, (size_t)len); +#elif defined(Z_USE_AVX2) return (HsInt)validate_utf8_fast_avx(p, (size_t)len); #elif defined(__SSE2__) return (HsInt)validate_utf8_fast(p, (size_t)len); diff --git a/third_party/fastbase64/src/fastavx512bwbase64.c b/third_party/fastbase64/src/fastavx512bwbase64.c new file mode 100644 --- /dev/null +++ b/third_party/fastbase64/src/fastavx512bwbase64.c @@ -0,0 +1,196 @@ +#include "fastavx512bwbase64.h" + +#include +#include + +static inline __m512i enc_reshuffle(const __m512i input) { + + // from https://github.com/WojciechMula/base64simd/blob/master/encode/encode.avx512bw.cpp + + // place each 12-byte subarray in seprate 128-bit lane + // tmp1 = [?? ?? ?? ??|D2 D1 D0 C2|C1 C0 B2 B1|B0 A2 A1 A0] x 4 + // ignored + const __m512i tmp1 = _mm512_permutexvar_epi32( + _mm512_set_epi32(-1, 11, 10, 9, -1, 8, 7, 6, -1, 5, 4, 3, -1, 2, 1, 0), + input + ); + + // reshuffle bytes within 128-bit lanes to format required by + // AVX512BW unpack procedure + // tmp2 = [D1 D2 D0 D1|C1 C2 C0 C1|B1 B2 B0 B1|A1 A2 A0 A1] x 4 + // 10 11 9 10 7 8 6 7 4 5 3 4 1 2 0 1 + const __m512i tmp2 = _mm512_shuffle_epi8( + tmp1, + _mm512_set4_epi32(0x0a0b090a, 0x07080607, 0x04050304, 0x01020001) + ); + + return tmp2; +} + +static inline __m512i enc_translate(const __m512i input) { + + // from https://github.com/WojciechMula/base64simd/blob/master/encode/lookup.avx512bw.cpp + + // reduce 0..51 -> 0 + // 52..61 -> 1 .. 10 + // 62 -> 11 + // 63 -> 12 + __m512i result = _mm512_subs_epu8(input, _mm512_set1_epi8(51)); + + // distinguish between ranges 0..25 and 26..51: + // 0 .. 25 -> remains 0 + // 26 .. 51 -> becomes 13 + const __mmask64 less = _mm512_cmpgt_epi8_mask(_mm512_set1_epi8(26), input); + result = _mm512_mask_mov_epi8(result, less, _mm512_set1_epi8(13)); + + /* the SSE lookup + const __m128i shift_LUT = _mm_setr_epi8( + 'a' - 26, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, + '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '+' - 62, + '/' - 63, 'A', 0, 0 + ); + which is: + 0x47, 0xfc, 0xfc, 0xfc, + 0xfc, 0xfc, 0xfc, 0xfc, + 0xfc, 0xfc, 0xfc, 0xed, + 0xf0, 0x41, 0x00, 0x00 + + Note that the order of above list is reserved (due to _mm_setr_epi8), + so the invocation _mm512_set4_epi32 looks... odd. + */ + const __m512i shift_LUT = _mm512_set4_epi32( + 0x000041f0, + 0xedfcfcfc, + 0xfcfcfcfc, + 0xfcfcfc47 + ); + + // read shift + result = _mm512_shuffle_epi8(shift_LUT, result); + + return _mm512_add_epi8(result, input); +} + +static inline __m512i dec_reshuffle(__m512i input) { + + // from https://github.com/WojciechMula/base64simd/blob/master/decode/pack.avx512bw.cpp + const __m512i merge_ab_and_bc = _mm512_maddubs_epi16(input, _mm512_set1_epi32(0x01400140)); + + return _mm512_madd_epi16(merge_ab_and_bc, _mm512_set1_epi32(0x00011000)); +} + + +size_t fast_avx512bw_base64_encode(char* dest, const char* str, size_t len) { + const char* const dest_orig = dest; + __m512i inputvector; + while (len >= 64) { + inputvector = _mm512_loadu_si512((__m512i *)(str)); + inputvector = enc_reshuffle(inputvector); + inputvector = enc_translate(inputvector); + _mm512_storeu_si512((__m512i *)dest, inputvector); + str += 48; + dest += 64; + len -= 48; + } + size_t scalarret = chromium_base64_encode(dest, str, len); + if(scalarret == MODP_B64_ERROR) return MODP_B64_ERROR; + return (dest - dest_orig) + scalarret; +} + +#define build_dword(b0, b1, b2, b3) \ + (((uint32_t)((uint8_t)(b0)) << 0*8) \ + | ((uint32_t)((uint8_t)(b1)) << 1*8) \ + | ((uint32_t)((uint8_t)(b2)) << 2*8) \ + | ((uint32_t)((uint8_t)(b3)) << 3*8)) + +#define _mm512_set4lanes_epi8(b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15) \ + _mm512_setr4_epi32( \ + build_dword( b0, b1, b2, b3), \ + build_dword( b4, b5, b6, b7), \ + build_dword( b8, b9, b10, b11), \ + build_dword(b12, b13, b14, b15)) + +size_t fast_avx512bw_base64_decode(char *out, const char *src, size_t srclen) { + char* out_orig = out; + while (srclen >= 88) { + + // load + const __m512i input = _mm512_loadu_si512((const __m512i*)(src)); + + // translate from ASCII + // and https://github.com/WojciechMula/base64simd/blob/master/decode/lookup.avx512bw.cpp + const __m512i higher_nibble = _mm512_and_si512(_mm512_srli_epi32(input, 4), _mm512_set1_epi8(0x0f)); + const __m512i lower_nibble = _mm512_and_si512(input, _mm512_set1_epi8(0x0f)); + + const __m512i shiftLUT = _mm512_set4lanes_epi8( + 0, 0, 19, 4, -65, -65, -71, -71, + 0, 0, 0, 0, 0, 0, 0, 0); + + const __m512i maskLUT = _mm512_set4lanes_epi8( + /* 0 : 0b1010_1000*/ 0xa8, + /* 1 .. 9 : 0b1111_1000*/ 0xf8, 0xf8, 0xf8, 0xf8, + 0xf8, 0xf8, 0xf8, 0xf8, + 0xf8, + /* 10 : 0b1111_0000*/ 0xf0, + /* 11 : 0b0101_0100*/ 0x54, + /* 12 .. 14 : 0b0101_0000*/ 0x50, 0x50, 0x50, + /* 15 : 0b0101_0100*/ 0x54 + ); + + const __m512i bitposLUT = _mm512_set4lanes_epi8( + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ); + + const __m512i sh = _mm512_shuffle_epi8(shiftLUT, higher_nibble); + const __mmask64 eq_2f = _mm512_cmpeq_epi8_mask(input, _mm512_set1_epi8(0x2f)); + const __m512i shift = _mm512_mask_mov_epi8(sh, eq_2f, _mm512_set1_epi8(16)); + + const __m512i M = _mm512_shuffle_epi8(maskLUT, lower_nibble); + const __m512i bit = _mm512_shuffle_epi8(bitposLUT, higher_nibble); + + const uint64_t match = _mm512_test_epi8_mask(M, bit); + + if (match != (uint64_t)(-1)) { + // some characters do not match the valid range + return MODP_B64_ERROR; + } + + const __m512i translated = _mm512_add_epi8(input, shift); + + const __m512i packed = dec_reshuffle(translated); + + // and https://github.com/WojciechMula/base64simd/blob/master/decode/decode.avx512bw.cpp + // + const __m512i t1 = _mm512_shuffle_epi8( + packed, + _mm512_set4lanes_epi8( + 2, 1, 0, + 6, 5, 4, + 10, 9, 8, + 14, 13, 12, + -1, -1, -1, -1) + ); + + // shuffle bytes + const __m512i s6 = _mm512_setr_epi32( + 0, 1, 2, + 4, 5, 6, + 8, 9, 10, + 12, 13, 14, + // unused + 0, 0, 0, 0); + + const __m512i t2 = _mm512_permutexvar_epi32(s6, t1); + + _mm512_storeu_si512((__m512i*)(out), t2); + + srclen -= 64; + src += 64; + out += 48; + } + size_t scalarret = chromium_base64_decode(out, src, srclen); + if(scalarret == MODP_B64_ERROR) return MODP_B64_ERROR; + return (out - out_orig) + scalarret; +} +