Z-Data 0.8.6.1 → 0.8.7.0
raw patch · 5 files changed
+260/−24 lines, 5 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
Files
- ChangeLog.md +4/−0
- Z-Data.cabal +32/−13
- cbits/bytes.c +12/−6
- cbits/text.c +16/−5
- third_party/fastbase64/src/fastavx512bwbase64.c +196/−0
ChangeLog.md view
@@ -1,5 +1,9 @@ # Revision history for Z-Data +## 0.8.7.0 -- 2021-06-12++* Add `use-avx2` and `use-avx512` flags(default to False) to enable avx2 or avx512 base64 codec, UTF8 validator, etc.+ ## 0.8.6.1 -- 2021-06-09 * Change SIMD base64 codec to https://github.com/lemire/fastbase64.
Z-Data.cabal view
@@ -1,6 +1,6 @@ cabal-version: 2.4 name: Z-Data-version: 0.8.6.1+version: 0.8.7.0 synopsis: Array, vector and text description: This package provides array, slice and text operations license: BSD-3-Clause@@ -110,15 +110,21 @@ default: False manual: True -flag no-avx+flag use-avx2 description:- Do not use AVX instructions(utf8 validation, base64 codec, etc), this is added for- making some cached code runnable on VMs where AVX is not available.+ Use AVX2 instructions(utf8 validation, base64 codec, etc). - default: True+ default: False manual: True +flag use-avx512+ description:+ Use AVX512 instructions(utf8 validation, base64 codec, etc). + default: False+ manual: True++ library exposed-modules: Z.Data.Array@@ -201,10 +207,16 @@ utf8rewind.h chromiumbase64.h - if !flag(no-avx)+ if flag(use-avx2) includes: fastavxbase64.h + if flag(use-avx512)+ includes:+ simdasciicheck_avx512.h+ simdutf8check_avx512.h+ fastavx512bwbase64.h+ c-sources: cbits/bytes.c cbits/dtoa.c@@ -221,10 +233,14 @@ third_party/utf8rewind/source/unicodedatabase.c third_party/utf8rewind/source/utf8rewind.c - if !flag(no-avx)+ if flag(use-avx2) c-sources: third_party/fastbase64/src/fastavxbase64.c + if flag(use-avx512)+ c-sources:+ third_party/fastbase64/src/fastavx512bwbase64.c+ cxx-sources: cbits/regex.cc third_party/re2/re2/bitstate.cc@@ -253,10 +269,16 @@ default-language: Haskell2010 build-tool-depends: hsc2hs:hsc2hs -any - cc-options: -std=c11 -march=native -Wno-sign-compare- if flag(no-avx)- cc-options: -DNO_AVX+ cc-options: -std=c11 -Wno-sign-compare + if flag(use-avx512)+ cc-options: -march=cannonlake -DZ_USE_AVX512+ else+ if flag(use-avx2)+ cc-options: -mavx2 -DZ_USE_AVX2+ else+ cc-options: -msse4.2+ cxx-options: -std=c++11 if os(windows)@@ -379,6 +401,3 @@ else cpp-options: -DINTEGER_GMP build-depends: integer-gmp >=0.2 && <1.2-- if flag(no-avx)- cc-options: -DNO_AVX
cbits/bytes.c view
@@ -36,7 +36,9 @@ #if defined(__x86_64__) #include <x86intrin.h> #endif-#if defined(__AVX2__) && !defined(NO_AVX)+#if defined(Z_USE_AVX512)+#include <fastavx512bwbase64.h>+#elif defined(Z_USE_AVX2) #include <fastavxbase64.h> #endif @@ -291,7 +293,9 @@ // Base64 codec void hs_base64_encode(char* output, HsInt output_off, const char* input, HsInt off, HsInt len){-#if defined(__AVX2__) && !defined(NO_AVX)+#if defined(Z_USE_AVX512)+ fast_avx512bw_base64_encode(output+output_off, input+off, len);+#elif defined(Z_USE_AVX2) fast_avx2_base64_encode(output+output_off, input+off, len); #else chromium_base64_encode(output+output_off, input+off, len);@@ -299,7 +303,9 @@ } HsInt hs_base64_decode(char* output, const char* input, HsInt off, HsInt len){-#if defined(__AVX2__) && !defined(NO_AVX)+#if defined(Z_USE_AVX512)+ size_t r = fast_avx512bw_base64_decode(output, input+off, len);+#elif defined(Z_USE_AVX2) size_t r = fast_avx2_base64_decode(output, input+off, len); #else size_t r = chromium_base64_decode(output, input+off, len);@@ -346,7 +352,7 @@ // this function should be called with large enough len, otherwise underflow will happen. size_t hs_count_simd(unsigned char *str, size_t len, unsigned char w) {-#if defined(__AVX2__) && !defined(NO_AVX)+#if defined(Z_USE_AVX2) __m256i pat = _mm256_set1_epi8(w); size_t prefix = 0, res = 0; size_t i = 0;@@ -451,7 +457,7 @@ } HsInt hs_count_ba(unsigned char *str, HsInt off, HsInt len, unsigned char w) {-#if (defined(__AVX2__) && !defined(NO_AVX)) || defined(__SSE2__)+#if defined(__SSE4_2__) || defined(Z_USE_AVX2) || defined(Z_USE_AVX512) if (len >= 1024) return (HsInt)hs_count_simd(str+off, len, w); else { #endif@@ -460,7 +466,7 @@ for (res = 0; len-- != 0; ++str) res += *str == w; return res;-#if (defined(__AVX2__) && !defined(NO_AVX)) || defined(__SSE2__)+#if defined(__SSE4_2__) || defined(Z_USE_AVX2) || defined(Z_USE_AVX512) } #endif }
cbits/text.c view
@@ -30,14 +30,19 @@ #include <utf8rewind.h> #include <codepoint.h> -#ifdef __SSE2__+#if defined(Z_USE_AVX512)+#include <simdasciicheck_avx512.h>+#include <simdutf8check_avx512.h>+#else #include <simdasciicheck.h> #include <simdutf8check.h> #endif HsInt ascii_validate(const char* p, HsInt off, HsInt len){ const char* q = p + off;-#if defined(__AVX2__) && !defined(NO_AVX)+#if defined(Z_USE_AVX512)+ return (HsInt)validate_ascii_fast_avx512(q, (size_t)len);+#elif defined(Z_USE_AVX2) return (HsInt)validate_ascii_fast_avx(q, (size_t)len); #elif defined(__SSE2__) return (HsInt)validate_ascii_fast(q, (size_t)len);@@ -48,7 +53,9 @@ // for some reason unknown, on windows we have to supply a seperated version of ascii_validate // otherwise we got segfault if we import the same FFI with different type (Addr# vs ByteArray#) HsInt ascii_validate_addr(const char* p, HsInt len){-#if defined(__AVX2__) && !defined(NO_AVX)+#if defined(Z_USE_AVX512)+ return (HsInt)validate_ascii_fast_avx512(p, (size_t)len);+#elif defined(Z_USE_AVX2) return (HsInt)validate_ascii_fast_avx(p, (size_t)len); #elif defined(__SSE2__) return (HsInt)validate_ascii_fast(p, (size_t)len);@@ -59,7 +66,9 @@ HsInt utf8_validate(const char* p, HsInt off, HsInt len){ const char* q = p + off;-#if defined(__AVX2__) && !defined(NO_AVX)+#if defined(Z_USE_AVX512)+ return (HsInt)validate_utf8_fast_avx512(q, (size_t)len);+#elif defined(Z_USE_AVX2) return (HsInt)validate_utf8_fast_avx(q, (size_t)len); #elif defined(__SSE2__) return (HsInt)validate_utf8_fast(q, (size_t)len);@@ -70,7 +79,9 @@ // for some reason unknown, on windows we have to supply a seperated version of utf8_validate // otherwise we got segfault if we import the same FFI with different type (Addr# vs ByteArray#) HsInt utf8_validate_addr(const char* p, HsInt len){-#if defined(__AVX2__) && !defined(NO_AVX)+#if defined(Z_USE_AVX512)+ return (HsInt)validate_utf8_fast_avx512(p, (size_t)len);+#elif defined(Z_USE_AVX2) return (HsInt)validate_utf8_fast_avx(p, (size_t)len); #elif defined(__SSE2__) return (HsInt)validate_utf8_fast(p, (size_t)len);
+ third_party/fastbase64/src/fastavx512bwbase64.c view
@@ -0,0 +1,196 @@+#include "fastavx512bwbase64.h"++#include <x86intrin.h>+#include <stdbool.h>++static inline __m512i enc_reshuffle(const __m512i input) {++ // from https://github.com/WojciechMula/base64simd/blob/master/encode/encode.avx512bw.cpp++ // place each 12-byte subarray in seprate 128-bit lane+ // tmp1 = [?? ?? ?? ??|D2 D1 D0 C2|C1 C0 B2 B1|B0 A2 A1 A0] x 4+ // ignored+ const __m512i tmp1 = _mm512_permutexvar_epi32(+ _mm512_set_epi32(-1, 11, 10, 9, -1, 8, 7, 6, -1, 5, 4, 3, -1, 2, 1, 0),+ input+ );++ // reshuffle bytes within 128-bit lanes to format required by+ // AVX512BW unpack procedure+ // tmp2 = [D1 D2 D0 D1|C1 C2 C0 C1|B1 B2 B0 B1|A1 A2 A0 A1] x 4+ // 10 11 9 10 7 8 6 7 4 5 3 4 1 2 0 1+ const __m512i tmp2 = _mm512_shuffle_epi8(+ tmp1,+ _mm512_set4_epi32(0x0a0b090a, 0x07080607, 0x04050304, 0x01020001)+ );++ return tmp2;+}++static inline __m512i enc_translate(const __m512i input) {++ // from https://github.com/WojciechMula/base64simd/blob/master/encode/lookup.avx512bw.cpp++ // reduce 0..51 -> 0+ // 52..61 -> 1 .. 10+ // 62 -> 11+ // 63 -> 12+ __m512i result = _mm512_subs_epu8(input, _mm512_set1_epi8(51));++ // distinguish between ranges 0..25 and 26..51:+ // 0 .. 25 -> remains 0+ // 26 .. 51 -> becomes 13+ const __mmask64 less = _mm512_cmpgt_epi8_mask(_mm512_set1_epi8(26), input);+ result = _mm512_mask_mov_epi8(result, less, _mm512_set1_epi8(13));++ /* the SSE lookup+ const __m128i shift_LUT = _mm_setr_epi8(+ 'a' - 26, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52,+ '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '+' - 62,+ '/' - 63, 'A', 0, 0+ );+ which is:+ 0x47, 0xfc, 0xfc, 0xfc,+ 0xfc, 0xfc, 0xfc, 0xfc,+ 0xfc, 0xfc, 0xfc, 0xed,+ 0xf0, 0x41, 0x00, 0x00++ Note that the order of above list is reserved (due to _mm_setr_epi8),+ so the invocation _mm512_set4_epi32 looks... odd.+ */+ const __m512i shift_LUT = _mm512_set4_epi32(+ 0x000041f0,+ 0xedfcfcfc,+ 0xfcfcfcfc,+ 0xfcfcfc47+ );++ // read shift+ result = _mm512_shuffle_epi8(shift_LUT, result);++ return _mm512_add_epi8(result, input);+}++static inline __m512i dec_reshuffle(__m512i input) {++ // from https://github.com/WojciechMula/base64simd/blob/master/decode/pack.avx512bw.cpp+ const __m512i merge_ab_and_bc = _mm512_maddubs_epi16(input, _mm512_set1_epi32(0x01400140));++ return _mm512_madd_epi16(merge_ab_and_bc, _mm512_set1_epi32(0x00011000));+}+++size_t fast_avx512bw_base64_encode(char* dest, const char* str, size_t len) {+ const char* const dest_orig = dest;+ __m512i inputvector;+ while (len >= 64) {+ inputvector = _mm512_loadu_si512((__m512i *)(str));+ inputvector = enc_reshuffle(inputvector);+ inputvector = enc_translate(inputvector);+ _mm512_storeu_si512((__m512i *)dest, inputvector);+ str += 48;+ dest += 64;+ len -= 48;+ }+ size_t scalarret = chromium_base64_encode(dest, str, len);+ if(scalarret == MODP_B64_ERROR) return MODP_B64_ERROR;+ return (dest - dest_orig) + scalarret;+}++#define build_dword(b0, b1, b2, b3) \+ (((uint32_t)((uint8_t)(b0)) << 0*8) \+ | ((uint32_t)((uint8_t)(b1)) << 1*8) \+ | ((uint32_t)((uint8_t)(b2)) << 2*8) \+ | ((uint32_t)((uint8_t)(b3)) << 3*8))+ +#define _mm512_set4lanes_epi8(b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15) \+ _mm512_setr4_epi32( \+ build_dword( b0, b1, b2, b3), \+ build_dword( b4, b5, b6, b7), \+ build_dword( b8, b9, b10, b11), \+ build_dword(b12, b13, b14, b15))++size_t fast_avx512bw_base64_decode(char *out, const char *src, size_t srclen) {+ char* out_orig = out;+ while (srclen >= 88) {++ // load+ const __m512i input = _mm512_loadu_si512((const __m512i*)(src));+ + // translate from ASCII+ // and https://github.com/WojciechMula/base64simd/blob/master/decode/lookup.avx512bw.cpp+ const __m512i higher_nibble = _mm512_and_si512(_mm512_srli_epi32(input, 4), _mm512_set1_epi8(0x0f));+ const __m512i lower_nibble = _mm512_and_si512(input, _mm512_set1_epi8(0x0f));++ const __m512i shiftLUT = _mm512_set4lanes_epi8(+ 0, 0, 19, 4, -65, -65, -71, -71,+ 0, 0, 0, 0, 0, 0, 0, 0);++ const __m512i maskLUT = _mm512_set4lanes_epi8(+ /* 0 : 0b1010_1000*/ 0xa8,+ /* 1 .. 9 : 0b1111_1000*/ 0xf8, 0xf8, 0xf8, 0xf8,+ 0xf8, 0xf8, 0xf8, 0xf8,+ 0xf8,+ /* 10 : 0b1111_0000*/ 0xf0,+ /* 11 : 0b0101_0100*/ 0x54,+ /* 12 .. 14 : 0b0101_0000*/ 0x50, 0x50, 0x50,+ /* 15 : 0b0101_0100*/ 0x54+ );++ const __m512i bitposLUT = _mm512_set4lanes_epi8(+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00+ );++ const __m512i sh = _mm512_shuffle_epi8(shiftLUT, higher_nibble);+ const __mmask64 eq_2f = _mm512_cmpeq_epi8_mask(input, _mm512_set1_epi8(0x2f));+ const __m512i shift = _mm512_mask_mov_epi8(sh, eq_2f, _mm512_set1_epi8(16));++ const __m512i M = _mm512_shuffle_epi8(maskLUT, lower_nibble);+ const __m512i bit = _mm512_shuffle_epi8(bitposLUT, higher_nibble);++ const uint64_t match = _mm512_test_epi8_mask(M, bit);++ if (match != (uint64_t)(-1)) {+ // some characters do not match the valid range+ return MODP_B64_ERROR;+ }++ const __m512i translated = _mm512_add_epi8(input, shift);++ const __m512i packed = dec_reshuffle(translated);++ // and https://github.com/WojciechMula/base64simd/blob/master/decode/decode.avx512bw.cpp+ // + const __m512i t1 = _mm512_shuffle_epi8(+ packed,+ _mm512_set4lanes_epi8(+ 2, 1, 0,+ 6, 5, 4,+ 10, 9, 8,+ 14, 13, 12,+ -1, -1, -1, -1)+ );++ // shuffle bytes+ const __m512i s6 = _mm512_setr_epi32(+ 0, 1, 2,+ 4, 5, 6,+ 8, 9, 10,+ 12, 13, 14,+ // unused+ 0, 0, 0, 0);++ const __m512i t2 = _mm512_permutexvar_epi32(s6, t1);++ _mm512_storeu_si512((__m512i*)(out), t2);++ srclen -= 64;+ src += 64;+ out += 48;+ }+ size_t scalarret = chromium_base64_decode(out, src, srclen);+ if(scalarret == MODP_B64_ERROR) return MODP_B64_ERROR;+ return (out - out_orig) + scalarret;+}+