hw-dsv 0.3.2 → 0.3.3
raw patch · 6 files changed
+31/−143 lines, 6 filesdep ~hw-simd
Dependency ranges changed: hw-simd
Files
- README.md +7/−3
- app/App/Commands/QueryLazy.hs +9/−1
- app/App/Commands/QueryStrict.hs +9/−1
- cbits/simd.c +0/−112
- cbits/simd.h +0/−13
- hw-dsv.cabal +6/−13
README.md view
@@ -52,7 +52,7 @@ ```text $ stack install-$ cat 7g.csv | pv -t -e -b -a | hw-dsv query-lazy -k 0 -k 1 -d , -e '|' > /dev/null+$ cat 7g.csv | pv -t -e -b -a | hw-dsv query-lazy -k 1 -k 2 -d , -e '|' > /dev/null 7.08GiB 0:07:25 [16.3MiB/s] ``` @@ -60,7 +60,7 @@ ```text $ stack install --flag bits-extra:bmi2 --flag hw-bits:bmi2 --flag hw-rankselect-base:bmi2 --flag hw-rankselect:bmi2 --flag hw-dsv:bmi2 --flag hw-dsv:avx2-$ cat 7gb.csv | pv -t -e -b -a | hw-dsv query-lazy -k 0 -k 1 -d , -e '|' > /dev/null+$ cat 7gb.csv | pv -t -e -b -a | hw-dsv query-lazy -k 1 -k 2 -d , -e '|' > /dev/null 7.08GiB 0:00:39 [ 181MiB/s] ``` @@ -68,9 +68,13 @@ ```text $ stack install --flag bits-extra:bmi2 --flag hw-bits:bmi2 --flag hw-rankselect-base:bmi2 --flag hw-rankselect:bmi2 --flag hw-dsv:bmi2-$ cat 7gb.csv | pv -t -e -b -a | hw-dsv query-lazy -k 0 -k 1 -d , -e '|' > /dev/null+$ cat 7gb.csv | pv -t -e -b -a | hw-dsv query-lazy -k 1 -k 2 -d , -e '|' > /dev/null 7.08GiB 0:00:43 [ 165MiB/s] ```++## `hw-dsv` command line options++The `hw-dsv` application accepts 1-based column indexes rather than 0-based. The library is 0-based. ## Using `hw-dsv` as a library
app/App/Commands/QueryLazy.hs view
@@ -16,6 +16,7 @@ import Data.List import Data.Semigroup ((<>)) import Options.Applicative hiding (columns)+import Text.Read (readEither) import qualified App.IO as IO import qualified App.Lens as L@@ -105,10 +106,17 @@ cmdQueryLazy :: Mod CommandFields (IO ()) cmdQueryLazy = command "query-lazy" $ flip info idm $ runQueryLazy <$> optsQueryLazy +nonZeroOneBased :: Mod OptionFields Int -> Parser Int+nonZeroOneBased = option $ eitherReader $ \s -> do+ a <- readEither s+ if a == 0+ then Left "cannot index column 0"+ else Right (a - 1)+ optsQueryLazy :: Parser QueryLazyOptions optsQueryLazy = QueryLazyOptions <$> many- ( option auto+ ( nonZeroOneBased ( long "column" <> short 'k' <> help "Column to select"
app/App/Commands/QueryStrict.hs view
@@ -16,6 +16,7 @@ import Data.List import Data.Semigroup ((<>)) import Options.Applicative+import Text.Read (readEither) import qualified App.IO as IO import qualified App.Lens as L@@ -52,10 +53,17 @@ cmdQueryStrict :: Mod CommandFields (IO ()) cmdQueryStrict = command "query-strict" $ flip info idm $ runQueryStrict <$> optsQueryStrict +nonZeroOneBased :: Mod OptionFields Int -> Parser Int+nonZeroOneBased = option $ eitherReader $ \s -> do+ a <- readEither s+ if a == 0+ then Left "cannot index column 0"+ else Right (a - 1)+ optsQueryStrict :: Parser QueryStrictOptions optsQueryStrict = QueryStrictOptions <$> many- ( option auto+ ( nonZeroOneBased ( long "column" <> short 'k' <> help "Column to select"
− cbits/simd.c
@@ -1,112 +0,0 @@-#include "simd.h"--#include <immintrin.h>-#include <mmintrin.h>-#include <stdint.h>-#include <stdio.h>-#include <stdlib.h>-#include <string.h>-#include <unistd.h>-#include <ctype.h>--typedef uint8_t v32si __attribute__ ((vector_size (32)));-typedef uint8_t v16si __attribute__ ((vector_size (16)));--void print_bits_16(uint16_t word) {- size_t i;-- putc('|', stdout);-- for (i = 0; i < 16; ++i) {- putc((word & (1L << i)) ? '1' : '0', stdout);- }- printf("|\n");-}--void system_memcpy(- char *target,- char *source,- size_t len) {- memcpy(target, source, len);-}--void avx2_memcpy(- uint8_t *target,- uint8_t *source,- size_t len) {-#if defined(AVX2_ENABLED)- size_t aligned_len = (len / 32) * 32;- size_t remaining_len = len - aligned_len;-- size_t i;-- for (i = 0; i < aligned_len; i += 32) {- __m128i v0 = *(__m128i*)(source + i );- __m128i v1 = *(__m128i*)(source + i + 16);-- *(__m128i*)(target + i ) = v0;- *(__m128i*)(target + i + 16 ) = v1;- }-- memcpy(target + aligned_len, source + aligned_len, remaining_len);-#endif-}---void sse_cmpeq8(- uint8_t byte,- uint64_t *target,- size_t target_length,- uint8_t *source) {- uint16_t *target16 = (uint16_t *)target;-- __m128i v_comparand = _mm_set1_epi8(byte);-- uint16_t *out_mask = (uint16_t*)target;- size_t i;-- for (i = 0; i < target_length * 4; ++i) {- __m128i v_data_a = *(__m128i*)(source + (i * 16));- __m128i v_results_a = _mm_cmpeq_epi8(v_data_a, v_comparand);- uint16_t mask = (uint16_t)_mm_movemask_epi8(v_results_a);- target16[i] = mask;- }-}--void avx2_cmpeq8(- uint8_t byte,- uint64_t *target,- size_t target_length,- uint8_t *source) {-#if defined(AVX2_ENABLED)- uint32_t *target32 = (uint32_t *)target;-- __m256i v_comparand = _mm256_set1_epi8(byte);-- uint32_t *out_mask = (uint32_t*)target;-- size_t i;-- for (i = 0; i < target_length * 2; ++i) {- __m256i v_data_a = *(__m256i*)(source + (i * 32));- __m256i v_results_a = _mm256_cmpeq_epi8(v_data_a, v_comparand);- uint32_t mask = (uint32_t)_mm256_movemask_epi8(v_results_a);- target32[i] = mask;- }-#endif-}--int example_main() {- uint8_t source[32] = "01234567890123456789012345678901";- uint8_t target[33];- avx2_memcpy(target, source, 32);- target[32] = 0;- printf("%s\n", target);-- // uint8_t source2[64] = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";- // uint64_t target2[1] = {0};- // avx2_cmpeq8('0', target2, 1, source2);- // printf("%llu\n", target2[0]);-- return 0;-}
− cbits/simd.h
@@ -1,13 +0,0 @@-#include <unistd.h>-#include <stdint.h>--void avx2_memcpy(- uint8_t *target,- uint8_t *source,- size_t len);--void avx2_cmpeq8(- uint8_t byte,- uint64_t *target,- size_t target_length,- uint8_t *source);
hw-dsv.cabal view
@@ -3,7 +3,7 @@ -- see: https://github.com/sol/hpack name: hw-dsv-version: 0.3.2+version: 0.3.3 synopsis: Unbelievably fast streaming DSV file parser description: Please see the README on Github at <https://github.com/haskell-works/hw-dsv#readme> category: Text, CSV, SIMD, Succinct Data Structures, Data Structures@@ -19,7 +19,6 @@ cabal-version: >= 1.10 extra-source-files:- cbits/simd.h ChangeLog.md data/bench/data-0001000.csv README.md@@ -47,10 +46,6 @@ hs-source-dirs: src ghc-options: -O2 -Wall- include-dirs:- cbits- c-sources:- cbits/simd.c build-depends: base >= 4.7 && < 5 , bits-extra >= 0.0.1.2 && < 0.1@@ -61,10 +56,8 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7- , hw-simd >= 0.1.1.2 && < 0.2+ , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13- build-tools:- c2hs if flag(sse42) ghc-options: -msse4.2 cc-options: -msse4.2@@ -115,7 +108,7 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7- , hw-simd >= 0.1.1.2 && < 0.2+ , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13 , hw-dsv , hedgehog >= 0.5 && < 0.7@@ -170,7 +163,7 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7- , hw-simd >= 0.1.1.2 && < 0.2+ , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13 , cassava >= 0.5.1.0 && < 0.6 , hw-dsv@@ -211,7 +204,7 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7- , hw-simd >= 0.1.1.2 && < 0.2+ , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13 , directory >= 1.2.2 && < 1.4 , hw-dsv@@ -261,7 +254,7 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7- , hw-simd >= 0.1.1.2 && < 0.2+ , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13 , cassava >= 0.5.1.0 && < 0.6 , criterion >= 1.4.1.0 && < 1.6