diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ ```text $ stack install -$ cat 7g.csv | pv -t -e -b -a | hw-dsv query-lazy -k 0 -k 1 -d , -e '|' > /dev/null +$ cat 7g.csv | pv -t -e -b -a | hw-dsv query-lazy -k 1 -k 2 -d , -e '|' > /dev/null 7.08GiB 0:07:25 [16.3MiB/s] ``` @@ -60,7 +60,7 @@ ```text $ stack install --flag bits-extra:bmi2 --flag hw-bits:bmi2 --flag hw-rankselect-base:bmi2 --flag hw-rankselect:bmi2 --flag hw-dsv:bmi2 --flag hw-dsv:avx2 -$ cat 7gb.csv | pv -t -e -b -a | hw-dsv query-lazy -k 0 -k 1 -d , -e '|' > /dev/null +$ cat 7gb.csv | pv -t -e -b -a | hw-dsv query-lazy -k 1 -k 2 -d , -e '|' > /dev/null 7.08GiB 0:00:39 [ 181MiB/s] ``` @@ -68,9 +68,13 @@ ```text $ stack install --flag bits-extra:bmi2 --flag hw-bits:bmi2 --flag hw-rankselect-base:bmi2 --flag hw-rankselect:bmi2 --flag hw-dsv:bmi2 -$ cat 7gb.csv | pv -t -e -b -a | hw-dsv query-lazy -k 0 -k 1 -d , -e '|' > /dev/null +$ cat 7gb.csv | pv -t -e -b -a | hw-dsv query-lazy -k 1 -k 2 -d , -e '|' > /dev/null 7.08GiB 0:00:43 [ 165MiB/s] ``` + +## `hw-dsv` command line options + +The `hw-dsv` application accepts 1-based column indexes rather than 0-based. The library is 0-based. ## Using `hw-dsv` as a library diff --git a/app/App/Commands/QueryLazy.hs b/app/App/Commands/QueryLazy.hs --- a/app/App/Commands/QueryLazy.hs +++ b/app/App/Commands/QueryLazy.hs @@ -16,6 +16,7 @@ import Data.List import Data.Semigroup ((<>)) import Options.Applicative hiding (columns) +import Text.Read (readEither) import qualified App.IO as IO import qualified App.Lens as L @@ -105,10 +106,17 @@ cmdQueryLazy :: Mod CommandFields (IO ()) cmdQueryLazy = command "query-lazy" $ flip info idm $ runQueryLazy <$> optsQueryLazy +nonZeroOneBased :: Mod OptionFields Int -> Parser Int +nonZeroOneBased = option $ eitherReader $ \s -> do + a <- readEither s + if a == 0 + then Left "cannot index column 0" + else Right (a - 1) + optsQueryLazy :: Parser QueryLazyOptions optsQueryLazy = QueryLazyOptions <$> many - ( option auto + ( nonZeroOneBased ( long "column" <> short 'k' <> help "Column to select" diff --git a/app/App/Commands/QueryStrict.hs b/app/App/Commands/QueryStrict.hs --- a/app/App/Commands/QueryStrict.hs +++ b/app/App/Commands/QueryStrict.hs @@ -16,6 +16,7 @@ import Data.List import Data.Semigroup ((<>)) import Options.Applicative +import Text.Read (readEither) import qualified App.IO as IO import qualified App.Lens as L @@ -52,10 +53,17 @@ cmdQueryStrict :: Mod CommandFields (IO ()) cmdQueryStrict = command "query-strict" $ flip info idm $ runQueryStrict <$> optsQueryStrict +nonZeroOneBased :: Mod OptionFields Int -> Parser Int +nonZeroOneBased = option $ eitherReader $ \s -> do + a <- readEither s + if a == 0 + then Left "cannot index column 0" + else Right (a - 1) + optsQueryStrict :: Parser QueryStrictOptions optsQueryStrict = QueryStrictOptions <$> many - ( option auto + ( nonZeroOneBased ( long "column" <> short 'k' <> help "Column to select" diff --git a/cbits/simd.c b/cbits/simd.c deleted file mode 100644 --- a/cbits/simd.c +++ /dev/null @@ -1,112 +0,0 @@ -#include "simd.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -typedef uint8_t v32si __attribute__ ((vector_size (32))); -typedef uint8_t v16si __attribute__ ((vector_size (16))); - -void print_bits_16(uint16_t word) { - size_t i; - - putc('|', stdout); - - for (i = 0; i < 16; ++i) { - putc((word & (1L << i)) ? '1' : '0', stdout); - } - printf("|\n"); -} - -void system_memcpy( - char *target, - char *source, - size_t len) { - memcpy(target, source, len); -} - -void avx2_memcpy( - uint8_t *target, - uint8_t *source, - size_t len) { -#if defined(AVX2_ENABLED) - size_t aligned_len = (len / 32) * 32; - size_t remaining_len = len - aligned_len; - - size_t i; - - for (i = 0; i < aligned_len; i += 32) { - __m128i v0 = *(__m128i*)(source + i ); - __m128i v1 = *(__m128i*)(source + i + 16); - - *(__m128i*)(target + i ) = v0; - *(__m128i*)(target + i + 16 ) = v1; - } - - memcpy(target + aligned_len, source + aligned_len, remaining_len); -#endif -} - - -void sse_cmpeq8( - uint8_t byte, - uint64_t *target, - size_t target_length, - uint8_t *source) { - uint16_t *target16 = (uint16_t *)target; - - __m128i v_comparand = _mm_set1_epi8(byte); - - uint16_t *out_mask = (uint16_t*)target; - size_t i; - - for (i = 0; i < target_length * 4; ++i) { - __m128i v_data_a = *(__m128i*)(source + (i * 16)); - __m128i v_results_a = _mm_cmpeq_epi8(v_data_a, v_comparand); - uint16_t mask = (uint16_t)_mm_movemask_epi8(v_results_a); - target16[i] = mask; - } -} - -void avx2_cmpeq8( - uint8_t byte, - uint64_t *target, - size_t target_length, - uint8_t *source) { -#if defined(AVX2_ENABLED) - uint32_t *target32 = (uint32_t *)target; - - __m256i v_comparand = _mm256_set1_epi8(byte); - - uint32_t *out_mask = (uint32_t*)target; - - size_t i; - - for (i = 0; i < target_length * 2; ++i) { - __m256i v_data_a = *(__m256i*)(source + (i * 32)); - __m256i v_results_a = _mm256_cmpeq_epi8(v_data_a, v_comparand); - uint32_t mask = (uint32_t)_mm256_movemask_epi8(v_results_a); - target32[i] = mask; - } -#endif -} - -int example_main() { - uint8_t source[32] = "01234567890123456789012345678901"; - uint8_t target[33]; - avx2_memcpy(target, source, 32); - target[32] = 0; - printf("%s\n", target); - - // uint8_t source2[64] = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; - // uint64_t target2[1] = {0}; - // avx2_cmpeq8('0', target2, 1, source2); - // printf("%llu\n", target2[0]); - - return 0; -} diff --git a/cbits/simd.h b/cbits/simd.h deleted file mode 100644 --- a/cbits/simd.h +++ /dev/null @@ -1,13 +0,0 @@ -#include -#include - -void avx2_memcpy( - uint8_t *target, - uint8_t *source, - size_t len); - -void avx2_cmpeq8( - uint8_t byte, - uint64_t *target, - size_t target_length, - uint8_t *source); diff --git a/hw-dsv.cabal b/hw-dsv.cabal --- a/hw-dsv.cabal +++ b/hw-dsv.cabal @@ -3,7 +3,7 @@ -- see: https://github.com/sol/hpack name: hw-dsv -version: 0.3.2 +version: 0.3.3 synopsis: Unbelievably fast streaming DSV file parser description: Please see the README on Github at category: Text, CSV, SIMD, Succinct Data Structures, Data Structures @@ -19,7 +19,6 @@ cabal-version: >= 1.10 extra-source-files: - cbits/simd.h ChangeLog.md data/bench/data-0001000.csv README.md @@ -47,10 +46,6 @@ hs-source-dirs: src ghc-options: -O2 -Wall - include-dirs: - cbits - c-sources: - cbits/simd.c build-depends: base >= 4.7 && < 5 , bits-extra >= 0.0.1.2 && < 0.1 @@ -61,10 +56,8 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7 - , hw-simd >= 0.1.1.2 && < 0.2 + , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13 - build-tools: - c2hs if flag(sse42) ghc-options: -msse4.2 cc-options: -msse4.2 @@ -115,7 +108,7 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7 - , hw-simd >= 0.1.1.2 && < 0.2 + , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13 , hw-dsv , hedgehog >= 0.5 && < 0.7 @@ -170,7 +163,7 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7 - , hw-simd >= 0.1.1.2 && < 0.2 + , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13 , cassava >= 0.5.1.0 && < 0.6 , hw-dsv @@ -211,7 +204,7 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7 - , hw-simd >= 0.1.1.2 && < 0.2 + , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13 , directory >= 1.2.2 && < 1.4 , hw-dsv @@ -261,7 +254,7 @@ , hw-rankselect >= 0.12.0.2 && < 0.13 , hw-rankselect-base >= 0.3.2.0 && < 0.4 , hw-prim >= 0.6.2.14 && < 0.7 - , hw-simd >= 0.1.1.2 && < 0.2 + , hw-simd >= 0.1.1.3 && < 0.2 , vector >= 0.12.0.1 && < 0.13 , cassava >= 0.5.1.0 && < 0.6 , criterion >= 1.4.1.0 && < 1.6