diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Revision history for dataframe +## 1.1.0.0 +### Breaking changes +* Remove `OptionalColumn` constructor; fold nullability into `BoxedColumn`/`UnboxedColumn` via bit-packed bitmap. +* Remove `NFData` instance from `Columnable` constraint. + +### New features +* Add `toCsv` and `toSeparated` for converting a DataFrame to CSV/delimited text without writing to a file. +* `safeRead` now defaults reading columns to `Maybe a`. +* Split SIMD CSV reader into a separate `dataframe-fastcsv` package. + +### Bug fixes +* Fix joins for missing key columns (#187). +* Fix single column not found error when using typed dataframe. +* Fix Synthesis to use `SafeLookup` constraint. +* Fix `writeSeparated` ignoring separator parameter (was hardcoded to comma). + +### Internal +* Slice groups now use custom backpermute instead of converting unboxed vectors. +* Reuse comparison operators in Subset. +* Refactor `getRowAsText` for readability using pattern guards. + ## 1.0.0.1 * toMarkdownTable is now toMarkdown (mostly used internally) * Provide toMarkdown' that outputs string diff --git a/app/Synthesis.hs b/app/Synthesis.hs --- a/app/Synthesis.hs +++ b/app/Synthesis.hs @@ -1,53 +1,51 @@ +{-# LANGUAGE DataKinds #-} {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE TypeApplications #-} +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE TypeOperators #-} +import Data.Char import qualified Data.Text as T import qualified DataFrame as D +import DataFrame.DecisionTree import qualified DataFrame.Functions as F +import DataFrame.Operators +import qualified DataFrame.Typed as DT +import System.Random -import Data.Char -import DataFrame.DecisionTree -import DataFrame.Operators hiding (name) +$( DT.deriveSchemaFromCsvFileWith + D.defaultReadOptions{D.safeRead = True} + "TrainSchema" + "./data/titanic/train.csv" + ) +$( DT.deriveSchemaFromCsvFileWith + D.defaultReadOptions{D.safeRead = True} + "TestSchema" + "./data/titanic/test.csv" + ) -import System.Random +-- Survived is Maybe Int (safeRead = True); prediction is Int (model output). +type RawPredSchema = + '[DT.Column "Survived" (Maybe Int), DT.Column "prediction" Int] -$(F.declareColumnsFromCsvFile "./data/titanic/train.csv") +prediction :: D.Expr Int +prediction = F.col @Int "prediction" main :: IO () main = do - train <- D.readCsv "./data/titanic/train.csv" - test <- D.readCsv "./data/titanic/test.csv" + rawTrain <- D.readCsv "./data/titanic/train.csv" + rawTest <- D.readCsv "./data/titanic/test.csv" - -- Apply the same transformations to training and test. - let combined = - (train <> test) - |> D.deriveMany - [ "Ticket" .= F.lift (T.filter isAlpha) ticket - , "Name" .= F.match "\\s*([A-Za-z]+)\\." name - , "Cabin" .= F.whenPresent (T.take 1) cabin - ] - |> D.renameMany - [ (F.name name, "title") - , (F.name cabin, "cabin_prefix") - , (F.name pclass, "passenger_class") - , (F.name sibsp, "number_of_siblings_and_spouses") - , (F.name parch, "number_of_parents_and_children") - ] - print combined + train <- + maybe (fail "train.csv schema mismatch") pure (DT.freeze @TrainSchema rawTrain) + test <- + maybe (fail "test.csv schema mismatch") pure (DT.freeze @TestSchema rawTest) - let (train', validation) = - D.take - (D.nRows train) - combined - |> D.filterJust (F.name survived) - |> D.randomSplit (mkStdGen 4232) 0.7 - -- Split the test out again. - test' = - D.drop - (D.nRows train) - combined + let (trainDf, validDf) = + D.randomSplit (mkStdGen 4232) 0.7 (DT.thaw (clean train)) + testDf = DT.thaw (clean test) model = fitDecisionTree @@ -61,70 +59,84 @@ { complexityPenalty = 0.1 , maxExprDepth = 3 , disallowedCombinations = - [ (F.name age, F.name fare) + [ ("Age", "Fare") , ("passenger_class", "number_of_siblings_and_spouses") , ("passenger_class", "number_of_parents_and_children") ] } } ) - survived -- Label to predict - ( train' - |> D.exclude [F.name passengerid] - ) + (F.fromMaybe 0 (F.col @(Maybe Int) "Survived")) + (trainDf |> D.exclude ["PassengerId"]) print model putStrLn "Training accuracy: " - print $ - computeAccuracy - (train' |> D.derive (F.name prediction) model) + print $ computeAccuracy (trainDf |> D.derive (F.name prediction) model) putStrLn "Validation accuracy: " - print $ - computeAccuracy - ( validation - |> D.derive (F.name prediction) model - ) + print $ computeAccuracy (validDf |> D.derive (F.name prediction) model) - let predictions = D.derive (F.name survived) model test' D.writeCsv "./predictions.csv" - (predictions |> D.select [F.name passengerid, F.name survived]) + ( testDf + |> D.derive "Survived" model + |> D.select ["PassengerId", "Survived"] + ) -prediction :: D.Expr Int -prediction = F.col @Int "prediction" +clean :: + ( DT.AssertPresent "Ticket" cols + , DT.SafeLookup "Ticket" cols ~ Maybe T.Text + , DT.AssertPresent "Name" cols + , DT.SafeLookup "Name" cols ~ Maybe T.Text + , DT.AssertPresent "Cabin" cols + , DT.SafeLookup "Cabin" cols ~ Maybe T.Text + ) => + DT.TypedDataFrame cols -> + DT.TypedDataFrame + ( DT.RenameManyInSchema + '[ '("Name", "title") + , '("Cabin", "cabin_prefix") + , '("Pclass", "passenger_class") + , '("SibSp", "number_of_siblings_and_spouses") + , '("Parch", "number_of_parents_and_children") + ] + cols + ) +clean tdf = + tdf + |> DT.replaceColumn @"Ticket" (DT.nullLift (T.filter isAlpha) (DT.col @"Ticket")) + |> DT.replaceColumn @"Name" (DT.nullLift extractTitle (DT.col @"Name")) + |> DT.replaceColumn @"Cabin" (DT.nullLift (T.take 1) (DT.col @"Cabin")) + |> DT.renameMany + @'[ '("Name", "title") + , '("Cabin", "cabin_prefix") + , '("Pclass", "passenger_class") + , '("SibSp", "number_of_siblings_and_spouses") + , '("Parch", "number_of_parents_and_children") + ] +-- | Extract title (e.g. "Mr", "Mrs") from a full Titanic passenger name. +extractTitle :: T.Text -> T.Text +extractTitle name = + case filter (T.isSuffixOf ".") (T.words name) of + (w : _) -> T.dropEnd 1 w + [] -> "" + +{- | Compute binary classification accuracy from a DataFrame containing + "Survived" and "prediction" columns. +-} computeAccuracy :: D.DataFrame -> Double computeAccuracy df = - let - tp = - fromIntegral $ - D.nRows - ( D.filterWhere - (survived .== F.lit (1 :: Int) .&& prediction .== F.lit (1 :: Int)) - df - ) - tn = - fromIntegral $ - D.nRows - ( D.filterWhere - (survived .== F.lit (0 :: Int) .&& prediction .== F.lit (0 :: Int)) - df - ) - fp = - fromIntegral $ - D.nRows - ( D.filterWhere - (survived .== F.lit (0 :: Int) .&& prediction .== F.lit (1 :: Int)) - df - ) - fn = - fromIntegral $ - D.nRows - ( D.filterWhere - (survived .== F.lit (1 :: Int) .&& prediction .== F.lit (0 :: Int)) - df - ) - in - (tp + tn) / (tp + tn + fp + fn) + let tdf = + DT.impute @"Survived" 0 $ + DT.unsafeFreeze @RawPredSchema $ + df |> D.select ["Survived", "prediction"] + survived = DT.col @"Survived" + pred = DT.col @"prediction" + count expr = fromIntegral (DT.nRows (DT.filterWhere expr tdf)) + tp = count ((survived DT..==. DT.lit 1) DT..&&. (pred DT..==. DT.lit 1)) + tn = count ((survived DT..==. DT.lit 0) DT..&&. (pred DT..==. DT.lit 0)) + fp = count ((survived DT..==. DT.lit 0) DT..&&. (pred DT..==. DT.lit 1)) + fn = count ((survived DT..==. DT.lit 1) DT..&&. (pred DT..==. DT.lit 0)) + in (tp + tn) / (tp + tn + fp + fn) diff --git a/benchmark/Main.hs b/benchmark/Main.hs --- a/benchmark/Main.hs +++ b/benchmark/Main.hs @@ -43,7 +43,7 @@ groupByHaskell :: IO () groupByHaskell = do - df <- D.fastReadCsvUnstable "./data/housing.csv" + df <- D.readCsv "./data/housing.csv" print $ df |> D.groupBy ["ocean_proximity"] @@ -84,12 +84,6 @@ parseFile :: String -> IO () parseFile = void . D.readCsv -parseFileUnstable :: String -> IO () -parseFileUnstable = void . D.readCsvUnstable - -parseFileUnstableSIMD :: String -> IO () -parseFileUnstableSIMD = void . D.fastReadCsvUnstable - parseHousingCSV :: IO () parseHousingCSV = parseFile "./data/housing.csv" @@ -215,22 +209,12 @@ , bgroup "housing.csv (1.4 MB)" [ bench "Attoparsec" $ nfIO $ parseFile "./data/housing.csv" - , bench "Native Haskell" $ nfIO $ parseFileUnstable "./data/housing.csv" - , bench "SIMD" $ nfIO $ parseFileUnstableSIMD "./data/housing.csv" ] , bgroup "effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv (9.1 MB)" [ bench "Attoparsec" $ nfIO $ parseFile - "./data/effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv" - , bench "Native Haskell" $ - nfIO $ - parseFileUnstable - "./data/effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv" - , bench "SIMD" $ - nfIO $ - parseFileUnstableSIMD "./data/effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv" ] , bgroup diff --git a/cbits/process_csv.c b/cbits/process_csv.c deleted file mode 100644 --- a/cbits/process_csv.c +++ /dev/null @@ -1,198 +0,0 @@ -#include "process_csv.h" -#include -#include - -/* - * compile with clang -O3 - * in cabal use - * cc-options: -O3 - * - * Produce an array of field delimiter indices - * Fields can be delimited by commas and newlines - * TODO: allow the user to provide a custom delimiter - * character to replace commas. - * This should work with UTF-8, so long as the delimiter - * character is a single byte. - * - * Delimiters can be escaped inside of quotes. Quotes - * can also be placed inside quotes by double quoting. - * For the purposes of this parser we can ignore double - * quotes inside quotes, thereby treating the first quote - * as the closing of the string and the next one the - * immediate opening of a new one - * - * We can find the quoted regions by first finding - * the positions of the quotes (cmpeq and then movemask) - * and then using the carryless multiplication operation - * to know the regions that are quoted. We can then simply - * and the inverse of the quotemask to exclude commas and - * newlines inside quotes - * - */ - -// if the character is found at a particular -// position in the array of bytes, the -// corresponding bit in the returned uint64_t should -// be turned on. -// Example: searching for commas in -// input: one, two, three -// result: 000100001000000 -#ifdef HAS_SIMD_CSV -static uint64_t find_character_in_chunk(uint8_t *in, uint8_t c) { -#ifdef USE_AVX2 - // AVX2 implementation: load two 32-byte chunks - __m256i v0 = _mm256_loadu_si256((const __m256i *)(in)); - __m256i v1 = _mm256_loadu_si256((const __m256i *)(in + 32)); - __m256i b = _mm256_set1_epi8((char)c); - __m256i m0 = _mm256_cmpeq_epi8(v0, b); - __m256i m1 = _mm256_cmpeq_epi8(v1, b); - uint32_t lo = (uint32_t)_mm256_movemask_epi8(m0); - uint32_t hi = (uint32_t)_mm256_movemask_epi8(m1); - return ((uint64_t)hi << 32) | (uint64_t)lo; -#else // USE_NEON - // ARM NEON implementation: load 64 bytes deinterleaved - uint8x16x4_t src = vld4q_u8(in); - uint8x16_t mask = vmovq_n_u8(c); - uint8x16_t cmp0 = vceqq_u8(src.val[0], mask); - uint8x16_t cmp1 = vceqq_u8(src.val[1], mask); - uint8x16_t cmp2 = vceqq_u8(src.val[2], mask); - uint8x16_t cmp3 = vceqq_u8(src.val[3], mask); - - // For an explanation of how to do movemask in - // NEON, see: https://branchfree.org/2019/04/01/fitting-my-head-through-the-arm-holes-or-two-sequences-to-substitute-for-the-missing-pmovmskb-instruction-on-arm-neon/ - // The specific implementation below is owed to the - // user 'aqrit' in a comment on the blog above - // There's also https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon - // - // The input to the move mask must be de-interleaved. - // That is, for a string after vceqq_u8 we have, - // cmp0 = aaaaaaaa / eeeeeeee / ... - // cmp1 = bbbbbbbb / ffffffff / ... - // cmp2 = cccccccc / gggggggg / ... - // cmp3 = dddddddd / hhhhhhhh / ... - // Luckily vld4q_u8 does this for us. Now we want - // to interleave this into a 64bit integer with bits - // abcdefgh... - // cmp0 holds bits for positions - // 0,4,8,..., cmp1 holds bits for 1,5,9,..., and so on. - // So to bring together the bits for different positions - // we right shift and combine with vsriq_n_u8 - // - // vsriq_n_u8 shifts each byte of the first operand - // right by n bits, and combines it with the bits of - // the second operand. - // example: - // uint8_t mask = 0xFF >> n; // if n = 1, 0111 1111 - // uint8_t shifted = operand1 >> n // 0bbb, bbbb - // operand1 = (operand2 & (!mask)) | shifted - // (aaaa aaaa & 1000 0000) | 0bbb bbbb = abbb bbbb - // - // So we first bring together the bits the first two - // rows and the next two rows (so to speak) - // t0 = abbbbbbb/efffffff/... - uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1); - // t1 = cddddddd/ghhhhhhh/... - uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1); - // Now we must combine each of our combined rows - // so that we get back our column interleaved - // t2 = abcddddd/efghhhhh/... - uint8x16_t t2 = vsriq_n_u8(t1, t0, 2); - // Then to get rid of the repeated bits in the upper half - // t3 = abcdabcd/efghefgh/... - uint8x16_t t3 = vsriq_n_u8(t2, t2, 4); - // and now it's the relatively simple matter of getting - // rid of half the bits. We combine our 8bit words into 16 - // bit words for this step, and then we shift right by 4 - // and turn the result into an 8 bit word - // afterreinterpert: abcdabcdefghefgh/... - // afterrightshift: 0000abcdabcdefgh/... - // take the lower bits: abcdefgh/... - uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4); - // Finally we recombine them into a 64 bit integer - // (vreinterpret_u64_u8 here does uint8x8 -> uint64x1 - // and vget_lane_u64 does uint64x1 -> uint64) - return vget_lane_u64(vreinterpret_u64_u8(t4), 0); -#endif -} -#endif - -// I owe a debt to https://github.com/geofflangdale/simdcsv -// Let's go ahead and assume `in` will only ever get 64 bytes -// initial_quoted will be either all_ones ~0ULL or all_zeros 0ULL -#ifdef HAS_SIMD_CSV -static uint64_t parse_chunk(uint8_t *in, uint8_t separator, uint64_t *initial_quoted) { - uint64_t quotebits = find_character_in_chunk(in, QUOTE_CHAR); - // See https://wunkolo.github.io/post/2020/05/pclmulqdq-tricks/ - // Also, section 3.1.1 of Parsing Gigabytes of JSON per Second, - // Geoff Langdale, Daniel Lemire, https://arxiv.org/pdf/1902.08318 -#ifdef USE_AVX2 - // Use PCLMUL for carryless multiplication on x86 - __m128i a = _mm_set_epi64x(0, (int64_t)ALL_ONES_MASK); - __m128i b = _mm_set_epi64x(0, (int64_t)quotebits); - __m128i result = _mm_clmulepi64_si128(a, b, 0); - uint64_t quotemask = (uint64_t)_mm_cvtsi128_si64(result); -#else // USE_NEON - // Use vmull_p64 (PMULL) for carryless multiplication on ARM - // Requires ARM crypto extensions (compile: __ARM_FEATURE_AES, runtime: pmull flag) - uint64_t quotemask = vmull_p64(ALL_ONES_MASK, quotebits); -#endif - quotemask ^= (*initial_quoted); - // Find out if the chunk ends in a quoted region by looking - // at the last bit - (*initial_quoted) = (uint64_t)((int64_t)quotemask >> 63); - - uint64_t commabits = find_character_in_chunk(in, separator); - uint64_t newlinebits = find_character_in_chunk(in, NEWLINE_CHAR); - - uint64_t delimiter_bits = (commabits | newlinebits) & ~quotemask; - return delimiter_bits; -} -#endif - -#ifdef HAS_SIMD_CSV -static size_t find_one_indices(size_t start_index, uint64_t bits, size_t *indices, size_t *base) { - size_t position = 0; - uint64_t bitset = bits; - while (bitset != 0) { - // temp only has the least significant bit of - // bitset turned on. - // In twos complement: 0 - x = ~ x + 1 - uint64_t temp = bitset & -bitset; - // count trailing zeros - size_t r = __builtin_ctzll(bitset); - indices[(*base) + position] = start_index + r; - position++; - - bitset ^= temp; - } - *base += position; - return position; -} -#endif - -size_t get_delimiter_indices(uint8_t *buf, size_t len, uint8_t separator, size_t *indices) { - // Recall we padded our file with 64 empty bytes. - // So if, for example, we had a file of 187 bytes - // We pad it with zeros and so we have 251 bytes - // The chunks we have are ptr + 0, ptr + 64, and pt - // (we don't do ptr + 192 since we do len - 64 - // below. This way, in ptr + 128 we have 59 bytes of - // actual data and 5 bytes of zeros. If we didn't do this - // we'd be reading past the end of file on the last row -#ifdef HAS_SIMD_CSV - size_t unpaddedLen = len < 64 ? 0 : len - 64; - uint64_t initial_quoted = 0ULL; - size_t base = 0; - for (size_t i = 0; i < unpaddedLen; i += 64) { - uint8_t *in = buf + i; - uint64_t delimiter_bits = parse_chunk(in, separator, &initial_quoted); - find_one_indices(i, delimiter_bits, indices, &base); - } - return base; -#else - // SIMD not available or carryless multiplication not supported. - // Signal fallback to Haskell implementation. - (void)buf; (void)len; (void)indices; - return (size_t)-1; -#endif -} diff --git a/cbits/process_csv.h b/cbits/process_csv.h deleted file mode 100644 --- a/cbits/process_csv.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef PROCESS_CSV -#define PROCESS_CSV - -#include -#include - -// Define feature macros for SIMD support with carryless multiplication -// We need both SIMD instructions AND carryless multiplication for the CSV parser -#if defined(__AVX2__) && defined(__PCLMUL__) - #define HAS_SIMD_CSV 1 - #define USE_AVX2 1 - #include - #include -#elif defined(__ARM_NEON) && (defined(__ARM_FEATURE_AES) || defined(__ARM_FEATURE_CRYPTO)) - // Note: __ARM_FEATURE_CRYPTO is deprecated; prefer __ARM_FEATURE_AES - // We need polynomial multiply (vmull_p64/PMULL) for carryless multiplication - // Runtime check: 'pmull' flag in /proc/cpuinfo on Linux - // We support both macros for compatibility with older compilers - #define HAS_SIMD_CSV 1 - #define USE_NEON 1 - #include -#endif - -// CSV parsing constants -#define COMMA_CHAR 0x2C -#define NEWLINE_CHAR 0x0A -#define QUOTE_CHAR 0x22 -#define ALL_ONES_MASK ~0ULL -#define ALL_ZEROS_MASK 0ULL - -size_t get_delimiter_indices(uint8_t *buf, size_t len, uint8_t separator, size_t* indices); - -#endif diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe -version: 1.0.0.1 +version: 1.1.0.0 synopsis: A fast, safe, and intuitive DataFrame library. @@ -16,10 +16,12 @@ category: Data tested-with: GHC ==9.4.8 || ==9.6.7 || ==9.8.4 || ==9.10.3 || ==9.12.2 extra-doc-files: CHANGELOG.md README.md -extra-source-files: cbits/process_csv.h - cbits/arrow_abi.h +extra-source-files: cbits/arrow_abi.h cbits/dataframe_arrow.h cbits/rts_init.c + tests/data/typing/texts.txt + tests/data/typing/texts_with_empties.txt + tests/data/typing/texts_with_empties_and_nullish.txt data/titanic/*.csv data/sharded/*.parquet tests/data/*.csv @@ -82,7 +84,6 @@ DataFrame.Display.Terminal.Plot, DataFrame.IO.CSV, DataFrame.IO.JSON, - DataFrame.IO.Unstable.CSV, DataFrame.IO.Parquet, DataFrame.IO.Parquet.Binary, DataFrame.IO.Parquet.Dictionary, @@ -140,8 +141,6 @@ vector-algorithms ^>= 0.9, zlib >= 0.5 && < 1, zstd >= 0.1.2.0 && < 0.2, - mmap >= 0.5.8 && < 0.6, - parallel >= 3.2.2.0 && < 5, stm >= 2.5 && < 3, filepath >= 1.4 && < 2, Glob >= 0.10 && < 1, @@ -149,10 +148,6 @@ streamly-core >= 0.2.3 && < 0.4, streamly-bytestring >= 0.2.0 && < 0.4 hs-source-dirs: src - c-sources: cbits/process_csv.c - include-dirs: cbits - includes: process_csv.h - install-includes: process_csv.h default-language: Haskell2010 foreign-library dataframe-arrow @@ -163,7 +158,7 @@ DataFrame.IR build-depends: base >= 4 && < 5, - dataframe ^>= 1, + dataframe ^>= 1.1, text >= 2.0 && < 3, aeson >= 0.11 && < 3, bytestring >= 0.11 && < 0.13, @@ -180,7 +175,7 @@ import: warnings main-is: Benchmark.hs build-depends: base >= 4 && < 5, - dataframe ^>= 1, + dataframe ^>= 1.1, random >= 1 && < 2, time >= 1.12 && < 2, vector ^>= 0.13, @@ -192,7 +187,7 @@ import: warnings main-is: Synthesis.hs build-depends: base >= 4 && < 5, - dataframe ^>= 1, + dataframe ^>= 1.1, random >= 1 && < 2, text >= 2.0 && < 3 hs-source-dirs: app @@ -219,7 +214,7 @@ build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.13, containers >= 0.6.7 && < 0.9, - dataframe ^>= 1, + dataframe ^>= 1.1, directory >= 1.3.0.0 && < 2, random >= 1 && < 2, text >= 2.0 && < 3, @@ -236,7 +231,7 @@ build-depends: base >= 4 && < 5, criterion >= 1 && < 2, process >= 1.6 && < 2, - dataframe ^>= 1, + dataframe ^>= 1.1, random >= 1 && < 2, default-language: Haskell2010 ghc-options: @@ -266,6 +261,7 @@ Operations.Nullable, Operations.Provenance, Operations.ReadCsv, + Operations.WriteCsv, Operations.Shuffle, Operations.Sort, Operations.Subset, @@ -274,13 +270,12 @@ Operations.Typing, LazyParquet, Parquet, + ParquetTestData, Properties, Monad build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.13, - dataframe ^>= 1, - bytestring >= 0.11 && < 0.13, - directory >= 1.3.0.0 && < 2, + dataframe ^>= 1.1, HUnit ^>= 1.6, QuickCheck >= 2 && < 3, random-shuffle >= 0.0.4 && < 1, diff --git a/ffi/DataFrame/IO/Arrow.hs b/ffi/DataFrame/IO/Arrow.hs --- a/ffi/DataFrame/IO/Arrow.hs +++ b/ffi/DataFrame/IO/Arrow.hs @@ -25,7 +25,7 @@ import qualified DataFrame.Internal.Column as DI import Control.Monad (foldM_, forM, join, void, when, zipWithM_) -import Data.Bits (setBit, testBit) +import Data.Bits (popCount, setBit, testBit) import Data.Int (Int32, Int64) import Data.Maybe (fromMaybe, isNothing) import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) @@ -188,25 +188,28 @@ p `at` _arrayPrivateData $ castStablePtrToPtr cleanup return p -buildBitmap :: V.Vector (Maybe a) -> IO (Ptr Word8, Int) -buildBitmap vec = do - let n = V.length vec - bitmapBytes = max 1 ((n + 7) `div` 8) - nullCount = V.length (V.filter isNothing vec) - bitmapPtr <- mallocBytes bitmapBytes :: IO (Ptr Word8) - mapM_ (\i -> pokeElemOff bitmapPtr i (0 :: Word8)) [0 .. bitmapBytes - 1] - V.iforM_ vec $ \i mv -> - case mv of - Nothing -> return () - Just _ -> do - let byteIdx = i `div` 8 - bitIdx = i `mod` 8 - b <- peekElemOff bitmapPtr byteIdx - pokeElemOff bitmapPtr byteIdx (setBit b bitIdx) +{- | Allocate an Arrow-format validity bitmap from a 'DI.Bitmap'. +Returns (ptr, nullCount). Caller must 'free' the pointer. +-} +bitmapToPtr :: Int -> DI.Bitmap -> IO (Ptr Word8, Int) +bitmapToPtr n bm = do + let numBytes = max 1 ((n + 7) `div` 8) + validCount = VU.foldl' (\acc b -> acc + popCount b) 0 bm + nullCount = n - validCount + bitmapPtr <- mallocBytes numBytes :: IO (Ptr Word8) + VU.imapM_ (pokeElemOff bitmapPtr) bm + when (VU.length bm < numBytes) $ + mapM_ + (\i -> pokeElemOff bitmapPtr i (0 :: Word8)) + [VU.length bm .. numBytes - 1] return (bitmapPtr, nullCount) +-- | Read an Arrow validity bitmap into a 'DI.Bitmap'. +readArrowBitmap :: Ptr Word8 -> Int -> IO DI.Bitmap +readArrowBitmap bitmapPtr n = VU.generateM ((n + 7) `div` 8) (peekElemOff bitmapPtr) + columnToArrow :: T.Text -> Column -> IO (Ptr ArrowSchema, Ptr ArrowArray) -columnToArrow colName (UnboxedColumn (vec :: VU.Vector a)) +columnToArrow colName (UnboxedColumn _ (vec :: VU.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = do let n = VU.length vec dataPtr <- mallocArray (max 1 n) :: IO (Ptr Int64) @@ -214,7 +217,7 @@ sPtr <- makeLeafSchema "l" colName aPtr <- makeLeafArray n 0 [nullPtr, castPtr dataPtr] (free dataPtr) return (sPtr, aPtr) -columnToArrow colName (UnboxedColumn (vec :: VU.Vector a)) +columnToArrow colName (UnboxedColumn _ (vec :: VU.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = do let n = VU.length vec dataPtr <- mallocArray (max 1 n) :: IO (Ptr Double) @@ -222,7 +225,7 @@ sPtr <- makeLeafSchema "g" colName aPtr <- makeLeafArray n 0 [nullPtr, castPtr dataPtr] (free dataPtr) return (sPtr, aPtr) -columnToArrow colName (BoxedColumn (vec :: V.Vector a)) +columnToArrow colName (BoxedColumn _ (vec :: V.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @T.Text) = do let n = V.length vec bss = map TE.encodeUtf8 (V.toList vec) @@ -250,7 +253,7 @@ [nullPtr, castPtr offPtr, castPtr charsPtr] (free offPtr >> free charsPtr) return (sPtr, aPtr) -columnToArrow colName (BoxedColumn (vec :: V.Vector a)) +columnToArrow colName (BoxedColumn _ (vec :: V.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = do let n = V.length vec dataPtr <- mallocArray (max 1 n) :: IO (Ptr Double) @@ -258,7 +261,7 @@ sPtr <- makeLeafSchema "g" colName aPtr <- makeLeafArray n 0 [nullPtr, castPtr dataPtr] (free dataPtr) return (sPtr, aPtr) -columnToArrow colName (BoxedColumn (vec :: V.Vector a)) +columnToArrow colName (BoxedColumn _ (vec :: V.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = do let n = V.length vec dataPtr <- mallocArray (max 1 n) :: IO (Ptr Int64) @@ -266,16 +269,13 @@ sPtr <- makeLeafSchema "l" colName aPtr <- makeLeafArray n 0 [nullPtr, castPtr dataPtr] (free dataPtr) return (sPtr, aPtr) -columnToArrow colName (OptionalColumn (vec :: V.Vector (Maybe a))) +-- Nullable Int (UnboxedColumn with bitmap) +columnToArrow colName (UnboxedColumn (Just bm) (vec :: VU.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = do - let n = V.length vec - (bitmapPtr, nullCount) <- buildBitmap vec + let n = VU.length vec + (bitmapPtr, nullCount) <- bitmapToPtr n bm dataPtr <- mallocArray (max 1 n) :: IO (Ptr Int64) - V.imapM_ - ( \i mv -> - pokeElemOff dataPtr i (fromIntegral (fromMaybe 0 mv) :: Int64) - ) - vec + VU.imapM_ (\i v -> pokeElemOff dataPtr i (fromIntegral v :: Int64)) vec sPtr <- makeLeafSchema "l" colName aPtr <- makeLeafArray @@ -284,16 +284,13 @@ [castPtr bitmapPtr, castPtr dataPtr] (free bitmapPtr >> free dataPtr) return (sPtr, aPtr) -columnToArrow colName (OptionalColumn (vec :: V.Vector (Maybe a))) +-- Nullable Double (UnboxedColumn with bitmap) +columnToArrow colName (UnboxedColumn (Just bm) (vec :: VU.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = do - let n = V.length vec - (bitmapPtr, nullCount) <- buildBitmap vec + let n = VU.length vec + (bitmapPtr, nullCount) <- bitmapToPtr n bm dataPtr <- mallocArray (max 1 n) :: IO (Ptr Double) - V.imapM_ - ( \i mv -> - pokeElemOff dataPtr i (maybe 0.0 realToFrac mv :: Double) - ) - vec + VU.imapM_ (\i v -> pokeElemOff dataPtr i (realToFrac v :: Double)) vec sPtr <- makeLeafSchema "g" colName aPtr <- makeLeafArray @@ -302,14 +299,18 @@ [castPtr bitmapPtr, castPtr dataPtr] (free bitmapPtr >> free dataPtr) return (sPtr, aPtr) -columnToArrow colName (OptionalColumn (vec :: V.Vector (Maybe a))) +-- Nullable Text (BoxedColumn with bitmap) +columnToArrow colName (BoxedColumn (Just bm) (vec :: V.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @T.Text) = do let n = V.length vec - texts = V.toList vec - bss = map (maybe BS.empty TE.encodeUtf8) texts + -- For null positions, use empty BS (null placeholder in vec is never evaluated) + bss = + map + (\i -> if DI.bitmapTestBit bm i then TE.encodeUtf8 (vec V.! i) else BS.empty) + [0 .. n - 1] cumOff = scanl (+) 0 (map BS.length bss) total = last cumOff - (bitmapPtr, nullCount) <- buildBitmap vec + (bitmapPtr, nullCount) <- bitmapToPtr n bm offPtr <- mallocArray (n + 1) :: IO (Ptr Int32) zipWithM_ (\i o -> pokeElemOff offPtr i (fromIntegral o :: Int32)) @@ -353,7 +354,7 @@ let nRows = case colsInOrder of [] -> 0 - _ -> DI.columnLength (snd (head colsInOrder)) + (_, col) : _ -> DI.columnLength col topSchema <- mallocBytes arrowSchemaSize fmtStr <- newCString "+s" nameStr <- newCString "" @@ -463,19 +464,12 @@ if nullCnt > 0 then do let bitmapPtr = castPtr bitmapVoid :: Ptr Word8 - vec <- V.generateM n $ \i -> do - valid <- bitmapIsSet bitmapPtr i - if valid - then do - v <- peekElemOff dataPtr i - return (Just (fromIntegral v :: Int)) - else return Nothing - return $ OptionalColumn (vec :: V.Vector (Maybe Int)) + bm <- readArrowBitmap bitmapPtr n + vec <- VU.generateM n $ \i -> fmap fromIntegral (peekElemOff dataPtr i :: IO Int64) + return $ UnboxedColumn (Just bm) (vec :: VU.Vector Int) else do - vec <- VU.generateM n $ \i -> do - v <- peekElemOff dataPtr i - return (fromIntegral v :: Int) - return $ UnboxedColumn (vec :: VU.Vector Int) + vec <- VU.generateM n $ \i -> fmap fromIntegral (peekElemOff dataPtr i :: IO Int64) + return $ UnboxedColumn Nothing (vec :: VU.Vector Int) readInt32Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column readInt32Col n nullCnt bufArr = do @@ -485,19 +479,12 @@ if nullCnt > 0 then do let bitmapPtr = castPtr bitmapVoid :: Ptr Word8 - vec <- V.generateM n $ \i -> do - valid <- bitmapIsSet bitmapPtr i - if valid - then do - v <- peekElemOff dataPtr i - return (Just (fromIntegral v :: Int)) - else return Nothing - return $ OptionalColumn (vec :: V.Vector (Maybe Int)) + bm <- readArrowBitmap bitmapPtr n + vec <- VU.generateM n $ \i -> fmap fromIntegral (peekElemOff dataPtr i :: IO Int32) + return $ UnboxedColumn (Just bm) (vec :: VU.Vector Int) else do - vec <- VU.generateM n $ \i -> do - v <- peekElemOff dataPtr i - return (fromIntegral v :: Int) - return $ UnboxedColumn (vec :: VU.Vector Int) + vec <- VU.generateM n $ \i -> fmap fromIntegral (peekElemOff dataPtr i :: IO Int32) + return $ UnboxedColumn Nothing (vec :: VU.Vector Int) readFloat64Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column readFloat64Col n nullCnt bufArr = do @@ -507,15 +494,12 @@ if nullCnt > 0 then do let bitmapPtr = castPtr bitmapVoid :: Ptr Word8 - vec <- V.generateM n $ \i -> do - valid <- bitmapIsSet bitmapPtr i - if valid - then Just <$> (peekElemOff dataPtr i :: IO Double) - else return Nothing - return $ OptionalColumn (vec :: V.Vector (Maybe Double)) + bm <- readArrowBitmap bitmapPtr n + vec <- VU.generateM n (peekElemOff dataPtr) + return $ UnboxedColumn (Just bm) (vec :: VU.Vector Double) else do vec <- VU.generateM n (peekElemOff dataPtr) - return $ UnboxedColumn (vec :: VU.Vector Double) + return $ UnboxedColumn Nothing (vec :: VU.Vector Double) readFloat32Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column readFloat32Col n nullCnt bufArr = do @@ -525,19 +509,12 @@ if nullCnt > 0 then do let bitmapPtr = castPtr bitmapVoid :: Ptr Word8 - vec <- V.generateM n $ \i -> do - valid <- bitmapIsSet bitmapPtr i - if valid - then do - v <- peekElemOff dataPtr i - return (Just (realToFrac v :: Double)) - else return Nothing - return $ OptionalColumn (vec :: V.Vector (Maybe Double)) + bm <- readArrowBitmap bitmapPtr n + vec <- VU.generateM n $ \i -> fmap (realToFrac :: Float -> Double) (peekElemOff dataPtr i) + return $ UnboxedColumn (Just bm) (vec :: VU.Vector Double) else do - vec <- VU.generateM n $ \i -> do - v <- peekElemOff dataPtr i - return (realToFrac v :: Double) - return $ UnboxedColumn (vec :: VU.Vector Double) + vec <- VU.generateM n $ \i -> fmap (realToFrac :: Float -> Double) (peekElemOff dataPtr i) + return $ UnboxedColumn Nothing (vec :: VU.Vector Double) -- | Read a large_string (format "U") column with int64 offsets. readLargeUtf8Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column @@ -547,30 +524,20 @@ charVoid <- peekElemOff bufArr 2 let offsetPtr = castPtr offsetVoid :: Ptr Int64 charPtr = castPtr charVoid :: Ptr Word8 + let readText i = do + start <- fromIntegral <$> peekElemOff offsetPtr i + end <- fromIntegral <$> peekElemOff offsetPtr (i + 1) + TE.decodeUtf8 + <$> BS.packCStringLen (castPtr (charPtr `plusPtr` start), end - start) if nullCnt > 0 then do let bitmapPtr = castPtr bitmapVoid :: Ptr Word8 - vec <- V.generateM n $ \i -> do - valid <- bitmapIsSet bitmapPtr i - if valid - then do - start <- fromIntegral <$> peekElemOff offsetPtr i - end <- fromIntegral <$> peekElemOff offsetPtr (i + 1) - bs <- - BS.packCStringLen - (castPtr (charPtr `plusPtr` start), end - start) - return $ Just (TE.decodeUtf8 bs) - else return Nothing - return $ OptionalColumn vec + bm <- readArrowBitmap bitmapPtr n + vec <- V.generateM n readText + return $ BoxedColumn (Just bm) vec else do - vec <- V.generateM n $ \i -> do - start <- fromIntegral <$> peekElemOff offsetPtr i - end <- fromIntegral <$> peekElemOff offsetPtr (i + 1) - bs <- - BS.packCStringLen - (castPtr (charPtr `plusPtr` start), end - start) - return $ TE.decodeUtf8 bs - return $ BoxedColumn vec + vec <- V.generateM n readText + return $ BoxedColumn Nothing vec -- | Read a utf8 (format "u") column with int32 offsets. readUtf8Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column @@ -580,27 +547,17 @@ charVoid <- peekElemOff bufArr 2 let offsetPtr = castPtr offsetVoid :: Ptr Int32 charPtr = castPtr charVoid :: Ptr Word8 + readText i = do + start <- fromIntegral <$> peekElemOff offsetPtr i + end <- fromIntegral <$> peekElemOff offsetPtr (i + 1) + TE.decodeUtf8 + <$> BS.packCStringLen (castPtr (charPtr `plusPtr` start), end - start) if nullCnt > 0 then do let bitmapPtr = castPtr bitmapVoid :: Ptr Word8 - vec <- V.generateM n $ \i -> do - valid <- bitmapIsSet bitmapPtr i - if valid - then do - start <- fromIntegral <$> peekElemOff offsetPtr i - end <- fromIntegral <$> peekElemOff offsetPtr (i + 1) - bs <- - BS.packCStringLen - (castPtr (charPtr `plusPtr` start), end - start) - return $ Just (TE.decodeUtf8 bs) - else return Nothing - return $ OptionalColumn vec + bm <- readArrowBitmap bitmapPtr n + vec <- V.generateM n readText + return $ BoxedColumn (Just bm) vec else do - vec <- V.generateM n $ \i -> do - start <- fromIntegral <$> peekElemOff offsetPtr i - end <- fromIntegral <$> peekElemOff offsetPtr (i + 1) - bs <- - BS.packCStringLen - (castPtr (charPtr `plusPtr` start), end - start) - return $ TE.decodeUtf8 bs - return $ BoxedColumn vec + vec <- V.generateM n readText + return $ BoxedColumn Nothing vec diff --git a/ffi/DataFrame/IR.hs b/ffi/DataFrame/IR.hs --- a/ffi/DataFrame/IR.hs +++ b/ffi/DataFrame/IR.hs @@ -113,12 +113,14 @@ -- | Build a SortOrder from a column's runtime type. mkSortOrder :: Bool -> T.Text -> Column -> SortOrder -mkSortOrder True n (UnboxedColumn (_ :: VU.Vector a)) = Asc (Col @a n) -mkSortOrder False n (UnboxedColumn (_ :: VU.Vector a)) = Desc (Col @a n) -mkSortOrder True n (BoxedColumn (_ :: V.Vector a)) = Asc (Col @a n) -mkSortOrder False n (BoxedColumn (_ :: V.Vector a)) = Desc (Col @a n) -mkSortOrder True n (OptionalColumn (_ :: V.Vector (Maybe a))) = Asc (Col @(Maybe a) n) -mkSortOrder False n (OptionalColumn (_ :: V.Vector (Maybe a))) = Desc (Col @(Maybe a) n) +mkSortOrder True n (UnboxedColumn Nothing (_ :: VU.Vector a)) = Asc (Col @a n) +mkSortOrder False n (UnboxedColumn Nothing (_ :: VU.Vector a)) = Desc (Col @a n) +mkSortOrder True n (UnboxedColumn (Just _) (_ :: VU.Vector a)) = Asc (Col @(Maybe a) n) +mkSortOrder False n (UnboxedColumn (Just _) (_ :: VU.Vector a)) = Desc (Col @(Maybe a) n) +mkSortOrder True n (BoxedColumn Nothing (_ :: V.Vector a)) = Asc (Col @a n) +mkSortOrder False n (BoxedColumn Nothing (_ :: V.Vector a)) = Desc (Col @a n) +mkSortOrder True n (BoxedColumn (Just _) (_ :: V.Vector a)) = Asc (Col @(Maybe a) n) +mkSortOrder False n (BoxedColumn (Just _) (_ :: V.Vector a)) = Desc (Col @(Maybe a) n) -- | Dispatch aggregation by fn name and runtime column type. buildNamedExpr :: DataFrame -> AggSpec -> IO NamedExpr @@ -133,22 +135,28 @@ "DataFrame.IR: unknown aggregation '" ++ T.unpack other ++ "'" countExpr :: T.Text -> T.Text -> Column -> IO NamedExpr -countExpr name colName (UnboxedColumn (_ :: VU.Vector a)) = return $ name .= count (Col @a colName) -countExpr name colName (BoxedColumn (_ :: V.Vector a)) = return $ name .= count (Col @a colName) -countExpr name colName (OptionalColumn (_ :: V.Vector (Maybe a))) = return $ name .= count (Col @(Maybe a) colName) +countExpr name colName (UnboxedColumn Nothing (_ :: VU.Vector a)) = return $ name .= count (Col @a colName) +countExpr name colName (UnboxedColumn (Just _) (_ :: VU.Vector a)) = return $ name .= count (Col @(Maybe a) colName) +countExpr name colName (BoxedColumn Nothing (_ :: V.Vector a)) = return $ name .= count (Col @a colName) +countExpr name colName (BoxedColumn (Just _) (_ :: V.Vector a)) = return $ name .= count (Col @(Maybe a) colName) sumExpr :: T.Text -> T.Text -> Column -> IO NamedExpr -sumExpr name colName (UnboxedColumn (_ :: VU.Vector a)) +sumExpr name colName (UnboxedColumn Nothing (_ :: VU.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = return $ name .= Functions.sum (Col @Int colName) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = return $ name .= Functions.sum (Col @Double colName) -sumExpr name colName (BoxedColumn (_ :: V.Vector a)) +sumExpr name colName (UnboxedColumn (Just _) (_ :: VU.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = + return $ name .= sumMaybe (Col @(Maybe Int) colName) + | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = + return $ name .= sumMaybe (Col @(Maybe Double) colName) +sumExpr name colName (BoxedColumn Nothing (_ :: V.Vector a)) + | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = return $ name .= Functions.sum (Col @Int colName) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = return $ name .= Functions.sum (Col @Double colName) -sumExpr name colName (OptionalColumn (_ :: V.Vector (Maybe a))) +sumExpr name colName (BoxedColumn (Just _) (_ :: V.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = return $ name .= sumMaybe (Col @(Maybe Int) colName) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = @@ -159,15 +167,20 @@ "DataFrame.IR: sum: unsupported column type for '" ++ T.unpack colName ++ "'" meanExpr :: T.Text -> T.Text -> Column -> IO NamedExpr -meanExpr name colName (UnboxedColumn (_ :: VU.Vector a)) +meanExpr name colName (UnboxedColumn Nothing (_ :: VU.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = return $ name .= mean (Col @Int colName) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = return $ name .= mean (Col @Double colName) -meanExpr name colName (BoxedColumn (_ :: V.Vector a)) +meanExpr name colName (UnboxedColumn (Just _) (_ :: VU.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = + return $ name .= meanMaybe (Col @(Maybe Double) colName) + | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = + return $ name .= meanMaybe (Col @(Maybe Int) colName) +meanExpr name colName (BoxedColumn Nothing (_ :: V.Vector a)) + | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = return $ name .= mean (Col @Double colName) -meanExpr name colName (OptionalColumn (_ :: V.Vector (Maybe a))) +meanExpr name colName (BoxedColumn (Just _) (_ :: V.Vector a)) | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = return $ name .= meanMaybe (Col @(Maybe Double) colName) | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = diff --git a/src/DataFrame.hs b/src/DataFrame.hs --- a/src/DataFrame.hs +++ b/src/DataFrame.hs @@ -216,7 +216,6 @@ -- * I/O module CSV, - module UnstableCSV, module Parquet, -- * Type conversion @@ -266,12 +265,6 @@ readParquetFilesWithOpts, readParquetWithOpts, ) -import DataFrame.IO.Unstable.CSV as UnstableCSV ( - fastReadCsvUnstable, - fastReadTsvUnstable, - readCsvUnstable, - readTsvUnstable, - ) import DataFrame.Internal.Column as Column ( Column, fromList, @@ -288,8 +281,10 @@ GroupedDataFrame, empty, null, + toCsv, toMarkdown, toMarkdown', + toSeparated, ) import DataFrame.Internal.Expression as Expression (Expr, prettyPrint) import DataFrame.Internal.Row as Row ( diff --git a/src/DataFrame/DecisionTree.hs b/src/DataFrame/DecisionTree.hs --- a/src/DataFrame/DecisionTree.hs +++ b/src/DataFrame/DecisionTree.hs @@ -638,18 +638,24 @@ numericCols df = concatMap extract (columnNames df) where extract col = case unsafeGetColumn col df of - UnboxedColumn (_ :: VU.Vector b) -> + UnboxedColumn Nothing (_ :: VU.Vector b) -> case testEquality (typeRep @b) (typeRep @Double) of Just Refl -> [NDouble (Col col)] Nothing -> case sIntegral @b of STrue -> [NDouble (F.toDouble (Col @b col))] SFalse -> [] - OptionalColumn (_ :: V.Vector (Maybe b)) -> + BoxedColumn (Just _) (_ :: V.Vector b) -> case testEquality (typeRep @b) (typeRep @Double) of Just Refl -> [NMaybeDouble (Col @(Maybe b) col)] Nothing -> case sIntegral @b of STrue -> [NMaybeDouble (F.whenPresent (realToFrac @b @Double) (Col @(Maybe b) col))] SFalse -> [] + UnboxedColumn (Just _) (_ :: VU.Vector b) -> + case testEquality (typeRep @b) (typeRep @Double) of + Just Refl -> [NMaybeDouble (Col @(Maybe b) col)] + Nothing -> case sIntegral @b of + STrue -> [NMaybeDouble (F.whenPresent (realToFrac @b @Double) (Col @(Maybe b) col))] + SFalse -> [] _ -> [] numericExprs :: @@ -697,18 +703,18 @@ let genConds :: T.Text -> [Expr Bool] genConds colName = case unsafeGetColumn colName df of - (BoxedColumn (col :: V.Vector a)) -> + (BoxedColumn Nothing (col :: V.Vector a)) -> let ps = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99] in map (F.lift2 (==) (Col @a colName)) ps - (OptionalColumn (col :: V.Vector (Maybe a))) -> case sFloating @a of + (BoxedColumn (Just _) (col :: V.Vector a)) -> case sFloating @a of STrue -> [] -- handled by numericCols / numericExprs SFalse -> case sIntegral @a of STrue -> [] -- handled by numericCols / numericExprs SFalse -> map - (F.lift2 (==) (Col @(Maybe a) colName) . Lit . (`percentileOrd'` col)) + (F.lift2 (==) (Col @(Maybe a) colName) . Lit . Just . (`percentileOrd'` col)) [1, 25, 75, 99] - (UnboxedColumn (_ :: VU.Vector a)) -> [] + (UnboxedColumn _ (_ :: VU.Vector a)) -> [] columnConds = concatMap @@ -724,13 +730,15 @@ ] where colConds (!l, !r) = case (unsafeGetColumn l df, unsafeGetColumn r df) of - (BoxedColumn (col1 :: V.Vector a), BoxedColumn (_ :: V.Vector b)) -> - case testEquality (typeRep @a) (typeRep @b) of - Nothing -> [] - Just Refl -> [F.lift2 (==) (Col @a l) (Col @a r)] - (UnboxedColumn (_ :: VU.Vector a), UnboxedColumn (_ :: VU.Vector b)) -> [] - ( OptionalColumn (_ :: V.Vector (Maybe a)) - , OptionalColumn (_ :: V.Vector (Maybe b)) + ( BoxedColumn Nothing (col1 :: V.Vector a) + , BoxedColumn Nothing (_ :: V.Vector b) + ) -> + case testEquality (typeRep @a) (typeRep @b) of + Nothing -> [] + Just Refl -> [F.lift2 (==) (Col @a l) (Col @a r)] + (UnboxedColumn _ (_ :: VU.Vector a), UnboxedColumn _ (_ :: VU.Vector b)) -> [] + ( BoxedColumn (Just _) (_ :: V.Vector a) + , BoxedColumn (Just _) (_ :: V.Vector b) ) -> case testEquality (typeRep @a) (typeRep @b) of Nothing -> [] Just Refl -> case testEquality (typeRep @a) (typeRep @T.Text) of diff --git a/src/DataFrame/Display/Terminal/Plot.hs b/src/DataFrame/Display/Terminal/Plot.hs --- a/src/DataFrame/Display/Terminal/Plot.hs +++ b/src/DataFrame/Display/Terminal/Plot.hs @@ -339,15 +339,12 @@ Just idx -> let col = columns df V.! idx in case col of - BoxedColumn vec -> + BoxedColumn _ vec -> let counts = countValues vec in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts] - UnboxedColumn vec -> + UnboxedColumn _ vec -> let counts = countValuesUnboxed vec in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts] - OptionalColumn vec -> - let counts = countValues vec - in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts] where countValues :: (Ord a, Show a) => V.Vector a -> [(a, Int)] countValues vec = M.toList $ V.foldr' (\x acc -> M.insertWith (+) x 1 acc) M.empty vec @@ -365,9 +362,8 @@ Just idx -> let col = columns df V.! idx in case col of - BoxedColumn vec -> V.toList $ V.map (T.pack . show) vec - UnboxedColumn vec -> V.toList $ VG.map (T.pack . show) (VG.convert vec) - OptionalColumn vec -> V.toList $ V.map (T.pack . show) vec + BoxedColumn _ vec -> V.toList $ V.map (T.pack . show) vec + UnboxedColumn _ vec -> V.toList $ VG.map (T.pack . show) (VG.convert vec) extractNumericColumn :: (HasCallStack) => T.Text -> DataFrame -> [Double] extractNumericColumn colName df = @@ -376,9 +372,8 @@ Just idx -> let col = columns df V.! idx in case col of - BoxedColumn vec -> vectorToDoubles vec - UnboxedColumn vec -> unboxedVectorToDoubles vec - _ -> [] + BoxedColumn _ vec -> vectorToDoubles vec + UnboxedColumn _ vec -> unboxedVectorToDoubles vec vectorToDoubles :: forall a. (Columnable a, Show a) => V.Vector a -> [Double] vectorToDoubles vec = diff --git a/src/DataFrame/Display/Web/Plot.hs b/src/DataFrame/Display/Web/Plot.hs --- a/src/DataFrame/Display/Web/Plot.hs +++ b/src/DataFrame/Display/Web/Plot.hs @@ -850,13 +850,10 @@ Just idx -> let col = columns df V.! idx in case col of - BoxedColumn (vec :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of + BoxedColumn _ (vec :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> V.toList vec Nothing -> V.toList $ V.map (T.pack . show) vec - UnboxedColumn vec -> V.toList $ VG.map (T.pack . show) (VG.convert vec) - OptionalColumn (vec :: V.Vector (Maybe a)) -> case testEquality (typeRep @a) (typeRep @T.Text) of - Nothing -> V.toList $ V.map (T.pack . show) vec - Just Refl -> V.toList $ V.map (maybe "Nothing" ("Just " <>)) vec + UnboxedColumn _ vec -> V.toList $ VG.map (T.pack . show) (VG.convert vec) extractNumericColumn :: (HasCallStack) => T.Text -> DataFrame -> [Double] extractNumericColumn colName df = @@ -865,9 +862,8 @@ Just idx -> let col = columns df V.! idx in case col of - BoxedColumn vec -> vectorToDoubles vec - UnboxedColumn vec -> unboxedVectorToDoubles vec - _ -> [] + BoxedColumn _ vec -> vectorToDoubles vec + UnboxedColumn _ vec -> unboxedVectorToDoubles vec vectorToDoubles :: forall a. (Columnable a, Show a) => V.Vector a -> [Double] vectorToDoubles vec = @@ -898,21 +894,14 @@ Just idx -> let col = columns df V.! idx in case col of - BoxedColumn (vec :: V.Vector a) -> + BoxedColumn _ (vec :: V.Vector a) -> let counts = countValues vec in case testEquality (typeRep @a) (typeRep @T.Text) of Nothing -> Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts] Just Refl -> Just [(k, fromIntegral v) | (k, v) <- counts] - UnboxedColumn vec -> + UnboxedColumn _ vec -> let counts = countValuesUnboxed vec in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts] - OptionalColumn (vec :: V.Vector (Maybe a)) -> - let counts = countValues vec - in case testEquality (typeRep @a) (typeRep @T.Text) of - Nothing -> Just [((T.pack . show) k, fromIntegral v) | (k, v) <- counts] - Just Refl -> - Just - [(maybe "Nothing" ("Just " <>) k, fromIntegral v) | (k, v) <- counts] where countValues :: (Ord a, Show a) => V.Vector a -> [(a, Int)] countValues vec = M.toList $ V.foldr' (\x acc -> M.insertWith (+) x 1 acc) M.empty vec diff --git a/src/DataFrame/Errors.hs b/src/DataFrame/Errors.hs --- a/src/DataFrame/Errors.hs +++ b/src/DataFrame/Errors.hs @@ -11,6 +11,7 @@ import Control.Exception import Data.Array +import qualified Data.List as L import Data.Typeable (Typeable) import DataFrame.Display.Terminal.Colours import Type.Reflection (TypeRep) @@ -29,7 +30,7 @@ TypeErrorContext a b -> DataFrameException AggregatedAndNonAggregatedException :: T.Text -> T.Text -> DataFrameException - ColumnNotFoundException :: T.Text -> T.Text -> [T.Text] -> DataFrameException + ColumnsNotFoundException :: [T.Text] -> T.Text -> [T.Text] -> DataFrameException EmptyDataSetException :: T.Text -> DataFrameException InternalException :: T.Text -> DataFrameException NonColumnReferenceException :: T.Text -> DataFrameException @@ -51,7 +52,7 @@ (errorColumnName context) (callingFunctionName context) errorString - show (ColumnNotFoundException columnName callPoint availableColumns) = columnNotFound columnName callPoint availableColumns + show (ColumnsNotFoundException columnNames callPoint availableColumns) = columnsNotFound columnNames callPoint availableColumns show (EmptyDataSetException callPoint) = emptyDataSetError callPoint show (WrongQuantileNumberException q) = wrongQuantileNumberError q show (WrongQuantileIndexException qs q) = wrongQuantileIndexError qs q @@ -65,15 +66,46 @@ ++ T.unpack expr2 columnNotFound :: T.Text -> T.Text -> [T.Text] -> String -columnNotFound name callPoint columns = +columnNotFound missingColumn = columnsNotFound [missingColumn] + +columnsNotFound :: [T.Text] -> T.Text -> [T.Text] -> String +columnsNotFound missingColumns callPoint availableColumns = red "\n\n[ERROR] " - ++ "Column not found: " - ++ T.unpack name + ++ missingColumnsLabel missingColumns + ++ ": " + ++ T.unpack (T.intercalate ", " missingColumns) ++ " for operation " ++ T.unpack callPoint - ++ "\n\tDid you mean " - ++ T.unpack (guessColumnName name columns) - ++ "?\n\n" + ++ formatSuggestions missingColumns availableColumns + ++ "\n\n" + where + missingColumnsLabel [_] = "Column not found" + missingColumnsLabel _ = "Columns not found" + + formatSuggestions [missingColumn] columns = + case guessColumnName missingColumn columns of + "" -> "" + guessed -> + "\n\tDid you mean " + ++ T.unpack guessed + ++ "?" + formatSuggestions names columns = + case traverse (`suggestColumnName` columns) names of + Just guessedColumns + | not (null guessedColumns) -> + "\n\tDid you mean " + ++ formatColumnSuggestions guessedColumns + ++ "?" + _ -> "" + + suggestColumnName missingColumn columns = case guessColumnName missingColumn columns of + "" -> Nothing + guessed -> Just guessed + + formatColumnSuggestions guessedColumns = + "[" + ++ L.intercalate ", " (map (show . T.unpack) guessedColumns) + ++ "]" typeMismatchError :: String -> String -> String typeMismatchError givenType expectedType = diff --git a/src/DataFrame/Functions.hs b/src/DataFrame/Functions.hs --- a/src/DataFrame/Functions.hs +++ b/src/DataFrame/Functions.hs @@ -628,7 +628,8 @@ in fmap concat $ forM specs $ \(raw, nm, tyStr) -> do ty <- typeFromString (words tyStr) - trace (T.unpack (nm <> " :: Expr " <> T.pack tyStr)) pure () + let tyDisplay = if ' ' `elem` tyStr then "(" <> T.pack tyStr <> ")" else T.pack tyStr + trace (T.unpack (nm <> " :: Expr " <> tyDisplay)) pure () let n = mkName (T.unpack nm) sig <- sigD n [t|Expr $(pure ty)|] val <- valD (varP n) (normalB [|col $(TH.lift raw)|]) [] diff --git a/src/DataFrame/IO/CSV.hs b/src/DataFrame/IO/CSV.hs --- a/src/DataFrame/IO/CSV.hs +++ b/src/DataFrame/IO/CSV.hs @@ -39,7 +39,7 @@ import Data.Type.Equality (TestEquality (testEquality)) import Data.Word (Word8) import DataFrame.Internal.Column -import DataFrame.Internal.DataFrame (DataFrame (..)) +import DataFrame.Internal.DataFrame (DataFrame (..), toSeparated) import DataFrame.Internal.Parsing import DataFrame.Internal.Schema import DataFrame.Operations.Typing @@ -220,7 +220,7 @@ ReadOptions { headerSpec = UseFirstRow , typeSpec = InferFromSample 100 - , safeRead = True + , safeRead = False , dateFormat = "%Y-%m-%d" , columnSeparator = ',' , numColumns = Nothing @@ -389,30 +389,33 @@ vec <- freezePagedUnboxedVector gv valid <- freezePagedUnboxedVector validRef if VU.all (== 1) valid - then return $! UnboxedColumn vec + then return $! UnboxedColumn Nothing vec else constructOptional vec valid freezeBuilderColumn (BuilderDouble gv validRef) = do vec <- freezePagedUnboxedVector gv valid <- freezePagedUnboxedVector validRef if VU.all (== 1) valid - then return $! UnboxedColumn vec + then return $! UnboxedColumn Nothing vec else constructOptional vec valid freezeBuilderColumn (BuilderText gv validRef) = do vec <- freezePagedVector gv valid <- freezePagedUnboxedVector validRef if VU.all (== 1) valid - then return $! BoxedColumn vec + then return $! BoxedColumn Nothing vec else constructOptionalBoxed vec valid freezeBuilderColumn (BuilderBS _ _) = error "freezeBuilderColumn: BuilderBS must be finalized via finalizeBuilderColumn" finalizeBuilderColumn :: ReadOptions -> BuilderColumn -> IO Column -finalizeBuilderColumn opts (BuilderBS gv validRef) = do - vec <- freezePagedVector gv - valid <- freezePagedUnboxedVector validRef - return $! inferColumnFromBS opts vec valid -finalizeBuilderColumn _ bc = freezeBuilderColumn bc +finalizeBuilderColumn opts bc = do + col <- case bc of + BuilderBS gv validRef -> do + vec <- freezePagedVector gv + valid <- freezePagedUnboxedVector validRef + return $! inferColumnFromBS opts vec valid + _ -> freezeBuilderColumn bc + return $! if safeRead opts then ensureOptional col else col inferColumnFromBS :: ReadOptions -> V.Vector BS.ByteString -> VU.Vector Word8 -> Column @@ -523,23 +526,13 @@ constructOptional :: (VU.Unbox a, Columnable a) => VU.Vector a -> VU.Vector Word8 -> IO Column constructOptional vec valid = do - let size = VU.length vec - mvec <- VM.new size - forM_ [0 .. size - 1] $ \i -> - if (valid VU.! i) == 0 - then VM.write mvec i Nothing - else VM.write mvec i (Just (vec VU.! i)) - OptionalColumn <$> V.freeze mvec + let bm = buildBitmapFromValid valid + pure $ UnboxedColumn (Just bm) vec constructOptionalBoxed :: V.Vector T.Text -> VU.Vector Word8 -> IO Column constructOptionalBoxed vec valid = do - let size = V.length vec - mvec <- VM.new size - forM_ [0 .. size - 1] $ \i -> - if (valid VU.! i) == 0 - then VM.write mvec i Nothing - else VM.write mvec i (Just (vec V.! i)) - OptionalColumn <$> V.freeze mvec + let bm = buildBitmapFromValid valid + pure $ BoxedColumn (Just bm) vec writeCsv :: FilePath -> DataFrame -> IO () writeCsv = writeSeparated ',' @@ -554,61 +547,7 @@ FilePath -> DataFrame -> IO () -writeSeparated c filepath df = withFile filepath WriteMode $ \handle -> do - let (rows, _) = dataframeDimensions df - let headers = map fst (L.sortBy (compare `on` snd) (M.toList (columnIndices df))) - TIO.hPutStrLn handle (T.intercalate "," headers) - forM_ [0 .. (rows - 1)] $ \i -> do - let row = getRowAsText df i - TIO.hPutStrLn handle (T.intercalate "," row) - -getRowAsText :: DataFrame -> Int -> [T.Text] -getRowAsText df i = V.ifoldr go [] (columns df) - where - indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df)) - go k (BoxedColumn (c :: V.Vector a)) acc = case c V.!? i of - Just e -> textRep : acc - where - textRep = case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> e - Nothing -> case typeRep @a of - App t1 t2 -> case eqTypeRep t1 (typeRep @Maybe) of - Just HRefl -> case testEquality t2 (typeRep @T.Text) of - Just Refl -> fromMaybe "null" e - Nothing -> (fromOptional . T.pack . show) e - where - fromOptional s - | T.isPrefixOf "Just " s = T.drop (T.length "Just ") s - | otherwise = "null" - Nothing -> (T.pack . show) e - _ -> (T.pack . show) e - Nothing -> - error $ - "Column " - ++ T.unpack (indexMap M.! k) - ++ " has less items than " - ++ "the other columns at index " - ++ show i - go k (UnboxedColumn c) acc = case c VU.!? i of - Just e -> T.pack (show e) : acc - Nothing -> - error $ - "Column " - ++ T.unpack (indexMap M.! k) - ++ " has less items than " - ++ "the other columns at index " - ++ show i - go k (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of - Just e -> case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> fromMaybe T.empty e : acc - Nothing -> maybe T.empty (T.pack . show) e : acc - Nothing -> - error $ - "Column " - ++ T.unpack (indexMap M.! k) - ++ " has less items than " - ++ "the other columns at index " - ++ show i +writeSeparated c filepath df = TIO.writeFile filepath (toSeparated c df) stripQuotes :: T.Text -> T.Text stripQuotes txt = diff --git a/src/DataFrame/IO/Parquet.hs b/src/DataFrame/IO/Parquet.hs --- a/src/DataFrame/IO/Parquet.hs +++ b/src/DataFrame/IO/Parquet.hs @@ -19,10 +19,11 @@ import Data.Text.Encoding import Data.Time import Data.Time.Clock.POSIX (posixSecondsToUTCTime) -import DataFrame.Errors (DataFrameException (ColumnNotFoundException)) +import qualified Data.Vector as V +import DataFrame.Errors (DataFrameException (ColumnsNotFoundException)) import DataFrame.Internal.Binary (littleEndianWord32) import qualified DataFrame.Internal.Column as DI -import DataFrame.Internal.DataFrame (DataFrame) +import DataFrame.Internal.DataFrame (DataFrame, columns) import DataFrame.Internal.Expression (Expr, getColumns) import qualified DataFrame.Operations.Core as DI import DataFrame.Operations.Merge () @@ -75,6 +76,8 @@ -- ^ Optional row filter expression applied before projection. , rowRange :: Maybe (Int, Int) -- ^ Optional row slice @(start, end)@ with start-inclusive/end-exclusive semantics. + , safeColumns :: Bool + -- ^ When True, every column is promoted to OptionalColumn after read, regardless of nullability in the schema. } deriving (Eq, Show) @@ -87,6 +90,7 @@ { selectedColumns = Nothing , predicate = Nothing , rowRange = Nothing + , safeColumns = False } @ -} @@ -96,6 +100,7 @@ { selectedColumns = Nothing , predicate = Nothing , rowRange = Nothing + , safeColumns = False } -- Public API -------------------------------------------------------------- @@ -179,16 +184,15 @@ in unless (L.null missing) ( throw - ( ColumnNotFoundException - (T.pack $ show missing) + ( ColumnsNotFoundException + missing "readParquetWithOpts" availableSelectedColumns ) ) - let totalRows = sum (map (fromIntegral . rowGroupNumRows) (rowGroups fileMetadata)) :: Int - colMutMap <- newIORef (M.empty :: M.Map T.Text DI.MutableColumn) - colOffMap <- newIORef (M.empty :: M.Map T.Text Int) + -- Collect per-column chunk lists; concatenate at the end to preserve bitmaps. + colListMap <- newIORef (M.empty :: M.Map T.Text [DI.Column]) lTypeMap <- newIORef (M.empty :: M.Map T.Text LogicalType) let schemaElements = schema fileMetadata @@ -261,22 +265,13 @@ maybeTypeLength lType - mutMapSnap <- readIORef colMutMap - case M.lookup colFullName mutMapSnap of - Nothing -> do - mc <- DI.newMutableColumn totalRows column - DI.copyIntoMutableColumn mc 0 column - modifyIORef' colMutMap (M.insert colFullName mc) - modifyIORef' colOffMap (M.insert colFullName (DI.columnLength column)) - Just mc -> do - off <- (M.! colFullName) <$> readIORef colOffMap - DI.copyIntoMutableColumn mc off column - modifyIORef' colOffMap (M.adjust (+ DI.columnLength column) colFullName) + modifyIORef' colListMap (M.insertWith (++) colFullName [column]) modifyIORef' lTypeMap (M.insert colFullName lType) - finalMutMap <- readIORef colMutMap - finalColMap <- - M.traverseWithKey (\_ mc -> DI.freezeMutableColumn mc) finalMutMap + finalListMap <- readIORef colListMap + -- Reverse the accumulated lists (they were prepended) and concat columns per-name, + -- preserving bitmaps correctly via concatManyColumns. + let finalColMap = M.map (DI.concatManyColumns . reverse) finalListMap finalLTypeMap <- readIORef lTypeMap let orderedColumns = map @@ -349,9 +344,15 @@ applyPredicate opts df = maybe df (`DS.filterWhere` df) (predicate opts) +applySafeRead :: ParquetReadOptions -> DataFrame -> DataFrame +applySafeRead opts df + | safeColumns opts = df{columns = V.map DI.ensureOptional (columns df)} + | otherwise = df + applyReadOptions :: ParquetReadOptions -> DataFrame -> DataFrame applyReadOptions opts = - applyRowRange opts + applySafeRead opts + . applyRowRange opts . applySelectedColumns opts . applyPredicate opts diff --git a/src/DataFrame/IO/Parquet/Thrift.hs b/src/DataFrame/IO/Parquet/Thrift.hs --- a/src/DataFrame/IO/Parquet/Thrift.hs +++ b/src/DataFrame/IO/Parquet/Thrift.hs @@ -68,9 +68,8 @@ let colType :: TType colType = case unsafeGetColumn colName df of - (DI.BoxedColumn (col :: V.Vector a)) -> haskellToTType @a - (DI.UnboxedColumn (col :: VU.Vector a)) -> haskellToTType @a - (DI.OptionalColumn (col :: V.Vector (Maybe a))) -> haskellToTType @a + (DI.BoxedColumn _ (col :: V.Vector a)) -> haskellToTType @a + (DI.UnboxedColumn _ (col :: VU.Vector a)) -> haskellToTType @a lType = if DI.hasElemType @T.Text (unsafeGetColumn colName df) || DI.hasElemType @(Maybe T.Text) (unsafeGetColumn colName df) diff --git a/src/DataFrame/IO/Unstable/CSV.hs b/src/DataFrame/IO/Unstable/CSV.hs deleted file mode 100644 --- a/src/DataFrame/IO/Unstable/CSV.hs +++ /dev/null @@ -1,352 +0,0 @@ -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE CApiFFI #-} -{-# LANGUAGE ForeignFunctionInterface #-} - -module DataFrame.IO.Unstable.CSV ( - fastReadCsvUnstable, - readCsvUnstable, - fastReadTsvUnstable, - readTsvUnstable, - getDelimiterIndices, -) where - -import qualified Data.Vector as Vector -import qualified Data.Vector.Storable as VS -import Data.Vector.Storable.Mutable ( - grow, - unsafeFromForeignPtr, - ) -import qualified Data.Vector.Storable.Mutable as VSM -import System.IO.MMap ( - Mode (WriteCopy), - mmapFileForeignPtr, - ) - -import Foreign ( - Ptr, - castForeignPtr, - castPtr, - mallocArray, - newForeignPtr_, - ) -import Foreign.C.Types - -import qualified Data.ByteString as BS -import Data.ByteString.Internal (ByteString (PS)) -import qualified Data.Map as M -import Data.Text (Text) -import qualified Data.Text as Text -import qualified Data.Text.Encoding as TextEncoding -import Data.Word (Word8) - -import Control.Parallel.Strategies (parList, rpar, using) -import Data.Array.IArray (array, (!)) -import Data.Array.Unboxed (UArray) -import Data.Ix (range) - -import DataFrame.IO.CSV ( - HeaderSpec (..), - ReadOptions (..), - defaultReadOptions, - shouldInferFromSample, - stripQuotes, - typeInferenceSampleSize, - ) -import DataFrame.Internal.DataFrame (DataFrame (..)) -import DataFrame.Operations.Typing (ParseOptions (..), parseFromExamples) - -readSeparatedDefaultFast :: Word8 -> FilePath -> IO DataFrame -readSeparatedDefaultFast separator = - readSeparated - separator - defaultReadOptions - getDelimiterIndices - -readSeparatedDefault :: Word8 -> FilePath -> IO DataFrame -readSeparatedDefault separator = - readSeparated - separator - defaultReadOptions - ( \separator originalLen v -> do - indices <- mallocArray originalLen - getDelimiterIndices_ separator originalLen v indices - ) - -fastReadCsvUnstable :: FilePath -> IO DataFrame -fastReadCsvUnstable = readSeparatedDefaultFast comma - -readCsvUnstable :: FilePath -> IO DataFrame -readCsvUnstable = readSeparatedDefault comma - -fastReadTsvUnstable :: FilePath -> IO DataFrame -fastReadTsvUnstable = readSeparatedDefaultFast tab - -readTsvUnstable :: FilePath -> IO DataFrame -readTsvUnstable = readSeparatedDefault tab - -readSeparated :: - Word8 -> - ReadOptions -> - (Word8 -> Int -> VS.Vector Word8 -> IO (VS.Vector CSize)) -> - FilePath -> - IO DataFrame -readSeparated separator opts delimiterIndices filePath = do - -- We use write copy mode so that we can append - -- padding to the end of the memory space - (bufferPtr, offset, len) <- - mmapFileForeignPtr - filePath - WriteCopy - Nothing - let mutableFile = unsafeFromForeignPtr bufferPtr offset len - paddedMutableFile <- grow mutableFile 64 - paddedCSVFile <- VS.unsafeFreeze paddedMutableFile - indices <- delimiterIndices separator len paddedCSVFile - let numCol = countColumnsInFirstRow paddedCSVFile indices - totalRows = VS.length indices `div` numCol - extractField' = extractField paddedCSVFile indices - (columnNames, dataStartRow) = case headerSpec opts of - NoHeader -> - ( Vector.fromList $ - map (Text.pack . show) [0 .. numCol - 1] - , 0 - ) - UseFirstRow -> - ( Vector.fromList $ - map (stripQuotes . extractField') [0 .. numCol - 1] - , 1 - ) - ProvideNames ns -> - (Vector.fromList ns, 0) - numRow = totalRows - dataStartRow - parseTypes col = - let n = - if shouldInferFromSample (typeSpec opts) - then typeInferenceSampleSize (typeSpec opts) - else 0 - parseOpts = - ParseOptions - { missingValues = missingIndicators opts - , sampleSize = n - , parseSafe = safeRead opts - , parseDateFormat = dateFormat opts - } - in parseFromExamples parseOpts col - generateColumn col = - parseTypes $ - Vector.fromListN - numRow - ( map - ( \row -> - (stripQuotes . extractField') - (row * numCol + col) - ) - [dataStartRow .. totalRows - 1] - ) - columns = - Vector.fromListN - numCol - ( map generateColumn [0 .. numCol - 1] - `using` parList rpar - ) - columnIndices = - M.fromList $ - zip (Vector.toList columnNames) [0 ..] - dataframeDimensions = (numRow, numCol) - return $ - DataFrame columns columnIndices dataframeDimensions M.empty - -{-# INLINE extractField #-} -extractField :: - VS.Vector Word8 -> - VS.Vector CSize -> - Int -> - Text -extractField file indices position = - Text.strip - . TextEncoding.decodeUtf8Lenient - . unsafeToByteString - $ VS.slice - previous - (next - previous) - file - where - previous = - if position == 0 - then 0 - else 1 + fromIntegral (indices VS.! (position - 1)) - next = fromIntegral $ indices VS.! position - unsafeToByteString :: VS.Vector Word8 -> BS.ByteString - unsafeToByteString v = PS (castForeignPtr ptr) 0 len - where - (ptr, len) = VS.unsafeToForeignPtr0 v - -foreign import capi "process_csv.h get_delimiter_indices" - get_delimiter_indices :: - Ptr CUChar -> -- input - CSize -> -- input size - CUChar -> -- separator character - Ptr CSize -> -- result array - IO CSize -- occupancy of result array - -{-# INLINE getDelimiterIndices #-} -getDelimiterIndices :: - Word8 -> - Int -> - VS.Vector Word8 -> - IO (VS.Vector CSize) -getDelimiterIndices separator originalLen csvFile = - VS.unsafeWith csvFile $ \buffer -> do - let paddedLen = VS.length csvFile - -- GC-managed pinned memory: freed automatically, no leak in streaming use. - resultMV <- VSM.unsafeNew paddedLen - num_fields <- - VSM.unsafeWith resultMV $ \indicesPtr -> - get_delimiter_indices - (castPtr buffer) - (fromIntegral paddedLen) - (fromIntegral separator) - (castPtr indicesPtr) - if num_fields == -1 - then do - -- Haskell state-machine fallback, writing directly into resultMV. - let trans = stateTransitionTable separator - processChar (!state, !idx) i byte = - case state of - UnEscaped -> - if byte == lf || byte == separator - then do - VSM.unsafeWrite resultMV idx (fromIntegral i) - return (toEnum (trans ! (fromEnum state, byte)), idx + 1) - else return (toEnum (trans ! (fromEnum state, byte)), idx) - Escaped -> - return (toEnum (trans ! (fromEnum state, byte)), idx) - (_, finalIdx) <- VS.ifoldM' processChar (UnEscaped, 0 :: Int) csvFile - finalLen <- - if originalLen > 0 && csvFile VS.! (originalLen - 1) /= lf - then do - VSM.unsafeWrite resultMV finalIdx (fromIntegral originalLen) - return (finalIdx + 1) - else return finalIdx - VS.unsafeFreeze (VSM.slice 0 finalLen resultMV) - else do - let n = fromIntegral num_fields - finalLen <- - if originalLen > 0 && csvFile VS.! (originalLen - 1) /= lf - then do - VSM.write resultMV n (fromIntegral originalLen) - return (n + 1) - else return n - VS.unsafeFreeze (VSM.slice 0 finalLen resultMV) - --- We have a Native version in case the C version --- cannot be used. For example if neither ARM_NEON --- nor AVX2 are available - -lf, cr, comma, tab, quote :: Word8 -lf = 0x0A -cr = 0x0D -comma = 0x2C -tab = 0x09 -quote = 0x22 - --- We parse using a state machine -data State - = UnEscaped -- non quoted - | Escaped -- quoted - deriving (Enum) - -{-# INLINE stateTransitionTable #-} -stateTransitionTable :: Word8 -> UArray (Int, Word8) Int -stateTransitionTable separator = array ((0, 0), (1, 255)) [(i, f i) | i <- range ((0, 0), (1, 255))] - where - f (0, character) - -- Unescaped newline - | character == 0x0A = fromEnum UnEscaped - -- Unescaped separator - | character == separator = fromEnum UnEscaped - -- Unescaped quote - | character == 0x22 = fromEnum Escaped - | otherwise = fromEnum UnEscaped - -- Escaped quote - -- escaped quote in fields are dealt as - -- consecutive quoted sections of a field - -- example: If we have - -- field1, "abc""def""ghi, field3 - -- we end up processing abc, def, and ghi - -- as consecutive quoted strings. - f (1, 0x22) = fromEnum UnEscaped - -- Everything else - f (state, _) = state - -{-# INLINE getDelimiterIndices_ #-} -getDelimiterIndices_ :: - Word8 -> - Int -> - VS.Vector Word8 -> - Ptr CSize -> - IO (VS.Vector CSize) -getDelimiterIndices_ separator originalLen csvFile resultPtr = do - resultVector <- resultVectorM - (_, resultLen) <- - VS.ifoldM' - (processCharacter resultVector) - (UnEscaped, 0) - csvFile - -- Handle the case where the file doesn't end with a newline - -- We need to add a final delimiter for the last field - finalResultLen <- - if originalLen > 0 && csvFile VS.! (originalLen - 1) /= lf - then do - VSM.write resultVector resultLen (fromIntegral originalLen) - return (resultLen + 1) - else return resultLen - VS.unsafeFreeze $ VSM.slice 0 finalResultLen resultVector - where - paddedLen = VS.length csvFile - resultVectorM = do - resultForeignPtr <- newForeignPtr_ resultPtr - return $ VSM.unsafeFromForeignPtr0 resultForeignPtr paddedLen - transitionTable = stateTransitionTable separator - processCharacter :: - VSM.IOVector CSize -> - (State, Int) -> - Int -> - Word8 -> - IO (State, Int) - processCharacter - resultVector - (!state, !resultIndex) - index - character = - case state of - UnEscaped -> - if character == lf || character == separator - then do - VSM.write - resultVector - resultIndex - (fromIntegral index) - return (newState, resultIndex + 1) - else return (newState, resultIndex) - Escaped -> return (newState, resultIndex) - where - newState = - toEnum $ - transitionTable - ! (fromEnum state, character) - -{-# INLINE countColumnsInFirstRow #-} -countColumnsInFirstRow :: - VS.Vector Word8 -> - VS.Vector CSize -> - Int -countColumnsInFirstRow file indices - | VS.length indices == 0 = 0 - | otherwise = - 1 - + VS.length - ( VS.takeWhile - (\i -> file VS.! fromIntegral i /= lf) - indices - ) diff --git a/src/DataFrame/Internal/Column.hs b/src/DataFrame/Internal/Column.hs --- a/src/DataFrame/Internal/Column.hs +++ b/src/DataFrame/Internal/Column.hs @@ -28,1329 +28,1698 @@ import Control.DeepSeq (NFData (..), rnf) import Control.Exception (throw) -import Control.Monad.ST (runST) -import Data.Kind (Type) -import Data.Maybe -import Data.These -import Data.Type.Equality (TestEquality (..)) -import DataFrame.Errors -import DataFrame.Internal.Parsing -import DataFrame.Internal.Types -import System.IO.Unsafe (unsafePerformIO) -import Type.Reflection - -{- | Our representation of a column is a GADT that can store data based on the underlying data. - -This allows us to pattern match on data kinds and limit some operations to only some -kinds of vectors. E.g. operations for missing data only happen in an OptionalColumn. --} -data Column where - BoxedColumn :: (Columnable a) => VB.Vector a -> Column - UnboxedColumn :: (Columnable a, VU.Unbox a) => VU.Vector a -> Column - OptionalColumn :: (Columnable a) => VB.Vector (Maybe a) -> Column - -{- | A mutable companion struct to dataframe columns. - -Used mostly as an intermediate structure for I/O. --} -data MutableColumn where - MBoxedColumn :: (Columnable a) => VBM.IOVector a -> MutableColumn - MUnboxedColumn :: (Columnable a, VU.Unbox a) => VUM.IOVector a -> MutableColumn - MOptionalColumn :: (Columnable a) => VBM.IOVector (Maybe a) -> MutableColumn - -{- | A TypedColumn is a wrapper around our type-erased column. -It is used to type check expressions on columns. - -Note: there is no guarantee that the Phanton type is the -same as the underlying vector type. --} -data TypedColumn a where - TColumn :: (Columnable a) => Column -> TypedColumn a - -instance (Eq a) => Eq (TypedColumn a) where - (==) :: (Eq a) => TypedColumn a -> TypedColumn a -> Bool - (==) (TColumn a) (TColumn b) = a == b - -instance (Ord a) => Ord (TypedColumn a) where - compare :: (Ord a) => TypedColumn a -> TypedColumn a -> Ordering - compare (TColumn a) (TColumn b) = compare a b - --- | Gets the underlying value from a TypedColumn. -unwrapTypedColumn :: TypedColumn a -> Column -unwrapTypedColumn (TColumn value) = value - --- | Gets the underlying vector from a TypedColumn. -vectorFromTypedColumn :: TypedColumn a -> VB.Vector a -vectorFromTypedColumn (TColumn value) = either throw id (toVector value) - --- | Checks if a column contains missing values. -hasMissing :: Column -> Bool -hasMissing (OptionalColumn column) = True -hasMissing _ = False - --- | Checks if a column contains only missing values. -allMissing :: Column -> Bool -allMissing (OptionalColumn column) = VB.length (VB.filter isNothing column) == VB.length column -allMissing _ = False - --- | Checks if a column contains numeric values. -isNumeric :: Column -> Bool -isNumeric (UnboxedColumn (vec :: VU.Vector a)) = case sNumeric @a of - STrue -> True - _ -> False -isNumeric (BoxedColumn (vec :: VB.Vector a)) = case testEquality (typeRep @a) (typeRep @Integer) of - Nothing -> False - Just Refl -> True -isNumeric _ = False - --- | Checks if a column is of a given type values. -hasElemType :: forall a. (Columnable a) => Column -> Bool -hasElemType = \case - BoxedColumn (column :: VB.Vector b) -> check (typeRep @b) - UnboxedColumn (column :: VU.Vector b) -> check (typeRep @b) - OptionalColumn (column :: VB.Vector b) -> check (typeRep @b) - where - check :: forall (b :: Type). TypeRep b -> Bool - check = isJust . testEquality (typeRep @a) - --- | An internal/debugging function to get the column type of a column. -columnVersionString :: Column -> String -columnVersionString column = case column of - BoxedColumn _ -> "Boxed" - UnboxedColumn _ -> "Unboxed" - OptionalColumn _ -> "Optional" - -{- | An internal/debugging function to get the type stored in the outermost vector -of a column. --} -columnTypeString :: Column -> String -columnTypeString column = case column of - BoxedColumn (column :: VB.Vector a) -> show (typeRep @a) - UnboxedColumn (column :: VU.Vector a) -> show (typeRep @a) - OptionalColumn (column :: VB.Vector a) -> show (typeRep @a) - -instance (Show a) => Show (TypedColumn a) where - show :: (Show a) => TypedColumn a -> String - show (TColumn col) = show col - -instance NFData Column where - rnf (BoxedColumn (v :: VB.Vector a)) = VB.foldl' (const (`seq` ())) () v - rnf (UnboxedColumn v) = v `seq` () - rnf (OptionalColumn (v :: VB.Vector (Maybe a))) = VB.foldl' (const (`seq` ())) () v - -instance Show Column where - show :: Column -> String - show (BoxedColumn column) = show column - show (UnboxedColumn column) = show column - show (OptionalColumn column) = show column - -instance Eq Column where - (==) :: Column -> Column -> Bool - (==) (BoxedColumn (a :: VB.Vector t1)) (BoxedColumn (b :: VB.Vector t2)) = - case testEquality (typeRep @t1) (typeRep @t2) of - Nothing -> False - Just Refl -> a == b - (==) (OptionalColumn (a :: VB.Vector t1)) (OptionalColumn (b :: VB.Vector t2)) = - case testEquality (typeRep @t1) (typeRep @t2) of - Nothing -> False - Just Refl -> a == b - (==) (UnboxedColumn (a :: VU.Vector t1)) (UnboxedColumn (b :: VU.Vector t2)) = - case testEquality (typeRep @t1) (typeRep @t2) of - Nothing -> False - Just Refl -> a == b - (==) _ _ = False - --- Generalised LEQ that does reflection. -generalLEQ :: - forall a b. (Typeable a, Typeable b, Ord a, Ord b) => a -> b -> Bool -generalLEQ x y = case testEquality (typeRep @a) (typeRep @b) of - Nothing -> False - Just Refl -> x <= y - -instance Ord Column where - (<=) :: Column -> Column -> Bool - (<=) (BoxedColumn (a :: VB.Vector t1)) (BoxedColumn (b :: VB.Vector t2)) = generalLEQ a b - (<=) (OptionalColumn (a :: VB.Vector t1)) (OptionalColumn (b :: VB.Vector t2)) = generalLEQ a b - (<=) (UnboxedColumn (a :: VU.Vector t1)) (UnboxedColumn (b :: VU.Vector t2)) = generalLEQ a b - (<=) _ _ = False - -{- | A class for converting a vector to a column of the appropriate type. -Given each Rep we tell the `toColumnRep` function which Column type to pick. --} -class ColumnifyRep (r :: Rep) a where - toColumnRep :: VB.Vector a -> Column - --- | Constraint synonym for what we can put into columns. -type Columnable a = - ( Columnable' a - , ColumnifyRep (KindOf a) a - , UnboxIf a - , IntegralIf a - , FloatingIf a - , SBoolI (Unboxable a) - , SBoolI (Numeric a) - , SBoolI (IntegralTypes a) - , SBoolI (FloatingTypes a) - ) - -instance - (Columnable a, VU.Unbox a) => - ColumnifyRep 'RUnboxed a - where - toColumnRep :: (Columnable a, VUM.Unbox a) => VB.Vector a -> Column - toColumnRep = UnboxedColumn . VU.convert - -instance - (Columnable a) => - ColumnifyRep 'RBoxed a - where - toColumnRep :: (Columnable a) => VB.Vector a -> Column - toColumnRep = BoxedColumn - -instance - (Columnable a) => - ColumnifyRep 'ROptional (Maybe a) - where - toColumnRep = OptionalColumn - -{- | O(n) Convert a vector to a column. Automatically picks the best representation of a vector to store the underlying data in. - -__Examples:__ - -@ -> import qualified Data.Vector as V -> fromVector (VB.fromList [(1 :: Int), 2, 3, 4]) -[1,2,3,4] -@ --} -fromVector :: - forall a. - (Columnable a, ColumnifyRep (KindOf a) a) => - VB.Vector a -> Column -fromVector = toColumnRep @(KindOf a) - -{- | O(n) Convert an unboxed vector to a column. This avoids the extra conversion if you already have the data in an unboxed vector. - -__Examples:__ - -@ -> import qualified Data.Vector.Unboxed as V -> fromUnboxedVector (VB.fromList [(1 :: Int), 2, 3, 4]) -[1,2,3,4] -@ --} -fromUnboxedVector :: - forall a. (Columnable a, VU.Unbox a) => VU.Vector a -> Column -fromUnboxedVector = UnboxedColumn - -{- | O(n) Convert a list to a column. Automatically picks the best representation of a vector to store the underlying data in. - -__Examples:__ - -@ -> fromList [(1 :: Int), 2, 3, 4] -[1,2,3,4] -@ --} -fromList :: - forall a. - (Columnable a, ColumnifyRep (KindOf a) a) => - [a] -> Column -fromList = toColumnRep @(KindOf a) . VB.fromList - --- An internal helper for type errors -throwTypeMismatch :: - forall (a :: Type) (b :: Type). - (Typeable a, Typeable b) => Either DataFrameException Column -throwTypeMismatch = - Left $ - TypeMismatchException - MkTypeErrorContext - { userType = Right (typeRep @b) - , expectedType = Right (typeRep @a) - , callingFunctionName = Nothing - , errorColumnName = Nothing - } - --- | An internal function to map a function over the values of a column. -mapColumn :: - forall b c. - (Columnable b, Columnable c) => - (b -> c) -> Column -> Either DataFrameException Column -mapColumn f = \case - BoxedColumn (col :: VB.Vector a) -> run col - OptionalColumn (col :: VB.Vector (Maybe a)) -> runOpt col - UnboxedColumn (col :: VU.Vector a) -> runUnboxed col - where - run :: forall a. (Typeable a) => VB.Vector a -> Either DataFrameException Column - run col = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> Right (fromVector @c (VB.map f col)) - Nothing -> throwTypeMismatch @a @b - - -- For OptionalColumn: first try exact match (b = Maybe a, user maps over Maybe - -- values directly), then try lenient fmap fallback (b = a, auto-fmap over inner). - runOpt :: - forall a. - (Columnable a) => VB.Vector (Maybe a) -> Either DataFrameException Column - runOpt col = case testEquality (typeRep @(Maybe a)) (typeRep @b) of - Just Refl -> Right (fromVector @c (VB.map f col)) - Nothing -> case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> Right (OptionalColumn (VB.map (fmap f) col)) - Nothing -> throwTypeMismatch @(Maybe a) @b - - runUnboxed :: - forall a. - (Typeable a, VU.Unbox a) => VU.Vector a -> Either DataFrameException Column - runUnboxed col = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> Right $ case sUnbox @c of - STrue -> UnboxedColumn (VU.map f col) - SFalse -> fromVector @c (VB.generate (VU.length col) (f . VU.unsafeIndex col)) - Nothing -> throwTypeMismatch @a @b -{-# INLINEABLE mapColumn #-} - --- | Applies a function that returns an unboxed result to an unboxed vector, storing the result in a column. -imapColumn :: - forall b c. - (Columnable b, Columnable c) => - (Int -> b -> c) -> Column -> Either DataFrameException Column -imapColumn f = \case - BoxedColumn (col :: VB.Vector a) -> run col - OptionalColumn (col :: VB.Vector (Maybe a)) -> runOpt col - UnboxedColumn (col :: VU.Vector a) -> runUnboxed col - where - run :: forall a. (Typeable a) => VB.Vector a -> Either DataFrameException Column - run col = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> Right (fromVector @c (VB.imap f col)) - Nothing -> throwTypeMismatch @a @b - - runOpt :: - forall a. - (Columnable a) => VB.Vector (Maybe a) -> Either DataFrameException Column - runOpt col = case testEquality (typeRep @(Maybe a)) (typeRep @b) of - Just Refl -> Right (fromVector @c (VB.imap f col)) - Nothing -> case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> Right (OptionalColumn (VB.imap (fmap . f) col)) - Nothing -> throwTypeMismatch @(Maybe a) @b - - runUnboxed :: - forall a. - (Typeable a, VU.Unbox a) => VU.Vector a -> Either DataFrameException Column - runUnboxed col = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> Right $ case sUnbox @c of - STrue -> UnboxedColumn (VU.imap f col) - SFalse -> BoxedColumn (VB.imap f (VG.convert col)) - Nothing -> throwTypeMismatch @a @b - --- | O(1) Gets the number of elements in the column. -columnLength :: Column -> Int -columnLength (BoxedColumn xs) = VB.length xs -columnLength (UnboxedColumn xs) = VU.length xs -columnLength (OptionalColumn xs) = VB.length xs -{-# INLINE columnLength #-} - --- | O(n) Gets the number of elements in the column. -numElements :: Column -> Int -numElements (BoxedColumn xs) = VB.length xs -numElements (UnboxedColumn xs) = VU.length xs -numElements (OptionalColumn xs) = VB.foldl' (\acc x -> acc + fromEnum (isJust x)) 0 xs -{-# INLINE numElements #-} - --- | O(n) Takes the first n values of a column. -takeColumn :: Int -> Column -> Column -takeColumn n (BoxedColumn xs) = BoxedColumn $ VG.take n xs -takeColumn n (UnboxedColumn xs) = UnboxedColumn $ VG.take n xs -takeColumn n (OptionalColumn xs) = OptionalColumn $ VG.take n xs -{-# INLINE takeColumn #-} - --- | O(n) Takes the last n values of a column. -takeLastColumn :: Int -> Column -> Column -takeLastColumn n column = sliceColumn (columnLength column - n) n column -{-# INLINE takeLastColumn #-} - --- | O(n) Takes n values after a given column index. -sliceColumn :: Int -> Int -> Column -> Column -sliceColumn start n (BoxedColumn xs) = BoxedColumn $ VG.slice start n xs -sliceColumn start n (UnboxedColumn xs) = UnboxedColumn $ VG.slice start n xs -sliceColumn start n (OptionalColumn xs) = OptionalColumn $ VG.slice start n xs -{-# INLINE sliceColumn #-} - --- | O(n) Selects the elements at a given set of indices. Does not change the order. -atIndicesStable :: VU.Vector Int -> Column -> Column -atIndicesStable indexes (BoxedColumn column) = - BoxedColumn $ - VB.generate - (VU.length indexes) - (\i -> column `VB.unsafeIndex` (indexes `VU.unsafeIndex` i)) -atIndicesStable indexes (UnboxedColumn column) = UnboxedColumn $ VU.unsafeBackpermute column indexes -atIndicesStable indexes (OptionalColumn column) = - OptionalColumn $ - VB.generate - (VU.length indexes) - (\i -> column `VB.unsafeIndex` (indexes `VU.unsafeIndex` i)) -{-# INLINE atIndicesStable #-} - -{- | Like 'atIndicesStable' but treats negative indices as null, -producing an 'OptionalColumn'. Keeps the index vector fully -unboxed (no @VB.Vector (Maybe Int)@). --} -gatherWithSentinel :: VU.Vector Int -> Column -> Column -gatherWithSentinel indices col = - let !n = VU.length indices - in case col of - BoxedColumn v -> - OptionalColumn $ - VB.generate n $ \i -> - let !idx = indices `VU.unsafeIndex` i - in if idx < 0 then Nothing else Just (v `VB.unsafeIndex` idx) - UnboxedColumn v -> - OptionalColumn $ - VB.generate n $ \i -> - let !idx = indices `VU.unsafeIndex` i - in if idx < 0 then Nothing else Just (v `VU.unsafeIndex` idx) - OptionalColumn v -> - OptionalColumn $ - VB.generate n $ \i -> - let !idx = indices `VU.unsafeIndex` i - in if idx < 0 then Nothing else v `VB.unsafeIndex` idx -{-# INLINE gatherWithSentinel #-} - --- | Internal helper to get indices in a boxed vector. -getIndices :: VU.Vector Int -> VB.Vector a -> VB.Vector a -getIndices indices xs = VB.generate (VU.length indices) (\i -> xs VB.! (indices VU.! i)) -{-# INLINE getIndices #-} - --- | Internal helper to get indices in an unboxed vector. -getIndicesUnboxed :: (VU.Unbox a) => VU.Vector Int -> VU.Vector a -> VU.Vector a -getIndicesUnboxed indices xs = VU.generate (VU.length indices) (\i -> xs VU.! (indices VU.! i)) -{-# INLINE getIndicesUnboxed #-} - -findIndices :: - forall a. - (Columnable a) => - (a -> Bool) -> - Column -> - Either DataFrameException (VU.Vector Int) -findIndices pred = \case - BoxedColumn (v :: VB.Vector b) -> run v VG.convert - OptionalColumn (v :: VB.Vector b) -> run v VG.convert - UnboxedColumn (v :: VU.Vector b) -> run v id - where - run :: - forall b v. - (Typeable b, VG.Vector v b, VG.Vector v Int) => - v b -> - (v Int -> VU.Vector Int) -> - Either DataFrameException (VU.Vector Int) - run column finalize = case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> Right . finalize $ VG.findIndices pred column - Nothing -> - Left $ - TypeMismatchException - MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @b) - , callingFunctionName = Just "findIndices" - , errorColumnName = Nothing - } - --- | An internal function that returns a vector of how indexes change after a column is sorted. -sortedIndexes :: Bool -> Column -> VU.Vector Int -sortedIndexes asc = \case - BoxedColumn column -> sortWorker VG.convert column - UnboxedColumn column -> sortWorker id column - OptionalColumn column -> sortWorker VG.convert column - where - sortWorker :: - (VG.Vector v a, Ord a, VG.Vector v (Int, a), VG.Vector v Int) => - (v Int -> VU.Vector Int) -> v a -> VU.Vector Int - sortWorker finalize column = runST $ do - withIndexes <- VG.thaw $ VG.indexed column - let cmp = if asc then compare else flip compare - VA.sortBy (\(_, b) (_, b') -> cmp b b') withIndexes - sorted <- VG.unsafeFreeze withIndexes - return $ finalize $ VG.map fst sorted -{-# INLINE sortedIndexes #-} - --- | Fold (right) column with index. -ifoldrColumn :: - forall a b. - (Columnable a, Columnable b) => - (Int -> a -> b -> b) -> b -> Column -> Either DataFrameException b -ifoldrColumn f acc = \case - BoxedColumn column -> foldrWorker column - UnboxedColumn column -> foldrWorker column - OptionalColumn column -> foldrWorker column - where - foldrWorker :: - forall c v. - (Typeable c, VG.Vector v c) => - v c -> - Either DataFrameException b - foldrWorker vec = case testEquality (typeRep @a) (typeRep @c) of - Just Refl -> pure $ VG.ifoldr f acc vec - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @c) - , callingFunctionName = Just "ifoldrColumn" - , errorColumnName = Nothing - } - ) - -foldlColumn :: - forall a b. - (Columnable a, Columnable b) => - (b -> a -> b) -> b -> Column -> Either DataFrameException b -foldlColumn f acc = \case - BoxedColumn column -> foldlWorker column - UnboxedColumn column -> foldlWorker column - OptionalColumn column -> foldlWorker column - where - foldlWorker :: - forall c v. - (Typeable c, VG.Vector v c) => - v c -> - Either DataFrameException b - foldlWorker vec = case testEquality (typeRep @a) (typeRep @c) of - Just Refl -> pure $ VG.foldl' f acc vec - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @c) - , callingFunctionName = Just "ifoldrColumn" - , errorColumnName = Nothing - } - ) - -foldl1Column :: - forall a. - (Columnable a) => - (a -> a -> a) -> Column -> Either DataFrameException a -foldl1Column f = \case - BoxedColumn column -> foldl1Worker column - UnboxedColumn column -> foldl1Worker column - OptionalColumn column -> foldl1Worker column - where - foldl1Worker :: - forall c v. - (Typeable c, VG.Vector v c) => - v c -> - Either DataFrameException a - foldl1Worker vec = case testEquality (typeRep @a) (typeRep @c) of - Just Refl -> pure $ VG.foldl1' f vec - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @c) - , callingFunctionName = Just "foldl1Column" - , errorColumnName = Nothing - } - ) - -{- | O(n) Seedless fold over groups using the first element of each group as seed. -Like 'foldDirectGroups' but for the case where no initial accumulator is available. --} -foldl1DirectGroups :: - forall a. - (Columnable a) => - (a -> a -> a) -> - Column -> - VU.Vector Int -> - VU.Vector Int -> - Either DataFrameException Column -foldl1DirectGroups f col valueIndices offsets - | VU.length offsets <= 1 = pure $ fromVector @a VB.empty - | otherwise = case col of - UnboxedColumn (vec :: VU.Vector d) -> UnboxedColumn <$> foldl1Worker vec - BoxedColumn (vec :: VB.Vector d) -> BoxedColumn <$> foldl1Worker vec - OptionalColumn (vec :: VB.Vector (Maybe d)) -> OptionalColumn <$> foldl1Worker vec - where - foldl1Worker :: - forall c v. - (Typeable c, VG.Vector v c) => - v c -> - Either DataFrameException (v c) - foldl1Worker vec = case testEquality (typeRep @a) (typeRep @c) of - Just Refl -> - Right $ - VG.generate (VU.length offsets - 1) foldGroup - where - foldGroup k = - let !s = VU.unsafeIndex offsets k - !e = VU.unsafeIndex offsets (k + 1) - !seed = VG.unsafeIndex vec (VU.unsafeIndex valueIndices s) - in go (s + 1) e seed - go !i !e !acc - | i >= e = acc - | otherwise = - go (i + 1) e $! - f acc (VG.unsafeIndex vec (VU.unsafeIndex valueIndices i)) - Nothing -> - Left $ - TypeMismatchException - MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @c) - , callingFunctionName = Just "foldl1DirectGroups" - , errorColumnName = Nothing - } -{-# INLINEABLE foldl1DirectGroups #-} - -{- | O(n) fold over groups by scanning the column LINEARLY. -rowToGroup[i] = group index for row i. -Avoids random column reads; random writes go to the accumulator array which is -small (nGroups entries) and typically cache-resident. -When @acc@ is unboxable, uses an unboxed mutable vector for the accumulator -array, eliminating pointer indirection on every read/write. --} -foldLinearGroups :: - forall b acc. - (Columnable b, Columnable acc) => - (acc -> b -> acc) -> - acc -> - Column -> - VU.Vector Int -> -- rowToGroup (length n) - Int -> -- nGroups - Either DataFrameException Column -foldLinearGroups f seed col rowToGroup nGroups - | nGroups == 0 = Right (fromVector @acc VB.empty) - | otherwise = case col of - UnboxedColumn (vec :: VU.Vector d) -> foldLinearWorker vec - BoxedColumn (vec :: VB.Vector d) -> foldLinearWorker vec - OptionalColumn (vec :: VB.Vector (Maybe d)) -> foldLinearWorker vec - where - foldLinearWorker :: - forall c v. - (Typeable c, VG.Vector v c) => - v c -> - Either DataFrameException Column - foldLinearWorker vec = case testEquality (typeRep @b) (typeRep @c) of - Just Refl -> - Right $ - unsafePerformIO $ - runWith - ( \readAt writeAt -> - VG.iforM_ vec $ \row x -> do - let !k = VG.unsafeIndex rowToGroup row - cur <- readAt k - writeAt k $! f cur x - ) - Nothing -> - Left $ - TypeMismatchException - MkTypeErrorContext - { userType = Right (typeRep @b) - , expectedType = Right (typeRep @c) - , callingFunctionName = Just "foldLinearGroups" - , errorColumnName = Nothing - } - - -- \| Allocate accumulators, run the traversal, return a frozen Column. - -- When @acc@ is unboxable, uses an unboxed mutable vector (no pointer - -- indirection per read/write) and returns UnboxedColumn directly — - -- avoiding a round-trip through VB.Vector. - runWith :: ((Int -> IO acc) -> (Int -> acc -> IO ()) -> IO ()) -> IO Column - runWith body = case sUnbox @acc of - STrue -> do - accs <- VUM.replicate nGroups seed - body (VUM.unsafeRead accs) (VUM.unsafeWrite accs) - UnboxedColumn <$> VU.unsafeFreeze accs - SFalse -> do - accs <- VBM.replicate nGroups seed - body (VBM.unsafeRead accs) (VBM.unsafeWrite accs) - fromVector @acc <$> VB.unsafeFreeze accs - {-# INLINE runWith #-} -{-# INLINEABLE foldLinearGroups #-} - -headColumn :: forall a. (Columnable a) => Column -> Either DataFrameException a -headColumn = \case - BoxedColumn col -> headWorker col - UnboxedColumn col -> headWorker col - OptionalColumn col -> headWorker col - where - headWorker :: - forall c v. - (Typeable c, VG.Vector v c) => - v c -> - Either DataFrameException a - headWorker vec = case testEquality (typeRep @a) (typeRep @c) of - Just Refl -> - if VG.null vec - then Left (EmptyDataSetException "headColumn") - else pure (VG.head vec) - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @c) - , callingFunctionName = Just "headColumn" - , errorColumnName = Nothing - } - ) - --- | An internal, column version of zip. -zipColumns :: Column -> Column -> Column -zipColumns (BoxedColumn column) (BoxedColumn other) = BoxedColumn (VG.zip column other) -zipColumns (BoxedColumn column) (UnboxedColumn other) = - BoxedColumn - ( VB.generate - (min (VG.length column) (VG.length other)) - (\i -> (column VG.! i, other VG.! i)) - ) -zipColumns (BoxedColumn column) (OptionalColumn optcolumn) = BoxedColumn (VG.zip (VB.convert column) optcolumn) -zipColumns (UnboxedColumn column) (BoxedColumn other) = - BoxedColumn - ( VB.generate - (min (VG.length column) (VG.length other)) - (\i -> (column VG.! i, other VG.! i)) - ) -zipColumns (UnboxedColumn column) (UnboxedColumn other) = UnboxedColumn (VG.zip column other) -zipColumns (UnboxedColumn column) (OptionalColumn optcolumn) = BoxedColumn (VG.zip (VB.convert column) optcolumn) -zipColumns (OptionalColumn optcolumn) (BoxedColumn column) = BoxedColumn (VG.zip optcolumn (VB.convert column)) -zipColumns (OptionalColumn optcolumn) (UnboxedColumn column) = BoxedColumn (VG.zip optcolumn (VB.convert column)) -zipColumns (OptionalColumn optcolumn) (OptionalColumn optother) = BoxedColumn (VG.zip optcolumn optother) -{-# INLINE zipColumns #-} - --- | Merge two columns using `These`. -mergeColumns :: Column -> Column -> Column -mergeColumns colA colB = case (colA, colB) of - (OptionalColumn c1, OptionalColumn c2) -> - OptionalColumn $ mkVec c1 c2 $ \v1 v2 -> - case (v1, v2) of - (Nothing, Nothing) -> Nothing - (Just x, Nothing) -> Just (This x) - (Nothing, Just y) -> Just (That y) - (Just x, Just y) -> Just (These x y) - (OptionalColumn c1, BoxedColumn c2) -> optReq c1 c2 - (OptionalColumn c1, UnboxedColumn c2) -> optReq c1 c2 - (BoxedColumn c1, OptionalColumn c2) -> reqOpt c1 c2 - (UnboxedColumn c1, OptionalColumn c2) -> reqOpt c1 c2 - (BoxedColumn c1, BoxedColumn c2) -> reqReq c1 c2 - (BoxedColumn c1, UnboxedColumn c2) -> reqReq c1 c2 - (UnboxedColumn c1, BoxedColumn c2) -> reqReq c1 c2 - (UnboxedColumn c1, UnboxedColumn c2) -> reqReq c1 c2 - where - mkVec c1 c2 combineElements = - VB.generate - (min (VG.length c1) (VG.length c2)) - (\i -> combineElements (c1 VG.! i) (c2 VG.! i)) - {-# INLINE mkVec #-} - - reqReq c1 c2 = BoxedColumn $ mkVec c1 c2 These - - reqOpt c1 c2 = BoxedColumn $ mkVec c1 c2 $ \v1 v2 -> - case v2 of - Nothing -> This v1 - Just y -> These v1 y - - optReq c1 c2 = BoxedColumn $ mkVec c1 c2 $ \v1 v2 -> - case v1 of - Nothing -> That v2 - Just x -> These x v2 -{-# INLINE mergeColumns #-} - --- | An internal, column version of zipWith. -zipWithColumns :: - forall a b c. - (Columnable a, Columnable b, Columnable c) => - (a -> b -> c) -> Column -> Column -> Either DataFrameException Column -zipWithColumns f (UnboxedColumn (column :: VU.Vector d)) (UnboxedColumn (other :: VU.Vector e)) = case testEquality (typeRep @a) (typeRep @d) of - Just Refl -> case testEquality (typeRep @b) (typeRep @e) of - Just Refl -> pure $ case sUnbox @c of - STrue -> UnboxedColumn (VU.zipWith f column other) - SFalse -> fromVector $ VB.zipWith f (VG.convert column) (VG.convert other) - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @b) - , expectedType = Right (typeRep @e) - , callingFunctionName = Just "zipWithColumns" - , errorColumnName = Nothing - } - ) - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @d) - , callingFunctionName = Just "zipWithColumns" - , errorColumnName = Nothing - } - ) --- TODO: mchavinda - reuse pattern from interpret where we augment the --- error at the end. -zipWithColumns f left right = case toVector @a left of - Left (TypeMismatchException context) -> - Left $ - TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"}) - Left e -> Left e - Right left' -> case toVector @b right of - Left (TypeMismatchException context) -> - Left $ - TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"}) - Left e -> Left e - Right right' -> pure $ fromVector $ VB.zipWith f left' right' -{-# INLINE zipWithColumns #-} - --- Functions for mutable columns (intended for IO). -writeColumn :: Int -> T.Text -> MutableColumn -> IO (Either T.Text Bool) -writeColumn i value (MBoxedColumn (col :: VBM.IOVector a)) = - let - in case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> - ( if isNullish value - then VBM.unsafeWrite col i "" >> return (Left $! value) - else VBM.unsafeWrite col i value >> return (Right True) - ) - Nothing -> return (Left value) -writeColumn i value (MOptionalColumn (col :: VBM.IOVector (Maybe a))) = - let - in case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> - ( if isNullish value - then VBM.unsafeWrite col i Nothing >> return (Left $! value) - else VBM.unsafeWrite col i (Just value) >> return (Right True) - ) - Nothing -> return (Left value) -writeColumn i value (MUnboxedColumn (col :: VUM.IOVector a)) = - case testEquality (typeRep @a) (typeRep @Int) of - Just Refl -> case readInt value of - Just v -> VUM.unsafeWrite col i v >> return (Right True) - Nothing -> VUM.unsafeWrite col i 0 >> return (Left value) - Nothing -> case testEquality (typeRep @a) (typeRep @Double) of - Nothing -> return (Left $! value) - Just Refl -> case readDouble value of - Just v -> VUM.unsafeWrite col i v >> return (Right True) - Nothing -> VUM.unsafeWrite col i 0 >> return (Left $! value) -{-# INLINE writeColumn #-} - -freezeColumn' :: [(Int, T.Text)] -> MutableColumn -> IO Column -freezeColumn' nulls (MOptionalColumn col) = OptionalColumn <$> VB.unsafeFreeze col -freezeColumn' nulls (MBoxedColumn col) - | null nulls = BoxedColumn <$> VB.unsafeFreeze col - | all (isNullish . snd) nulls = - OptionalColumn - . VB.imap (\i v -> if i `elem` map fst nulls then Nothing else Just v) - <$> VB.unsafeFreeze col - | otherwise = - BoxedColumn - . VB.imap - ( \i v -> - if i `elem` map fst nulls - then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) - else Right v - ) - <$> VB.unsafeFreeze col -freezeColumn' nulls (MUnboxedColumn col) - | null nulls = UnboxedColumn <$> VU.unsafeFreeze col - | all (isNullish . snd) nulls = - VU.unsafeFreeze col >>= \c -> - return $ - OptionalColumn $ - VB.generate - (VU.length c) - (\i -> if i `elem` map fst nulls then Nothing else Just (c VU.! i)) - | otherwise = - VU.unsafeFreeze col >>= \c -> - return $ - BoxedColumn $ - VB.generate - (VU.length c) - ( \i -> - if i `elem` map fst nulls - then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) - else Right (c VU.! i) - ) -{-# INLINE freezeColumn' #-} - --- | Fills the end of a column, up to n, with Nothing. Does nothing if column has length greater than n. -expandColumn :: Int -> Column -> Column -expandColumn n (OptionalColumn col) = OptionalColumn $ col <> VB.replicate (n - VG.length col) Nothing -expandColumn n column@(BoxedColumn col) - | n > VG.length col = - OptionalColumn $ VB.map Just col <> VB.replicate (n - VG.length col) Nothing - | otherwise = column -expandColumn n column@(UnboxedColumn col) - | n > VG.length col = - OptionalColumn $ - VB.map Just (VU.convert col) <> VB.replicate (n - VG.length col) Nothing - | otherwise = column - --- | Fills the beginning of a column, up to n, with Nothing. Does nothing if column has length greater than n. -leftExpandColumn :: Int -> Column -> Column -leftExpandColumn n column@(OptionalColumn col) - | n > VG.length col = - OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> col - | otherwise = column -leftExpandColumn n column@(BoxedColumn col) - | n > VG.length col = - OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> VG.map Just col - | otherwise = column -leftExpandColumn n column@(UnboxedColumn col) - | n > VG.length col = - OptionalColumn $ - VG.replicate (n - VG.length col) Nothing <> VG.map Just (VU.convert col) - | otherwise = column - -{- | Concatenates two columns. -Returns Nothing if the columns are of different types. --} -concatColumns :: Column -> Column -> Either DataFrameException Column -concatColumns left right = case (left, right) of - (OptionalColumn l, OptionalColumn r) -> case testEquality (typeOf l) (typeOf r) of - Just Refl -> pure (OptionalColumn $ l <> r) - Nothing -> Left (mismatchErr (typeOf r) (typeOf l)) - (BoxedColumn l, BoxedColumn r) -> case testEquality (typeOf l) (typeOf r) of - Just Refl -> pure (BoxedColumn $ l <> r) - Nothing -> Left (mismatchErr (typeOf r) (typeOf l)) - (UnboxedColumn l, UnboxedColumn r) -> case testEquality (typeOf l) (typeOf r) of - Just Refl -> pure (UnboxedColumn $ l <> r) - Nothing -> Left (mismatchErr (typeOf r) (typeOf l)) - _ -> Left (mismatchErr (typeOf right) (typeOf left)) - where - mismatchErr :: - forall (x :: Type) (y :: Type). TypeRep x -> TypeRep y -> DataFrameException - mismatchErr ta tb = - withTypeable ta $ - withTypeable tb $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right ta - , expectedType = Right tb - , callingFunctionName = Just "concatColumns" - , errorColumnName = Nothing - } - ) - -{- | Concatenates two columns. - -Works similar to 'concatColumns', but unlike that function, it will also combine columns of different types -by wrapping the values in an Either. - -E.g. combining Column containing [1,2] with Column containing ["a","b"] -will result in a Column containing [Left 1, Left 2, Right "a", Right "b"]. --} - -{- | O(n) Concatenate a list of same-type columns in a single allocation. -All columns must have the same constructor and element type (as they will -within a single Parquet column). Calls 'error' on mismatch. --} -concatManyColumns :: [Column] -> Column -concatManyColumns [] = fromList ([] :: [Maybe Int]) -concatManyColumns [c] = c -concatManyColumns (c0 : cs) = case c0 of - OptionalColumn v0 -> - let getVec (OptionalColumn v) = case testEquality (typeOf v0) (typeOf v) of - Just Refl -> v - Nothing -> error "concatManyColumns: OptionalColumn type mismatch" - getVec _ = error "concatManyColumns: column constructor mismatch" - in OptionalColumn (VB.concat (v0 : map getVec cs)) - BoxedColumn v0 -> - let getVec (BoxedColumn v) = case testEquality (typeOf v0) (typeOf v) of - Just Refl -> v - Nothing -> error "concatManyColumns: BoxedColumn type mismatch" - getVec _ = error "concatManyColumns: column constructor mismatch" - in BoxedColumn (VB.concat (v0 : map getVec cs)) - UnboxedColumn v0 -> - let getVec (UnboxedColumn v) = case testEquality (typeOf v0) (typeOf v) of - Just Refl -> v - Nothing -> error "concatManyColumns: UnboxedColumn type mismatch" - getVec _ = error "concatManyColumns: column constructor mismatch" - in UnboxedColumn (VU.concat (v0 : map getVec cs)) - -concatColumnsEither :: Column -> Column -> Column -concatColumnsEither (OptionalColumn left) (OptionalColumn right) = case testEquality (typeOf left) (typeOf right) of - Nothing -> - OptionalColumn $ fmap (fmap Left) left <> fmap (fmap Right) right - Just Refl -> - OptionalColumn $ left <> right -concatColumnsEither (BoxedColumn left) (BoxedColumn right) = case testEquality (typeOf left) (typeOf right) of - Nothing -> - BoxedColumn $ fmap Left left <> fmap Right right - Just Refl -> - BoxedColumn $ left <> right -concatColumnsEither (UnboxedColumn left) (UnboxedColumn right) = case testEquality (typeOf left) (typeOf right) of - Nothing -> - BoxedColumn $ fmap Left (VG.convert left) <> fmap Right (VG.convert right) - Just Refl -> - UnboxedColumn $ left <> right -concatColumnsEither (BoxedColumn left) (UnboxedColumn right) = - BoxedColumn $ fmap Left left <> fmap Right (VG.convert right) -concatColumnsEither (UnboxedColumn left) (BoxedColumn right) = - BoxedColumn $ fmap Left (VG.convert left) <> fmap Right right -concatColumnsEither (OptionalColumn (left :: VB.Vector (Maybe a))) (BoxedColumn (right :: VB.Vector b)) = - case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> OptionalColumn $ left <> fmap Just right - Nothing -> OptionalColumn $ fmap (fmap Left) left <> fmap (Just . Right) right -concatColumnsEither (BoxedColumn (left :: VB.Vector a)) (OptionalColumn (right :: VB.Vector (Maybe b))) = - case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> OptionalColumn $ fmap Just left <> right - Nothing -> OptionalColumn $ fmap (Just . Left) left <> fmap (fmap Right) right -concatColumnsEither (OptionalColumn (left :: VB.Vector (Maybe a))) (UnboxedColumn (right :: VU.Vector b)) = - case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> OptionalColumn $ left <> fmap Just (VG.convert right) - Nothing -> - OptionalColumn $ fmap (fmap Left) left <> fmap (Just . Right) (VG.convert right) -concatColumnsEither (UnboxedColumn (left :: VU.Vector a)) (OptionalColumn (right :: VB.Vector (Maybe b))) = - case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> OptionalColumn $ fmap Just (VG.convert left) <> right - Nothing -> - OptionalColumn $ fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right - --- | Allocate a mutable column of size @n@ matching the constructor/type of the given column. -newMutableColumn :: Int -> Column -> IO MutableColumn -newMutableColumn n (OptionalColumn (_ :: VB.Vector (Maybe a))) = - MOptionalColumn <$> (VBM.new n :: IO (VBM.IOVector (Maybe a))) -newMutableColumn n (BoxedColumn (_ :: VB.Vector a)) = - MBoxedColumn <$> (VBM.new n :: IO (VBM.IOVector a)) -newMutableColumn n (UnboxedColumn (_ :: VU.Vector a)) = - MUnboxedColumn <$> (VUM.new n :: IO (VUM.IOVector a)) - --- | Copy a column chunk into a mutable column starting at offset @off@. -copyIntoMutableColumn :: MutableColumn -> Int -> Column -> IO () -copyIntoMutableColumn (MOptionalColumn (mv :: VBM.IOVector (Maybe b))) off (OptionalColumn (v :: VB.Vector (Maybe a))) = - case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> VG.imapM_ (\i x -> VBM.unsafeWrite mv (off + i) x) v - Nothing -> error "copyIntoMutableColumn: Optional type mismatch" -copyIntoMutableColumn (MBoxedColumn (mv :: VBM.IOVector b)) off (BoxedColumn (v :: VB.Vector a)) = - case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> VG.imapM_ (\i x -> VBM.unsafeWrite mv (off + i) x) v - Nothing -> error "copyIntoMutableColumn: Boxed type mismatch" -copyIntoMutableColumn (MUnboxedColumn (mv :: VUM.IOVector b)) off (UnboxedColumn (v :: VU.Vector a)) = - case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> VG.imapM_ (\i x -> VUM.unsafeWrite mv (off + i) x) v - Nothing -> error "copyIntoMutableColumn: Unboxed type mismatch" -copyIntoMutableColumn _ _ _ = - error "copyIntoMutableColumn: constructor mismatch" - --- | Freeze a mutable column into an immutable column. -freezeMutableColumn :: MutableColumn -> IO Column -freezeMutableColumn (MOptionalColumn mv) = OptionalColumn <$> VB.unsafeFreeze mv -freezeMutableColumn (MBoxedColumn mv) = BoxedColumn <$> VB.unsafeFreeze mv -freezeMutableColumn (MUnboxedColumn mv) = UnboxedColumn <$> VU.unsafeFreeze mv - -{- | O(n) Converts a column to a list. Throws an exception if the wrong type is specified. - -__Examples:__ - -@ -> column = fromList [(1 :: Int), 2, 3, 4] -> toList @Int column -[1,2,3,4] -> toList @Double column -exception: ... -@ --} -toList :: forall a. (Columnable a) => Column -> [a] -toList xs = case toVector @a xs of - Left err -> throw err - Right val -> VB.toList val - -{- | Converts a column to a vector of a specific type. - -This is a type-safe conversion that requires the column's element type -to exactly match the requested type. You must specify the desired type -via type applications. - -==== __Type Parameters__ - -[@a@] The element type to convert to -[@v@] The vector type (e.g., 'VU.Vector', 'VB.Vector') - -==== __Examples__ - ->>> toVector @Int @VU.Vector column -Right (unboxed vector of Ints) - ->>> toVector @Text @VB.Vector column -Right (boxed vector of Text) - -==== __Returns__ - -* 'Right' - The converted vector if types match -* 'Left' 'TypeMismatchException' - If the column's type doesn't match the requested type - -==== __See also__ - -For numeric conversions with automatic type coercion, see 'toDoubleVector', -'toFloatVector', and 'toIntVector'. --} -toVector :: - forall a v. - (VG.Vector v a, Columnable a) => Column -> Either DataFrameException (v a) -toVector = \case - OptionalColumn col -> toVectorWorker col - BoxedColumn col -> toVectorWorker col - UnboxedColumn col -> toVectorWorker col - where - toVectorWorker :: - forall c w. - (Typeable c, VG.Vector w c) => - w c -> - Either DataFrameException (v a) - toVectorWorker vec = case testEquality (typeRep @a) (typeRep @c) of - Just Refl -> Right $ VG.convert vec - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Right (typeRep @c) - , callingFunctionName = Just "toVector" - , errorColumnName = Nothing - } - ) - --- Some common types we will use for numerical computing. - -{- | Converts a column to an unboxed vector of 'Double' values. - -This function performs intelligent type coercion for numeric types: - -* If the column is already 'Double', returns it directly -* If the column contains other floating-point types, converts via 'realToFrac' -* If the column contains integral types, converts via 'fromIntegral' (beware of overflow if the type is `Integer`). - -==== __Optional column handling__ - -For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number). -This allows optional numeric data to be represented in the resulting vector. - -==== __Returns__ - -* 'Right' - The converted 'Double' vector -* 'Left' 'TypeMismatchException' - If the column is not numeric --} -toDoubleVector :: Column -> Either DataFrameException (VU.Vector Double) -toDoubleVector column = - case column of - UnboxedColumn (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Double) of - Just Refl -> Right f - Nothing -> case sFloating @a of - STrue -> Right (VU.map realToFrac f) - SFalse -> case sIntegral @a of - STrue -> Right (VU.map fromIntegral f) - SFalse -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Double) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "toDoubleVector" - , errorColumnName = Nothing - } - ) - OptionalColumn (f :: VB.Vector (Maybe a)) -> case testEquality (typeRep @a) (typeRep @Double) of - Just Refl -> Right (VB.convert $ VB.map (fromMaybe (read @Double "NaN")) f) - Nothing -> case sFloating @a of - STrue -> - Right - (VB.convert $ VB.map (maybe (read @Double "NaN") realToFrac) f) - SFalse -> case sIntegral @a of - STrue -> - Right - (VB.convert $ VB.map (maybe (read @Double "NaN") fromIntegral) f) - SFalse -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Double) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "toDoubleVector" - , errorColumnName = Nothing - } - ) - BoxedColumn (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of - Just Refl -> Right (VB.convert $ VB.map fromIntegral f) - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Double) - , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) - , callingFunctionName = Just "toDoubleVector" - , errorColumnName = Nothing - } - ) - -{- | Converts a column to an unboxed vector of 'Float' values. - -This function performs intelligent type coercion for numeric types: - -* If the column is already 'Float', returns it directly -* If the column contains other floating-point types, converts via 'realToFrac' -* If the column contains integral types, converts via 'fromIntegral' -* If the column is boxed 'Integer', converts via 'fromIntegral' (beware of overflow for 64-bit integers and `Integer`) - -==== __Optional column handling__ - -For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number). -This allows optional numeric data to be represented in the resulting vector. - -==== __Returns__ - -* 'Right' - The converted 'Float' vector -* 'Left' 'TypeMismatchException' - If the column is not numeric - -==== __Precision warning__ - -Converting from 'Double' to 'Float' may result in loss of precision. --} -toFloatVector :: Column -> Either DataFrameException (VU.Vector Float) -toFloatVector column = - case column of - UnboxedColumn (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Float) of - Just Refl -> Right f - Nothing -> case sFloating @a of - STrue -> Right (VU.map realToFrac f) - SFalse -> case sIntegral @a of - STrue -> Right (VU.map fromIntegral f) - SFalse -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Float) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "toFloatVector" - , errorColumnName = Nothing - } - ) - OptionalColumn (f :: VB.Vector (Maybe a)) -> case testEquality (typeRep @a) (typeRep @Float) of - Just Refl -> Right (VB.convert $ VB.map (fromMaybe (read @Float "NaN")) f) - Nothing -> case sFloating @a of - STrue -> - Right - (VB.convert $ VB.map (maybe (read @Float "NaN") realToFrac) f) - SFalse -> case sIntegral @a of - STrue -> - Right - (VB.convert $ VB.map (maybe (read @Float "NaN") fromIntegral) f) - SFalse -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Float) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "toFloatVector" - , errorColumnName = Nothing - } - ) - BoxedColumn (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of - Just Refl -> Right (VB.convert $ VB.map fromIntegral f) - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Float) - , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) - , callingFunctionName = Just "toFloatVector" - , errorColumnName = Nothing - } - ) - -{- | Converts a column to an unboxed vector of 'Int' values. - -This function performs intelligent type coercion for numeric types: - -* If the column is already 'Int', returns it directly -* If the column contains floating-point types, rounds via 'round' and converts -* If the column contains other integral types, converts via 'fromIntegral' -* If the column is boxed 'Integer', converts via 'fromIntegral' - -==== __Returns__ - -* 'Right' - The converted 'Int' vector -* 'Left' 'TypeMismatchException' - If the column is not numeric - -==== __Note__ - -Unlike 'toDoubleVector' and 'toFloatVector', this function does NOT support -'OptionalColumn'. Optional columns must be handled separately. - -==== __Rounding behavior__ - -Floating-point values are rounded to the nearest integer using 'round'. -For example: 2.5 rounds to 2, 3.5 rounds to 4 (banker's rounding). --} -toIntVector :: Column -> Either DataFrameException (VU.Vector Int) -toIntVector column = - case column of - UnboxedColumn (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of - Just Refl -> Right f - Nothing -> case sFloating @a of - STrue -> Right (VU.map (round . realToFrac) f) - SFalse -> case sIntegral @a of - STrue -> Right (VU.map fromIntegral f) - SFalse -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Int) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "toIntVector" - , errorColumnName = Nothing - } - ) - BoxedColumn (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of - Just Refl -> Right (VB.convert $ VB.map fromIntegral f) - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Int) - , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) - , callingFunctionName = Just "toIntVector" - , errorColumnName = Nothing - } - ) - _ -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Int) - , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) - , callingFunctionName = Just "toIntVector" - , errorColumnName = Nothing - } - ) - -toUnboxedVector :: - forall a. - (Columnable a, VU.Unbox a) => Column -> Either DataFrameException (VU.Vector a) -toUnboxedVector column = - case column of - UnboxedColumn (f :: VU.Vector b) -> case testEquality (typeRep @a) (typeRep @b) of - Just Refl -> Right f - Nothing -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @Int) - , expectedType = Right (typeRep @a) - , callingFunctionName = Just "toUnboxedVector" - , errorColumnName = Nothing - } - ) - _ -> - Left $ - TypeMismatchException - ( MkTypeErrorContext - { userType = Right (typeRep @a) - , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) - , callingFunctionName = Just "toUnboxedVector" - , errorColumnName = Nothing - } - ) -{-# SPECIALIZE toUnboxedVector :: - Column -> Either DataFrameException (VU.Vector Double) - #-} +import Control.Monad (forM_, when) +import Control.Monad.ST (runST) +import Data.Bits ( + complement, + popCount, + setBit, + shiftL, + shiftR, + testBit, + (.&.), + ) +import Data.Kind (Type) +import Data.Maybe +import Data.These +import Data.Type.Equality (TestEquality (..)) +import Data.Word (Word8) +import DataFrame.Errors +import DataFrame.Internal.Parsing +import DataFrame.Internal.Types +import System.IO.Unsafe (unsafePerformIO) +import Type.Reflection + +-- | A bit-packed validity bitmap. Bit @i@ = 1 means row @i@ is valid (not null). +type Bitmap = VU.Vector Word8 + +{- | Our representation of a column is a GADT that can store data based on the underlying data. + +This allows us to pattern match on data kinds and limit some operations to only some +kinds of vectors. Nullability is represented via an optional bit-packed 'Bitmap': +@Nothing@ = no nulls; @Just bm@ = bit @i@ of @bm@ is 1 iff row @i@ is valid. +-} +data Column where + BoxedColumn :: (Columnable a) => Maybe Bitmap -> VB.Vector a -> Column + UnboxedColumn :: + (Columnable a, VU.Unbox a) => Maybe Bitmap -> VU.Vector a -> Column + +{- | A mutable companion struct to dataframe columns. + +Used mostly as an intermediate structure for I/O. +-} +data MutableColumn where + MBoxedColumn :: (Columnable a) => VBM.IOVector a -> MutableColumn + MUnboxedColumn :: (Columnable a, VU.Unbox a) => VUM.IOVector a -> MutableColumn + +-- --------------------------------------------------------------------------- +-- Bitmap helpers +-- --------------------------------------------------------------------------- + +-- | Test whether row @i@ is valid (not null) in a bitmap. +bitmapTestBit :: Bitmap -> Int -> Bool +bitmapTestBit bm i = testBit (VU.unsafeIndex bm (i `shiftR` 3)) (i .&. 7) +{-# INLINE bitmapTestBit #-} + +-- | Build a fully-valid bitmap for @n@ rows (all bits set). +allValidBitmap :: Int -> Bitmap +allValidBitmap n = + let bytes = (n + 7) `shiftR` 3 + lastBits = n .&. 7 + full = VU.replicate (bytes - 1) 0xFF + lastByte = if lastBits == 0 then 0xFF else (1 `shiftL` lastBits) - 1 + in if bytes == 0 then VU.empty else VU.snoc full lastByte +{-# INLINE allValidBitmap #-} + +{- | Build a bitmap from a @VU.Vector Word8@ validity vector +(1 = valid, 0 = null), as produced by Arrow / Parquet decoders. +-} +buildBitmapFromValid :: VU.Vector Word8 -> Bitmap +buildBitmapFromValid valid = + let n = VU.length valid + bytes = (n + 7) `shiftR` 3 + in VU.generate bytes $ \b -> + let base = b `shiftL` 3 + setBitIf acc bit = + let idx = base + bit + in if idx < n && VU.unsafeIndex valid idx /= 0 + then setBit acc bit + else acc + in foldl setBitIf (0 :: Word8) [0 .. 7] + +{- | Build a bitmap from a list of null-row indices. +@nullIdxs@ are the positions that are NULL. +-} +buildBitmapFromNulls :: Int -> [Int] -> Bitmap +buildBitmapFromNulls n nullIdxs = + let base = allValidBitmap n + in VU.modify + ( \mv -> + forM_ nullIdxs $ \i -> do + let byteIdx = i `shiftR` 3 + bitIdx = i .&. 7 + v <- VUM.unsafeRead mv byteIdx + VUM.unsafeWrite mv byteIdx (clearBit8 v bitIdx) + ) + base + where + clearBit8 :: Word8 -> Int -> Word8 + clearBit8 b bit = b .&. complement (1 `shiftL` bit) + +-- | Slice a bitmap for rows @[start .. start+len-1]@. +bitmapSlice :: Int -> Int -> Bitmap -> Bitmap +bitmapSlice start len bm + | start .&. 7 == 0 = + -- byte-aligned: simple slice; clamp so we never ask for more bytes than exist + let startByte = start `shiftR` 3 + bytes = min ((len + 7) `shiftR` 3) (VU.length bm - startByte) + in VU.slice startByte bytes bm + | otherwise = + -- non-aligned: unpack bit-by-bit and repack + let n = min len (VU.length bm `shiftL` 3 - start) + in buildBitmapFromValid $ + VU.generate n $ + \i -> if bitmapTestBit bm (start + i) then 1 else 0 + +-- | Concatenate two bitmaps covering @n1@ and @n2@ rows respectively. +bitmapConcat :: Int -> Bitmap -> Int -> Bitmap -> Bitmap +bitmapConcat n1 bm1 n2 bm2 = + buildBitmapFromValid $ + VU.generate (n1 + n2) $ \i -> + if i < n1 + then if bitmapTestBit bm1 i then 1 else 0 + else if bitmapTestBit bm2 (i - n1) then 1 else 0 + +-- | Combine two bitmaps with AND (both must be valid for result to be valid). +mergeBitmaps :: Bitmap -> Bitmap -> Bitmap +mergeBitmaps = VU.zipWith (.&.) + +{- | Materialize a nullable column from @VB.Vector (Maybe a)@. +When @a@ is unboxable, creates an 'UnboxedColumn' (more compact). +Otherwise creates a 'BoxedColumn'. +Always attaches a bitmap so the column is recognized as nullable even when +no 'Nothing' values are present (preserves the Maybe type marker). +-} +fromMaybeVec :: forall a. (Columnable a) => VB.Vector (Maybe a) -> Column +fromMaybeVec v = case sUnbox @a of + STrue -> fromMaybeVecUnboxed v + SFalse -> + let n = VB.length v + nullIdxs = [i | i <- [0 .. n - 1], isNothing (VB.unsafeIndex v i)] + bm = if null nullIdxs then allValidBitmap n else buildBitmapFromNulls n nullIdxs + dat = VB.map (fromMaybe (errorWithoutStackTrace "fromMaybeVec: Nothing slot")) v + in BoxedColumn (Just bm) dat + +{- | Materialize a nullable 'UnboxedColumn' to @VB.Vector (Maybe a)@ using runST. +Always attaches a bitmap so the column is recognized as nullable even when +no 'Nothing' values are present (preserves the Maybe type marker). +-} +fromMaybeVecUnboxed :: + forall a. (Columnable a, VU.Unbox a) => VB.Vector (Maybe a) -> Column +fromMaybeVecUnboxed v = + let n = VB.length v + nullIdxs = [i | i <- [0 .. n - 1], isNothing (VB.unsafeIndex v i)] + bm = if null nullIdxs then allValidBitmap n else buildBitmapFromNulls n nullIdxs + dat = runST $ do + mv <- VUM.new n + VG.iforM_ v $ \i mx -> forM_ mx (VUM.unsafeWrite mv i) + VU.unsafeFreeze mv + in UnboxedColumn (Just bm) dat + +-- | Materialize an element from a column at index @i@, respecting the bitmap. +columnElemIsNull :: Column -> Int -> Bool +columnElemIsNull (BoxedColumn (Just bm) _) i = not (bitmapTestBit bm i) +columnElemIsNull (UnboxedColumn (Just bm) _) i = not (bitmapTestBit bm i) +columnElemIsNull _ _ = False + +-- | Return the 'Maybe Bitmap' from a column. +columnBitmap :: Column -> Maybe Bitmap +columnBitmap (BoxedColumn bm _) = bm +columnBitmap (UnboxedColumn bm _) = bm + +-- --------------------------------------------------------------------------- +-- End bitmap helpers +-- --------------------------------------------------------------------------- + +{- | A TypedColumn is a wrapper around our type-erased column. +It is used to type check expressions on columns. + +Note: there is no guarantee that the Phanton type is the +same as the underlying vector type. +-} +data TypedColumn a where + TColumn :: (Columnable a) => Column -> TypedColumn a + +instance (Eq a) => Eq (TypedColumn a) where + (==) :: (Eq a) => TypedColumn a -> TypedColumn a -> Bool + (==) (TColumn a) (TColumn b) = a == b + +instance (Ord a) => Ord (TypedColumn a) where + compare :: (Ord a) => TypedColumn a -> TypedColumn a -> Ordering + compare (TColumn a) (TColumn b) = compare a b + +-- | Gets the underlying value from a TypedColumn. +unwrapTypedColumn :: TypedColumn a -> Column +unwrapTypedColumn (TColumn value) = value + +-- | Gets the underlying vector from a TypedColumn. +vectorFromTypedColumn :: TypedColumn a -> VB.Vector a +vectorFromTypedColumn (TColumn value) = either throw id (toVector value) + +-- | Checks if a column contains missing values (has a bitmap). +hasMissing :: Column -> Bool +hasMissing (BoxedColumn (Just _) _) = True +hasMissing (UnboxedColumn (Just _) _) = True +hasMissing _ = False + +-- | Checks if a column contains only missing values. +allMissing :: Column -> Bool +allMissing (BoxedColumn (Just bm) col) = VU.all (== 0) bm && not (VB.null col) +allMissing (UnboxedColumn (Just bm) col) = VU.all (== 0) bm && not (VU.null col) +allMissing _ = False + +-- | Checks if a column contains numeric values. +isNumeric :: Column -> Bool +isNumeric (UnboxedColumn _ (vec :: VU.Vector a)) = case sNumeric @a of + STrue -> True + _ -> False +isNumeric (BoxedColumn _ (vec :: VB.Vector a)) = case testEquality (typeRep @a) (typeRep @Integer) of + Nothing -> False + Just Refl -> True + +{- | Checks if a column is of a given type values. +For nullable columns (@BoxedColumn (Just _)@ or @UnboxedColumn (Just _)@), +also returns @True@ when @a = Maybe b@ and the column stores @b@ internally. +-} +hasElemType :: forall a. (Columnable a) => Column -> Bool +hasElemType = \case + BoxedColumn bm (column :: VB.Vector b) -> checkBoxed bm (typeRep @b) + UnboxedColumn bm (column :: VU.Vector b) -> checkUnboxed bm (typeRep @b) + where + -- Direct type match + directMatch :: forall (b :: Type). TypeRep b -> Bool + directMatch = isJust . testEquality (typeRep @a) + -- For a nullable column (has bitmap), also accept a = Maybe b + checkMaybe :: forall (b :: Type). TypeRep b -> Bool + checkMaybe tb = case typeRep @a of + App tMaybe tInner -> case eqTypeRep tMaybe (typeRep @Maybe) of + Just HRefl -> isJust (testEquality tInner tb) + Nothing -> False + _ -> False + checkBoxed :: forall (b :: Type). Maybe Bitmap -> TypeRep b -> Bool + checkBoxed bm tb = directMatch tb || (isJust bm && checkMaybe tb) + checkUnboxed :: forall (b :: Type). Maybe Bitmap -> TypeRep b -> Bool + checkUnboxed bm tb = directMatch tb || (isJust bm && checkMaybe tb) + +-- | An internal/debugging function to get the column type of a column. +columnVersionString :: Column -> String +columnVersionString column = case column of + BoxedColumn Nothing _ -> "Boxed" + BoxedColumn (Just _) _ -> "NullableBoxed" + UnboxedColumn Nothing _ -> "Unboxed" + UnboxedColumn (Just _) _ -> "NullableUnboxed" + +{- | An internal/debugging function to get the type stored in the outermost vector +of a column. +-} +columnTypeString :: Column -> String +columnTypeString column = case column of + BoxedColumn Nothing (_ :: VB.Vector a) -> show (typeRep @a) + BoxedColumn (Just _) (_ :: VB.Vector a) -> showMaybeType @a + UnboxedColumn Nothing (_ :: VU.Vector a) -> show (typeRep @a) + UnboxedColumn (Just _) (_ :: VU.Vector a) -> showMaybeType @a + where + showMaybeType :: forall a. (Typeable a) => String + showMaybeType = + let s = show (typeRep @a) + in "Maybe " ++ if ' ' `elem` s then "(" ++ s ++ ")" else s + +instance (Show a) => Show (TypedColumn a) where + show :: (Show a) => TypedColumn a -> String + show (TColumn col) = show col + +instance NFData Column where + rnf (BoxedColumn Nothing (v :: VB.Vector a)) = VB.foldl' (const (`seq` ())) () v + rnf (BoxedColumn (Just bm) (v :: VB.Vector a)) = + let n = VB.length v + go !i + | i >= n = () + | bitmapTestBit bm i = VB.unsafeIndex v i `seq` go (i + 1) + | otherwise = go (i + 1) + in go 0 + rnf (UnboxedColumn _ v) = v `seq` () + +instance Show Column where + show :: Column -> String + show (BoxedColumn Nothing column) = show column + show (BoxedColumn (Just bm) column) = + let n = VB.length column + elems = + [ if bitmapTestBit bm i then show (VB.unsafeIndex column i) else "null" + | i <- [0 .. n - 1] + ] + in "[" ++ foldl (\acc e -> if null acc then e else acc ++ "," ++ e) "" elems ++ "]" + show (UnboxedColumn Nothing column) = show column + show (UnboxedColumn (Just bm) column) = + let n = VU.length column + elems = + [ if bitmapTestBit bm i then show (VU.unsafeIndex column i) else "null" + | i <- [0 .. n - 1] + ] + in "[" ++ foldl (\acc e -> if null acc then e else acc ++ "," ++ e) "" elems ++ "]" + +{- | Compare two nullable boxed columns element by element, skipping null slots. +Uses a manual loop to avoid stream fusion forcing null-slot error thunks. +-} +eqBoxedCols :: + (Eq a) => Maybe Bitmap -> VB.Vector a -> Maybe Bitmap -> VB.Vector a -> Bool +eqBoxedCols bm1 a bm2 b + | VB.length a /= VB.length b = False + | otherwise = go 0 + where + !n = VB.length a + go !i + | i >= n = True + | nullA || nullB = (nullA == nullB) && go (i + 1) + | VB.unsafeIndex a i == VB.unsafeIndex b i = go (i + 1) + | otherwise = False + where + nullA = maybe False (\bm -> not (bitmapTestBit bm i)) bm1 + nullB = maybe False (\bm -> not (bitmapTestBit bm i)) bm2 +{-# INLINE eqBoxedCols #-} + +instance Eq Column where + (==) :: Column -> Column -> Bool + (==) (BoxedColumn bm1 (a :: VB.Vector t1)) (BoxedColumn bm2 (b :: VB.Vector t2)) = + case testEquality (typeRep @t1) (typeRep @t2) of + Nothing -> False + Just Refl -> eqBoxedCols bm1 a bm2 b + (==) (UnboxedColumn bm1 (a :: VU.Vector t1)) (UnboxedColumn bm2 (b :: VU.Vector t2)) = + case testEquality (typeRep @t1) (typeRep @t2) of + Nothing -> False + Just Refl -> + VU.length a == VU.length b + && VU.and + ( VU.imap + ( \i x -> + let nullA = maybe False (\bm -> not (bitmapTestBit bm i)) bm1 + nullB = maybe False (\bm -> not (bitmapTestBit bm i)) bm2 + in if nullA || nullB then nullA == nullB else x == VU.unsafeIndex b i + ) + a + ) + (==) _ _ = False + +-- Generalised LEQ that does reflection. +generalLEQ :: + forall a b. (Typeable a, Typeable b, Ord a, Ord b) => a -> b -> Bool +generalLEQ x y = case testEquality (typeRep @a) (typeRep @b) of + Nothing -> False + Just Refl -> x <= y + +instance Ord Column where + (<=) :: Column -> Column -> Bool + (<=) (BoxedColumn _ (a :: VB.Vector t1)) (BoxedColumn _ (b :: VB.Vector t2)) = generalLEQ a b + (<=) (UnboxedColumn _ (a :: VU.Vector t1)) (UnboxedColumn _ (b :: VU.Vector t2)) = generalLEQ a b + (<=) _ _ = False + +{- | A class for converting a vector to a column of the appropriate type. +Given each Rep we tell the `toColumnRep` function which Column type to pick. +-} +class ColumnifyRep (r :: Rep) a where + toColumnRep :: VB.Vector a -> Column + +-- | Constraint synonym for what we can put into columns. +type Columnable a = + ( Columnable' a + , ColumnifyRep (KindOf a) a + , UnboxIf a + , IntegralIf a + , FloatingIf a + , SBoolI (Unboxable a) + , SBoolI (Numeric a) + , SBoolI (IntegralTypes a) + , SBoolI (FloatingTypes a) + ) + +instance + (Columnable a, VU.Unbox a) => + ColumnifyRep 'RUnboxed a + where + toColumnRep :: (Columnable a, VUM.Unbox a) => VB.Vector a -> Column + toColumnRep v = UnboxedColumn Nothing (VU.convert v) + +instance + (Columnable a) => + ColumnifyRep 'RBoxed a + where + toColumnRep :: (Columnable a) => VB.Vector a -> Column + toColumnRep = BoxedColumn Nothing + +instance + (Columnable a) => + ColumnifyRep 'RNullableBoxed (Maybe a) + where + toColumnRep :: (Columnable a) => VB.Vector (Maybe a) -> Column + toColumnRep = fromMaybeVec + +{- | O(n) Convert a vector to a column. Automatically picks the best representation of a vector to store the underlying data in. + +__Examples:__ + +@ +> import qualified Data.Vector as V +> fromVector (VB.fromList [(1 :: Int), 2, 3, 4]) +[1,2,3,4] +@ +-} +fromVector :: + forall a. + (Columnable a, ColumnifyRep (KindOf a) a) => + VB.Vector a -> Column +fromVector = toColumnRep @(KindOf a) + +{- | O(n) Convert an unboxed vector to a column. This avoids the extra conversion if you already have the data in an unboxed vector. + +__Examples:__ + +@ +> import qualified Data.Vector.Unboxed as V +> fromUnboxedVector (VB.fromList [(1 :: Int), 2, 3, 4]) +[1,2,3,4] +@ +-} +fromUnboxedVector :: + forall a. (Columnable a, VU.Unbox a) => VU.Vector a -> Column +fromUnboxedVector = UnboxedColumn Nothing + +{- | O(n) Convert a list to a column. Automatically picks the best representation of a vector to store the underlying data in. + +__Examples:__ + +@ +> fromList [(1 :: Int), 2, 3, 4] +[1,2,3,4] +@ +-} +fromList :: + forall a. + (Columnable a, ColumnifyRep (KindOf a) a) => + [a] -> Column +fromList = toColumnRep @(KindOf a) . VB.fromList + +-- An internal helper for type errors +throwTypeMismatch :: + forall (a :: Type) (b :: Type). + (Typeable a, Typeable b) => Either DataFrameException Column +throwTypeMismatch = + Left $ + TypeMismatchException + MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @a) + , callingFunctionName = Nothing + , errorColumnName = Nothing + } + +-- | An internal function to map a function over the values of a column. +mapColumn :: + forall b c. + (Columnable b, Columnable c) => + (b -> c) -> Column -> Either DataFrameException Column +mapColumn f = \case + BoxedColumn bm (col :: VB.Vector a) -> runBoxed bm col + UnboxedColumn bm (col :: VU.Vector a) -> runUnboxed bm col + where + runBoxed :: + forall a. + (Columnable a) => + Maybe Bitmap -> VB.Vector a -> Either DataFrameException Column + runBoxed bm col = case testEquality (typeRep @b) (typeRep @(Maybe a)) of + -- user maps over Maybe a (nullable column as Maybe) + Just Refl -> + let !n = VB.length col + in -- Build result directly without intermediate Maybe vector to avoid + -- fusion forcing null slots via VU.convert. + Right $ case sUnbox @c of + STrue -> UnboxedColumn Nothing $ + VU.generate n $ \i -> + f + ( if maybe True (`bitmapTestBit` i) bm + then Just (VB.unsafeIndex col i) + else Nothing + ) + SFalse -> fromVector @c $ + VB.generate n $ \i -> + f + ( if maybe True (`bitmapTestBit` i) bm + then Just (VB.unsafeIndex col i) + else Nothing + ) + Nothing -> case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> + -- user maps over inner type a; preserve bitmap + Right $ case sUnbox @c of + STrue -> UnboxedColumn bm (VU.generate (VB.length col) (f . VB.unsafeIndex col)) + SFalse -> BoxedColumn bm (VB.map f col) + Nothing -> throwTypeMismatch @a @b + + runUnboxed :: + forall a. + (Columnable a, VU.Unbox a) => + Maybe Bitmap -> VU.Vector a -> Either DataFrameException Column + runUnboxed bm col = case testEquality (typeRep @b) (typeRep @(Maybe a)) of + Just Refl -> + let !n = VU.length col + in Right $ case sUnbox @c of + STrue -> UnboxedColumn Nothing $ + VU.generate n $ \i -> + f + ( if maybe True (`bitmapTestBit` i) bm + then Just (VU.unsafeIndex col i) + else Nothing + ) + SFalse -> fromVector @c $ + VB.generate n $ \i -> + f + ( if maybe True (`bitmapTestBit` i) bm + then Just (VU.unsafeIndex col i) + else Nothing + ) + Nothing -> case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right $ case sUnbox @c of + STrue -> UnboxedColumn bm (VU.map f col) + SFalse -> BoxedColumn bm (VB.generate (VU.length col) (f . VU.unsafeIndex col)) + Nothing -> throwTypeMismatch @a @b +{-# INLINEABLE mapColumn #-} + +-- | Applies a function that returns an unboxed result to an unboxed vector, storing the result in a column. +imapColumn :: + forall b c. + (Columnable b, Columnable c) => + (Int -> b -> c) -> Column -> Either DataFrameException Column +imapColumn f = \case + BoxedColumn bm (col :: VB.Vector a) -> runBoxed bm col + UnboxedColumn bm (col :: VU.Vector a) -> runUnboxed bm col + where + runBoxed :: + forall a. + (Columnable a) => + Maybe Bitmap -> VB.Vector a -> Either DataFrameException Column + runBoxed bm col = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right $ case sUnbox @c of + STrue -> + UnboxedColumn + bm + (VU.generate (VB.length col) (\i -> f i (VB.unsafeIndex col i))) + SFalse -> BoxedColumn bm (VB.imap f col) + Nothing -> throwTypeMismatch @a @b + + runUnboxed :: + forall a. + (Columnable a, VU.Unbox a) => + Maybe Bitmap -> VU.Vector a -> Either DataFrameException Column + runUnboxed bm col = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right $ case sUnbox @c of + STrue -> UnboxedColumn bm (VU.imap f col) + SFalse -> BoxedColumn bm (VB.imap f (VG.convert col)) + Nothing -> throwTypeMismatch @a @b + +-- | O(1) Gets the number of elements in the column. +columnLength :: Column -> Int +columnLength (BoxedColumn _ xs) = VB.length xs +columnLength (UnboxedColumn _ xs) = VU.length xs +{-# INLINE columnLength #-} + +-- | O(n) Gets the number of non-null elements in the column. +numElements :: Column -> Int +numElements (BoxedColumn Nothing xs) = VB.length xs +numElements (BoxedColumn (Just bm) xs) = VU.foldl' (\acc b -> acc + popCount b) 0 bm +numElements (UnboxedColumn Nothing xs) = VU.length xs +numElements (UnboxedColumn (Just bm) xs) = VU.foldl' (\acc b -> acc + popCount b) 0 bm +{-# INLINE numElements #-} + +-- | O(n) Takes the first n values of a column. +takeColumn :: Int -> Column -> Column +takeColumn n (BoxedColumn bm xs) = + BoxedColumn (fmap (bitmapSlice 0 n) bm) (VG.take n xs) +takeColumn n (UnboxedColumn bm xs) = + UnboxedColumn (fmap (bitmapSlice 0 n) bm) (VG.take n xs) +{-# INLINE takeColumn #-} + +-- | O(n) Takes the last n values of a column. +takeLastColumn :: Int -> Column -> Column +takeLastColumn n column = sliceColumn (columnLength column - n) n column +{-# INLINE takeLastColumn #-} + +-- | O(n) Takes n values after a given column index. +sliceColumn :: Int -> Int -> Column -> Column +sliceColumn start n (BoxedColumn bm xs) = + BoxedColumn (fmap (bitmapSlice start n) bm) (VG.slice start n xs) +sliceColumn start n (UnboxedColumn bm xs) = + UnboxedColumn (fmap (bitmapSlice start n) bm) (VG.slice start n xs) +{-# INLINE sliceColumn #-} + +-- | O(n) Selects the elements at a given set of indices. Does not change the order. +atIndicesStable :: VU.Vector Int -> Column -> Column +atIndicesStable indexes (BoxedColumn bm column) = + BoxedColumn + ( fmap + ( \bm0 -> + buildBitmapFromValid $ + VU.map (\i -> if bitmapTestBit bm0 i then 1 else 0) indexes + ) + bm + ) + ( VB.generate + (VU.length indexes) + ((column `VB.unsafeIndex`) . (indexes `VU.unsafeIndex`)) + ) +atIndicesStable indexes (UnboxedColumn bm column) = + UnboxedColumn + ( fmap + ( \bm0 -> + buildBitmapFromValid $ + VU.map (\i -> if bitmapTestBit bm0 i then 1 else 0) indexes + ) + bm + ) + (VU.unsafeBackpermute column indexes) +{-# INLINE atIndicesStable #-} + +{- | Like 'atIndicesStable' but treats negative indices as null. +Keeps the index vector fully unboxed (no @VB.Vector (Maybe Int)@). +-} +gatherWithSentinel :: VU.Vector Int -> Column -> Column +gatherWithSentinel indices col = + let !n = VU.length indices + newBm = buildBitmapFromValid $ VU.generate n $ \i -> + if VU.unsafeIndex indices i < 0 then 0 else 1 + in case col of + BoxedColumn srcBm v -> + let dat = VB.generate n $ \i -> + let !idx = VU.unsafeIndex indices i + in if idx < 0 then VB.unsafeIndex v 0 else VB.unsafeIndex v idx + bm = case srcBm of + Nothing -> Just newBm + Just sb -> + Just + ( mergeBitmaps + newBm + ( buildBitmapFromValid $ VU.generate n $ \i -> + let idx = VU.unsafeIndex indices i + in if idx >= 0 && bitmapTestBit sb idx then 1 else 0 + ) + ) + in BoxedColumn bm dat + UnboxedColumn srcBm v -> + let dat = runST $ do + mv <- VUM.new n + VG.iforM_ indices $ \i idx -> + when (idx >= 0) $ VUM.unsafeWrite mv i (VU.unsafeIndex v idx) + VU.unsafeFreeze mv + bm = case srcBm of + Nothing -> Just newBm + Just sb -> + Just + ( mergeBitmaps + newBm + ( buildBitmapFromValid $ VU.generate n $ \i -> + let idx = VU.unsafeIndex indices i + in if idx >= 0 && bitmapTestBit sb idx then 1 else 0 + ) + ) + in UnboxedColumn bm dat +{-# INLINE gatherWithSentinel #-} + +-- | Internal helper to get indices in a boxed vector. +getIndices :: VU.Vector Int -> VB.Vector a -> VB.Vector a +getIndices indices xs = VB.generate (VU.length indices) (\i -> xs VB.! (indices VU.! i)) +{-# INLINE getIndices #-} + +-- | Internal helper to get indices in an unboxed vector. +getIndicesUnboxed :: (VU.Unbox a) => VU.Vector Int -> VU.Vector a -> VU.Vector a +getIndicesUnboxed indices xs = VU.generate (VU.length indices) (\i -> xs VU.! (indices VU.! i)) +{-# INLINE getIndicesUnboxed #-} + +findIndices :: + forall a. + (Columnable a) => + (a -> Bool) -> + Column -> + Either DataFrameException (VU.Vector Int) +findIndices pred = \case + BoxedColumn _ (v :: VB.Vector b) -> run v VG.convert + UnboxedColumn _ (v :: VU.Vector b) -> run v id + where + run :: + forall b v. + (Typeable b, VG.Vector v b, VG.Vector v Int) => + v b -> + (v Int -> VU.Vector Int) -> + Either DataFrameException (VU.Vector Int) + run column finalize = case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right . finalize $ VG.findIndices pred column + Nothing -> + Left $ + TypeMismatchException + MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @b) + , callingFunctionName = Just "findIndices" + , errorColumnName = Nothing + } + +-- | An internal function that returns a vector of how indexes change after a column is sorted. +sortedIndexes :: Bool -> Column -> VU.Vector Int +sortedIndexes asc = \case + BoxedColumn _ column -> sortWorker VG.convert column + UnboxedColumn _ column -> sortWorker id column + where + sortWorker :: + (VG.Vector v a, Ord a, VG.Vector v (Int, a), VG.Vector v Int) => + (v Int -> VU.Vector Int) -> v a -> VU.Vector Int + sortWorker finalize column = runST $ do + withIndexes <- VG.thaw $ VG.indexed column + let cmp = if asc then compare else flip compare + VA.sortBy (\(_, b) (_, b') -> cmp b b') withIndexes + sorted <- VG.unsafeFreeze withIndexes + return $ finalize $ VG.map fst sorted +{-# INLINE sortedIndexes #-} + +-- | Fold (right) column with index. +ifoldrColumn :: + forall a b. + (Columnable a, Columnable b) => + (Int -> a -> b -> b) -> b -> Column -> Either DataFrameException b +ifoldrColumn f acc = \case + BoxedColumn _ column -> foldrWorker column + UnboxedColumn _ column -> foldrWorker column + where + foldrWorker :: + forall c v. + (Typeable c, VG.Vector v c) => + v c -> + Either DataFrameException b + foldrWorker vec = case testEquality (typeRep @a) (typeRep @c) of + Just Refl -> pure $ VG.ifoldr f acc vec + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @c) + , callingFunctionName = Just "ifoldrColumn" + , errorColumnName = Nothing + } + ) + +foldlColumn :: + forall a b. + (Columnable a, Columnable b) => + (b -> a -> b) -> b -> Column -> Either DataFrameException b +foldlColumn f acc = \case + BoxedColumn _ column -> foldlWorker column + UnboxedColumn _ column -> foldlWorker column + where + foldlWorker :: + forall c v. + (Typeable c, VG.Vector v c) => + v c -> + Either DataFrameException b + foldlWorker vec = case testEquality (typeRep @a) (typeRep @c) of + Just Refl -> pure $ VG.foldl' f acc vec + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @c) + , callingFunctionName = Just "ifoldrColumn" + , errorColumnName = Nothing + } + ) + +foldl1Column :: + forall a. + (Columnable a) => + (a -> a -> a) -> Column -> Either DataFrameException a +foldl1Column f = \case + BoxedColumn _ column -> foldl1Worker column + UnboxedColumn _ column -> foldl1Worker column + where + foldl1Worker :: + forall c v. + (Typeable c, VG.Vector v c) => + v c -> + Either DataFrameException a + foldl1Worker vec = case testEquality (typeRep @a) (typeRep @c) of + Just Refl -> pure $ VG.foldl1' f vec + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @c) + , callingFunctionName = Just "foldl1Column" + , errorColumnName = Nothing + } + ) + +{- | O(n) Seedless fold over groups using the first element of each group as seed. +Like 'foldDirectGroups' but for the case where no initial accumulator is available. +-} +foldl1DirectGroups :: + forall a. + (Columnable a) => + (a -> a -> a) -> + Column -> + VU.Vector Int -> + VU.Vector Int -> + Either DataFrameException Column +foldl1DirectGroups f col valueIndices offsets + | VU.length offsets <= 1 = pure $ fromVector @a VB.empty + | otherwise = case col of + UnboxedColumn _ (vec :: VU.Vector d) -> UnboxedColumn Nothing <$> foldl1Worker vec + BoxedColumn _ (vec :: VB.Vector d) -> BoxedColumn Nothing <$> foldl1Worker vec + where + foldl1Worker :: + forall c v. + (Typeable c, VG.Vector v c) => + v c -> + Either DataFrameException (v c) + foldl1Worker vec = case testEquality (typeRep @a) (typeRep @c) of + Just Refl -> + Right $ + VG.generate (VU.length offsets - 1) foldGroup + where + foldGroup k = + let !s = VU.unsafeIndex offsets k + !e = VU.unsafeIndex offsets (k + 1) + !seed = VG.unsafeIndex vec (VU.unsafeIndex valueIndices s) + in go (s + 1) e seed + go !i !e !acc + | i >= e = acc + | otherwise = + go (i + 1) e $! + f acc (VG.unsafeIndex vec (VU.unsafeIndex valueIndices i)) + Nothing -> + Left $ + TypeMismatchException + MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @c) + , callingFunctionName = Just "foldl1DirectGroups" + , errorColumnName = Nothing + } +{-# INLINEABLE foldl1DirectGroups #-} + +{- | O(n) fold over groups by scanning the column LINEARLY. +rowToGroup[i] = group index for row i. +Avoids random column reads; random writes go to the accumulator array which is +small (nGroups entries) and typically cache-resident. +When @acc@ is unboxable, uses an unboxed mutable vector for the accumulator +array, eliminating pointer indirection on every read/write. +-} +foldLinearGroups :: + forall b acc. + (Columnable b, Columnable acc) => + (acc -> b -> acc) -> + acc -> + Column -> + VU.Vector Int -> -- rowToGroup (length n) + Int -> -- nGroups + Either DataFrameException Column +foldLinearGroups f seed col rowToGroup nGroups + | nGroups == 0 = Right (fromVector @acc VB.empty) + | otherwise = case col of + UnboxedColumn _ (vec :: VU.Vector d) -> foldLinearWorker vec + BoxedColumn _ (vec :: VB.Vector d) -> foldLinearWorker vec + where + foldLinearWorker :: + forall c v. + (Typeable c, VG.Vector v c) => + v c -> + Either DataFrameException Column + foldLinearWorker vec = case testEquality (typeRep @b) (typeRep @c) of + Just Refl -> + Right $ + unsafePerformIO $ + runWith + ( \readAt writeAt -> + VG.iforM_ vec $ \row x -> do + let !k = VG.unsafeIndex rowToGroup row + cur <- readAt k + writeAt k $! f cur x + ) + Nothing -> + Left $ + TypeMismatchException + MkTypeErrorContext + { userType = Right (typeRep @b) + , expectedType = Right (typeRep @c) + , callingFunctionName = Just "foldLinearGroups" + , errorColumnName = Nothing + } + + -- \| Allocate accumulators, run the traversal, return a frozen Column. + -- When @acc@ is unboxable, uses an unboxed mutable vector (no pointer + -- indirection per read/write) and returns UnboxedColumn directly — + -- avoiding a round-trip through VB.Vector. + runWith :: ((Int -> IO acc) -> (Int -> acc -> IO ()) -> IO ()) -> IO Column + runWith body = case sUnbox @acc of + STrue -> do + accs <- VUM.replicate nGroups seed + body (VUM.unsafeRead accs) (VUM.unsafeWrite accs) + UnboxedColumn Nothing <$> VU.unsafeFreeze accs + SFalse -> do + accs <- VBM.replicate nGroups seed + body (VBM.unsafeRead accs) (VBM.unsafeWrite accs) + fromVector @acc <$> VB.unsafeFreeze accs + {-# INLINE runWith #-} +{-# INLINEABLE foldLinearGroups #-} + +headColumn :: forall a. (Columnable a) => Column -> Either DataFrameException a +headColumn = \case + BoxedColumn _ col -> headWorker col + UnboxedColumn _ col -> headWorker col + where + headWorker :: + forall c v. + (Typeable c, VG.Vector v c) => + v c -> + Either DataFrameException a + headWorker vec = case testEquality (typeRep @a) (typeRep @c) of + Just Refl -> + if VG.null vec + then Left (EmptyDataSetException "headColumn") + else pure (VG.head vec) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @c) + , callingFunctionName = Just "headColumn" + , errorColumnName = Nothing + } + ) + +-- | An internal, column version of zip. +zipColumns :: Column -> Column -> Column +zipColumns (BoxedColumn _ column) (BoxedColumn _ other) = BoxedColumn Nothing (VG.zip column other) +zipColumns (BoxedColumn _ column) (UnboxedColumn _ other) = + BoxedColumn + Nothing + ( VB.generate + (min (VG.length column) (VG.length other)) + (\i -> (column VG.! i, other VG.! i)) + ) +zipColumns (UnboxedColumn _ column) (BoxedColumn _ other) = + BoxedColumn + Nothing + ( VB.generate + (min (VG.length column) (VG.length other)) + (\i -> (column VG.! i, other VG.! i)) + ) +zipColumns (UnboxedColumn _ column) (UnboxedColumn _ other) = UnboxedColumn Nothing (VG.zip column other) +{-# INLINE zipColumns #-} + +-- | Merge two columns using `These`. +mergeColumns :: Column -> Column -> Column +mergeColumns colA colB = case (colA, colB) of + (BoxedColumn bmA c1, BoxedColumn bmB c2) -> case (bmA, bmB) of + (Just ba, Just bb) -> + BoxedColumn Nothing $ mkVec c1 c2 $ \i v1 v2 -> + let nullA = not (bitmapTestBit ba i) + nullB = not (bitmapTestBit bb i) + in case (nullA, nullB) of + (True, True) -> error "mergeColumns: both null" + (False, True) -> This v1 + (True, False) -> That v2 + (False, False) -> These v1 v2 + (Just ba, Nothing) -> + BoxedColumn Nothing $ mkVec c1 c2 $ \i v1 v2 -> + if not (bitmapTestBit ba i) then That v2 else These v1 v2 + (Nothing, Just bb) -> + BoxedColumn Nothing $ mkVec c1 c2 $ \i v1 v2 -> + if not (bitmapTestBit bb i) then This v1 else These v1 v2 + (Nothing, Nothing) -> + BoxedColumn Nothing $ mkVecSimple c1 c2 These + (BoxedColumn _ c1, UnboxedColumn _ c2) -> + BoxedColumn Nothing $ mkVecSimple c1 c2 These + (UnboxedColumn _ c1, BoxedColumn _ c2) -> + BoxedColumn Nothing $ mkVecSimple c1 c2 These + (UnboxedColumn _ c1, UnboxedColumn _ c2) -> + BoxedColumn Nothing $ mkVecSimple c1 c2 These + where + mkVec c1 c2 combineElements = + VB.generate + (min (VG.length c1) (VG.length c2)) + (\i -> combineElements i (c1 VG.! i) (c2 VG.! i)) + {-# INLINE mkVec #-} + + mkVecSimple c1 c2 f = + VB.generate + (min (VG.length c1) (VG.length c2)) + (\i -> f (c1 VG.! i) (c2 VG.! i)) + {-# INLINE mkVecSimple #-} +{-# INLINE mergeColumns #-} + +-- | An internal, column version of zipWith. +zipWithColumns :: + forall a b c. + (Columnable a, Columnable b, Columnable c) => + (a -> b -> c) -> Column -> Column -> Either DataFrameException Column +zipWithColumns f (UnboxedColumn bmL (column :: VU.Vector d)) (UnboxedColumn bmR (other :: VU.Vector e)) = case testEquality (typeRep @a) (typeRep @d) of + Just Refl -> case testEquality (typeRep @b) (typeRep @e) of + Just Refl + -- Fast path: both plain unboxed, no bitmaps involved in the output type + | isNothing bmL + , isNothing bmR -> + pure $ case sUnbox @c of + STrue -> UnboxedColumn Nothing (VU.zipWith f column other) + SFalse -> fromVector $ VB.zipWith f (VG.convert column) (VG.convert other) + -- Type mismatch or bitmap involvement: fall through to general toVector path + _ -> zipWithColumnsGeneral f (UnboxedColumn bmL column) (UnboxedColumn bmR other) + Nothing -> zipWithColumnsGeneral f (UnboxedColumn bmL column) (UnboxedColumn bmR other) +-- TODO: mchavinda - reuse pattern from interpret where we augment the +-- error at the end. +zipWithColumns f left right = zipWithColumnsGeneral f left right + +zipWithColumnsGeneral :: + forall a b c. + (Columnable a, Columnable b, Columnable c) => + (a -> b -> c) -> Column -> Column -> Either DataFrameException Column +zipWithColumnsGeneral f left right = case toVector @a left of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"}) + Left e -> Left e + Right left' -> case toVector @b right of + Left (TypeMismatchException context) -> + Left $ + TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"}) + Left e -> Left e + Right right' -> pure $ fromVector $ VB.zipWith f left' right' +{-# INLINE zipWithColumnsGeneral #-} +{-# INLINE zipWithColumns #-} + +-- Functions for mutable columns (intended for IO). +writeColumn :: Int -> T.Text -> MutableColumn -> IO (Either T.Text Bool) +writeColumn i value (MBoxedColumn (col :: VBM.IOVector a)) = + case testEquality (typeRep @a) (typeRep @T.Text) of + Just Refl -> + if isNullish value + then VBM.unsafeWrite col i "" >> return (Left $! value) + else VBM.unsafeWrite col i value >> return (Right True) + Nothing -> return (Left value) +writeColumn i value (MUnboxedColumn (col :: VUM.IOVector a)) = + case testEquality (typeRep @a) (typeRep @Int) of + Just Refl -> case readInt value of + Just v -> VUM.unsafeWrite col i v >> return (Right True) + Nothing -> VUM.unsafeWrite col i 0 >> return (Left value) + Nothing -> case testEquality (typeRep @a) (typeRep @Double) of + Nothing -> return (Left $! value) + Just Refl -> case readDouble value of + Just v -> VUM.unsafeWrite col i v >> return (Right True) + Nothing -> VUM.unsafeWrite col i 0 >> return (Left $! value) +{-# INLINE writeColumn #-} + +freezeColumn' :: [(Int, T.Text)] -> MutableColumn -> IO Column +freezeColumn' nulls (MBoxedColumn col) + | null nulls = BoxedColumn Nothing <$> VB.unsafeFreeze col + | all (isNullish . snd) nulls = do + frozen <- VB.unsafeFreeze col + let n = VB.length frozen + bm = buildBitmapFromNulls n (map fst nulls) + return $ BoxedColumn (Just bm) frozen + | otherwise = + BoxedColumn Nothing + . VB.imap + ( \i v -> + if i `elem` map fst nulls + then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) + else Right v + ) + <$> VB.unsafeFreeze col +freezeColumn' nulls (MUnboxedColumn col) + | null nulls = UnboxedColumn Nothing <$> VU.unsafeFreeze col + | all (isNullish . snd) nulls = do + c <- VU.unsafeFreeze col + let n = VU.length c + bm = buildBitmapFromNulls n (map fst nulls) + return $ UnboxedColumn (Just bm) c + | otherwise = do + c <- VU.unsafeFreeze col + return $ + BoxedColumn Nothing $ + VB.generate + (VU.length c) + ( \i -> + if i `elem` map fst nulls + then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) + else Right (c VU.! i) + ) +{-# INLINE freezeColumn' #-} + +{- | Promote a non-nullable column to a nullable one (add an all-valid bitmap). +No-op when already nullable. +-} +ensureOptional :: Column -> Column +ensureOptional c@(BoxedColumn (Just _) _) = c +ensureOptional (BoxedColumn Nothing col) = + BoxedColumn (Just (allValidBitmap (VB.length col))) col +ensureOptional c@(UnboxedColumn (Just _) _) = c +ensureOptional (UnboxedColumn Nothing col) = + UnboxedColumn (Just (allValidBitmap (VU.length col))) col + +-- | Fills the end of a column, up to n, with null rows. Does nothing if column has length >= n. +expandColumn :: Int -> Column -> Column +expandColumn n column@(BoxedColumn bm col) + | n <= VG.length col = column + | otherwise = + let extra = n - VG.length col + newBm = case bm of + Nothing -> Just (buildBitmapFromNulls n [VG.length col .. n - 1]) + Just b -> + Just + (bitmapConcat (VG.length col) b extra (VU.replicate ((extra + 7) `shiftR` 3) 0)) + -- pad data with default (undefined slot, protected by bitmap) + newCol = col <> VB.replicate extra (errorWithoutStackTrace "expandColumn: null slot") + in BoxedColumn newBm newCol +expandColumn n column@(UnboxedColumn bm col) + | n <= VG.length col = column + | otherwise = + let extra = n - VG.length col + newBm = case bm of + Nothing -> Just (buildBitmapFromNulls n [VG.length col .. n - 1]) + Just b -> + Just + (bitmapConcat (VG.length col) b extra (VU.replicate ((extra + 7) `shiftR` 3) 0)) + newCol = runST $ do + mv <- VUM.new n + VU.imapM_ (VUM.unsafeWrite mv) col + VU.unsafeFreeze mv + in UnboxedColumn newBm newCol + +-- | Fills the beginning of a column, up to n, with null rows. Does nothing if column has length >= n. +leftExpandColumn :: Int -> Column -> Column +leftExpandColumn n column@(BoxedColumn bm col) + | n <= VG.length col = column + | otherwise = + let extra = n - VG.length col + origLen = VG.length col + newBm = case bm of + Nothing -> Just (buildBitmapFromNulls n [0 .. extra - 1]) + Just b -> + let nullPart = VU.replicate ((extra + 7) `shiftR` 3) 0 + in Just (bitmapConcat extra nullPart origLen b) + newCol = + VB.replicate extra (errorWithoutStackTrace "leftExpandColumn: null slot") <> col + in BoxedColumn newBm newCol +leftExpandColumn n column@(UnboxedColumn bm col) + | n <= VG.length col = column + | otherwise = + let extra = n - VG.length col + origLen = VG.length col + newBm = case bm of + Nothing -> Just (buildBitmapFromNulls n [0 .. extra - 1]) + Just b -> + let nullPart = VU.replicate ((extra + 7) `shiftR` 3) 0 + in Just (bitmapConcat extra nullPart origLen b) + newCol = runST $ do + mv <- VUM.new n + VU.imapM_ (\i x -> VUM.unsafeWrite mv (extra + i) x) col + VU.unsafeFreeze mv + in UnboxedColumn newBm newCol + +{- | Concatenates two columns. +Returns Nothing if the columns are of different types. +-} +concatColumns :: Column -> Column -> Either DataFrameException Column +concatColumns left right = case (left, right) of + (BoxedColumn bmL l, BoxedColumn bmR r) -> case testEquality (typeOf l) (typeOf r) of + Just Refl -> + let newBm = case (bmL, bmR) of + (Nothing, Nothing) -> Nothing + (Just bl, Nothing) -> + Just + (bitmapConcat (VB.length l) bl (VB.length r) (allValidBitmap (VB.length r))) + (Nothing, Just br) -> + Just + (bitmapConcat (VB.length l) (allValidBitmap (VB.length l)) (VB.length r) br) + (Just bl, Just br) -> Just (bitmapConcat (VB.length l) bl (VB.length r) br) + in pure (BoxedColumn newBm (l <> r)) + Nothing -> Left (mismatchErr (typeOf r) (typeOf l)) + (UnboxedColumn bmL l, UnboxedColumn bmR r) -> case testEquality (typeOf l) (typeOf r) of + Just Refl -> + let newBm = case (bmL, bmR) of + (Nothing, Nothing) -> Nothing + (Just bl, Nothing) -> + Just + (bitmapConcat (VU.length l) bl (VU.length r) (allValidBitmap (VU.length r))) + (Nothing, Just br) -> + Just + (bitmapConcat (VU.length l) (allValidBitmap (VU.length l)) (VU.length r) br) + (Just bl, Just br) -> Just (bitmapConcat (VU.length l) bl (VU.length r) br) + in pure (UnboxedColumn newBm (l <> r)) + Nothing -> Left (mismatchErr (typeOf r) (typeOf l)) + _ -> Left (mismatchErr (typeOf right) (typeOf left)) + where + mismatchErr :: + forall (x :: Type) (y :: Type). TypeRep x -> TypeRep y -> DataFrameException + mismatchErr ta tb = + withTypeable ta $ + withTypeable tb $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right ta + , expectedType = Right tb + , callingFunctionName = Just "concatColumns" + , errorColumnName = Nothing + } + ) + +{- | Concatenates two columns. + +Works similar to 'concatColumns', but unlike that function, it will also combine columns of different types +by wrapping the values in an Either. + +E.g. combining Column containing [1,2] with Column containing ["a","b"] +will result in a Column containing [Left 1, Left 2, Right "a", Right "b"]. +-} + +{- | O(n) Concatenate a list of same-type columns in a single allocation. +All columns must have the same constructor and element type (as they will +within a single Parquet column). Calls 'error' on mismatch. +-} +concatManyColumns :: [Column] -> Column +concatManyColumns [] = fromList ([] :: [Maybe Int]) +concatManyColumns [c] = c +concatManyColumns (c0 : cs) = case c0 of + BoxedColumn bm0 v0 -> + let getCol (BoxedColumn bm v) = case testEquality (typeOf v0) (typeOf v) of + Just Refl -> (bm, v) + Nothing -> error "concatManyColumns: BoxedColumn type mismatch" + getCol _ = error "concatManyColumns: column constructor mismatch" + rest = map getCol cs + allVecs = v0 : map snd rest + allBms = bm0 : map fst rest + newBm + | all isNothing allBms = Nothing + | otherwise = + let pairs = zip allVecs allBms + expandedBms = map (\(v, mb) -> fromMaybe (allValidBitmap (VB.length v)) mb) pairs + go b1 n1 b2 n2 = bitmapConcat n1 b1 n2 b2 + concatBms [] = VU.empty + concatBms [(b, v)] = b + concatBms ((b1, v1) : (b2, v2) : rest') = + let merged = go b1 (VB.length v1) b2 (VB.length v2) + in concatBms ((merged, v1 <> v2) : rest') + in Just $ concatBms (zip expandedBms allVecs) + in BoxedColumn newBm (VB.concat allVecs) + UnboxedColumn bm0 v0 -> + let getCol (UnboxedColumn bm v) = case testEquality (typeOf v0) (typeOf v) of + Just Refl -> (bm, v) + Nothing -> error "concatManyColumns: UnboxedColumn type mismatch" + getCol _ = error "concatManyColumns: column constructor mismatch" + rest = map getCol cs + allVecs = v0 : map snd rest + allBms = bm0 : map fst rest + newBm + | all isNothing allBms = Nothing + | otherwise = + let pairs = zip allVecs allBms + expandedBms = map (\(v, mb) -> fromMaybe (allValidBitmap (VU.length v)) mb) pairs + go b1 n1 b2 n2 = bitmapConcat n1 b1 n2 b2 + concatBms [] = VU.empty + concatBms [(b, _)] = b + concatBms ((b1, v1) : (b2, v2) : rest') = + let merged = go b1 (VU.length v1) b2 (VU.length v2) + in concatBms ((merged, v1 <> v2) : rest') + in Just $ concatBms (zip expandedBms allVecs) + in UnboxedColumn newBm (VU.concat allVecs) + +concatColumnsEither :: Column -> Column -> Column +concatColumnsEither (BoxedColumn bmL left) (BoxedColumn bmR right) = case testEquality (typeOf left) (typeOf right) of + Nothing -> + BoxedColumn Nothing $ fmap Left left <> fmap Right right + Just Refl -> + let newBm = case (bmL, bmR) of + (Nothing, Nothing) -> Nothing + (Just bl, Nothing) -> + Just + ( bitmapConcat + (VB.length left) + bl + (VB.length right) + (allValidBitmap (VB.length right)) + ) + (Nothing, Just br) -> + Just + ( bitmapConcat + (VB.length left) + (allValidBitmap (VB.length left)) + (VB.length right) + br + ) + (Just bl, Just br) -> Just (bitmapConcat (VB.length left) bl (VB.length right) br) + in BoxedColumn newBm $ left <> right +concatColumnsEither (UnboxedColumn bmL left) (UnboxedColumn bmR right) = case testEquality (typeOf left) (typeOf right) of + Nothing -> + BoxedColumn Nothing $ + fmap Left (VG.convert left) <> fmap Right (VG.convert right) + Just Refl -> + let newBm = case (bmL, bmR) of + (Nothing, Nothing) -> Nothing + (Just bl, Nothing) -> + Just + ( bitmapConcat + (VU.length left) + bl + (VU.length right) + (allValidBitmap (VU.length right)) + ) + (Nothing, Just br) -> + Just + ( bitmapConcat + (VU.length left) + (allValidBitmap (VU.length left)) + (VU.length right) + br + ) + (Just bl, Just br) -> Just (bitmapConcat (VU.length left) bl (VU.length right) br) + in UnboxedColumn newBm $ left <> right +concatColumnsEither (BoxedColumn _ left) (UnboxedColumn _ right) = + BoxedColumn Nothing $ fmap Left left <> fmap Right (VG.convert right) +concatColumnsEither (UnboxedColumn _ left) (BoxedColumn _ right) = + BoxedColumn Nothing $ fmap Left (VG.convert left) <> fmap Right right + +-- | Allocate a mutable column of size @n@ matching the constructor/type of the given column. +newMutableColumn :: Int -> Column -> IO MutableColumn +newMutableColumn n (BoxedColumn _ (_ :: VB.Vector a)) = + MBoxedColumn <$> (VBM.new n :: IO (VBM.IOVector a)) +newMutableColumn n (UnboxedColumn _ (_ :: VU.Vector a)) = + MUnboxedColumn <$> (VUM.new n :: IO (VUM.IOVector a)) + +-- | Copy a column chunk into a mutable column starting at offset @off@. +copyIntoMutableColumn :: MutableColumn -> Int -> Column -> IO () +copyIntoMutableColumn (MBoxedColumn (mv :: VBM.IOVector b)) off (BoxedColumn _ (v :: VB.Vector a)) = + case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> VG.imapM_ (\i x -> VBM.unsafeWrite mv (off + i) x) v + Nothing -> error "copyIntoMutableColumn: Boxed type mismatch" +copyIntoMutableColumn (MUnboxedColumn (mv :: VUM.IOVector b)) off (UnboxedColumn _ (v :: VU.Vector a)) = + case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> VG.imapM_ (\i x -> VUM.unsafeWrite mv (off + i) x) v + Nothing -> error "copyIntoMutableColumn: Unboxed type mismatch" +copyIntoMutableColumn _ _ _ = + error "copyIntoMutableColumn: constructor mismatch" + +-- | Freeze a mutable column into an immutable column. +freezeMutableColumn :: MutableColumn -> IO Column +freezeMutableColumn (MBoxedColumn mv) = BoxedColumn Nothing <$> VB.unsafeFreeze mv +freezeMutableColumn (MUnboxedColumn mv) = UnboxedColumn Nothing <$> VU.unsafeFreeze mv + +{- | O(n) Converts a column to a list. Throws an exception if the wrong type is specified. + +__Examples:__ + +@ +> column = fromList [(1 :: Int), 2, 3, 4] +> toList @Int column +[1,2,3,4] +> toList @Double column +exception: ... +@ +-} +toList :: forall a. (Columnable a) => Column -> [a] +toList xs = case toVector @a xs of + Left err -> throw err + Right val -> VB.toList val + +{- | Converts a column to a vector of a specific type. + +This is a type-safe conversion that requires the column's element type +to exactly match the requested type. You must specify the desired type +via type applications. + +==== __Type Parameters__ + +[@a@] The element type to convert to +[@v@] The vector type (e.g., 'VU.Vector', 'VB.Vector') + +==== __Examples__ + +>>> toVector @Int @VU.Vector column +Right (unboxed vector of Ints) + +>>> toVector @Text @VB.Vector column +Right (boxed vector of Text) + +==== __Returns__ + +* 'Right' - The converted vector if types match +* 'Left' 'TypeMismatchException' - If the column's type doesn't match the requested type + +==== __See also__ + +For numeric conversions with automatic type coercion, see 'toDoubleVector', +'toFloatVector', and 'toIntVector'. +-} +toVector :: + forall a v. + (VG.Vector v a, Columnable a) => Column -> Either DataFrameException (v a) +toVector col = case col of + BoxedColumn bm (inner :: VB.Vector c) -> + -- Check if user wants Maybe c (nullable) or c directly + case testEquality (typeRep @a) (typeRep @c) of + Just Refl -> Right $ VG.convert inner + Nothing -> + -- Try: a = Maybe c + case testEquality (typeRep @a) (typeRep @(Maybe c)) of + Just Refl -> + -- Use VB.generate to avoid fusion forcing null slots + let !n = VB.length inner + maybeVec = case bm of + Nothing -> VB.generate n (Just . VB.unsafeIndex inner) + Just bitmap -> VB.generate n $ \i -> + if bitmapTestBit bitmap i then Just (VB.unsafeIndex inner i) else Nothing + in Right $ VG.convert maybeVec + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @c) + , callingFunctionName = Just "toVector" + , errorColumnName = Nothing + } + ) + UnboxedColumn bm (inner :: VU.Vector c) -> + case testEquality (typeRep @a) (typeRep @c) of + Just Refl -> Right $ VG.convert inner + Nothing -> + case testEquality (typeRep @a) (typeRep @(Maybe c)) of + Just Refl -> + let maybeVec = case bm of + Nothing -> VB.generate (VU.length inner) (Just . VU.unsafeIndex inner) + Just bitmap -> VB.generate (VU.length inner) $ \i -> + if bitmapTestBit bitmap i then Just (VU.unsafeIndex inner i) else Nothing + in Right $ VG.convert maybeVec + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Right (typeRep @c) + , callingFunctionName = Just "toVector" + , errorColumnName = Nothing + } + ) + +-- Some common types we will use for numerical computing. + +{- | Converts a column to an unboxed vector of 'Double' values. + +This function performs intelligent type coercion for numeric types: + +* If the column is already 'Double', returns it directly +* If the column contains other floating-point types, converts via 'realToFrac' +* If the column contains integral types, converts via 'fromIntegral' (beware of overflow if the type is `Integer`). + +==== __Optional column handling__ + +For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number). +This allows optional numeric data to be represented in the resulting vector. + +==== __Returns__ + +* 'Right' - The converted 'Double' vector +* 'Left' 'TypeMismatchException' - If the column is not numeric +-} +toDoubleVector :: Column -> Either DataFrameException (VU.Vector Double) +toDoubleVector column = + case column of + UnboxedColumn bm (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Double) of + Just Refl -> case bm of + Nothing -> Right f + Just bitmap -> Right $ VU.imap (\i x -> if bitmapTestBit bitmap i then x else read "NaN") f + Nothing -> case sFloating @a of + STrue -> + Right + ( VU.imap + ( \i x -> case bm of + Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN" + _ -> realToFrac x + ) + f + ) + SFalse -> case sIntegral @a of + STrue -> + Right + ( VU.imap + ( \i x -> case bm of + Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN" + _ -> fromIntegral x + ) + f + ) + SFalse -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @Double) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "toDoubleVector" + , errorColumnName = Nothing + } + ) + BoxedColumn bm (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of + Just Refl -> + Right + ( VB.convert $ + VB.imap + ( \i x -> case bm of + Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN" + _ -> fromIntegral x + ) + f + ) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @Double) + , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) + , callingFunctionName = Just "toDoubleVector" + , errorColumnName = Nothing + } + ) + +{- | Converts a column to an unboxed vector of 'Float' values. + +This function performs intelligent type coercion for numeric types: + +* If the column is already 'Float', returns it directly +* If the column contains other floating-point types, converts via 'realToFrac' +* If the column contains integral types, converts via 'fromIntegral' +* If the column is boxed 'Integer', converts via 'fromIntegral' (beware of overflow for 64-bit integers and `Integer`) + +==== __Optional column handling__ + +For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number). +This allows optional numeric data to be represented in the resulting vector. + +==== __Returns__ + +* 'Right' - The converted 'Float' vector +* 'Left' 'TypeMismatchException' - If the column is not numeric + +==== __Precision warning__ + +Converting from 'Double' to 'Float' may result in loss of precision. +-} +toFloatVector :: Column -> Either DataFrameException (VU.Vector Float) +toFloatVector column = + case column of + UnboxedColumn bm (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Float) of + Just Refl -> case bm of + Nothing -> Right f + Just bitmap -> Right $ VU.imap (\i x -> if bitmapTestBit bitmap i then x else read "NaN") f + Nothing -> case sFloating @a of + STrue -> + Right + ( VU.imap + ( \i x -> case bm of + Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN" + _ -> realToFrac x + ) + f + ) + SFalse -> case sIntegral @a of + STrue -> + Right + ( VU.imap + ( \i x -> case bm of + Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN" + _ -> fromIntegral x + ) + f + ) + SFalse -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @Float) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "toFloatVector" + , errorColumnName = Nothing + } + ) + BoxedColumn bm (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of + Just Refl -> + Right + ( VB.convert $ + VB.imap + ( \i x -> case bm of + Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN" + _ -> fromIntegral x + ) + f + ) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @Float) + , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) + , callingFunctionName = Just "toFloatVector" + , errorColumnName = Nothing + } + ) + +{- | Converts a column to an unboxed vector of 'Int' values. + +This function performs intelligent type coercion for numeric types: + +* If the column is already 'Int', returns it directly +* If the column contains floating-point types, rounds via 'round' and converts +* If the column contains other integral types, converts via 'fromIntegral' +* If the column is boxed 'Integer', converts via 'fromIntegral' + +==== __Returns__ + +* 'Right' - The converted 'Int' vector +* 'Left' 'TypeMismatchException' - If the column is not numeric + +==== __Note__ + +Unlike 'toDoubleVector' and 'toFloatVector', this function does NOT support +'OptionalColumn'. Optional columns must be handled separately. + +==== __Rounding behavior__ + +Floating-point values are rounded to the nearest integer using 'round'. +For example: 2.5 rounds to 2, 3.5 rounds to 4 (banker's rounding). +-} +toIntVector :: Column -> Either DataFrameException (VU.Vector Int) +toIntVector column = + case column of + UnboxedColumn _ (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of + Just Refl -> Right f + Nothing -> case sFloating @a of + STrue -> Right (VU.map (round . realToFrac) f) + SFalse -> case sIntegral @a of + STrue -> Right (VU.map fromIntegral f) + SFalse -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @Int) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "toIntVector" + , errorColumnName = Nothing + } + ) + BoxedColumn _ (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of + Just Refl -> Right (VB.convert $ VB.map fromIntegral f) + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @Int) + , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) + , callingFunctionName = Just "toIntVector" + , errorColumnName = Nothing + } + ) + +toUnboxedVector :: + forall a. + (Columnable a, VU.Unbox a) => Column -> Either DataFrameException (VU.Vector a) +toUnboxedVector column = + case column of + UnboxedColumn _ (f :: VU.Vector b) -> case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> Right f + Nothing -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @Int) + , expectedType = Right (typeRep @a) + , callingFunctionName = Just "toUnboxedVector" + , errorColumnName = Nothing + } + ) + _ -> + Left $ + TypeMismatchException + ( MkTypeErrorContext + { userType = Right (typeRep @a) + , expectedType = Left (columnTypeString column) :: Either String (TypeRep ()) + , callingFunctionName = Just "toUnboxedVector" + , errorColumnName = Nothing + } + ) {-# INLINE toUnboxedVector #-} diff --git a/src/DataFrame/Internal/DataFrame.hs b/src/DataFrame/Internal/DataFrame.hs --- a/src/DataFrame/Internal/DataFrame.hs +++ b/src/DataFrame/Internal/DataFrame.hs @@ -1,8 +1,10 @@ +{-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE InstanceSigs #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} @@ -17,13 +19,18 @@ import Control.Exception (throw) import Data.Function (on) import Data.List (sortBy, transpose, (\\)) -import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) +import Data.Maybe (fromMaybe) +import Data.Type.Equality ( + TestEquality (testEquality), + type (:~:) (Refl), + type (:~~:) (HRefl), + ) import DataFrame.Display.Terminal.PrettyPrint import DataFrame.Errors import DataFrame.Internal.Column import DataFrame.Internal.Expression import Text.Printf -import Type.Reflection (typeRep) +import Type.Reflection (Typeable, eqTypeRep, typeRep, pattern App) import Prelude hiding (null) data DataFrame = DataFrame @@ -110,19 +117,34 @@ let header = map fst (sortBy (compare `on` snd) $ M.toList (columnIndices d)) types = V.toList $ V.filter (/= "") $ V.map getType (columns d) getType :: Column -> T.Text - getType (BoxedColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a) - getType (UnboxedColumn (column :: VU.Vector a)) = T.pack $ show (typeRep @a) - getType (OptionalColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a) + showMaybeType :: forall a. (Typeable a) => String + showMaybeType = + let s = show (typeRep @a) + in "Maybe " <> if ' ' `elem` s then "(" <> s <> ")" else s + getType (BoxedColumn Nothing (_ :: V.Vector a)) = T.pack $ show (typeRep @a) + getType (BoxedColumn (Just _) (_ :: V.Vector a)) = T.pack $ showMaybeType @a + getType (UnboxedColumn Nothing (_ :: VU.Vector a)) = T.pack $ show (typeRep @a) + getType (UnboxedColumn (Just _) (_ :: VU.Vector a)) = T.pack $ showMaybeType @a -- Separate out cases dynamically so we don't end up making round trip string -- copies. get :: Maybe Column -> V.Vector T.Text - get (Just (BoxedColumn (column :: V.Vector a))) = case testEquality (typeRep @a) (typeRep @T.Text) of + get (Just (BoxedColumn (Just bm) (column :: V.Vector a))) = + V.generate (V.length column) $ \i -> + if bitmapTestBit bm i + then T.pack (show (Just (V.unsafeIndex column i))) + else "Nothing" + get (Just (BoxedColumn Nothing (column :: V.Vector a))) = case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> column Nothing -> case testEquality (typeRep @a) (typeRep @String) of Just Refl -> V.map T.pack column Nothing -> V.map (T.pack . show) column - get (Just (UnboxedColumn column)) = V.map (T.pack . show) (V.convert column) - get (Just (OptionalColumn column)) = V.map (T.pack . show) column + get (Just (UnboxedColumn (Just bm) column)) = + let col = V.convert column + in V.generate (V.length col) $ \i -> + if bitmapTestBit bm i + then T.pack (show (Just (V.unsafeIndex col i))) + else "Nothing" + get (Just (UnboxedColumn Nothing column)) = V.map (T.pack . show) (V.convert column) get Nothing = V.empty getTextColumnFromFrame df (i, name) = get $ (V.!?) (columns d) ((M.!) (columnIndices d) name) rows = @@ -161,16 +183,16 @@ {- | Retrieves a column by name from the dataframe, throwing an exception if not found. -This is an unsafe version of 'getColumn' that throws 'ColumnNotFoundException' +This is an unsafe version of 'getColumn' that throws 'ColumnsNotFoundException' if the column does not exist. Use this when you are certain the column exists. ==== __Throws__ -* 'ColumnNotFoundException' - if the column with the given name does not exist +* 'ColumnsNotFoundException' - if the column with the given name does not exist -} unsafeGetColumn :: T.Text -> DataFrame -> Column unsafeGetColumn name df = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df) + Nothing -> throw $ ColumnsNotFoundException [name] "" (M.keys $ columnIndices df) Just col -> col {- | Checks if the dataframe is empty (has no columns). @@ -180,3 +202,40 @@ -} null :: DataFrame -> Bool null df = V.null (columns df) + +-- | Convert a DataFrame to a CSV (comma-separated) text. +toCsv :: DataFrame -> T.Text +toCsv = toSeparated ',' + +-- | Convert a DataFrame to a text representation with a custom separator. +toSeparated :: Char -> DataFrame -> T.Text +toSeparated sep df + | null df = T.empty + | otherwise = + let (rows, _) = dataframeDimensions df + headers = map fst (sortBy (compare `on` snd) (M.toList (columnIndices df))) + sepText = T.singleton sep + headerLine = T.intercalate sepText headers + dataLines = map (T.intercalate sepText . getRowAsText df) [0 .. rows - 1] + in T.unlines (headerLine : dataLines) + +getRowAsText :: DataFrame -> Int -> [T.Text] +getRowAsText df i = map (`showElement` i) (V.toList (columns df)) + +showElement :: Column -> Int -> T.Text +showElement (BoxedColumn _ (c :: V.Vector a)) i = case c V.!? i of + Nothing -> error $ "Column index out of bounds at row " ++ show i + Just e + | Just Refl <- testEquality (typeRep @a) (typeRep @T.Text) -> e + | App t1 t2 <- typeRep @a + , Just HRefl <- eqTypeRep t1 (typeRep @Maybe) -> + case testEquality t2 (typeRep @T.Text) of + Just Refl -> fromMaybe "null" e + Nothing -> stripJust (T.pack (show e)) + | otherwise -> T.pack (show e) +showElement (UnboxedColumn _ c) i = case c VU.!? i of + Nothing -> error $ "Column index out of bounds at row " ++ show i + Just e -> T.pack (show e) + +stripJust :: T.Text -> T.Text +stripJust = fromMaybe "null" . T.stripPrefix "Just " diff --git a/src/DataFrame/Internal/Interpreter.hs b/src/DataFrame/Internal/Interpreter.hs --- a/src/DataFrame/Internal/Interpreter.hs +++ b/src/DataFrame/Internal/Interpreter.hs @@ -212,18 +212,21 @@ -} sliceGroups :: Column -> VU.Vector Int -> VU.Vector Int -> V.Vector Column sliceGroups col os indices = case col of - BoxedColumn vec -> - let !sorted = V.unsafeBackpermute vec (V.convert indices) + BoxedColumn bm vec -> + let !sorted = + V.generate + (VU.length indices) + ((vec `V.unsafeIndex`) . (indices `VU.unsafeIndex`)) in V.generate nGroups $ \i -> - BoxedColumn (V.unsafeSlice (start i) (len i) sorted) - UnboxedColumn vec -> + BoxedColumn + (fmap (bitmapSlice (start i) (len i)) bm) + (V.unsafeSlice (start i) (len i) sorted) + UnboxedColumn bm vec -> let !sorted = VU.unsafeBackpermute vec indices in V.generate nGroups $ \i -> - UnboxedColumn (VU.unsafeSlice (start i) (len i) sorted) - OptionalColumn vec -> - let !sorted = V.unsafeBackpermute vec (V.convert indices) - in V.generate nGroups $ \i -> - OptionalColumn (V.unsafeSlice (start i) (len i) sorted) + UnboxedColumn + (fmap (bitmapSlice (start i) (len i)) bm) + (VU.unsafeSlice (start i) (len i) sorted) where !nGroups = VU.length os - 1 start i = os `VU.unsafeIndex` i @@ -272,7 +275,7 @@ (Columnable b) => (Either String Double -> b) -> Column -> Either DataFrameException Column promoteToDoubleWith onResult col = case col of - UnboxedColumn (v :: VU.Vector c) -> + UnboxedColumn Nothing (v :: VU.Vector c) -> case sFloating @c of STrue -> Right $ @@ -284,35 +287,34 @@ fromVector @b (V.map (onResult . Right . (fromIntegral :: c -> Double)) (VG.convert v)) SFalse -> castMismatch @c @b - OptionalColumn (v :: V.Vector (Maybe c)) -> + UnboxedColumn (Just bm) (v :: VU.Vector c) -> case sFloating @c of STrue -> Right $ fromVector @b - ( V.map - (maybe (onResult (Left "null")) (onResult . Right . (realToFrac :: c -> Double))) - v + ( V.generate (VU.length v) $ \i -> + if bitmapTestBit bm i + then onResult (Right (realToFrac (VU.unsafeIndex v i) :: Double)) + else onResult (Left "null") ) SFalse -> case sIntegral @c of STrue -> Right $ fromVector @b - ( V.map - ( maybe - (onResult (Left "null")) - (onResult . Right . (fromIntegral :: c -> Double)) - ) - v + ( V.generate (VU.length v) $ \i -> + if bitmapTestBit bm i + then onResult (Right (fromIntegral (VU.unsafeIndex v i) :: Double)) + else onResult (Left "null") ) - SFalse -> tryParseWith @Double onResult col - BoxedColumn _ -> tryParseWith @Double onResult col + SFalse -> castMismatch @c @b + BoxedColumn _ _ -> tryParseWith @Double onResult col promoteToFloatWith :: forall b. (Columnable b) => (Either String Float -> b) -> Column -> Either DataFrameException Column promoteToFloatWith onResult col = case col of - UnboxedColumn (v :: VU.Vector c) -> + UnboxedColumn Nothing (v :: VU.Vector c) -> case sFloating @c of STrue -> Right $ @@ -324,32 +326,34 @@ fromVector @b (V.map (onResult . Right . (fromIntegral :: c -> Float)) (VG.convert v)) SFalse -> castMismatch @c @b - OptionalColumn (v :: V.Vector (Maybe c)) -> + UnboxedColumn (Just bm) (v :: VU.Vector c) -> case sFloating @c of STrue -> Right $ fromVector @b - ( V.map - (maybe (onResult (Left "null")) (onResult . Right . (realToFrac :: c -> Float))) - v + ( V.generate (VU.length v) $ \i -> + if bitmapTestBit bm i + then onResult (Right (realToFrac (VU.unsafeIndex v i) :: Float)) + else onResult (Left "null") ) SFalse -> case sIntegral @c of STrue -> Right $ fromVector @b - ( V.map - (maybe (onResult (Left "null")) (onResult . Right . (fromIntegral :: c -> Float))) - v + ( V.generate (VU.length v) $ \i -> + if bitmapTestBit bm i + then onResult (Right (fromIntegral (VU.unsafeIndex v i) :: Float)) + else onResult (Left "null") ) - SFalse -> tryParseWith @Float onResult col - BoxedColumn _ -> tryParseWith @Float onResult col + SFalse -> castMismatch @c @b + BoxedColumn _ _ -> tryParseWith @Float onResult col promoteToIntWith :: forall b. (Columnable b) => (Either String Int -> b) -> Column -> Either DataFrameException Column promoteToIntWith onResult col = case col of - UnboxedColumn (v :: VU.Vector c) -> + UnboxedColumn Nothing (v :: VU.Vector c) -> case sFloating @c of STrue -> Right $ @@ -361,28 +365,27 @@ fromVector @b (V.map (onResult . Right . (fromIntegral :: c -> Int)) (VG.convert v)) SFalse -> castMismatch @c @b - OptionalColumn (v :: V.Vector (Maybe c)) -> + UnboxedColumn (Just bm) (v :: VU.Vector c) -> case sFloating @c of STrue -> Right $ fromVector @b - ( V.map - ( maybe - (onResult (Left "null")) - (onResult . Right . (round . (realToFrac :: c -> Double))) - ) - v + ( V.generate (VU.length v) $ \i -> + if bitmapTestBit bm i + then onResult (Right (round (realToFrac (VU.unsafeIndex v i) :: Double))) + else onResult (Left "null") ) SFalse -> case sIntegral @c of STrue -> Right $ fromVector @b - ( V.map - (maybe (onResult (Left "null")) (onResult . Right . (fromIntegral :: c -> Int))) - v + ( V.generate (VU.length v) $ \i -> + if bitmapTestBit bm i + then onResult (Right (fromIntegral (VU.unsafeIndex v i) :: Int)) + else onResult (Left "null") ) - SFalse -> tryParseWith @Int onResult col - BoxedColumn _ -> tryParseWith @Int onResult col + SFalse -> castMismatch @c @b + BoxedColumn _ _ -> tryParseWith @Int onResult col -- | Single parse primitive: apply @onResult@ to the result of 'reads'. parseWith :: (Read a) => (Either String a -> b) -> String -> b @@ -395,27 +398,34 @@ (Columnable a, Columnable b) => (Either String a -> b) -> Column -> Either DataFrameException Column tryParseWith onResult col = case col of - BoxedColumn (v :: V.Vector c) -> - case testEquality (typeRep @c) (typeRep @String) of - Just Refl -> Right $ fromVector @b $ V.map (parseWith onResult) v - Nothing -> - case testEquality (typeRep @c) (typeRep @T.Text) of - Just Refl -> Right $ fromVector @b $ V.map (parseWith onResult . T.unpack) v - Nothing -> castMismatch @c @b - OptionalColumn (v :: V.Vector (Maybe c)) -> + BoxedColumn bm (v :: V.Vector c) -> case testEquality (typeRep @c) (typeRep @String) of - Just Refl -> - Right $ - fromVector @b $ - V.map (maybe (onResult (Left "null")) (parseWith onResult)) v + Just Refl -> case bm of + Nothing -> Right $ fromVector @b $ V.map (parseWith onResult) v + Just bitmap -> + Right $ + fromVector @b $ + V.imap + ( \i x -> + if bitmapTestBit bitmap i then parseWith onResult x else onResult (Left "null") + ) + v Nothing -> case testEquality (typeRep @c) (typeRep @T.Text) of - Just Refl -> - Right $ - fromVector @b $ - V.map (maybe (onResult (Left "null")) (parseWith onResult . T.unpack)) v + Just Refl -> case bm of + Nothing -> Right $ fromVector @b $ V.map (parseWith onResult . T.unpack) v + Just bitmap -> + Right $ + fromVector @b $ + V.imap + ( \i x -> + if bitmapTestBit bitmap i + then parseWith onResult (T.unpack x) + else onResult (Left "null") + ) + v Nothing -> castMismatch @c @b - UnboxedColumn (_ :: VU.Vector c) -> castMismatch @c @b + UnboxedColumn _ (_ :: VU.Vector c) -> castMismatch @c @b {- | When the output type @b@ is @Maybe c@ (or @Maybe (Maybe c)@) and the column stores plain @c@ values, wrap each element in 'Just'. @@ -430,16 +440,15 @@ (Columnable a, Columnable b) => (Either String a -> b) -> Column -> Maybe (Either DataFrameException Column) tryMaybeWrap _onResult col = case col of - UnboxedColumn (v :: VU.Vector c) -> + UnboxedColumn Nothing (v :: VU.Vector c) -> let wrapped = V.map Just (VG.convert v) :: V.Vector (Maybe c) in case testEquality (typeRep @b) (typeRep @(Maybe c)) of Just Refl -> Just $ Right $ fromVector @b wrapped Nothing -> - -- join: b = Maybe (Maybe c) → produce Maybe c column case testEquality (typeRep @b) (typeRep @(Maybe (Maybe c))) of Just _ -> Just $ Right $ fromVector @(Maybe c) wrapped Nothing -> Nothing - BoxedColumn (v :: V.Vector c) -> + BoxedColumn Nothing (v :: V.Vector c) -> let wrapped = V.map Just v :: V.Vector (Maybe c) in case testEquality (typeRep @b) (typeRep @(Maybe c)) of Just Refl -> Just $ Right $ fromVector @b wrapped @@ -447,7 +456,6 @@ case testEquality (typeRep @b) (typeRep @(Maybe (Maybe c))) of Just _ -> Just $ Right $ fromVector @(Maybe c) wrapped Nothing -> Nothing - -- OptionalColumn and NullableColumn are already handled by the hasElemType guards above. _ -> Nothing castMismatch :: @@ -482,7 +490,7 @@ eval (FlatCtx df) (Col name) = case getColumn name df of Nothing -> - Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df) + Left $ ColumnsNotFoundException [name] "" (M.keys $ columnIndices df) Just c | hasElemType @a c -> Right (Flat c) | otherwise -> @@ -500,8 +508,8 @@ case getColumn name (fullDataframe gdf) of Nothing -> Left $ - ColumnNotFoundException - name + ColumnsNotFoundException + [name] "" (M.keys $ columnIndices $ fullDataframe gdf) Just c @@ -524,14 +532,14 @@ case getColumn name df of Nothing -> Left $ - ColumnNotFoundException name "" (M.keys $ columnIndices df) + ColumnsNotFoundException [name] "" (M.keys $ columnIndices df) Just c -> Flat <$> promoteColumnWith onResult c eval (GroupCtx gdf) (CastWith name _tag onResult) = case getColumn name (fullDataframe gdf) of Nothing -> Left $ - ColumnNotFoundException - name + ColumnsNotFoundException + [name] "" (M.keys $ columnIndices $ fullDataframe gdf) Just c -> do @@ -579,8 +587,8 @@ case getColumn name (fullDataframe gdf) of Nothing -> Left $ - ColumnNotFoundException - name + ColumnsNotFoundException + [name] "" (M.keys $ columnIndices $ fullDataframe gdf) Just col -> @@ -599,8 +607,8 @@ case getColumn name (fullDataframe gdf) of Nothing -> Left $ - ColumnNotFoundException - name + ColumnsNotFoundException + [name] "" (M.keys $ columnIndices $ fullDataframe gdf) Just col -> @@ -617,8 +625,8 @@ case getColumn name (fullDataframe gdf) of Nothing -> Left $ - ColumnNotFoundException - name + ColumnsNotFoundException + [name] "" (M.keys $ columnIndices $ fullDataframe gdf) Just col -> @@ -766,4 +774,4 @@ -- The Column payload is intentionally unused — the only -- call-site ('aggregate') immediately throws -- 'UnaggregatedException' on this constructor. - Right $ UnAggregated $ BoxedColumn @T.Text V.empty + Right $ UnAggregated $ BoxedColumn @T.Text Nothing V.empty diff --git a/src/DataFrame/Internal/Row.hs b/src/DataFrame/Internal/Row.hs --- a/src/DataFrame/Internal/Row.hs +++ b/src/DataFrame/Internal/Row.hs @@ -169,13 +169,12 @@ get name = case getColumn name df of Nothing -> throw $ - ColumnNotFoundException - name + ColumnsNotFoundException + [name] "[INTERNAL] mkRowFromArgs" (M.keys $ columnIndices df) - Just (BoxedColumn column) -> toAny (column V.! i) - Just (UnboxedColumn column) -> toAny (column VU.! i) - Just (OptionalColumn column) -> toAny (column V.! i) + Just (BoxedColumn _ column) -> toAny (column V.! i) + Just (UnboxedColumn _ column) -> toAny (column VU.! i) -- This function will return the items in the order that is specified -- by the user. For example, if the dataframe consists of the columns @@ -193,17 +192,14 @@ ++ "the other columns at index " ++ show i get name = case getColumn name df of - Just (BoxedColumn c) -> case c V.!? i of - Just e -> toAny e - Nothing -> throwError name - Just (OptionalColumn c) -> case c V.!? i of + Just (BoxedColumn _ c) -> case c V.!? i of Just e -> toAny e Nothing -> throwError name - Just (UnboxedColumn c) -> case c VU.!? i of + Just (UnboxedColumn _ c) -> case c VU.!? i of Just e -> toAny e Nothing -> throwError name Nothing -> - throw $ ColumnNotFoundException name "mkRowRep" (M.keys $ columnIndices df) + throw $ ColumnsNotFoundException [name] "mkRowRep" (M.keys $ columnIndices df) sortedIndexes' :: [Bool] -> V.Vector Row -> VU.Vector Int sortedIndexes' flipCompare rows = runST $ do diff --git a/src/DataFrame/Internal/Types.hs b/src/DataFrame/Internal/Types.hs --- a/src/DataFrame/Internal/Types.hs +++ b/src/DataFrame/Internal/Types.hs @@ -27,7 +27,7 @@ data Rep = RBoxed | RUnboxed - | ROptional + | RNullableBoxed -- | Type-level if statement. type family If (cond :: Bool) (yes :: k) (no :: k) :: k where @@ -68,9 +68,9 @@ Numeric Float = 'True Numeric _ = 'False --- | Compute the column representation tag for any ‘a’. +-- | Compute the column representation tag for any 'a'. type family KindOf a :: Rep where - KindOf (Maybe a) = 'ROptional + KindOf (Maybe a) = 'RNullableBoxed KindOf a = If (Unboxable a) 'RUnboxed 'RBoxed -- | Type-level boolean for constraint/type comparison. diff --git a/src/DataFrame/Lazy/IO/Binary.hs b/src/DataFrame/Lazy/IO/Binary.hs --- a/src/DataFrame/Lazy/IO/Binary.hs +++ b/src/DataFrame/Lazy/IO/Binary.hs @@ -51,11 +51,14 @@ import qualified Data.Vector.Unboxed as VU import Data.Bits (setBit, shiftL, testBit, (.|.)) -import Data.Maybe (fromMaybe, isJust) import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) import Data.Word (Word16, Word32, Word64, Word8) import qualified DataFrame.Internal.Binary as Binary -import DataFrame.Internal.Column (Column (..)) +import DataFrame.Internal.Column ( + Column (..), + bitmapTestBit, + buildBitmapFromValid, + ) import DataFrame.Internal.DataFrame (DataFrame (..)) import Foreign (ForeignPtr, castForeignPtr, plusForeignPtr, sizeOf) import System.Directory (getTemporaryDirectory, removeFile) @@ -110,48 +113,51 @@ nameLen = fromIntegral (BS.length nameBytes) :: Word16 columnTypeTag :: Column -> Word8 -columnTypeTag (UnboxedColumn (_ :: VU.Vector a)) = +columnTypeTag (UnboxedColumn Nothing (_ :: VU.Vector a)) = case testEquality (typeRep @a) (typeRep @Int) of Just Refl -> tagInt Nothing -> case testEquality (typeRep @a) (typeRep @Double) of Just Refl -> tagDouble Nothing -> error "spillToDisk: unsupported UnboxedColumn element type" -columnTypeTag (BoxedColumn _) = tagText -columnTypeTag (OptionalColumn (_ :: V.Vector (Maybe a))) = +columnTypeTag (UnboxedColumn (Just _) (_ :: VU.Vector a)) = case testEquality (typeRep @a) (typeRep @Int) of Just Refl -> tagMaybeInt Nothing -> case testEquality (typeRep @a) (typeRep @Double) of Just Refl -> tagMaybeDouble - Nothing -> tagMaybeText + Nothing -> error "spillToDisk: unsupported nullable UnboxedColumn element type" +columnTypeTag (BoxedColumn Nothing _) = tagText +columnTypeTag (BoxedColumn (Just _) _) = tagMaybeText buildColumnData :: Int -> Column -> BSB.Builder -buildColumnData _ (UnboxedColumn (v :: VU.Vector a)) = +buildColumnData _ (UnboxedColumn Nothing (v :: VU.Vector a)) = case testEquality (typeRep @a) (typeRep @Int) of Just Refl -> buildIntVector v Nothing -> case testEquality (typeRep @a) (typeRep @Double) of Just Refl -> buildDoubleVector v Nothing -> error "spillToDisk: unsupported UnboxedColumn element type" -buildColumnData _ (BoxedColumn (v :: V.Vector a)) = - case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> buildTextVector v - Nothing -> error "spillToDisk: unsupported BoxedColumn element type" -buildColumnData _ (OptionalColumn (v :: V.Vector (Maybe a))) = +buildColumnData _ (UnboxedColumn (Just bm) (v :: VU.Vector a)) = case testEquality (typeRep @a) (typeRep @Int) of Just Refl -> - buildNullBitmap (V.map isJust v) - <> buildIntVector (VU.convert (V.map (fromMaybe 0) v)) + buildNullBitmap (V.generate (VU.length v) (bitmapTestBit bm)) + <> buildIntVector v Nothing -> case testEquality (typeRep @a) (typeRep @Double) of Just Refl -> - buildNullBitmap (V.map isJust v) - <> buildDoubleVector (VU.convert (V.map (fromMaybe 0.0) v)) - Nothing -> - let showText x = case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> x - Nothing -> T.pack (show x) - texts = V.map (maybe T.empty showText) v - in buildNullBitmap (V.map isJust v) <> buildTextVector texts + buildNullBitmap (V.generate (VU.length v) (bitmapTestBit bm)) + <> buildDoubleVector v + Nothing -> error "spillToDisk: unsupported nullable UnboxedColumn element type" +buildColumnData _ (BoxedColumn Nothing (v :: V.Vector a)) = + case testEquality (typeRep @a) (typeRep @T.Text) of + Just Refl -> buildTextVector v + Nothing -> error "spillToDisk: unsupported BoxedColumn element type" +buildColumnData _ (BoxedColumn (Just bm) (v :: V.Vector a)) = + let isValidVec = V.generate (V.length v) (bitmapTestBit bm) + showText x = case testEquality (typeRep @a) (typeRep @T.Text) of + Just Refl -> x + Nothing -> T.pack (show x) + texts = V.imap (\i x -> if bitmapTestBit bm i then showText x else T.empty) v + in buildNullBitmap isValidVec <> buildTextVector texts {- | Bulk-encode an Int vector as 8-byte LE values (native layout on LE platforms). hPutBuilder flushes synchronously so the underlying ForeignPtr outlives the Builder. @@ -249,37 +255,28 @@ readColumnData bs off nrows tag | tag == tagInt = do (off', v) <- readIntColumn bs off nrows - return (off', UnboxedColumn v) + return (off', UnboxedColumn Nothing v) | tag == tagDouble = do (off', v) <- readDoubleColumn bs off nrows - return (off', UnboxedColumn v) + return (off', UnboxedColumn Nothing v) | tag == tagText = do (off', v) <- readTextColumn bs off nrows - return (off', BoxedColumn v) + return (off', BoxedColumn Nothing v) | tag == tagMaybeInt = do (off1, bitmap) <- readNullBitmap bs off nrows (off2, v) <- readIntColumn bs off1 nrows - let maybes = - V.fromList - (zipWith (\valid x -> if valid then Just x else Nothing) bitmap (VU.toList v)) :: - V.Vector (Maybe Int) - return (off2, OptionalColumn maybes) + let bm = buildBitmapFromValid (VU.fromList (map (\b -> if b then 1 else 0) bitmap)) + return (off2, UnboxedColumn (Just bm) v) | tag == tagMaybeDouble = do (off1, bitmap) <- readNullBitmap bs off nrows (off2, v) <- readDoubleColumn bs off1 nrows - let maybes = - V.fromList - (zipWith (\valid x -> if valid then Just x else Nothing) bitmap (VU.toList v)) :: - V.Vector (Maybe Double) - return (off2, OptionalColumn maybes) + let bm = buildBitmapFromValid (VU.fromList (map (\b -> if b then 1 else 0) bitmap)) + return (off2, UnboxedColumn (Just bm) v) | tag == tagMaybeText = do (off1, bitmap) <- readNullBitmap bs off nrows (off2, v) <- readTextColumn bs off1 nrows - let maybes = - V.fromList - (zipWith (\valid x -> if valid then Just x else Nothing) bitmap (V.toList v)) :: - V.Vector (Maybe T.Text) - return (off2, OptionalColumn maybes) + let bm = buildBitmapFromValid (VU.fromList (map (\b -> if b then 1 else 0) bitmap)) + return (off2, BoxedColumn (Just bm) v) | otherwise = Left ("unknown type tag " <> show tag) {- | Zero-copy Int column read: reuses the ByteString buffer's ForeignPtr. diff --git a/src/DataFrame/Lazy/IO/CSV.hs b/src/DataFrame/Lazy/IO/CSV.hs --- a/src/DataFrame/Lazy/IO/CSV.hs +++ b/src/DataFrame/Lazy/IO/CSV.hs @@ -30,6 +30,7 @@ Column (..), MutableColumn (..), columnLength, + ensureOptional, freezeColumn', writeColumn, ) @@ -60,7 +61,7 @@ ReadOptions { hasHeader = True , inferTypes = True - , safeRead = True + , safeRead = False , rowRange = Nothing , seekPos = Nothing , totalRows = Nothing @@ -218,7 +219,8 @@ IO Column freezeColumn mutableCols nulls opts colIndex = do col <- VM.unsafeRead mutableCols colIndex - freezeColumn' (nulls V.! colIndex) col + frozen <- freezeColumn' (nulls V.! colIndex) col + return $! if safeRead opts then ensureOptional frozen else frozen {-# INLINE freezeColumn #-} -- --------------------------------------------------------------------------- @@ -341,12 +343,6 @@ let val = TextEncoding.decodeUtf8Lenient bs in VM.unsafeWrite col i val >> return (Right True) Nothing -> return (Left (TextEncoding.decodeUtf8Lenient bs)) -writeColumnBs i bs (MOptionalColumn (col :: VM.IOVector (Maybe a))) = - case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> - let val = TextEncoding.decodeUtf8Lenient bs - in VM.unsafeWrite col i (Just val) >> return (Right True) - Nothing -> return (Left (TextEncoding.decodeUtf8Lenient bs)) writeColumnBs i bs (MUnboxedColumn (col :: VUM.IOVector a)) = case testEquality (typeRep @a) (typeRep @Double) of Just Refl -> case readByteStringDouble bs of @@ -417,7 +413,6 @@ sliceCol :: Int -> MutableColumn -> MutableColumn sliceCol n (MBoxedColumn col) = MBoxedColumn (VM.take n col) sliceCol n (MUnboxedColumn col) = MUnboxedColumn (VUM.take n col) -sliceCol n (MOptionalColumn col) = MOptionalColumn (VM.take n col) {- | Finds the index of the next unquoted newline (0x0A). Fast path: uses memchr (SIMD) and falls back to a quote-aware linear scan diff --git a/src/DataFrame/Operations/Aggregation.hs b/src/DataFrame/Operations/Aggregation.hs --- a/src/DataFrame/Operations/Aggregation.hs +++ b/src/DataFrame/Operations/Aggregation.hs @@ -30,6 +30,7 @@ Column (..), TypedColumn (..), atIndicesStable, + bitmapTestBit, ) import DataFrame.Internal.DataFrame (DataFrame (..), GroupedDataFrame (..)) import DataFrame.Internal.Expression @@ -49,8 +50,8 @@ groupBy names df | any (`notElem` columnNames df) names = throw $ - ColumnNotFoundException - (T.pack $ show $ names L.\\ columnNames df) + ColumnsNotFoundException + (names L.\\ columnNames df) "groupBy" (columnNames df) | nRows df == 0 = @@ -81,7 +82,7 @@ let selectedCols = map (columns df V.!) indicesToGroup forM_ selectedCols $ \case - UnboxedColumn (v :: VU.Vector a) -> + UnboxedColumn _ (v :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of Just Refl -> VU.imapM_ @@ -129,31 +130,28 @@ VUM.unsafeWrite mv i (i, hashWithSalt h x) ) v - BoxedColumn (v :: V.Vector a) -> + BoxedColumn bm (v :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> V.imapM_ ( \i t -> do (_, !h) <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (i, hashWithSalt h t) + let h' = case bm of + Just bm' | not (bitmapTestBit bm' i) -> hashWithSalt h (0 :: Int) -- null sentinel + _ -> hashWithSalt h t + VUM.unsafeWrite mv i (i, h') ) v Nothing -> V.imapM_ ( \i d -> do - let x = hash (show d) (_, !h) <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (i, hashWithSalt h x) + let h' = case bm of + Just bm' | not (bitmapTestBit bm' i) -> hashWithSalt h (0 :: Int) -- null sentinel + _ -> hashWithSalt h (hash (show d)) + VUM.unsafeWrite mv i (i, h') ) v - OptionalColumn v -> - V.imapM_ - ( \i d -> do - let x = hash (show d) - (_, !h) <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (i, hashWithSalt h x) - ) - v let numPasses = 4 bucketSize = 65536 @@ -197,7 +195,7 @@ let selectedCols = map (columns df V.!) indices forM_ selectedCols $ \case - UnboxedColumn (v :: VU.Vector a) -> + UnboxedColumn _ (v :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of Just Refl -> VU.imapM_ @@ -245,31 +243,28 @@ VUM.unsafeWrite mv i (hashWithSalt h x) ) v - BoxedColumn (v :: V.Vector a) -> + BoxedColumn bm (v :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> V.imapM_ ( \i (t :: T.Text) -> do h <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (hashWithSalt h t) + let h' = case bm of + Just bm' | not (bitmapTestBit bm' i) -> hashWithSalt h (0 :: Int) + _ -> hashWithSalt h t + VUM.unsafeWrite mv i h' ) v Nothing -> V.imapM_ ( \i d -> do - let x = hash (show d) + let x = case bm of + Just bm' | not (bitmapTestBit bm' i) -> 0 :: Int + _ -> hash (show d) h <- VUM.unsafeRead mv i VUM.unsafeWrite mv i (hashWithSalt h x) ) v - OptionalColumn v -> - V.imapM_ - ( \i d -> do - let x = hash (show d) - h <- VUM.unsafeRead mv i - VUM.unsafeWrite mv i (hashWithSalt h x) - ) - v VU.unsafeFreeze mv where diff --git a/src/DataFrame/Operations/Core.hs b/src/DataFrame/Operations/Core.hs --- a/src/DataFrame/Operations/Core.hs +++ b/src/DataFrame/Operations/Core.hs @@ -17,6 +17,7 @@ import qualified Data.Vector.Unboxed as VU import Control.Exception (throw) +import Data.Bits (popCount) import Data.Either import qualified Data.Foldable as Fold import Data.Function (on, (&)) @@ -306,7 +307,7 @@ -- | DataFrame to add the column to DataFrame -> DataFrame -insertUnboxedVector name xs = insertColumn name (UnboxedColumn xs) +insertUnboxedVector name xs = insertColumn name (UnboxedColumn Nothing xs) {- | /O(n)/ Add a column to the dataframe. @@ -394,7 +395,7 @@ | null df = throw (EmptyDataSetException "cloneColumn") | otherwise = fromMaybe ( throw $ - ColumnNotFoundException original "cloneColumn" (M.keys $ columnIndices df) + ColumnsNotFoundException [original] "cloneColumn" (M.keys $ columnIndices df) ) $ do column <- getColumn original df @@ -485,7 +486,7 @@ renameSafe orig new df | null df = throw (EmptyDataSetException "rename") | otherwise = fromMaybe - (Left $ ColumnNotFoundException orig "rename" (M.keys $ columnIndices df)) + (Left $ ColumnsNotFoundException [orig] "rename" (M.keys $ columnIndices df)) $ do columnIndex <- M.lookup orig (columnIndices df) let origRemoved = M.delete orig (columnIndices df) @@ -530,11 +531,11 @@ [ColumnInfo] indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df)) columnName i = M.lookup i indexMap - go acc i col@(OptionalColumn (c :: V.Vector a)) = + go acc i col@(BoxedColumn bm (c :: V.Vector a)) = let cname = columnName i countNulls = nulls col - columnType = T.pack $ show $ typeRep @a + columnType = T.pack $ columnTypeString col in if isNothing cname then acc @@ -545,43 +546,35 @@ countNulls columnType : acc - go acc i col@(BoxedColumn (c :: V.Vector a)) = + go acc i col@(UnboxedColumn bm c) = let cname = columnName i - columnType = T.pack $ show $ typeRep @a + countNulls = nulls col + columnType = T.pack $ columnTypeString col in if isNothing cname then acc else ColumnInfo (fromMaybe "" cname) - (columnLength col) - 0 + (columnLength col - countNulls) + countNulls columnType : acc - go acc i col@(UnboxedColumn c) = - let - cname = columnName i - columnType = T.pack $ columnTypeString col - in - -- Unboxed columns cannot have nulls since Maybe - -- is not an instance of Unbox a - if isNothing cname - then acc - else - ColumnInfo (fromMaybe "" cname) (columnLength col) 0 columnType : acc nulls :: Column -> Int -nulls (OptionalColumn xs) = VG.length $ VG.filter isNothing xs -nulls (BoxedColumn (xs :: V.Vector a)) = case testEquality (typeRep @a) (typeRep @T.Text) of +nulls (BoxedColumn (Just bm) xs) = + -- count null bits in bitmap + let n = VG.length xs + in n - VU.foldl' (\acc b -> acc + popCount b) 0 bm +nulls (BoxedColumn Nothing (xs :: V.Vector a)) = case testEquality (typeRep @a) (typeRep @T.Text) of Just Refl -> VG.length $ VG.filter isNullish xs Nothing -> case testEquality (typeRep @a) (typeRep @String) of Just Refl -> VG.length $ VG.filter (isNullish . T.pack) xs - Nothing -> case typeRep @a of - App t1 t2 -> case eqTypeRep t1 (typeRep @Maybe) of - Just HRefl -> VG.length $ VG.filter isNothing xs - Nothing -> 0 - _ -> 0 + Nothing -> 0 +nulls (UnboxedColumn (Just bm) xs) = + let n = VG.length xs + in n - VU.foldl' (\acc b -> acc + popCount b) 0 bm nulls _ = 0 {- | Creates a dataframe from a list of tuples with name and column. @@ -859,7 +852,8 @@ (Col name) -> case getColumn name df of Just col -> toVector col Nothing -> - Left $ ColumnNotFoundException name "columnAsVector" (M.keys $ columnIndices df) + Left $ + ColumnsNotFoundException [name] "columnAsVector" (M.keys $ columnIndices df) _ -> case interpret df expr of Left e -> throw e Right (TColumn col) -> toVector col @@ -876,7 +870,7 @@ Just col -> toIntVector col Nothing -> Left $ - ColumnNotFoundException name "columnAsIntVector" (M.keys $ columnIndices df) + ColumnsNotFoundException [name] "columnAsIntVector" (M.keys $ columnIndices df) columnAsIntVector expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> toIntVector col @@ -893,7 +887,10 @@ Just col -> toDoubleVector col Nothing -> Left $ - ColumnNotFoundException name "columnAsDoubleVector" (M.keys $ columnIndices df) + ColumnsNotFoundException + [name] + "columnAsDoubleVector" + (M.keys $ columnIndices df) columnAsDoubleVector expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> toDoubleVector col @@ -910,7 +907,10 @@ Just col -> toFloatVector col Nothing -> Left $ - ColumnNotFoundException name "columnAsFloatVector" (M.keys $ columnIndices df) + ColumnsNotFoundException + [name] + "columnAsFloatVector" + (M.keys $ columnIndices df) columnAsFloatVector expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> toFloatVector col @@ -923,7 +923,10 @@ Just col -> toUnboxedVector col Nothing -> Left $ - ColumnNotFoundException name "columnAsFloatVector" (M.keys $ columnIndices df) + ColumnsNotFoundException + [name] + "columnAsFloatVector" + (M.keys $ columnIndices df) columnAsUnboxedVector expr df = case interpret df expr of Left e -> throw e Right (TColumn col) -> toUnboxedVector col @@ -967,7 +970,8 @@ in map toNamedExpr names where identityUExpr name = case getColumn name df of - Just (BoxedColumn (_ :: V.Vector a)) -> UExpr (Col @a name) - Just (UnboxedColumn (_ :: VU.Vector a)) -> UExpr (Col @a name) - Just (OptionalColumn (_ :: V.Vector (Maybe a))) -> UExpr (Col @(Maybe a) name) + Just (BoxedColumn (Just _) (_ :: V.Vector a)) -> UExpr (Col @(Maybe a) name) + Just (BoxedColumn Nothing (_ :: V.Vector a)) -> UExpr (Col @a name) + Just (UnboxedColumn (Just _) (_ :: VU.Vector a)) -> UExpr (Col @(Maybe a) name) + Just (UnboxedColumn Nothing (_ :: VU.Vector a)) -> UExpr (Col @a name) Nothing -> error $ "showDerivedExpressions: column not found: " ++ T.unpack name diff --git a/src/DataFrame/Operations/Join.hs b/src/DataFrame/Operations/Join.hs --- a/src/DataFrame/Operations/Join.hs +++ b/src/DataFrame/Operations/Join.hs @@ -11,6 +11,7 @@ module DataFrame.Operations.Join where import Control.Applicative ((<|>)) +import Control.Exception (throw) import Control.Monad (forM_, when) import Control.Monad.ST (ST, runST) import qualified Data.HashMap.Strict as HM @@ -27,6 +28,9 @@ import qualified Data.Vector.Algorithms.Merge as VA import qualified Data.Vector.Unboxed as VU import qualified Data.Vector.Unboxed.Mutable as VUM +import DataFrame.Errors ( + DataFrameException (ColumnsNotFoundException), + ) import DataFrame.Internal.Column as D import DataFrame.Internal.DataFrame as D import DataFrame.Operations.Aggregation as D @@ -148,6 +152,15 @@ keyColIndices :: S.Set T.Text -> DataFrame -> [Int] keyColIndices csSet df = M.elems $ M.restrictKeys (D.columnIndices df) csSet +-- | Validate that all requested join keys exist, then return their indices. +validatedKeyColIndices :: T.Text -> S.Set T.Text -> DataFrame -> [Int] +validatedKeyColIndices callPoint csSet df = + let columnIdxs = D.columnIndices df + missingKeys = S.toAscList (csSet `S.difference` M.keysSet columnIdxs) + in case missingKeys of + [] -> M.elems $ M.restrictKeys columnIdxs csSet + _ -> throw (ColumnsNotFoundException missingKeys callPoint (M.keys columnIdxs)) + -- ============================================================ -- Inner Join -- ============================================================ @@ -175,36 +188,39 @@ innerJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame innerJoin cs left right | D.null right || D.null left = D.empty - | otherwise = - let - csSet = S.fromList cs - leftRows = fst (D.dimensions left) - rightRows = fst (D.dimensions right) + | otherwise = innerJoinNonEmpty cs left right - leftKeyIdxs = keyColIndices csSet left - rightKeyIdxs = keyColIndices csSet right - leftHashes = D.computeRowHashes leftKeyIdxs left - rightHashes = D.computeRowHashes rightKeyIdxs right +innerJoinNonEmpty :: [T.Text] -> DataFrame -> DataFrame -> DataFrame +innerJoinNonEmpty cs left right = + let + csSet = S.fromList cs + leftRows = fst (D.dimensions left) + rightRows = fst (D.dimensions right) - buildRows = min leftRows rightRows - (leftIxs, rightIxs) - | buildRows > joinStrategyThreshold = - sortMergeInnerKernel leftHashes rightHashes - | rightRows <= leftRows = - -- Build on right (smaller or equal), probe with left - hashInnerKernel leftHashes rightHashes - | otherwise = - -- Build on left (smaller), probe with right, swap result - let (!rIxs, !lIxs) = hashInnerKernel rightHashes leftHashes - in (lIxs, rIxs) - in - assembleInner csSet left right leftIxs rightIxs + leftKeyIdxs = validatedKeyColIndices "innerJoin" csSet left + rightKeyIdxs = validatedKeyColIndices "innerJoin" csSet right + leftHashes = D.computeRowHashes leftKeyIdxs left + rightHashes = D.computeRowHashes rightKeyIdxs right + buildRows = min leftRows rightRows + (leftIxs, rightIxs) + | buildRows > joinStrategyThreshold = + sortMergeInnerKernel leftHashes rightHashes + | rightRows <= leftRows = + -- Build on right (smaller or equal), probe with left + hashInnerKernel leftHashes rightHashes + | otherwise = + -- Build on left (smaller), probe with right, swap result + let (!rIxs, !lIxs) = hashInnerKernel rightHashes leftHashes + in (lIxs, rIxs) + in + assembleInner csSet left right leftIxs rightIxs + -- | Compute hashes for the given key column names in a DataFrame. buildHashColumn :: [T.Text] -> DataFrame -> VU.Vector Int buildHashColumn keys df = let csSet = S.fromList keys - keyIdxs = keyColIndices csSet df + keyIdxs = validatedKeyColIndices "buildHashColumn" csSet df in D.computeRowHashes keyIdxs df {- | Probe one batch of rows against a pre-built 'CompactIndex'. @@ -530,29 +546,36 @@ @ -} leftJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame -leftJoin cs left right +leftJoin = leftJoinWithCallPoint "leftJoin" + +leftJoinWithCallPoint :: + T.Text -> [T.Text] -> DataFrame -> DataFrame -> DataFrame +leftJoinWithCallPoint callPoint cs left right | D.null right || D.nRows right == 0 = left | D.null left || D.nRows left == 0 = D.empty - | otherwise = - let - csSet = S.fromList cs - rightRows = fst (D.dimensions right) + | otherwise = leftJoinNonEmpty callPoint cs left right - leftKeyIdxs = keyColIndices csSet left - rightKeyIdxs = keyColIndices csSet right - leftHashes = D.computeRowHashes leftKeyIdxs left - rightHashes = D.computeRowHashes rightKeyIdxs right +leftJoinNonEmpty :: T.Text -> [T.Text] -> DataFrame -> DataFrame -> DataFrame +leftJoinNonEmpty callPoint cs left right = + let + csSet = S.fromList cs + rightRows = fst (D.dimensions right) - -- Right is always the build side for left join - (leftIxs, rightIxs) - | rightRows > joinStrategyThreshold = - sortMergeLeftKernel leftHashes rightHashes - | otherwise = - hashLeftKernel leftHashes rightHashes - in - -- rightIxs uses -1 as sentinel for "no match" - assembleLeft csSet left right leftIxs rightIxs + leftKeyIdxs = validatedKeyColIndices callPoint csSet left + rightKeyIdxs = validatedKeyColIndices callPoint csSet right + leftHashes = D.computeRowHashes leftKeyIdxs left + rightHashes = D.computeRowHashes rightKeyIdxs right + -- Right is always the build side for left join + (leftIxs, rightIxs) + | rightRows > joinStrategyThreshold = + sortMergeLeftKernel leftHashes rightHashes + | otherwise = + hashLeftKernel leftHashes rightHashes + in + -- rightIxs uses -1 as sentinel for "no match" + assembleLeft csSet left right leftIxs rightIxs + {- | Hash-based left join kernel. Returns @(leftExpandedIndices, rightExpandedIndices)@ where right indices use @-1@ as sentinel for unmatched rows. @@ -801,34 +824,37 @@ -} rightJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame -rightJoin cs left right = leftJoin cs right left +rightJoin cs left right = leftJoinWithCallPoint "rightJoin" cs right left fullOuterJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame fullOuterJoin cs left right | D.null right || D.nRows right == 0 = left | D.null left || D.nRows left == 0 = right - | otherwise = - let - csSet = S.fromList cs - leftRows = fst (D.dimensions left) - rightRows = fst (D.dimensions right) + | otherwise = fullOuterJoinNonEmpty cs left right - leftKeyIdxs = keyColIndices csSet left - rightKeyIdxs = keyColIndices csSet right - leftHashes = D.computeRowHashes leftKeyIdxs left - rightHashes = D.computeRowHashes rightKeyIdxs right +fullOuterJoinNonEmpty :: [T.Text] -> DataFrame -> DataFrame -> DataFrame +fullOuterJoinNonEmpty cs left right = + let + csSet = S.fromList cs + leftRows = fst (D.dimensions left) + rightRows = fst (D.dimensions right) - -- Both sides can have nulls in full outer - (leftIxs, rightIxs) - | max leftRows rightRows > joinStrategyThreshold = - sortMergeFullOuterKernel leftHashes rightHashes - | otherwise = - hashFullOuterKernel leftHashes rightHashes - in - -- Both index vectors use -1 as sentinel - assembleFullOuter csSet left right leftIxs rightIxs + leftKeyIdxs = validatedKeyColIndices "fullOuterJoin" csSet left + rightKeyIdxs = validatedKeyColIndices "fullOuterJoin" csSet right + leftHashes = D.computeRowHashes leftKeyIdxs left + rightHashes = D.computeRowHashes rightKeyIdxs right + -- Both sides can have nulls in full outer + (leftIxs, rightIxs) + | max leftRows rightRows > joinStrategyThreshold = + sortMergeFullOuterKernel leftHashes rightHashes + | otherwise = + hashFullOuterKernel leftHashes rightHashes + in + -- Both index vectors use -1 as sentinel + assembleFullOuter csSet left right leftIxs rightIxs + {- | Hash-based full outer join kernel. Builds compact indices on both sides. Returns @(leftExpandedIndices, rightExpandedIndices)@ with @-1@ sentinels. @@ -1031,23 +1057,31 @@ insertIfPresent _ Nothing df = df insertIfPresent name (Just c) df = D.insertColumn name c df - -- Coalesce two OptionalColumns: take first non-Nothing per row, + -- Coalesce two nullable columns: take first non-Nothing per row, -- producing a non-optional column. coalesceKeyColumn :: Column -> Column -> Column coalesceKeyColumn - (OptionalColumn (lCol :: VB.Vector (Maybe a))) - (OptionalColumn (rCol :: VB.Vector (Maybe b))) = + (BoxedColumn lBm (lCol :: VB.Vector a)) + (BoxedColumn rBm (rCol :: VB.Vector b)) = case testEquality (typeRep @a) (typeRep @b) of Just Refl -> - D.fromVector $ - VB.zipWith - ( \l r -> - fromMaybe (error "fullOuterJoin: null on both sides of key column") (l <|> r) - ) - lCol - rCol + let asMaybe bm = + VB.imap + ( \i v -> case bm of + Just bm' -> if bitmapTestBit bm' i then Just v else Nothing + Nothing -> Just v + ) + lMaybe = asMaybe lBm lCol + rMaybe = asMaybe rBm rCol + in D.fromVector $ + VB.zipWith + ( \l r -> + fromMaybe (error "fullOuterJoin: null on both sides of key column") (l <|> r) + ) + lMaybe + rMaybe Nothing -> error "Cannot join columns of different types" - coalesceKeyColumn _ _ = error "fullOuterJoin: expected OptionalColumn for key columns" + coalesceKeyColumn _ _ = error "fullOuterJoin: expected nullable column for key columns" in D.fold ( \name df -> if S.member name csSet diff --git a/src/DataFrame/Operations/Permutation.hs b/src/DataFrame/Operations/Permutation.hs --- a/src/DataFrame/Operations/Permutation.hs +++ b/src/DataFrame/Operations/Permutation.hs @@ -53,8 +53,8 @@ sortBy sortOrds df | any (`notElem` columnNames df) names = throw $ - ColumnNotFoundException - (T.pack $ show $ names L.\\ columnNames df) + ColumnsNotFoundException + (names L.\\ columnNames df) "sortBy" (columnNames df) | otherwise = diff --git a/src/DataFrame/Operations/Statistics.hs b/src/DataFrame/Operations/Statistics.hs --- a/src/DataFrame/Operations/Statistics.hs +++ b/src/DataFrame/Operations/Statistics.hs @@ -213,7 +213,7 @@ _getColumnAsDouble :: T.Text -> DataFrame -> Maybe (VU.Vector Double) _getColumnAsDouble name df = case getColumn name df of - Just (UnboxedColumn (f :: VU.Vector a)) -> case testEquality (typeRep @a) (typeRep @Double) of + Just (UnboxedColumn _ (f :: VU.Vector a)) -> case testEquality (typeRep @a) (typeRep @Double) of Just Refl -> Just f Nothing -> case sIntegral @a of STrue -> Just (VU.map fromIntegral f) @@ -222,7 +222,7 @@ SFalse -> Nothing Nothing -> throw $ - ColumnNotFoundException name "_getColumnAsDouble" (M.keys $ columnIndices df) + ColumnsNotFoundException [name] "_getColumnAsDouble" (M.keys $ columnIndices df) _ -> Nothing -- Return a type mismatch error here. {-# INLINE _getColumnAsDouble #-} @@ -237,15 +237,12 @@ sum :: forall a. (Columnable a, Num a) => Expr a -> DataFrame -> a sum (Col name) df = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "sum" (M.keys $ columnIndices df) - Just ((UnboxedColumn (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of + Nothing -> throw $ ColumnsNotFoundException [name] "sum" (M.keys $ columnIndices df) + Just ((UnboxedColumn _ (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of Just Refl -> VG.sum column Nothing -> 0 - Just ((BoxedColumn (column :: V.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of + Just ((BoxedColumn _ (column :: V.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of Just Refl -> VG.sum column - Nothing -> 0 - Just ((OptionalColumn (column :: V.Vector (Maybe a')))) -> case testEquality (typeRep @a') (typeRep @a) of - Just Refl -> VG.sum (VG.map (fromMaybe 0) column) Nothing -> 0 sum expr df = case interpret df expr of Left e -> throw e diff --git a/src/DataFrame/Operations/Subset.hs b/src/DataFrame/Operations/Subset.hs --- a/src/DataFrame/Operations/Subset.hs +++ b/src/DataFrame/Operations/Subset.hs @@ -1,4 +1,6 @@ {-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE ConstraintKinds #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-} @@ -44,10 +46,23 @@ import DataFrame.Operations.Core import DataFrame.Operations.Merge () import DataFrame.Operations.Transformations (apply) +import DataFrame.Operators import System.Random import Type.Reflection import Prelude hiding (filter, take) +#if MIN_VERSION_random(1,3,0) +type SplittableGen g = (SplitGen g, RandomGen g) + +splitForStratified :: SplittableGen g => g -> (g, g) +splitForStratified = splitGen +#else +type SplittableGen g = RandomGen g + +splitForStratified :: SplittableGen g => g -> (g, g) +splitForStratified = split +#endif + -- | O(k * n) Take the first n rows of a DataFrame. take :: Int -> DataFrame -> DataFrame take n d = d{columns = V.map (takeColumn n') (columns d), dataframeDimensions = (n', c)} @@ -116,10 +131,33 @@ filter (Col filterColumnName) condition df = case getColumn filterColumnName df of Nothing -> throw $ - ColumnNotFoundException filterColumnName "filter" (M.keys $ columnIndices df) - Just (BoxedColumn (column :: V.Vector b)) -> filterByVector filterColumnName column condition df - Just (OptionalColumn (column :: V.Vector b)) -> filterByVector filterColumnName column condition df - Just (UnboxedColumn (column :: VU.Vector b)) -> filterByVector filterColumnName column condition df + ColumnsNotFoundException [filterColumnName] "filter" (M.keys $ columnIndices df) + Just col@(BoxedColumn bm (column :: V.Vector b)) -> + -- Check direct type match first, then try Maybe b match for nullable columns + case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> filterByVector filterColumnName column condition df + Nothing -> case (bm, typeRep @a) of + (Just bm', App tMaybe tInner) -> case eqTypeRep tMaybe (typeRep @Maybe) of + Just HRefl -> case testEquality tInner (typeRep @b) of + Just Refl -> + let maybeVec = V.imap (\i v -> if bitmapTestBit bm' i then Just v else Nothing) column + in filterByVector filterColumnName maybeVec condition df + Nothing -> filterByVector filterColumnName column condition df + Nothing -> filterByVector filterColumnName column condition df + _ -> filterByVector filterColumnName column condition df + Just col@(UnboxedColumn bm (column :: VU.Vector b)) -> + case testEquality (typeRep @a) (typeRep @b) of + Just Refl -> filterByVector filterColumnName column condition df + Nothing -> case (bm, typeRep @a) of + (Just bm', App tMaybe tInner) -> case eqTypeRep tMaybe (typeRep @Maybe) of + Just HRefl -> case testEquality tInner (typeRep @b) of + Just Refl -> + let maybeVec = V.generate (VU.length column) $ \i -> + if bitmapTestBit bm' i then Just (VU.unsafeIndex column i) else Nothing + in filterByVector filterColumnName maybeVec condition df + Nothing -> filterByVector filterColumnName column condition df + Nothing -> filterByVector filterColumnName column condition df + _ -> filterByVector filterColumnName column condition df filter expr condition df = let (TColumn col) = case interpret @a df (normalize expr) of @@ -193,9 +231,12 @@ filterJust :: T.Text -> DataFrame -> DataFrame filterJust name df = case getColumn name df of Nothing -> - throw $ ColumnNotFoundException name "filterJust" (M.keys $ columnIndices df) - Just column@(OptionalColumn (col :: V.Vector (Maybe a))) -> filter (Col @(Maybe a) name) isJust df & apply @(Maybe a) fromJust name - Just column -> df + throw $ ColumnsNotFoundException [name] "filterJust" (M.keys $ columnIndices df) + Just column | hasMissing column -> case column of + BoxedColumn (Just _) (col :: V.Vector a) -> filter (Col @(Maybe a) name) isJust df & apply @(Maybe a) fromJust name + UnboxedColumn (Just _) (col :: VU.Vector a) -> filter (Col @(Maybe a) name) isJust df & apply @(Maybe a) fromJust name + _ -> df + Just _ -> df {- | O(k) returns all rows with `Nothing` in a give column. @@ -204,8 +245,12 @@ filterNothing :: T.Text -> DataFrame -> DataFrame filterNothing name df = case getColumn name df of Nothing -> - throw $ ColumnNotFoundException name "filterNothing" (M.keys $ columnIndices df) - Just (OptionalColumn (col :: V.Vector (Maybe a))) -> filter (Col @(Maybe a) name) isNothing df + throw $ + ColumnsNotFoundException [name] "filterNothing" (M.keys $ columnIndices df) + Just column | hasMissing column -> case column of + BoxedColumn (Just _) (col :: V.Vector a) -> filter (Col @(Maybe a) name) isNothing df + UnboxedColumn (Just _) (col :: VU.Vector a) -> filter (Col @(Maybe a) name) isNothing df + _ -> df _ -> df {- | O(n * k) removes all rows with `Nothing` from the dataframe. @@ -242,8 +287,8 @@ | L.null cs = empty | any (`notElem` columnNames df) cs = throw $ - ColumnNotFoundException - (T.pack $ show $ cs L.\\ columnNames df) + ColumnsNotFoundException + (cs L.\\ columnNames df) "select" (columnNames df) | otherwise = @@ -345,23 +390,12 @@ sample pureGen p df = let rand = generateRandomVector pureGen (fst (dataframeDimensions df)) + cRand = col @Double "__rand__" in df - & insertUnboxedVector "__rand__" rand - & filterWhere - ( Binary - ( MkBinaryOp - { binaryFn = (>=) - , binaryName = "geq" - , binarySymbol = Just ">=" - , binaryCommutative = False - , binaryPrecedence = 1 - } - ) - (Col @Double "__rand__") - (Lit (1 - p)) - ) - & exclude ["__rand__"] + & insertUnboxedVector (name cRand) rand + & filterWhere (cRand .>=. Lit (1 - p)) + & exclude [name cRand] {- | Split a dataset into two. The first in the tuple gets a sample of p (0 <= p <= 1) and the second gets (1 - p). This is useful for creating test and train splits. @@ -377,38 +411,16 @@ randomSplit pureGen p df = let rand = generateRandomVector pureGen (fst (dataframeDimensions df)) - withRand = df & insertUnboxedVector "__rand__" rand + cRand = col @Double "__rand__" + withRand = df & insertUnboxedVector (name cRand) rand in ( withRand - & filterWhere - ( Binary - ( MkBinaryOp - { binaryFn = (<=) - , binaryName = "leq" - , binarySymbol = Just "<=" - , binaryCommutative = False - , binaryPrecedence = 1 - } - ) - (Col @Double "__rand__") - (Lit p) - ) - & exclude ["__rand__"] + & filterWhere (cRand .<=. Lit p) + & exclude [name cRand] , withRand & filterWhere - ( Binary - ( MkBinaryOp - { binaryFn = (>) - , binaryName = "gt" - , binarySymbol = Just ">" - , binaryCommutative = False - , binaryPrecedence = 1 - } - ) - (Col @Double "__rand__") - (Lit p) - ) - & exclude ["__rand__"] + (cRand .>. Lit p) + & exclude [name cRand] ) {- | Creates n folds of a dataframe. @@ -424,46 +436,20 @@ kFolds pureGen folds df = let rand = generateRandomVector pureGen (fst (dataframeDimensions df)) - withRand = df & insertUnboxedVector "__rand__" rand + cRand = col @Double "__rand__" + withRand = df & insertUnboxedVector (name cRand) rand partitionSize = 1 / fromIntegral folds singleFold n d = - d - & filterWhere - ( Binary - ( MkBinaryOp - { binaryFn = (>=) - , binaryName = "geq" - , binarySymbol = Just ">=" - , binaryCommutative = False - , binaryPrecedence = 1 - } - ) - (Col @Double "__rand__") - (Lit (fromIntegral n * partitionSize)) - ) + d & filterWhere (cRand .>=. Lit (fromIntegral n * partitionSize)) go (-1) _ = [] go n d = let d' = singleFold n d - d'' = - d - & filterWhere - ( Binary - ( MkBinaryOp - { binaryFn = (<) - , binaryName = "lt" - , binarySymbol = Just "<" - , binaryCommutative = False - , binaryPrecedence = 1 - } - ) - (Col @Double "__rand__") - (Lit (fromIntegral n * partitionSize)) - ) + d'' = d & filterWhere (cRand .<. Lit (fromIntegral n * partitionSize)) in d' : go (n - 1) d'' in - map (exclude ["__rand__"]) (go (folds - 1) withRand) + map (exclude [name cRand]) (go (folds - 1) withRand) generateRandomVector :: (RandomGen g) => g -> Int -> VU.Vector Double generateRandomVector pureGen k = VU.fromList $ go pureGen k @@ -477,12 +463,19 @@ -- | Convert any Column to a vector of Text labels (one per row). columnToTextVec :: Column -> V.Vector T.Text -columnToTextVec (BoxedColumn (col :: V.Vector a)) = - case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> col - Nothing -> V.map (T.pack . show) col -columnToTextVec (UnboxedColumn col) = V.map (T.pack . show) (V.convert col) -columnToTextVec (OptionalColumn col) = V.map (T.pack . show) col +columnToTextVec (BoxedColumn bm (col :: V.Vector a)) = + case bm of + Nothing -> case testEquality (typeRep @a) (typeRep @T.Text) of + Just Refl -> col + Nothing -> V.map (T.pack . show) col + Just bitmap -> + V.imap (\i x -> if bitmapTestBit bitmap i then T.pack (show x) else "null") col +columnToTextVec (UnboxedColumn bm col) = + case bm of + Nothing -> V.map (T.pack . show) (V.convert col) + Just bitmap -> + V.generate (VU.length col) $ \i -> + if bitmapTestBit bitmap i then T.pack (show (col VU.! i)) else "null" -- | Build a map from stringified label to row indices. groupByIndices :: Column -> M.Map T.Text (VU.Vector Int) @@ -513,7 +506,7 @@ -} stratifiedSample :: forall a g. - (SplitGen g, RandomGen g, Columnable a) => + (SplittableGen g, Columnable a) => g -> Double -> Expr a -> DataFrame -> DataFrame stratifiedSample gen p strataCol df = let col = case strataCol of @@ -523,7 +516,7 @@ go _ [] = mempty go g (ixs : rest) = let stratum = rowsAtIndices ixs df - (g1, g2) = splitGen g + (g1, g2) = splitForStratified g in sample g1 p stratum <> go g2 rest in go gen groups @@ -537,7 +530,7 @@ -} stratifiedSplit :: forall a g. - (SplitGen g, RandomGen g, Columnable a) => + (SplittableGen g, Columnable a) => g -> Double -> Expr a -> DataFrame -> (DataFrame, DataFrame) stratifiedSplit gen p strataCol df = let col = case strataCol of @@ -547,7 +540,7 @@ go _ [] = (mempty, mempty) go g (ixs : rest) = let stratum = rowsAtIndices ixs df - (g1, g2) = splitGen g + (g1, g2) = splitForStratified g (tr, va) = randomSplit g1 p stratum (trAcc, vaAcc) = go g2 rest in (tr <> trAcc, va <> vaAcc) diff --git a/src/DataFrame/Operations/Transformations.hs b/src/DataFrame/Operations/Transformations.hs --- a/src/DataFrame/Operations/Transformations.hs +++ b/src/DataFrame/Operations/Transformations.hs @@ -21,9 +21,9 @@ import Data.Maybe import DataFrame.Errors (DataFrameException (..), TypeErrorContext (..)) import DataFrame.Internal.Column ( - Column (..), Columnable, TypedColumn (..), + hasMissing, ifoldrColumn, imapColumn, mapColumn, @@ -63,7 +63,8 @@ DataFrame -> Either DataFrameException DataFrame safeApply f columnName d = case getColumn columnName d of - Nothing -> Left $ ColumnNotFoundException columnName "apply" (M.keys $ columnIndices d) + Nothing -> + Left $ ColumnsNotFoundException [columnName] "apply" (M.keys $ columnIndices d) Just column -> do column' <- mapColumn f column pure $ insertColumn columnName column' d @@ -163,8 +164,8 @@ applyWhere condition filterColumnName f columnName df = case getColumn filterColumnName df of Nothing -> throw $ - ColumnNotFoundException - filterColumnName + ColumnsNotFoundException + [filterColumnName] "applyWhere" (M.keys $ columnIndices df) Just column -> case ifoldrColumn @@ -193,7 +194,7 @@ applyAtIndex i f columnName df = case getColumn columnName df of Nothing -> throw $ - ColumnNotFoundException columnName "applyAtIndex" (M.keys $ columnIndices df) + ColumnsNotFoundException [columnName] "applyAtIndex" (M.keys $ columnIndices df) Just column -> case imapColumn (\index value -> if index == i then f value else value) column of Left e -> throw e Right column' -> insertColumn columnName column' df @@ -208,8 +209,9 @@ DataFrame imputeCore (Col columnName) value df = case getColumn columnName df of Nothing -> - throw $ ColumnNotFoundException columnName "impute" (M.keys $ columnIndices df) - Just (OptionalColumn _) -> case safeApply (fromMaybe value) columnName df of + throw $ + ColumnsNotFoundException [columnName] "impute" (M.keys $ columnIndices df) + Just col | hasMissing col -> case safeApply (fromMaybe value) columnName df of Left (TypeMismatchException context) -> throw $ TypeMismatchException (context{callingFunctionName = Just "impute"}) Left exception -> throw exception Right res -> res diff --git a/src/DataFrame/Operations/Typing.hs b/src/DataFrame/Operations/Typing.hs --- a/src/DataFrame/Operations/Typing.hs +++ b/src/DataFrame/Operations/Typing.hs @@ -11,11 +11,15 @@ import Control.Applicative (asum) import Control.Monad (join) -import Data.Maybe (fromMaybe) import qualified Data.Proxy as P import Data.Time import Data.Type.Equality (TestEquality (..)) -import DataFrame.Internal.Column (Column (..), fromVector) +import DataFrame.Internal.Column ( + Column (..), + bitmapTestBit, + ensureOptional, + fromVector, + ) import DataFrame.Internal.DataFrame (DataFrame (..), unsafeGetColumn) import DataFrame.Internal.Parsing import DataFrame.Internal.Schema @@ -55,19 +59,22 @@ parseDefaults opts df = df{columns = V.map (parseDefault opts) (columns df)} parseDefault :: ParseOptions -> Column -> Column -parseDefault opts (BoxedColumn (c :: V.Vector a)) = +parseDefault opts (BoxedColumn Nothing (c :: V.Vector a)) = case (typeRep @a) `testEquality` (typeRep @T.Text) of Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of Just Refl -> parseFromExamples opts (V.map T.pack c) - Nothing -> BoxedColumn c + Nothing -> BoxedColumn Nothing c Just Refl -> parseFromExamples opts c -parseDefault opts (OptionalColumn (c :: V.Vector (Maybe a))) = +parseDefault opts (BoxedColumn (Just bm) (c :: V.Vector a)) = case (typeRep @a) `testEquality` (typeRep @T.Text) of Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of Just Refl -> - parseFromExamples opts (V.map (T.pack . fromMaybe "") c) - Nothing -> BoxedColumn c - Just Refl -> parseFromExamples opts (V.map (fromMaybe "") c) + parseFromExamples + opts + (V.imap (\i x -> if bitmapTestBit bm i then T.pack x else "") c) + Nothing -> BoxedColumn (Just bm) c + Just Refl -> + parseFromExamples opts (V.imap (\i x -> if bitmapTestBit bm i then x else "") c) parseDefault _ column = column parseFromExamples :: ParseOptions -> V.Vector T.Text -> Column @@ -78,14 +85,16 @@ examples = V.map converter (V.take (sampleSize opts) cols) asMaybeText = V.map converter cols dfmt = parseDateFormat opts + result = + case makeParsingAssumption dfmt examples of + BoolAssumption -> handleBoolAssumption asMaybeText + IntAssumption -> handleIntAssumption asMaybeText + DoubleAssumption -> handleDoubleAssumption asMaybeText + TextAssumption -> handleTextAssumption asMaybeText + DateAssumption -> handleDateAssumption dfmt asMaybeText + NoAssumption -> handleNoAssumption dfmt asMaybeText in - case makeParsingAssumption dfmt examples of - BoolAssumption -> handleBoolAssumption asMaybeText - IntAssumption -> handleIntAssumption asMaybeText - DoubleAssumption -> handleDoubleAssumption asMaybeText - TextAssumption -> handleTextAssumption asMaybeText - DateAssumption -> handleDateAssumption dfmt asMaybeText - NoAssumption -> handleNoAssumption dfmt asMaybeText + if parseSafe opts then ensureOptional result else result handleBoolAssumption :: V.Vector (Maybe T.Text) -> Column handleBoolAssumption asMaybeText @@ -232,7 +241,7 @@ ts where asType :: SchemaType -> Column -> Column - asType (SType (_ :: P.Proxy a)) c@(BoxedColumn (col :: V.Vector b)) = case typeRep @a of + asType (SType (_ :: P.Proxy a)) c@(BoxedColumn _ (col :: V.Vector b)) = case typeRep @a of App t1 t2 -> case eqTypeRep t1 (typeRep @Maybe) of Just HRefl -> case testEquality (typeRep @a) (typeRep @b) of Just Refl -> c diff --git a/src/DataFrame/Synthesis.hs b/src/DataFrame/Synthesis.hs --- a/src/DataFrame/Synthesis.hs +++ b/src/DataFrame/Synthesis.hs @@ -177,7 +177,7 @@ Left e -> throw e Right v -> v hasInvalid = case res of - (TColumn (UnboxedColumn (col :: VU.Vector b))) -> case testEquality (typeRep @Double) (typeRep @b) of + (TColumn (UnboxedColumn _ (col :: VU.Vector b))) -> case testEquality (typeRep @Double) (typeRep @b) of Just Refl -> VU.any (\n -> isNaN n || isInfinite n) col Nothing -> False _ -> False diff --git a/src/DataFrame/Typed.hs b/src/DataFrame/Typed.hs --- a/src/DataFrame/Typed.hs +++ b/src/DataFrame/Typed.hs @@ -183,13 +183,16 @@ -- * Template Haskell deriveSchema, deriveSchemaFromCsvFile, + deriveSchemaFromCsvFileWith, -- * Schema type families (for advanced use) Lookup, + SafeLookup, HasName, SubsetSchema, ExcludeSchema, RenameInSchema, + RenameManyInSchema, RemoveColumn, Impute, Append, @@ -225,7 +228,11 @@ import DataFrame.Typed.Join (fullOuterJoin, innerJoin, leftJoin, rightJoin) import DataFrame.Typed.Operations import DataFrame.Typed.Schema -import DataFrame.Typed.TH (deriveSchema, deriveSchemaFromCsvFile) +import DataFrame.Typed.TH ( + deriveSchema, + deriveSchemaFromCsvFile, + deriveSchemaFromCsvFileWith, + ) import DataFrame.Typed.Types ( Column, TSortOrder (..), diff --git a/src/DataFrame/Typed/Access.hs b/src/DataFrame/Typed/Access.hs --- a/src/DataFrame/Typed/Access.hs +++ b/src/DataFrame/Typed/Access.hs @@ -21,7 +21,7 @@ import DataFrame.Internal.Column (Columnable) import DataFrame.Internal.Expression (Expr (Col)) import qualified DataFrame.Operations.Core as D -import DataFrame.Typed.Schema (AssertPresent, Lookup) +import DataFrame.Typed.Schema (AssertPresent, SafeLookup) import DataFrame.Typed.Types (TypedDataFrame (..)) {- | Retrieve a column as a boxed 'Vector', with the type determined by @@ -30,7 +30,7 @@ columnAsVector :: forall name cols a. ( KnownSymbol name - , a ~ Lookup name cols + , a ~ SafeLookup name cols , Columnable a , AssertPresent name cols ) => @@ -44,7 +44,7 @@ columnAsList :: forall name cols a. ( KnownSymbol name - , a ~ Lookup name cols + , a ~ SafeLookup name cols , Columnable a , AssertPresent name cols ) => diff --git a/src/DataFrame/Typed/Expr.hs b/src/DataFrame/Typed/Expr.hs --- a/src/DataFrame/Typed/Expr.hs +++ b/src/DataFrame/Typed/Expr.hs @@ -155,7 +155,7 @@ ) import DataFrame.Internal.Types (Promote, PromoteDiv) -import DataFrame.Typed.Schema (AssertPresent, Lookup) +import DataFrame.Typed.Schema (AssertPresent, SafeLookup) import DataFrame.Typed.Types (TExpr (..), TSortOrder (..)) import Prelude hiding (maximum, minimum, sum) @@ -172,7 +172,7 @@ col :: forall (name :: Symbol) cols a. ( KnownSymbol name - , a ~ Lookup name cols + , a ~ SafeLookup name cols , Columnable a , AssertPresent name cols ) => diff --git a/src/DataFrame/Typed/Freeze.hs b/src/DataFrame/Typed/Freeze.hs --- a/src/DataFrame/Typed/Freeze.hs +++ b/src/DataFrame/Typed/Freeze.hs @@ -18,6 +18,7 @@ import qualified Data.Text as T import Type.Reflection (SomeTypeRep) +import Data.List (stripPrefix) import qualified DataFrame.Internal.Column as C import qualified DataFrame.Internal.DataFrame as D import DataFrame.Operations.Core (columnNames) @@ -81,6 +82,17 @@ <> ", got " <> T.pack (C.columnTypeString col) --- | Check if a Column's element type matches the expected SomeTypeRep. +{- | Check if a Column's element type matches the expected SomeTypeRep. +For nullable columns (those with a bitmap), @Maybe a@ in the schema matches +a column whose inner type is @a@, since we store nullable data as +@BoxedColumn (Just bm) a@ or @UnboxedColumn (Just bm) a@ rather than +@Column (Maybe a)@. +-} matchesType :: SomeTypeRep -> C.Column -> Bool -matchesType expected col = T.pack (show expected) == T.pack (C.columnTypeString col) +matchesType expected col = + let expectedStr = show expected + colTypeStr = C.columnTypeString col + in expectedStr == colTypeStr + || ( C.hasMissing col -- nullable column: schema says "Maybe X", column stores "X" with a bitmap + && Just colTypeStr == stripPrefix "Maybe " expectedStr + ) diff --git a/src/DataFrame/Typed/Operations.hs b/src/DataFrame/Typed/Operations.hs --- a/src/DataFrame/Typed/Operations.hs +++ b/src/DataFrame/Typed/Operations.hs @@ -328,7 +328,7 @@ forall name a cols. ( KnownSymbol name , Columnable a - , a ~ Lookup name cols + , a ~ SafeLookup name cols , AssertPresent name cols ) => TExpr cols a -> TypedDataFrame cols -> TypedDataFrame cols diff --git a/src/DataFrame/Typed/Schema.hs b/src/DataFrame/Typed/Schema.hs --- a/src/DataFrame/Typed/Schema.hs +++ b/src/DataFrame/Typed/Schema.hs @@ -16,6 +16,7 @@ module DataFrame.Typed.Schema ( -- * Type families for schema manipulation Lookup, + SafeLookup, HasName, RemoveColumn, Impute, @@ -75,6 +76,15 @@ Lookup name '[] = TypeError ('Text "Column '" ':<>: 'Text name ':<>: 'Text "' not found in schema") + +{- | Like 'Lookup', but returns a harmless fallback ('Int') instead of +'TypeError' when the column is not found. Use together with +'AssertPresent' so the error fires exactly once. +-} +type family SafeLookup (name :: Symbol) (cols :: [Type]) :: Type where + SafeLookup name (Column name a ': _) = a + SafeLookup name (Column _ _ ': rest) = SafeLookup name rest + SafeLookup name '[] = Int -- | Unwrap a Maybe from a type after we impute values. type family Impute (name :: Symbol) (cols :: [Type]) :: [Type] where diff --git a/src/DataFrame/Typed/TH.hs b/src/DataFrame/Typed/TH.hs --- a/src/DataFrame/Typed/TH.hs +++ b/src/DataFrame/Typed/TH.hs @@ -8,6 +8,7 @@ -- * Schema inference deriveSchema, deriveSchemaFromCsvFile, + deriveSchemaFromCsvFileWith, -- * Re-export for TH splices TypedDataFrame, @@ -49,8 +50,11 @@ pure [TySynD synName [] schemaType] deriveSchemaFromCsvFile :: String -> String -> DecsQ -deriveSchemaFromCsvFile typeName path = do - df <- liftIO (D.readCsv path) +deriveSchemaFromCsvFile = deriveSchemaFromCsvFileWith D.defaultReadOptions + +deriveSchemaFromCsvFileWith :: D.ReadOptions -> String -> String -> DecsQ +deriveSchemaFromCsvFileWith opts typeName path = do + df <- liftIO (D.readSeparated opts path) deriveSchema typeName df getSchemaInfo :: D.DataFrame -> [(T.Text, String)] diff --git a/tests/DecisionTree.hs b/tests/DecisionTree.hs --- a/tests/DecisionTree.hs +++ b/tests/DecisionTree.hs @@ -439,7 +439,7 @@ [ ("label", DI.fromList (replicate 6 "pos" ++ replicate 6 "neg" :: [T.Text])) , ( "x" - , DI.OptionalColumn + , DI.fromVector ( V.fromList $ map (Just . fromIntegral) ([1 .. 6] :: [Int]) ++ map (Just . fromIntegral) ([7 .. 12] :: [Int]) :: @@ -455,7 +455,7 @@ [ ("label", DI.fromList (["pos", "pos", "pos", "neg", "neg", "neg"] :: [T.Text])) , ( "x" - , DI.OptionalColumn + , DI.fromVector ( V.fromList [Just 1.0, Nothing, Just 3.0, Just 7.0, Nothing, Just 9.0] :: V.Vector (Maybe Double) @@ -463,16 +463,16 @@ ) ] --- numericCols picks up OptionalColumn (Maybe Double) as NMaybeDouble. +-- numericCols picks up DI.fromVector (Maybe Double) as NMaybeDouble. numericColsNullableDoubleTest :: Test numericColsNullableDoubleTest = TestCase $ do let exprs = numericCols nullableSepDF hasMD = any (\case NMaybeDouble _ -> True; _ -> False) exprs assertBool - "numericCols finds NMaybeDouble for OptionalColumn (Maybe Double)" + "numericCols finds NMaybeDouble for DI.fromVector (Maybe Double)" hasMD --- numericCols picks up OptionalColumn (Maybe Int) as NMaybeDouble (via whenPresent). +-- numericCols picks up DI.fromVector (Maybe Int) as NMaybeDouble (via whenPresent). numericColsNullableIntTest :: Test numericColsNullableIntTest = TestCase $ do let df = @@ -480,18 +480,18 @@ [ ("label", DI.fromList (["pos", "neg"] :: [T.Text])) , ( "n" - , DI.OptionalColumn (V.fromList [Just (1 :: Int), Just 2] :: V.Vector (Maybe Int)) + , DI.fromVector (V.fromList [Just (1 :: Int), Just 2] :: V.Vector (Maybe Int)) ) ] hasMD = any (\case NMaybeDouble _ -> True; _ -> False) (numericCols df) - assertBool "numericCols finds NMaybeDouble for OptionalColumn (Maybe Int)" hasMD + assertBool "numericCols finds NMaybeDouble for DI.fromVector (Maybe Int)" hasMD --- generateNumericConds is non-empty for a DF with an OptionalColumn (Maybe Double). +-- generateNumericConds is non-empty for a DF with an DI.fromVector (Maybe Double). numericCondsNullableNonEmptyTest :: Test numericCondsNullableNonEmptyTest = TestCase $ assertBool - "generateNumericConds non-empty for OptionalColumn (Maybe Double)" + "generateNumericConds non-empty for DI.fromVector (Maybe Double)" (not (null (generateNumericConds defaultTreeConfig nullableSepDF))) -- Null values evaluate to False for threshold conditions (null rows route right). @@ -502,7 +502,7 @@ [ ("label", DI.fromList (["A", "B"] :: [T.Text])) , ( "x" - , DI.OptionalColumn + , DI.fromVector (V.fromList [Nothing, Just (5.0 :: Double)] :: V.Vector (Maybe Double)) ) ] @@ -541,14 +541,14 @@ (loss >= 0.0 && loss <= 1.0) -- numericExprsWithTerms produces cross-column combinations when one col is --- OptionalColumn (Maybe Double) and another is a plain UnboxedColumn Double. +-- DI.fromVector (Maybe Double) and another is a plain UnboxedColumn Double. numericExprsWithTermsMixedTest :: Test numericExprsWithTermsMixedTest = TestCase $ do let df = D.fromNamedColumns [ ( "x" - , DI.OptionalColumn + , DI.fromVector (V.fromList [Just 1.0, Just 2.0, Just 3.0] :: V.Vector (Maybe Double)) ) , ("y", DI.fromList ([4.0, 5.0, 6.0] :: [Double])) diff --git a/tests/GenDataFrame.hs b/tests/GenDataFrame.hs --- a/tests/GenDataFrame.hs +++ b/tests/GenDataFrame.hs @@ -14,9 +14,9 @@ genColumn :: Int -> Gen Column genColumn len = oneof - [ BoxedColumn . V.fromList <$> vectorOf len (arbitrary @Int) - , UnboxedColumn . VU.fromList <$> vectorOf len (arbitrary @Double) - , OptionalColumn . V.fromList <$> vectorOf len (arbitrary @(Maybe Int)) + [ BoxedColumn Nothing . V.fromList <$> vectorOf len (arbitrary @Int) + , UnboxedColumn Nothing . VU.fromList <$> vectorOf len (arbitrary @Double) + , fromVector . V.fromList <$> vectorOf len (arbitrary @(Maybe Int)) ] genDataFrame :: Gen DataFrame diff --git a/tests/Main.hs b/tests/Main.hs --- a/tests/Main.hs +++ b/tests/Main.hs @@ -33,6 +33,7 @@ import qualified Operations.Subset import qualified Operations.Take import qualified Operations.Typing +import qualified Operations.WriteCsv import qualified Parquet import qualified Properties @@ -53,6 +54,7 @@ ++ Operations.Nullable.tests ++ Operations.Provenance.tests ++ Operations.ReadCsv.tests + ++ Operations.WriteCsv.tests ++ Operations.Shuffle.tests ++ Operations.Sort.tests ++ Operations.Statistics.tests diff --git a/tests/Operations/Apply.hs b/tests/Operations/Apply.hs --- a/tests/Operations/Apply.hs +++ b/tests/Operations/Apply.hs @@ -40,7 +40,7 @@ TestCase ( assertEqual "Boxed apply unboxed when result is unboxed" - (Just $ DI.UnboxedColumn (VU.fromList (replicate 26 (1 :: Int)))) + (Just $ DI.UnboxedColumn Nothing (VU.fromList (replicate 26 (1 :: Int)))) (DI.getColumn "test2" $ D.apply @String (const (1 :: Int)) "test2" testData) ) @@ -49,7 +49,7 @@ TestCase ( assertEqual "Boxed apply remains in boxed vector" - (Just $ DI.BoxedColumn (V.fromList (replicate 26 (1 :: Integer)))) + (Just $ DI.BoxedColumn Nothing (V.fromList (replicate 26 (1 :: Integer)))) (DI.getColumn "test2" $ D.apply @String (const (1 :: Integer)) "test2" testData) ) @@ -204,6 +204,7 @@ "applyWhere works as intended" ( Just $ DI.UnboxedColumn + Nothing (VU.fromList (zipWith ($) (cycle [id, (+ 1)]) [(1 :: Int) .. 26])) ) ( D.getColumn "test5" $ @@ -223,7 +224,7 @@ TestCase ( assertEqual "impute fills Nothing with the given value" - (Just $ DI.UnboxedColumn (VU.fromList [1 :: Int, 0, 3])) + (Just $ DI.UnboxedColumn Nothing (VU.fromList [1 :: Int, 0, 3])) (DI.getColumn "opt" $ impute (F.col @(Maybe Int) "opt") 0 imputeData) ) diff --git a/tests/Operations/Derive.hs b/tests/Operations/Derive.hs --- a/tests/Operations/Derive.hs +++ b/tests/Operations/Derive.hs @@ -32,6 +32,7 @@ "derive works with column expression" ( Just $ DI.BoxedColumn + Nothing (V.fromList (zipWith (\n c -> show n ++ [c]) [1 .. 26] ['a' .. 'z'])) ) ( DI.getColumn "test4" $ diff --git a/tests/Operations/GroupBy.hs b/tests/Operations/GroupBy.hs --- a/tests/Operations/GroupBy.hs +++ b/tests/Operations/GroupBy.hs @@ -47,7 +47,7 @@ TestCase ( assertExpectException "[Error Case]" - (D.columnNotFound "[\"test0\"]" "groupBy" (D.columnNames testData)) + (D.columnsNotFound ["test0"] "groupBy" (D.columnNames testData)) (print $ D.groupBy ["test0"] testData) ) diff --git a/tests/Operations/InsertColumn.hs b/tests/Operations/InsertColumn.hs --- a/tests/Operations/InsertColumn.hs +++ b/tests/Operations/InsertColumn.hs @@ -31,7 +31,9 @@ TestCase ( assertEqual "Two columns should be equal" - (Just $ DI.BoxedColumn (V.fromList ["Thuba" :: T.Text, "Zodwa", "Themba"])) + ( Just $ + DI.BoxedColumn Nothing (V.fromList ["Thuba" :: T.Text, "Zodwa", "Themba"]) + ) ( DI.getColumn "new" $ D.insertVector "new" (V.fromList ["Thuba" :: T.Text, "Zodwa", "Themba"]) D.empty ) @@ -57,7 +59,7 @@ TestCase ( assertEqual "Value should be boxed" - (Just $ DI.UnboxedColumn (VU.fromList [1 :: Int, 2, 3])) + (Just $ DI.UnboxedColumn Nothing (VU.fromList [1 :: Int, 2, 3])) (DI.getColumn "new" $ D.insertVector "new" (V.fromList [1 :: Int, 2, 3]) D.empty) ) @@ -77,7 +79,7 @@ ( assertEqual "Missing values should be replaced with Nothing" ( Just $ - DI.OptionalColumn + DI.fromVector (V.fromList [Just "a" :: Maybe T.Text, Just "b", Just "c", Nothing, Nothing]) ) ( DI.getColumn "newer" $ @@ -92,7 +94,7 @@ ( assertEqual "Missing values should be replaced with Nothing" ( Just $ - DI.OptionalColumn + DI.fromVector (V.fromList [Just 1 :: Maybe Int, Just 2, Just 3, Nothing, Nothing]) ) ( DI.getColumn "newer" $ @@ -106,7 +108,7 @@ TestCase ( assertEqual "Missing values should be replaced with Nothing" - (Just $ DI.UnboxedColumn (VU.fromList [1 :: Int, 2, 3, 0, 0])) + (Just $ DI.UnboxedColumn Nothing (VU.fromList [1 :: Int, 2, 3, 0, 0])) ( DI.getColumn "newer" $ D.insertVectorWithDefault 0 "newer" (V.fromList [1 :: Int, 2, 3]) $ D.insertVector "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty @@ -118,7 +120,7 @@ TestCase ( assertEqual "Lists should be the same size" - (Just $ DI.UnboxedColumn (VU.fromList [(6 :: Int) .. 10])) + (Just $ DI.UnboxedColumn Nothing (VU.fromList [(6 :: Int) .. 10])) ( DI.getColumn "newer" $ D.insertVectorWithDefault 0 "newer" (V.fromList [(6 :: Int) .. 10]) $ D.insertVector "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty diff --git a/tests/Operations/Join.hs b/tests/Operations/Join.hs --- a/tests/Operations/Join.hs +++ b/tests/Operations/Join.hs @@ -4,7 +4,9 @@ module Operations.Join where -import Data.Text (Text) +import Assertions (assertExpectException) +import Control.Exception (evaluate) +import Data.Text (Text, unpack) import Data.These import qualified DataFrame as D import qualified DataFrame.Functions as F @@ -26,6 +28,21 @@ , ("B", D.fromList ["B0" :: Text, "B1", "B2"]) ] +assertMissingJoinColumns :: String -> [Text] -> D.DataFrame -> Assertion +assertMissingJoinColumns preface missingKeys result = do + assertExpectException + preface + (if length missingKeys == 1 then "Column not found" else "Columns not found") + (evaluate (D.nRows result)) + mapM_ + ( \missingKey -> + assertExpectException preface (unpack missingKey) (evaluate (D.nRows result)) + ) + missingKeys + +assertMissingJoinColumn :: String -> Text -> D.DataFrame -> Assertion +assertMissingJoinColumn preface missingKey = assertMissingJoinColumns preface [missingKey] + testInnerJoin :: Test testInnerJoin = TestCase @@ -253,6 +270,59 @@ (D.sortBy [D.Asc (F.col @Text "key")] (fullOuterJoin ["key"] dfL dfR)) ) +testInnerJoinMissingKey :: Test +testInnerJoinMissingKey = + TestCase $ + assertMissingJoinColumn + "Inner join should fail when the join key is missing" + "Cats" + (innerJoin ["Cats"] df1 df2) + +testLeftJoinMissingKey :: Test +testLeftJoinMissingKey = + TestCase $ + assertMissingJoinColumn + "Left join should fail when the join key is missing" + "Cats" + (leftJoin ["Cats"] df1 df2) + +testRightJoinMissingKey :: Test +testRightJoinMissingKey = + TestCase $ + assertMissingJoinColumn + "Right join should fail when the join key is missing" + "Animals" + (rightJoin ["Animals"] df1 df2) + +testFullOuterJoinMissingKey :: Test +testFullOuterJoinMissingKey = + TestCase $ + assertMissingJoinColumn + "Full outer join should fail when the join key is missing" + "Cats" + (fullOuterJoin ["Cats"] df1 df2) + +testInnerJoinMissingKeys :: Test +testInnerJoinMissingKeys = + TestCase $ + assertMissingJoinColumns + "Inner join should report every missing join key" + ["Animals", "Cats"] + (innerJoin ["Animals", "Cats"] df1 df2) + +testInnerJoinMissingKeysSuggestion :: Test +testInnerJoinMissingKeysSuggestion = + TestCase $ + let typoDf = + D.fromNamedColumns + [ ("hello", D.fromList ["H" :: Text]) + , ("world", D.fromList ["W" :: Text]) + ] + in assertExpectException + "Inner join should report consolidated suggestions for missing join keys" + "Did you mean [\"hello\", \"world\"]?" + (evaluate (D.nRows (innerJoin ["helo", "wrld"] typoDf typoDf))) + -- Empty DataFrame fixtures: same schema as df1/df2 but zero rows. emptyDf1 :: D.DataFrame emptyDf1 = @@ -353,6 +423,12 @@ , TestLabel "leftJoinWithCollisions" testLeftJoinWithCollisions , TestLabel "rightJoinWithCollisions" testRightJoinWithCollisions , TestLabel "outerJoinWithCollisions" testOuterJoinWithCollisions + , TestLabel "innerJoinMissingKey" testInnerJoinMissingKey + , TestLabel "leftJoinMissingKey" testLeftJoinMissingKey + , TestLabel "rightJoinMissingKey" testRightJoinMissingKey + , TestLabel "fullOuterJoinMissingKey" testFullOuterJoinMissingKey + , TestLabel "innerJoinMissingKeys" testInnerJoinMissingKeys + , TestLabel "innerJoinMissingKeysSuggestion" testInnerJoinMissingKeysSuggestion , TestLabel "innerJoinBothEmpty" testInnerJoinBothEmpty , TestLabel "innerJoinLeftEmpty" testInnerJoinLeftEmpty , TestLabel "innerJoinRightEmpty" testInnerJoinRightEmpty diff --git a/tests/Operations/Nullable.hs b/tests/Operations/Nullable.hs --- a/tests/Operations/Nullable.hs +++ b/tests/Operations/Nullable.hs @@ -27,7 +27,7 @@ testData = D.fromNamedColumns [ ("x", DI.fromList ([1, 2, 3] :: [Int])) - , ("y", DI.OptionalColumn (V.fromList [Just 10, Nothing, Just 30 :: Maybe Int])) + , ("y", DI.fromVector (V.fromList [Just 10, Nothing, Just 30 :: Maybe Int])) ] -- | col @Int .+ col @(Maybe Int) should give Maybe Int column @@ -36,7 +36,7 @@ TestCase ( assertEqual "Int .+ Maybe Int = Maybe Int" - (Just $ DI.OptionalColumn (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int])) + (Just $ DI.fromVector (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int])) ( DI.getColumn "result" $ D.derive "result" @@ -51,7 +51,7 @@ TestCase ( assertEqual "Maybe Int .+ Int = Maybe Int" - (Just $ DI.OptionalColumn (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int])) + (Just $ DI.fromVector (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int])) ( DI.getColumn "result" $ D.derive "result" @@ -81,7 +81,7 @@ TestCase ( assertEqual "Maybe Int .+ Maybe Int = Maybe Int" - (Just $ DI.OptionalColumn (V.fromList [Just 20, Nothing, Just 60 :: Maybe Int])) + (Just $ DI.fromVector (V.fromList [Just 20, Nothing, Just 60 :: Maybe Int])) ( DI.getColumn "result" $ D.derive "result" @@ -97,7 +97,7 @@ ( assertEqual "Int .- Maybe Int = Maybe Int" ( Just $ - DI.OptionalColumn (V.fromList [Just (-9), Nothing, Just (-27) :: Maybe Int]) + DI.fromVector (V.fromList [Just (-9), Nothing, Just (-27) :: Maybe Int]) ) ( DI.getColumn "result" $ D.derive @@ -114,7 +114,7 @@ ( assertEqual "Int .== Maybe Int = Maybe Bool" ( Just $ - DI.OptionalColumn (V.fromList [Just False, Nothing, Just False :: Maybe Bool]) + DI.fromVector (V.fromList [Just False, Nothing, Just False :: Maybe Bool]) ) ( DI.getColumn "result" $ D.derive @@ -150,7 +150,7 @@ ( assertEqual "nullLift negate (Maybe Int) propagates Nothing" ( Just $ - DI.OptionalColumn (V.fromList [Just (-10), Nothing, Just (-30) :: Maybe Int]) + DI.fromVector (V.fromList [Just (-10), Nothing, Just (-30) :: Maybe Int]) ) ( DI.getColumn "result" $ D.derive @@ -181,7 +181,7 @@ TestCase ( assertEqual "nullLift2 (+) Int (Maybe Int) = Maybe Int" - (Just $ DI.OptionalColumn (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int])) + (Just $ DI.fromVector (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int])) ( DI.getColumn "result" $ D.derive "result" @@ -196,7 +196,7 @@ TestCase ( assertEqual "nullLift2 (+) (Maybe Int) Int = Maybe Int" - (Just $ DI.OptionalColumn (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int])) + (Just $ DI.fromVector (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int])) ( DI.getColumn "result" $ D.derive "result" @@ -229,11 +229,11 @@ crossData = D.fromNamedColumns [ ("x", DI.fromList ([1, 2, 3] :: [Int])) - , ("y", DI.OptionalColumn (V.fromList [Just 10, Nothing, Just 30 :: Maybe Int])) + , ("y", DI.fromVector (V.fromList [Just 10, Nothing, Just 30 :: Maybe Int])) , ("d", DI.fromList ([1.5, 2.5, 3.5] :: [Double])) , ( "md" - , DI.OptionalColumn (V.fromList [Just 10.5, Nothing, Just 30.5 :: Maybe Double]) + , DI.fromVector (V.fromList [Just 10.5, Nothing, Just 30.5 :: Maybe Double]) ) ] @@ -268,7 +268,7 @@ ( assertEqual "Maybe Int .+ Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 11.5, Nothing, Just 33.5 :: Maybe Double]) + DI.fromVector (V.fromList [Just 11.5, Nothing, Just 33.5 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive "result" (F.col @(Maybe Int) "y" .+ F.col @Double "d") crossData @@ -282,7 +282,7 @@ ( assertEqual "Int .+ Maybe Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 11.5, Nothing, Just 33.5 :: Maybe Double]) + DI.fromVector (V.fromList [Just 11.5, Nothing, Just 33.5 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive "result" (F.col @Int "x" .+ F.col @(Maybe Double) "md") crossData @@ -296,7 +296,7 @@ ( assertEqual "Maybe Int .+ Maybe Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 20.5, Nothing, Just 60.5 :: Maybe Double]) + DI.fromVector (V.fromList [Just 20.5, Nothing, Just 60.5 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive @@ -349,7 +349,7 @@ ( assertEqual "Maybe Int .- Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 8.5, Nothing, Just 26.5 :: Maybe Double]) + DI.fromVector (V.fromList [Just 8.5, Nothing, Just 26.5 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive "result" (F.col @(Maybe Int) "y" .- F.col @Double "d") crossData @@ -363,7 +363,7 @@ ( assertEqual "Int .- Maybe Double = Maybe Double" ( Just $ - DI.OptionalColumn + DI.fromVector (V.fromList [Just (-9.5), Nothing, Just (-27.5) :: Maybe Double]) ) ( DI.getColumn "result" $ @@ -378,7 +378,7 @@ ( assertEqual "Maybe Int .- Maybe Double = Maybe Double" ( Just $ - DI.OptionalColumn + DI.fromVector (V.fromList [Just (-0.5), Nothing, Just (-0.5) :: Maybe Double]) ) ( DI.getColumn "result" $ @@ -408,7 +408,7 @@ ( assertEqual "Maybe Int .* Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 15.0, Nothing, Just 105.0 :: Maybe Double]) + DI.fromVector (V.fromList [Just 15.0, Nothing, Just 105.0 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive "result" (F.col @(Maybe Int) "y" .* F.col @Double "d") crossData @@ -422,7 +422,7 @@ ( assertEqual "Int .* Maybe Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 10.5, Nothing, Just 91.5 :: Maybe Double]) + DI.fromVector (V.fromList [Just 10.5, Nothing, Just 91.5 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive "result" (F.col @Int "x" .* F.col @(Maybe Double) "md") crossData @@ -436,7 +436,7 @@ ( assertEqual "Maybe Int .* Maybe Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 105.0, Nothing, Just 915.0 :: Maybe Double]) + DI.fromVector (V.fromList [Just 105.0, Nothing, Just 915.0 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive @@ -451,11 +451,11 @@ divData = D.fromNamedColumns [ ("x", DI.fromList ([2, 4, 6] :: [Int])) - , ("y", DI.OptionalColumn (V.fromList [Just 4, Nothing, Just 6 :: Maybe Int])) + , ("y", DI.fromVector (V.fromList [Just 4, Nothing, Just 6 :: Maybe Int])) , ("d", DI.fromList ([1.0, 2.0, 3.0] :: [Double])) , ( "md" - , DI.OptionalColumn (V.fromList [Just 1.0, Nothing, Just 3.0 :: Maybe Double]) + , DI.fromVector (V.fromList [Just 1.0, Nothing, Just 3.0 :: Maybe Double]) ) ] @@ -490,7 +490,7 @@ ( assertEqual "Maybe Int ./ Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 4.0, Nothing, Just 2.0 :: Maybe Double]) + DI.fromVector (V.fromList [Just 4.0, Nothing, Just 2.0 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive "result" (F.col @(Maybe Int) "y" ./ F.col @Double "d") divData @@ -504,7 +504,7 @@ ( assertEqual "Int ./ Maybe Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 2.0, Nothing, Just 2.0 :: Maybe Double]) + DI.fromVector (V.fromList [Just 2.0, Nothing, Just 2.0 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive "result" (F.col @Int "x" ./ F.col @(Maybe Double) "md") divData @@ -518,7 +518,7 @@ ( assertEqual "Maybe Int ./ Maybe Double = Maybe Double" ( Just $ - DI.OptionalColumn (V.fromList [Just 4.0, Nothing, Just 2.0 :: Maybe Double]) + DI.fromVector (V.fromList [Just 4.0, Nothing, Just 2.0 :: Maybe Double]) ) ( DI.getColumn "result" $ D.derive "result" (F.col @(Maybe Int) "y" ./ F.col @(Maybe Double) "md") divData @@ -705,7 +705,7 @@ TestCase ( assertEqual "apply negate to Maybe Int column fmaps" - (Just $ DI.OptionalColumn (V.fromList [Just (-10), Nothing, Just (-30 :: Int)])) + (Just $ DI.fromVector (V.fromList [Just (-10), Nothing, Just (-30 :: Int)])) ( DI.getColumn "y" $ D.apply @Int negate "y" testData ) diff --git a/tests/Operations/ReadCsv.hs b/tests/Operations/ReadCsv.hs --- a/tests/Operations/ReadCsv.hs +++ b/tests/Operations/ReadCsv.hs @@ -3,156 +3,29 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} --- Test fixtures inspired by csv-spectrum (https://github.com/max-mapper/csv-spectrum) - module Operations.ReadCsv where -import qualified Data.List as L -import qualified Data.Map as M import qualified Data.Text as T -import qualified Data.Text.IO as TIO -import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D -import Data.Function (on) -import Data.Maybe (fromMaybe) -import Data.Type.Equality (testEquality, (:~:) (Refl)) import DataFrame.Internal.Column (Column (..), columnTypeString) -import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame ( DataFrame (..), - columnIndices, - columns, dataframeDimensions, getColumn, ) -import System.Directory (removeFile) -import System.IO (IOMode (..), withFile) import Test.HUnit import Type.Reflection (typeRep) -fixtureDir :: FilePath -fixtureDir = "./tests/data/unstable_csv/" - -tempDir :: FilePath -tempDir = "./tests/data/unstable_csv/" +arbuthnotPath :: FilePath +arbuthnotPath = "./tests/data/arbuthnot.csv" readCsvNoInfer :: FilePath -> IO DataFrame readCsvNoInfer = D.readCsvWithOpts D.defaultReadOptions{D.typeSpec = D.NoInference} --------------------------------------------------------------------------------- --- Pretty-printer --------------------------------------------------------------------------------- - -prettyPrintCsv :: FilePath -> DataFrame -> IO () -prettyPrintCsv = prettyPrintSeparated ',' - -prettyPrintTsv :: FilePath -> DataFrame -> IO () -prettyPrintTsv = prettyPrintSeparated '\t' - -prettyPrintSeparated :: Char -> FilePath -> DataFrame -> IO () -prettyPrintSeparated sep filepath df = withFile filepath WriteMode $ \handle -> do - let (rows, _) = dataframeDimensions df - let headers = map fst (L.sortBy (compare `on` snd) (M.toList (columnIndices df))) - TIO.hPutStrLn - handle - (T.intercalate (T.singleton sep) (map (escapeField sep) headers)) - -- Write data rows - mapM_ - (TIO.hPutStrLn handle . T.intercalate (T.singleton sep) . getRowEscaped sep df) - [0 .. rows - 1] - --- Note: The unstable parser does not unescape doubled quotes ("" -> "), --- so we must not double-escape them here. We only wrap in quotes when needed. -escapeField :: Char -> T.Text -> T.Text -escapeField sep field - | needsQuoting = T.concat ["\"", field, "\""] - | otherwise = field - where - needsQuoting = - T.any (\c -> c == sep || c == '\n' || c == '\r' || c == '"') field - --- | Get a row from the DataFrame with all fields escaped -getRowEscaped :: Char -> DataFrame -> Int -> [T.Text] -getRowEscaped sep df i = V.ifoldr go [] (columns df) - where - go :: Int -> Column -> [T.Text] -> [T.Text] - go _ (BoxedColumn (c :: V.Vector a)) acc = case c V.!? i of - Just e -> escapeField sep textRep : acc - where - textRep = case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> e - Nothing -> T.pack (show e) - Nothing -> acc - go _ (UnboxedColumn c) acc = case c VU.!? i of - Just e -> escapeField sep (T.pack (show e)) : acc - Nothing -> acc - go _ (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of - Just e -> escapeField sep textRep : acc - where - textRep = case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> fromMaybe "" e - Nothing -> case e of - Just val -> T.pack (show val) - Nothing -> "" - Nothing -> acc - -testFastCsv :: String -> FilePath -> Test -testFastCsv name csvPath = TestLabel ("fast_roundtrip_" <> name) $ TestCase $ do - dfOriginal <- D.fastReadCsvUnstable csvPath - let tempPath = tempDir <> "temp_fast_" <> name <> ".csv" - prettyPrintCsv tempPath dfOriginal - dfRoundtrip <- D.fastReadCsvUnstable tempPath - assertEqual - ("Fast round-trip should produce equivalent DataFrame for " <> name) - dfOriginal - dfRoundtrip - removeFile tempPath - -testTsv :: String -> FilePath -> Test -testTsv name tsvPath = TestLabel ("roundtrip_tsv_" <> name) $ TestCase $ do - dfOriginal <- D.readTsvUnstable tsvPath - let tempPath = tempDir <> "temp_" <> name <> ".tsv" - prettyPrintTsv tempPath dfOriginal - dfRoundtrip <- D.readTsvUnstable tempPath - assertEqual - ("TSV round-trip should produce equivalent DataFrame for " <> name) - dfOriginal - dfRoundtrip - removeFile tempPath - --- Individual round-trip test cases for each fixture - -testSimpleFast :: Test -testSimpleFast = testFastCsv "simple" (fixtureDir <> "simple.csv") - -testCommaInQuotesFast :: Test -testCommaInQuotesFast = testFastCsv "comma_in_quotes" (fixtureDir <> "comma_in_quotes.csv") - -testEscapedQuotesFast :: Test -testEscapedQuotesFast = testFastCsv "escaped_quotes" (fixtureDir <> "escaped_quotes.csv") - -testNewlinesFast :: Test -testNewlinesFast = testFastCsv "newlines" (fixtureDir <> "newlines.csv") - -testUtf8Fast :: Test -testUtf8Fast = testFastCsv "utf8" (fixtureDir <> "utf8.csv") - -testQuotesAndNewlinesFast :: Test -testQuotesAndNewlinesFast = testFastCsv "quotes_and_newlines" (fixtureDir <> "quotes_and_newlines.csv") - -testEmptyValuesFast :: Test -testEmptyValuesFast = testFastCsv "empty_values" (fixtureDir <> "empty_values.csv") - -testJsonDataFast :: Test -testJsonDataFast = testFastCsv "json_data" (fixtureDir <> "json_data.csv") - -arbuthnotPath :: FilePath -arbuthnotPath = "./tests/data/arbuthnot.csv" - -- SpecifyTypes with NoInference fallback: named column is typed, rest stay Text specifyTypesNoInferenceFallback :: Test specifyTypesNoInferenceFallback = @@ -168,11 +41,11 @@ arbuthnotPath -- "year" must be Int case getColumn "year" df of - Just col@(UnboxedColumn _) -> assertEqual "year should be Int" "Int" (columnTypeString col) + Just col@(UnboxedColumn _ _) -> assertEqual "year should be Int" "Int" (columnTypeString col) _ -> assertFailure "expected UnboxedColumn for 'year'" -- "boys" unspecified + NoInference → stays Text case getColumn "boys" df of - Just col@(BoxedColumn _) -> assertEqual "boys should be Text" "Text" (columnTypeString col) + Just col@(BoxedColumn _ _) -> assertEqual "boys should be Text" "Text" (columnTypeString col) _ -> assertFailure "expected BoxedColumn for 'boys' with NoInference fallback" -- SpecifyTypes with InferFromSample fallback: named column typed, rest inferred @@ -190,11 +63,11 @@ arbuthnotPath -- "year" must be Int (explicitly specified) case getColumn "year" df of - Just col@(UnboxedColumn _) -> assertEqual "year should be Int" "Int" (columnTypeString col) + Just col@(UnboxedColumn _ _) -> assertEqual "year should be Int" "Int" (columnTypeString col) _ -> assertFailure "expected UnboxedColumn for 'year'" -- "boys" unspecified + InferFromSample → inferred as Int case getColumn "boys" df of - Just col@(UnboxedColumn _) -> assertEqual "boys should be Int" "Int" (columnTypeString col) + Just col@(UnboxedColumn _ _) -> assertEqual "boys should be Int" "Int" (columnTypeString col) _ -> assertFailure "expected UnboxedColumn for 'boys' with InferFromSample fallback" @@ -213,128 +86,17 @@ } arbuthnotPath case getColumn "girls" df of - Just col@(UnboxedColumn _) -> assertEqual "girls should be Int" "Int" (columnTypeString col) + Just col@(UnboxedColumn _ _) -> assertEqual "girls should be Int" "Int" (columnTypeString col) _ -> assertFailure "expected UnboxedColumn for 'girls' via fallback InferFromSample 10" -testCrlfCsv :: Test -testCrlfCsv = TestLabel "malformed_crlf_csv" $ TestCase $ do - df <- D.readCsvUnstable (fixtureDir <> "crlf.csv") - let (rows, cols) = dataframeDimensions df - assertEqual "crlf.csv: 2 data rows" 2 rows - assertEqual "crlf.csv: 2 columns" 2 cols - case getColumn "name" df of - Nothing -> assertFailure "crlf.csv: column 'name' missing" - Just col -> - assertEqual - "crlf.csv: name has no \\r" - (DI.fromList @T.Text ["Alice", "Bob"]) - col - -testCrlfTsv :: Test -testCrlfTsv = TestLabel "malformed_crlf_tsv" $ TestCase $ do - df <- D.readTsvUnstable (fixtureDir <> "crlf.tsv") - let (rows, _) = dataframeDimensions df - assertEqual "crlf.tsv: 1 data row" 1 rows - case getColumn "name" df of - Nothing -> assertFailure "crlf.tsv: column 'name' missing" - Just col -> - assertEqual - "crlf.tsv: name has no \\r" - (DI.fromList @T.Text ["Alice"]) - col - -testHeaderOnly :: Test -testHeaderOnly = TestLabel "malformed_header_only" $ TestCase $ do - df <- D.readCsvUnstable (fixtureDir <> "header_only.csv") - let (rows, cols) = dataframeDimensions df - assertEqual "header_only.csv: 0 data rows" 0 rows - assertEqual "header_only.csv: 3 columns" 3 cols - let names = map fst . L.sortBy (compare `on` snd) . M.toList $ columnIndices df - assertEqual "header_only.csv: column names" ["first", "second", "third"] names - -testTrailingBlankLine :: Test -testTrailingBlankLine = TestLabel "malformed_trailing_blank_line" $ TestCase $ do - df <- D.readCsvUnstable (fixtureDir <> "trailing_blank_line.csv") - -- blank line contributes 1 extra delimiter; (3+3+1) div 3 = 2, numRow=1 - assertEqual - "trailing_blank_line.csv: 1 data row visible" - 1 - (fst (dataframeDimensions df)) - -testAllEmptyRow :: Test -testAllEmptyRow = TestLabel "malformed_all_empty_row" $ TestCase $ do - df <- D.readCsvUnstable (fixtureDir <> "all_empty_row.csv") - assertEqual "all_empty_row.csv: 1 data row" 1 (fst (dataframeDimensions df)) - let checkEmpty colName = - case getColumn colName df of - Nothing -> assertFailure ("column '" <> T.unpack colName <> "' missing") - Just col -> - assertEqual - (T.unpack colName <> " is Nothing (empty field → null)") - (DI.fromList @(Maybe T.Text) [Nothing]) - col - mapM_ checkEmpty ["a", "b", "c"] - -testSingleCol :: Test -testSingleCol = TestLabel "malformed_single_col" $ TestCase $ do - df <- D.readCsvUnstable (fixtureDir <> "single_col.csv") - let (rows, cols) = dataframeDimensions df - assertEqual "single_col.csv: 3 data rows" 3 rows - assertEqual "single_col.csv: 1 column" 1 cols - case getColumn "name" df of - Nothing -> assertFailure "single_col.csv: column 'name' missing" - Just col -> - assertEqual - "single_col.csv: correct values" - (DI.fromList @T.Text ["Alice", "Bob", "Carol"]) - col - -testWhitespaceFields :: Test -testWhitespaceFields = TestLabel "malformed_whitespace_fields" $ TestCase $ do - df <- D.readCsvUnstable (fixtureDir <> "whitespace_fields.csv") - assertEqual - "whitespace_fields.csv: 2 data rows" - 2 - (fst (dataframeDimensions df)) - case getColumn "name" df of - Nothing -> assertFailure "whitespace_fields.csv: 'name' missing" - Just col -> assertEqual "name stripped" (DI.fromList @T.Text ["Alice", "Bob"]) col - case getColumn "city" df of - Nothing -> assertFailure "whitespace_fields.csv: 'city' missing" - Just col -> - assertEqual - "city stripped" - (DI.fromList @T.Text ["New York", "Los Angeles"]) - col - --- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full) --- Total delimiters: 3 (header) + 2 (short) + 3 (full) = 8 --- numCol=3, totalRows=8 div 3=2, numRow=1 --- Row-1 stride offsets 3,4,5 → fields "1","2","X" (X bleeds in from next row) -testMissingFields :: Test -testMissingFields = TestLabel "malformed_missing_fields" $ TestCase $ do - df <- D.readCsvUnstable (fixtureDir <> "missing_fields.csv") - assertEqual - "missing_fields.csv: integer-division gives 1 visible row" - 1 - (fst (dataframeDimensions df)) - case getColumn "c" df of - Nothing -> assertFailure "missing_fields.csv: column 'c' missing" - Just col -> - -- "X" bleeds from the start of the next row into the missing slot - assertEqual - "missing_fields.csv: col 'c' bleeds 'X' from next row" - (DI.fromList @T.Text ["X"]) - col - -- File: a,b,c header; row "1,2,3,EXTRA" -- Total delimiters: 3 + 4 = 7; 7 div 3 = 2, numRow=1 -- Row-1 strides 3,4,5 → "1","2","3" — EXTRA (stride 6) is never accessed testExtraFields :: Test testExtraFields = TestLabel "malformed_extra_fields" $ TestCase $ do - df <- readCsvNoInfer (fixtureDir <> "extra_fields.csv") + df <- readCsvNoInfer ("./tests/data/unstable_csv/" <> "extra_fields.csv") assertEqual "extra_fields.csv: 1 data row" 1 (fst (dataframeDimensions df)) assertEqual "extra_fields.csv: 3 columns" 3 (snd (dataframeDimensions df)) case getColumn "c" df of @@ -342,49 +104,13 @@ Just col -> assertEqual "extra_fields.csv: 'c' = '3' (EXTRA ignored)" - (DI.fromList @T.Text ["3"]) - col - -testNoTrailingNewline :: Test -testNoTrailingNewline = TestLabel "malformed_no_trailing_newline" $ TestCase $ do - df <- D.readCsvUnstable (fixtureDir <> "no_trailing_newline.csv") - assertEqual - "no_trailing_newline.csv: 1 data row" - 1 - (fst (dataframeDimensions df)) - case getColumn "name" df of - Nothing -> assertFailure "no_trailing_newline.csv: 'name' missing" - Just col -> assertEqual "name = Alice" (DI.fromList @T.Text ["Alice"]) col - case getColumn "city" df of - Nothing -> assertFailure "no_trailing_newline.csv: 'city' missing" - Just col -> - assertEqual - "city = London (synthetic delimiter worked)" - (DI.fromList @T.Text ["London"]) + (D.fromList @T.Text ["3"]) col tests :: [Test] tests = - [ testSimpleFast - , testCommaInQuotesFast - , testQuotesAndNewlinesFast - , testEscapedQuotesFast - , testNewlinesFast - , testUtf8Fast - , testQuotesAndNewlinesFast - , testEmptyValuesFast - , testJsonDataFast - , specifyTypesNoInferenceFallback + [ specifyTypesNoInferenceFallback , specifyTypesInferFallback , specifyTypesSampleSize - , testCrlfCsv - , testCrlfTsv - , testHeaderOnly - , testTrailingBlankLine - , testAllEmptyRow - , testSingleCol - , testWhitespaceFields - , testMissingFields , testExtraFields - , testNoTrailingNewline ] diff --git a/tests/Operations/Sort.hs b/tests/Operations/Sort.hs --- a/tests/Operations/Sort.hs +++ b/tests/Operations/Sort.hs @@ -85,7 +85,7 @@ TestCase ( assertExpectException "[Error Case]" - (D.columnNotFound "[\"test0\"]" "sortBy" (D.columnNames testData)) + (D.columnsNotFound ["test0"] "sortBy" (D.columnNames testData)) (print $ D.sortBy [D.Asc (F.col @Int "test0")] testData) ) diff --git a/tests/Operations/Statistics.hs b/tests/Operations/Statistics.hs --- a/tests/Operations/Statistics.hs +++ b/tests/Operations/Statistics.hs @@ -244,7 +244,7 @@ (abs (r - 1.0) < 1e-10) ) --- Requesting a missing column should throw ColumnNotFoundException +-- Requesting a missing column should throw ColumnsNotFoundException correlationMissingColumn :: Test correlationMissingColumn = TestCase diff --git a/tests/Operations/Typing.hs b/tests/Operations/Typing.hs --- a/tests/Operations/Typing.hs +++ b/tests/Operations/Typing.hs @@ -4,5125 +4,1108 @@ module Operations.Typing where import qualified Data.Text as T -import qualified Data.Vector as V -import qualified Data.Vector.Unboxed as VU -import qualified DataFrame as D -import qualified DataFrame.Internal.Column as DI -import qualified DataFrame.Operations.Typing as D - -import Data.Time (Day, fromGregorian) -import Test.HUnit (Test (TestCase, TestLabel), assertEqual) - -testData :: D.DataFrame -testData = - D.fromNamedColumns - [ ("test1", DI.fromList ([1 .. 26] :: [Int])) - , ("test2", DI.fromList ['a' .. 'z']) - ] - --- Dimensions -correctDimensions :: Test -correctDimensions = TestCase (assertEqual "should be (26, 2)" (26, 2) (D.dimensions testData)) - -emptyDataframeDimensions :: Test -emptyDataframeDimensions = TestCase (assertEqual "should be (0, 0)" (0, 0) (D.dimensions D.empty)) - -dimensionsTest :: [Test] -dimensionsTest = - [ TestLabel "dimensions_correctDimensions" correctDimensions - , TestLabel "dimensions_emptyDataframeDimensions" emptyDataframeDimensions - ] - --- PARSING TESTS -------- 1. SIMPLE CASES -parseBools :: Test -parseBools = - let afterParse :: [Bool] - afterParse = [True, True, True] ++ [False, False, False] - beforeParse :: [T.Text] - beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Bools without missing values as UnboxedColumn of Bools" - expected - actual - ) - -parseInts :: Test -parseInts = - let afterParse :: [Int] - afterParse = [1 .. 50] - beforeParse :: [T.Text] - beforeParse = T.pack . show <$> [1 .. 50] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints without missing values as UnboxedColumn of Ints" - expected - actual - ) - -parseDoubles :: Test -parseDoubles = - let afterParse :: [Double] - afterParse = [1.0 .. 50.0] ++ [3.14, 2.22, 8.55, 23.3, 12.22222235049450945049504950] - beforeParse :: [T.Text] - beforeParse = - T.pack . show - <$> [1.0 .. 50.0] ++ [3.14, 2.22, 8.55, 23.3, 12.22222235049450945049504950] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Doubles without missing values as UnboxedColumn of Doubles" - expected - actual - ) - -parseDates :: Test -parseDates = - let afterParse :: [Day] - afterParse = - [ fromGregorian 2020 02 12 - , fromGregorian 2020 02 13 - , fromGregorian 2020 02 14 - , fromGregorian 2020 02 15 - , fromGregorian 2020 02 16 - , fromGregorian 2020 02 17 - , fromGregorian 2020 02 18 - , fromGregorian 2020 02 19 - , fromGregorian 2020 02 20 - , fromGregorian 2020 02 21 - , fromGregorian 2020 02 22 - , fromGregorian 2020 02 23 - , fromGregorian 2020 02 24 - , fromGregorian 2020 02 25 - , fromGregorian 2020 02 26 - ] - beforeParse :: [T.Text] - beforeParse = - [ "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - , "2020-02-21" - , "2020-02-22" - , "2020-02-23" - , "2020-02-24" - , "2020-02-25" - , "2020-02-26" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Dates without missing values as BoxedColumn of Days" - expected - actual - ) - -parseTexts :: Test -parseTexts = - let afterParse :: [T.Text] - afterParse = - [ "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "To" - , "unite" - , "all" - , "people" - , "within" - , "our" - , "nation" - , "To" - , "denounce" - , "the" - , "evils" - , "of" - , "truth" - , "and" - , "love" - , "To" - , "extend" - , "our" - , "reach" - , "to" - , "the" - , "stars" - , "above" - , "JESSIE!" - , "JAMES!" - , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , "Surrender now or prepare to fight!" - , "Meowth, that's right!" - ] - beforeParse :: [T.Text] - beforeParse = - [ "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "To" - , "unite" - , "all" - , "people" - , "within" - , "our" - , "nation" - , "To" - , "denounce" - , "the" - , "evils" - , "of" - , "truth" - , "and" - , "love" - , "To" - , "extend" - , "our" - , "reach" - , "to" - , "the" - , "stars" - , "above" - , "JESSIE!" - , "JAMES!" - , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , "Surrender now or prepare to fight!" - , "Meowth, that's right!" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Text without missing values as BoxedColumn of Text" - expected - actual - ) - ---- 2. COMBINATION CASES -parseBoolsAndIntsAsTexts :: Test -parseBoolsAndIntsAsTexts = - let afterParse :: [T.Text] - afterParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] ++ ["1", "0", "1"] - beforeParse :: [T.Text] - beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] ++ ["1", "0", "1"] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses mixture of Bools and Ints as Text" - expected - actual - ) - -parseIntsAndDoublesAsDoubles :: Test -parseIntsAndDoublesAsDoubles = - let afterParse :: [Double] - afterParse = - [ 1 - , 2 - , 3 - , 4 - , 5 - , 6 - , 7 - , 8 - , 9 - , 10 - , 11 - , 12 - , 13 - , 14 - , 15 - , 16 - , 17 - , 18 - , 19 - , 20 - , 21 - , 22 - , 23 - , 24 - , 25 - , 26 - , 27 - , 28 - , 29 - , 30 - , 31 - , 32 - , 33 - , 34 - , 35 - , 36 - , 37 - , 38 - , 39 - , 40 - , 41 - , 42 - , 43 - , 44 - , 45 - , 46 - , 47 - , 48 - , 49 - , 50 - , 1.0 - , 2.0 - , 3.0 - , 4.0 - , 5.0 - , 6.0 - , 7.0 - , 8.0 - , 9.0 - , 10.0 - , 11.0 - , 12.0 - , 13.0 - , 14.0 - , 15.0 - , 16.0 - , 17.0 - , 18.0 - , 19.0 - , 20.0 - , 21.0 - , 22.0 - , 23.0 - , 24.0 - , 25.0 - , 26.0 - , 27.0 - , 28.0 - , 29.0 - , 30.0 - , 31.0 - , 32.0 - , 33.0 - , 34.0 - , 35.0 - , 36.0 - , 37.0 - , 38.0 - , 39.0 - , 40.0 - , 41.0 - , 42.0 - , 43.0 - , 44.0 - , 45.0 - , 46.0 - , 47.0 - , 48.0 - , 49.0 - , 50.0 - , 3.14 - , 2.22 - , 8.55 - , 23.3 - , 12.22222235049450945049504950 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - , "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles" - expected - actual - ) - -parseIntsAndDatesAsTexts :: Test -parseIntsAndDatesAsTexts = - let afterParse :: [T.Text] - afterParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Ints and Dates as BoxedColumn of Texts" - expected - actual - ) - -parseTextsAndDoublesAsTexts :: Test -parseTextsAndDoublesAsTexts = - let afterParse :: [T.Text] - afterParse = - [ "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - beforeParse :: [T.Text] - beforeParse = - [ "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Texts and Doubles as BoxedColumn of Texts" - expected - actual - ) - -parseDatesAndTextsAsTexts :: Test -parseDatesAndTextsAsTexts = - let afterParse :: [T.Text] - afterParse = - [ "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - , "2020-02-21" - , "2020-02-22" - , "2020-02-23" - , "2020-02-24" - , "2020-02-25" - , "2020-02-26" - , "Jessie" - , "James" - , "Meowth" - ] - beforeParse :: [T.Text] - beforeParse = - [ "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - , "2020-02-21" - , "2020-02-22" - , "2020-02-23" - , "2020-02-24" - , "2020-02-25" - , "2020-02-26" - , "Jessie" - , "James" - , "Meowth" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Dates and Texts as BoxedColumn of Texts" - expected - actual - ) - --- 3A. PARSING WITH SAFEREAD OFF - -parseBoolsWithoutSafeRead :: Test -parseBoolsWithoutSafeRead = - let afterParse :: [Bool] - afterParse = replicate 10 True ++ replicate 10 False - beforeParse :: [T.Text] - beforeParse = replicate 10 "true" ++ replicate 10 "false" - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Bools without missing values as UnboxedColumn of Bools, when safeRead is off" - expected - actual - ) - -parseIntsWithoutSafeRead :: Test -parseIntsWithoutSafeRead = - let afterParse :: [Int] - afterParse = - [ 1 - , 2 - , 3 - , 4 - , 5 - , 6 - , 7 - , 8 - , 9 - , 10 - , 11 - , 12 - , 13 - , 14 - , 15 - , 16 - , 17 - , 18 - , 19 - , 20 - , 21 - , 22 - , 23 - , 24 - , 25 - , 26 - , 27 - , 28 - , 29 - , 30 - , 31 - , 32 - , 33 - , 34 - , 35 - , 36 - , 37 - , 38 - , 39 - , 40 - , 41 - , 42 - , 43 - , 44 - , 45 - , 46 - , 47 - , 48 - , 49 - , 50 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - ] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints without missing values as UnboxedColumn of Ints, when safeRead is off" - expected - actual - ) - -parseDoublesWithoutSafeRead :: Test -parseDoublesWithoutSafeRead = - let afterParse :: [Double] - afterParse = - [ 1.0 - , 2.0 - , 3.0 - , 4.0 - , 5.0 - , 6.0 - , 7.0 - , 8.0 - , 9.0 - , 10.0 - , 11.0 - , 12.0 - , 13.0 - , 14.0 - , 15.0 - , 16.0 - , 17.0 - , 18.0 - , 19.0 - , 20.0 - , 21.0 - , 22.0 - , 23.0 - , 24.0 - , 25.0 - , 26.0 - , 27.0 - , 28.0 - , 29.0 - , 30.0 - , 31.0 - , 32.0 - , 33.0 - , 34.0 - , 35.0 - , 36.0 - , 37.0 - , 38.0 - , 39.0 - , 40.0 - , 41.0 - , 42.0 - , 43.0 - , 44.0 - , 45.0 - , 46.0 - , 47.0 - , 48.0 - , 49.0 - , 50.0 - , 3.14 - , 2.22 - , 8.55 - , 23.3 - , 12.22222235049450945049504950 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Doubles without missing values as UnboxedColumn of Doubles, when safeRead is off" - expected - actual - ) - -parseDatesWithoutSafeRead :: Test -parseDatesWithoutSafeRead = - let afterParse :: [Day] - afterParse = - [ fromGregorian 2020 02 12 - , fromGregorian 2020 02 13 - , fromGregorian 2020 02 14 - , fromGregorian 2020 02 15 - , fromGregorian 2020 02 16 - , fromGregorian 2020 02 17 - , fromGregorian 2020 02 18 - , fromGregorian 2020 02 19 - , fromGregorian 2020 02 20 - , fromGregorian 2020 02 21 - , fromGregorian 2020 02 22 - , fromGregorian 2020 02 23 - , fromGregorian 2020 02 24 - , fromGregorian 2020 02 25 - , fromGregorian 2020 02 26 - ] - beforeParse :: [T.Text] - beforeParse = - [ "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - , "2020-02-21" - , "2020-02-22" - , "2020-02-23" - , "2020-02-24" - , "2020-02-25" - , "2020-02-26" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Dates without missing values as BoxedColumn of Days" - expected - actual - ) - -parseTextsWithoutSafeRead :: Test -parseTextsWithoutSafeRead = - let afterParse :: [T.Text] - afterParse = - [ "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "To" - , "unite" - , "all" - , "people" - , "within" - , "our" - , "nation" - , "To" - , "denounce" - , "the" - , "evils" - , "of" - , "truth" - , "and" - , "love" - , "To" - , "extend" - , "our" - , "reach" - , "to" - , "the" - , "stars" - , "above" - , "JESSIE!" - , "JAMES!" - , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , "Surrender now or prepare to fight!" - , "Meowth, that's right!" - ] - beforeParse :: [T.Text] - beforeParse = - [ "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "To" - , "unite" - , "all" - , "people" - , "within" - , "our" - , "nation" - , "To" - , "denounce" - , "the" - , "evils" - , "of" - , "truth" - , "and" - , "love" - , "To" - , "extend" - , "our" - , "reach" - , "to" - , "the" - , "stars" - , "above" - , "JESSIE!" - , "JAMES!" - , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , "Surrender now or prepare to fight!" - , "Meowth, that's right!" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Text without missing values as BoxedColumn of Text" - expected - actual - ) - -parseBoolsAndEmptyStringsWithoutSafeRead :: Test -parseBoolsAndEmptyStringsWithoutSafeRead = - let afterParse :: [Maybe Bool] - afterParse = replicate 10 Nothing ++ replicate 10 (Just True) ++ replicate 10 (Just False) - beforeParse :: [T.Text] - beforeParse = replicate 10 "" ++ replicate 10 "true" ++ replicate 10 "false" - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Bools and empty Strings as OptionalColumn of Bools, when safeRead is off" - expected - actual - ) - -parseIntsAndEmptyStringsWithoutSafeRead :: Test -parseIntsAndEmptyStringsWithoutSafeRead = - let afterParse :: [Maybe Int] - afterParse = - [ Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Just 1 - , Just 2 - , Just 3 - , Just 4 - , Just 5 - , Just 6 - , Just 7 - , Just 8 - , Just 9 - , Just 10 - , Just 11 - , Just 12 - , Just 13 - , Just 14 - , Just 15 - , Just 16 - , Just 17 - , Just 18 - , Just 19 - , Just 20 - , Just 21 - , Just 22 - , Just 23 - , Just 24 - , Just 25 - , Just 26 - , Just 27 - , Just 28 - , Just 29 - , Just 30 - , Just 31 - , Just 32 - , Just 33 - , Just 34 - , Just 35 - , Just 36 - , Just 37 - , Just 38 - , Just 39 - , Just 40 - , Just 41 - , Just 42 - , Just 43 - , Just 44 - , Just 45 - , Just 46 - , Just 47 - , Just 48 - , Just 49 - , Just 50 - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "" - , "" - , "" - , "" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is off" - expected - actual - ) - -parseIntsAndDoublesAndEmptyStringsWithoutSafeRead :: Test -parseIntsAndDoublesAndEmptyStringsWithoutSafeRead = - let afterParse :: [Maybe Double] - afterParse = - [ Just 1.0 - , Just 2.0 - , Just 3.0 - , Just 4.0 - , Just 5.0 - , Just 6.0 - , Just 7.0 - , Just 8.0 - , Just 9.0 - , Just 10.0 - , Nothing - , Just 11.0 - , Just 12.0 - , Just 13.0 - , Just 14.0 - , Just 15.0 - , Just 16.0 - , Just 17.0 - , Just 18.0 - , Just 19.0 - , Just 20.0 - , Nothing - , Just 21.0 - , Just 22.0 - , Just 23.0 - , Just 24.0 - , Just 25.0 - , Just 26.0 - , Just 27.0 - , Just 28.0 - , Just 29.0 - , Just 30.0 - , Nothing - , Just 31.0 - , Just 32.0 - , Just 33.0 - , Just 34.0 - , Just 35.0 - , Just 36.0 - , Just 37.0 - , Just 38.0 - , Just 39.0 - , Just 40.0 - , Nothing - , Just 41.0 - , Just 42.0 - , Just 43.0 - , Just 44.0 - , Just 45.0 - , Just 46.0 - , Just 47.0 - , Just 48.0 - , Just 49.0 - , Just 50.0 - , Nothing - , Just 3.14 - , Just 2.22 - , Just 8.55 - , Just 23.3 - , Just 12.22222235049451 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" - expected - actual - ) - -parseDatesAndEmptyStringsWithoutSafeRead :: Test -parseDatesAndEmptyStringsWithoutSafeRead = - let afterParse :: [Maybe Day] - afterParse = - [ Just $ fromGregorian 2020 02 12 - , Just $ fromGregorian 2020 02 13 - , Just $ fromGregorian 2020 02 14 - , Nothing - , Just $ fromGregorian 2020 02 15 - , Just $ fromGregorian 2020 02 16 - , Just $ fromGregorian 2020 02 17 - , Nothing - , Just $ fromGregorian 2020 02 18 - , Just $ fromGregorian 2020 02 19 - , Just $ fromGregorian 2020 02 20 - , Nothing - , Just $ fromGregorian 2020 02 21 - , Just $ fromGregorian 2020 02 22 - , Just $ fromGregorian 2020 02 23 - , Nothing - , Just $ fromGregorian 2020 02 24 - , Just $ fromGregorian 2020 02 25 - , Just $ fromGregorian 2020 02 26 - , Nothing - ] - beforeParse :: [T.Text] - beforeParse = - [ "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - , "" - , "2020-02-21" - , "2020-02-22" - , "2020-02-23" - , "" - , "2020-02-24" - , "2020-02-25" - , "2020-02-26" - , "" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead off" - expected - actual - ) - -parseTextsAndEmptyStringsWithoutSafeRead :: Test -parseTextsAndEmptyStringsWithoutSafeRead = - let afterParse :: [Maybe T.Text] - afterParse = - [ Nothing - , Just "To" - , Just "protect" - , Just "the" - , Just "world" - , Just "from" - , Just "devastation" - , Nothing - , Just "To" - , Just "unite" - , Just "all" - , Just "people" - , Just "within" - , Just "our" - , Just "nation" - , Nothing - , Just "To" - , Just "denounce" - , Just "the" - , Just "evils" - , Just "of" - , Just "truth" - , Just "and" - , Just "love" - , Nothing - , Just "To" - , Just "extend" - , Just "our" - , Just "reach" - , Just "to" - , Just "the" - , Just "stars" - , Just "above" - , Nothing - , Just "JESSIE!" - , Just "JAMES!" - , Just "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , Nothing - , Just "Surrender now or prepare to fight!" - , Nothing - , Just "Meowth, that's right!" - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "" - , "To" - , "unite" - , "all" - , "people" - , "within" - , "our" - , "nation" - , "" - , "To" - , "denounce" - , "the" - , "evils" - , "of" - , "truth" - , "and" - , "love" - , "" - , "To" - , "extend" - , "our" - , "reach" - , "to" - , "the" - , "stars" - , "above" - , "" - , "JESSIE!" - , "JAMES!" - , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , "" - , "Surrender now or prepare to fight!" - , "" - , "Meowth, that's right!" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" - expected - actual - ) - -parseBoolsAndNullishStringsWithoutSafeRead :: Test -parseBoolsAndNullishStringsWithoutSafeRead = - let afterParse :: [T.Text] - afterParse = replicate 10 "N/A" ++ replicate 10 "True" ++ replicate 10 "False" - beforeParse :: [T.Text] - beforeParse = replicate 10 "N/A" ++ replicate 10 "True" ++ replicate 10 "False" - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Bools with nullish values as BoxedColumn of Texts, when safeRead is off" - expected - actual - ) - -parseIntsAndNullishStringsWithoutSafeRead :: Test -parseIntsAndNullishStringsWithoutSafeRead = - let afterParse :: [T.Text] - afterParse = - [ "N/A" - , "N/A" - , "N/A" - , "N/A" - , "N/A" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - ] - beforeParse :: [T.Text] - beforeParse = - [ "N/A" - , "N/A" - , "N/A" - , "N/A" - , "N/A" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints with nullish values as BoxedColumn of Texts, when safeRead is off" - expected - actual - ) - -parseIntsAndDoublesAndNullishStringsWithoutSafeRead :: Test -parseIntsAndDoublesAndNullishStringsWithoutSafeRead = - let afterParse :: [T.Text] - afterParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "Nothing" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "N/A" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "NULL" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "null" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "NAN" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "Nothing" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "N/A" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "NULL" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "null" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "NAN" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" - expected - actual - ) - -parseIntsAndNullishAndEmptyStringsWithoutSafeRead :: Test -parseIntsAndNullishAndEmptyStringsWithoutSafeRead = - let afterParse :: [Maybe T.Text] - afterParse = - [ Just "N/A" - , Just "N/A" - , Just "N/A" - , Just "N/A" - , Just "N/A" - , Nothing - , Just "1" - , Just "2" - , Just "3" - , Just "4" - , Just "5" - , Just "6" - , Just "7" - , Just "8" - , Just "9" - , Just "10" - , Nothing - , Just "11" - , Just "12" - , Just "13" - , Just "14" - , Just "15" - , Just "16" - , Just "17" - , Just "18" - , Just "19" - , Just "20" - , Nothing - , Just "21" - , Just "22" - , Just "23" - , Just "24" - , Just "25" - , Just "26" - , Just "27" - , Just "28" - , Just "29" - , Just "30" - , Nothing - , Just "31" - , Just "32" - , Just "33" - , Just "34" - , Just "35" - , Just "36" - , Just "37" - , Just "38" - , Just "39" - , Just "40" - , Nothing - , Just "41" - , Just "42" - , Just "43" - , Just "44" - , Just "45" - , Just "46" - , Just "47" - , Just "48" - , Just "49" - , Just "50" - , Nothing - ] - beforeParse :: [T.Text] - beforeParse = - [ "N/A" - , "N/A" - , "N/A" - , "N/A" - , "N/A" - , "" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - , "" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Texts, when safeRead is off" - expected - actual - ) - -parseTextsAndEmptyAndNullishStringsWithoutSafeRead :: Test -parseTextsAndEmptyAndNullishStringsWithoutSafeRead = - let afterParse :: [Maybe T.Text] - afterParse = - [ Nothing - , Just "To" - , Just "protect" - , Just "the" - , Just "world" - , Just "from" - , Just "devastation" - , Nothing - , Just "To" - , Just "unite" - , Just "all" - , Just "people" - , Just "within" - , Just "our" - , Just "nation" - , Nothing - , Just "To" - , Just "denounce" - , Just "the" - , Just "evils" - , Just "of" - , Just "truth" - , Just "and" - , Just "love" - , Nothing - , Just "To" - , Just "extend" - , Just "our" - , Just "reach" - , Just "to" - , Just "the" - , Just "stars" - , Just "above" - , Nothing - , Just "JESSIE!" - , Just "JAMES!" - , Just "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , Nothing - , Just "Surrender now or prepare to fight!" - , Nothing - , Just "Meowth, that's right!" - , Just "NaN" - , Just "Nothing" - , Just "N/A" - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "" - , "To" - , "unite" - , "all" - , "people" - , "within" - , "our" - , "nation" - , "" - , "To" - , "denounce" - , "the" - , "evils" - , "of" - , "truth" - , "and" - , "love" - , "" - , "To" - , "extend" - , "our" - , "reach" - , "to" - , "the" - , "stars" - , "above" - , "" - , "JESSIE!" - , "JAMES!" - , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , "" - , "Surrender now or prepare to fight!" - , "" - , "Meowth, that's right!" - , "NaN" - , "Nothing" - , "N/A" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" - expected - actual - ) - --- 3B. PARSING WITH SAFEREAD ON -parseBoolsAndEmptyStringsWithSafeRead :: Test -parseBoolsAndEmptyStringsWithSafeRead = - let afterParse :: [Maybe Bool] - afterParse = replicate 10 Nothing ++ replicate 10 (Just True) - beforeParse :: [T.Text] - beforeParse = replicate 10 "" ++ replicate 10 "true" - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Bools and empty strings as OptionalColumn of Bools, when safeRead is on" - expected - actual - ) - -parseIntsAndEmptyStringsWithSafeRead :: Test -parseIntsAndEmptyStringsWithSafeRead = - let afterParse :: [Maybe Int] - afterParse = - [ Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Just 1 - , Just 2 - , Just 3 - , Just 4 - , Just 5 - , Just 6 - , Just 7 - , Just 8 - , Just 9 - , Just 10 - , Just 11 - , Just 12 - , Just 13 - , Just 14 - , Just 15 - , Just 16 - , Just 17 - , Just 18 - , Just 19 - , Just 20 - , Just 21 - , Just 22 - , Just 23 - , Just 24 - , Just 25 - , Just 26 - , Just 27 - , Just 28 - , Just 29 - , Just 30 - , Just 31 - , Just 32 - , Just 33 - , Just 34 - , Just 35 - , Just 36 - , Just 37 - , Just 38 - , Just 39 - , Just 40 - , Just 41 - , Just 42 - , Just 43 - , Just 44 - , Just 45 - , Just 46 - , Just 47 - , Just 48 - , Just 49 - , Just 50 - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "" - , "" - , "" - , "" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is on" - expected - actual - ) - -parseIntsAndDoublesAndEmptyStringsWithSafeRead :: Test -parseIntsAndDoublesAndEmptyStringsWithSafeRead = - let afterParse :: [Maybe Double] - afterParse = - [ Just 1.0 - , Just 2.0 - , Just 3.0 - , Just 4.0 - , Just 5.0 - , Just 6.0 - , Just 7.0 - , Just 8.0 - , Just 9.0 - , Just 10.0 - , Nothing - , Just 11.0 - , Just 12.0 - , Just 13.0 - , Just 14.0 - , Just 15.0 - , Just 16.0 - , Just 17.0 - , Just 18.0 - , Just 19.0 - , Just 20.0 - , Nothing - , Just 21.0 - , Just 22.0 - , Just 23.0 - , Just 24.0 - , Just 25.0 - , Just 26.0 - , Just 27.0 - , Just 28.0 - , Just 29.0 - , Just 30.0 - , Nothing - , Just 31.0 - , Just 32.0 - , Just 33.0 - , Just 34.0 - , Just 35.0 - , Just 36.0 - , Just 37.0 - , Just 38.0 - , Just 39.0 - , Just 40.0 - , Nothing - , Just 41.0 - , Just 42.0 - , Just 43.0 - , Just 44.0 - , Just 45.0 - , Just 46.0 - , Just 47.0 - , Just 48.0 - , Just 49.0 - , Just 50.0 - , Nothing - , Just 3.14 - , Just 2.22 - , Just 8.55 - , Just 23.3 - , Just 12.22222235049451 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is on" - expected - actual - ) - -parseDatesAndEmptyStringsWithSafeRead :: Test -parseDatesAndEmptyStringsWithSafeRead = - let afterParse :: [Maybe Day] - afterParse = - [ Just $ fromGregorian 2020 02 12 - , Just $ fromGregorian 2020 02 13 - , Just $ fromGregorian 2020 02 14 - , Nothing - , Just $ fromGregorian 2020 02 15 - , Just $ fromGregorian 2020 02 16 - , Just $ fromGregorian 2020 02 17 - , Nothing - , Just $ fromGregorian 2020 02 18 - , Just $ fromGregorian 2020 02 19 - , Just $ fromGregorian 2020 02 20 - , Nothing - , Just $ fromGregorian 2020 02 21 - , Just $ fromGregorian 2020 02 22 - , Just $ fromGregorian 2020 02 23 - , Nothing - , Just $ fromGregorian 2020 02 24 - , Just $ fromGregorian 2020 02 25 - , Just $ fromGregorian 2020 02 26 - , Nothing - ] - beforeParse :: [T.Text] - beforeParse = - [ "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - , "" - , "2020-02-21" - , "2020-02-22" - , "2020-02-23" - , "" - , "2020-02-24" - , "2020-02-25" - , "2020-02-26" - , "" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead on" - expected - actual - ) - -parseTextsAndEmptyStringsWithSafeRead :: Test -parseTextsAndEmptyStringsWithSafeRead = - let afterParse :: [Maybe T.Text] - afterParse = - [ Nothing - , Just "To" - , Just "protect" - , Just "the" - , Just "world" - , Just "from" - , Just "devastation" - , Nothing - , Just "To" - , Just "unite" - , Just "all" - , Just "people" - , Just "within" - , Just "our" - , Just "nation" - , Nothing - , Just "To" - , Just "denounce" - , Just "the" - , Just "evils" - , Just "of" - , Just "truth" - , Just "and" - , Just "love" - , Nothing - , Just "To" - , Just "extend" - , Just "our" - , Just "reach" - , Just "to" - , Just "the" - , Just "stars" - , Just "above" - , Nothing - , Just "JESSIE!" - , Just "JAMES!" - , Just "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , Nothing - , Just "Surrender now or prepare to fight!" - , Nothing - , Just "Meowth, that's right!" - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "" - , "To" - , "unite" - , "all" - , "people" - , "within" - , "our" - , "nation" - , "" - , "To" - , "denounce" - , "the" - , "evils" - , "of" - , "truth" - , "and" - , "love" - , "" - , "To" - , "extend" - , "our" - , "reach" - , "to" - , "the" - , "stars" - , "above" - , "" - , "JESSIE!" - , "JAMES!" - , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , "" - , "Surrender now or prepare to fight!" - , "" - , "Meowth, that's right!" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead on" - expected - actual - ) - -parseIntsAndNullishStringsWithSafeRead :: Test -parseIntsAndNullishStringsWithSafeRead = - let afterParse :: [Maybe Int] - afterParse = - [ Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Just 1 - , Just 2 - , Just 3 - , Just 4 - , Just 5 - , Just 6 - , Just 7 - , Just 8 - , Just 9 - , Just 10 - , Just 11 - , Just 12 - , Just 13 - , Just 14 - , Just 15 - , Just 16 - , Just 17 - , Just 18 - , Just 19 - , Just 20 - , Just 21 - , Just 22 - , Just 23 - , Just 24 - , Just 25 - , Just 26 - , Just 27 - , Just 28 - , Just 29 - , Just 30 - , Just 31 - , Just 32 - , Just 33 - , Just 34 - , Just 35 - , Just 36 - , Just 37 - , Just 38 - , Just 39 - , Just 40 - , Just 41 - , Just 42 - , Just 43 - , Just 44 - , Just 45 - , Just 46 - , Just 47 - , Just 48 - , Just 49 - , Just 50 - ] - beforeParse :: [T.Text] - beforeParse = - [ "N/A" - , "N/A" - , "N/A" - , "N/A" - , "N/A" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints with nullish values as OptionalColumn of Ints, when safeRead is on" - expected - actual - ) - -parseIntsAndDoublesAndNullishStringsWithSafeRead :: Test -parseIntsAndDoublesAndNullishStringsWithSafeRead = - let afterParse :: [Maybe Double] - afterParse = - [ Just 1.0 - , Just 2.0 - , Just 3.0 - , Just 4.0 - , Just 5.0 - , Just 6.0 - , Just 7.0 - , Just 8.0 - , Just 9.0 - , Just 10.0 - , Nothing - , Just 11.0 - , Just 12.0 - , Just 13.0 - , Just 14.0 - , Just 15.0 - , Just 16.0 - , Just 17.0 - , Just 18.0 - , Just 19.0 - , Just 20.0 - , Nothing - , Just 21.0 - , Just 22.0 - , Just 23.0 - , Just 24.0 - , Just 25.0 - , Just 26.0 - , Just 27.0 - , Just 28.0 - , Just 29.0 - , Just 30.0 - , Nothing - , Just 31.0 - , Just 32.0 - , Just 33.0 - , Just 34.0 - , Just 35.0 - , Just 36.0 - , Just 37.0 - , Just 38.0 - , Just 39.0 - , Just 40.0 - , Nothing - , Just 41.0 - , Just 42.0 - , Just 43.0 - , Just 44.0 - , Just 45.0 - , Just 46.0 - , Just 47.0 - , Just 48.0 - , Just 49.0 - , Just 50.0 - , Nothing - , Just 3.14 - , Just 2.22 - , Just 8.55 - , Just 23.3 - , Just 12.03 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "Nothing" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "N/A" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "NULL" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "null" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "NAN" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.03" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" - expected - actual - ) - -parseIntsAndNullishAndEmptyStringsWithSafeRead :: Test -parseIntsAndNullishAndEmptyStringsWithSafeRead = - let afterParse :: [Maybe Int] - afterParse = - [ Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Just 1 - , Just 2 - , Just 3 - , Just 4 - , Just 5 - , Just 6 - , Just 7 - , Just 8 - , Just 9 - , Just 10 - , Nothing - , Just 11 - , Just 12 - , Just 13 - , Just 14 - , Just 15 - , Just 16 - , Just 17 - , Just 18 - , Just 19 - , Just 20 - , Nothing - , Just 21 - , Just 22 - , Just 23 - , Just 24 - , Just 25 - , Just 26 - , Just 27 - , Just 28 - , Just 29 - , Just 30 - , Nothing - , Just 31 - , Just 32 - , Just 33 - , Just 34 - , Just 35 - , Just 36 - , Just 37 - , Just 38 - , Just 39 - , Just 40 - , Nothing - , Just 41 - , Just 42 - , Just 43 - , Just 44 - , Just 45 - , Just 46 - , Just 47 - , Just 48 - , Just 49 - , Just 50 - , Nothing - ] - beforeParse :: [T.Text] - beforeParse = - [ "N/A" - , "N/A" - , "N/A" - , "N/A" - , "Nothing" - , "" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - , "" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Ints, when safeRead is on" - expected - actual - ) - -parseIntsAndDoublesAndNullishAndEmptyStringsWithSafeRead :: Test -parseIntsAndDoublesAndNullishAndEmptyStringsWithSafeRead = - let afterParse :: [Maybe Double] - afterParse = - [ Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Just 1 - , Just 2 - , Just 3 - , Just 4 - , Just 5 - , Just 6 - , Just 7 - , Just 8 - , Just 9 - , Just 10 - , Nothing - , Just 11 - , Just 12 - , Just 13 - , Just 14 - , Just 15 - , Just 16 - , Just 17 - , Just 18 - , Just 19 - , Just 20 - , Nothing - , Just 21 - , Just 22 - , Just 23 - , Just 24 - , Just 25 - , Just 26 - , Just 27 - , Just 28 - , Just 29 - , Just 30 - , Nothing - , Just 3.14 - ] - beforeParse :: [T.Text] - beforeParse = - [ "N/A" - , "N/A" - , "N/A" - , "N/A" - , "Nothing" - , "" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "" - , "3.14" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints and Doubles with nullish values AND empty strings as OptionalColumn of Doubles, when safeRead is on" - expected - actual - ) - -parseTextsAndEmptyAndNullishStringsWithSafeRead :: Test -parseTextsAndEmptyAndNullishStringsWithSafeRead = - let afterParse :: [Maybe T.Text] - afterParse = - [ Nothing - , Just "To" - , Just "protect" - , Just "the" - , Just "world" - , Just "from" - , Just "devastation" - , Nothing - , Just "To" - , Just "unite" - , Just "all" - , Just "people" - , Just "within" - , Just "our" - , Just "nation" - , Nothing - , Just "To" - , Just "denounce" - , Just "the" - , Just "evils" - , Just "of" - , Just "truth" - , Just "and" - , Just "love" - , Nothing - , Just "To" - , Just "extend" - , Just "our" - , Just "reach" - , Just "to" - , Just "the" - , Just "stars" - , Just "above" - , Nothing - , Just "JESSIE!" - , Just "JAMES!" - , Just "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , Nothing - , Just "Surrender now or prepare to fight!" - , Nothing - , Just "Meowth, that's right!" - , Nothing - , Nothing - , Nothing - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "To" - , "protect" - , "the" - , "world" - , "from" - , "devastation" - , "" - , "To" - , "unite" - , "all" - , "people" - , "within" - , "our" - , "nation" - , "" - , "To" - , "denounce" - , "the" - , "evils" - , "of" - , "truth" - , "and" - , "love" - , "" - , "To" - , "extend" - , "our" - , "reach" - , "to" - , "the" - , "stars" - , "above" - , "" - , "JESSIE!" - , "JAMES!" - , "TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT!" - , "" - , "Surrender now or prepare to fight!" - , "" - , "Meowth, that's right!" - , "NaN" - , "Nothing" - , "N/A" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" - expected - actual - ) - --- 4. PARSING SHOULD NOT DEPEND ON THE NUMBER OF EXAMPLES. -parseBoolsWithOneExample :: Test -parseBoolsWithOneExample = - let afterParse :: [Bool] - afterParse = False : replicate 50 True - beforeParse :: [T.Text] - beforeParse = "false" : replicate 50 "true" - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 1}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Bools without missing values as UnboxedColumn of Ints with only one example" - expected - actual - ) - -parseBoolsWithManyExamples :: Test -parseBoolsWithManyExamples = - let afterParse :: [Bool] - afterParse = False : replicate 50 True - beforeParse :: [T.Text] - beforeParse = "false" : replicate 50 "true" - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 49}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Bools without missing values as UnboxedColumn of Ints with only one example" - expected - actual - ) - -parseIntsWithOneExample :: Test -parseIntsWithOneExample = - let afterParse :: [Int] - afterParse = - [ 1 - , 2 - , 3 - , 4 - , 5 - , 6 - , 7 - , 8 - , 9 - , 10 - , 11 - , 12 - , 13 - , 14 - , 15 - , 16 - , 17 - , 18 - , 19 - , 20 - , 21 - , 22 - , 23 - , 24 - , 25 - , 26 - , 27 - , 28 - , 29 - , 30 - , 31 - , 32 - , 33 - , 34 - , 35 - , 36 - , 37 - , 38 - , 39 - , 40 - , 41 - , 42 - , 43 - , 44 - , 45 - , 46 - , 47 - , 48 - , 49 - , 50 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - ] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 1}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints without missing values as UnboxedColumn of Ints with only one example" - expected - actual - ) - -parseIntsWithTwentyFiveExamples :: Test -parseIntsWithTwentyFiveExamples = - let afterParse :: [Int] - afterParse = - [ 1 - , 2 - , 3 - , 4 - , 5 - , 6 - , 7 - , 8 - , 9 - , 10 - , 11 - , 12 - , 13 - , 14 - , 15 - , 16 - , 17 - , 18 - , 19 - , 20 - , 21 - , 22 - , 23 - , 24 - , 25 - , 26 - , 27 - , 28 - , 29 - , 30 - , 31 - , 32 - , 33 - , 34 - , 35 - , 36 - , 37 - , 38 - , 39 - , 40 - , 41 - , 42 - , 43 - , 44 - , 45 - , 46 - , 47 - , 48 - , 49 - , 50 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - ] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 25}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints without missing values as UnboxedColumn of Ints with some examples" - expected - actual - ) - -parseIntsWithFortyNineExamples :: Test -parseIntsWithFortyNineExamples = - let afterParse :: [Int] - afterParse = - [ 1 - , 2 - , 3 - , 4 - , 5 - , 6 - , 7 - , 8 - , 9 - , 10 - , 11 - , 12 - , 13 - , 14 - , 15 - , 16 - , 17 - , 18 - , 19 - , 20 - , 21 - , 22 - , 23 - , 24 - , 25 - , 26 - , 27 - , 28 - , 29 - , 30 - , 31 - , 32 - , 33 - , 34 - , 35 - , 36 - , 37 - , 38 - , 39 - , 40 - , 41 - , 42 - , 43 - , 44 - , 45 - , 46 - , 47 - , 48 - , 49 - , 50 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - ] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 49}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Ints without missing values as UnboxedColumn of Ints with many examples" - expected - actual - ) - -parseDatesWithOneExample :: Test -parseDatesWithOneExample = - let afterParse :: [Day] - afterParse = - [ fromGregorian 2020 02 12 - , fromGregorian 2020 02 13 - , fromGregorian 2020 02 14 - , fromGregorian 2020 02 15 - , fromGregorian 2020 02 16 - , fromGregorian 2020 02 17 - , fromGregorian 2020 02 18 - , fromGregorian 2020 02 19 - , fromGregorian 2020 02 20 - , fromGregorian 2020 02 21 - , fromGregorian 2020 02 22 - , fromGregorian 2020 02 23 - , fromGregorian 2020 02 24 - , fromGregorian 2020 02 25 - , fromGregorian 2020 02 26 - ] - beforeParse :: [T.Text] - beforeParse = - [ "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - , "2020-02-21" - , "2020-02-22" - , "2020-02-23" - , "2020-02-24" - , "2020-02-25" - , "2020-02-26" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 1}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Dates without missing values as BoxedColumn of Days with only one example" - expected - actual - ) - -parseDatesWithFifteenExamples :: Test -parseDatesWithFifteenExamples = - let afterParse :: [Day] - afterParse = - [ fromGregorian 2020 02 12 - , fromGregorian 2020 02 13 - , fromGregorian 2020 02 14 - , fromGregorian 2020 02 15 - , fromGregorian 2020 02 16 - , fromGregorian 2020 02 17 - , fromGregorian 2020 02 18 - , fromGregorian 2020 02 19 - , fromGregorian 2020 02 20 - , fromGregorian 2020 02 21 - , fromGregorian 2020 02 22 - , fromGregorian 2020 02 23 - , fromGregorian 2020 02 24 - , fromGregorian 2020 02 25 - , fromGregorian 2020 02 26 - ] - beforeParse :: [T.Text] - beforeParse = - [ "2020-02-12" - , "2020-02-13" - , "2020-02-14" - , "2020-02-15" - , "2020-02-16" - , "2020-02-17" - , "2020-02-18" - , "2020-02-19" - , "2020-02-20" - , "2020-02-21" - , "2020-02-22" - , "2020-02-23" - , "2020-02-24" - , "2020-02-25" - , "2020-02-26" - ] - expected = DI.BoxedColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 15}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses Dates without missing values as BoxedColumn of Days with many examples" - expected - actual - ) - -parseIntsAndDoublesAsDoublesWithOneExample :: Test -parseIntsAndDoublesAsDoublesWithOneExample = - let afterParse :: [Double] - afterParse = - [ 1 - , 2 - , 3 - , 4 - , 5 - , 6 - , 7 - , 8 - , 9 - , 10 - , 11 - , 12 - , 13 - , 14 - , 15 - , 16 - , 17 - , 18 - , 19 - , 20 - , 21 - , 22 - , 23 - , 24 - , 25 - , 26 - , 27 - , 28 - , 29 - , 30 - , 31 - , 32 - , 33 - , 34 - , 35 - , 36 - , 37 - , 38 - , 39 - , 40 - , 41 - , 42 - , 43 - , 44 - , 45 - , 46 - , 47 - , 48 - , 49 - , 50 - , 1.0 - , 2.0 - , 3.0 - , 4.0 - , 5.0 - , 6.0 - , 7.0 - , 8.0 - , 9.0 - , 10.0 - , 11.0 - , 12.0 - , 13.0 - , 14.0 - , 15.0 - , 16.0 - , 17.0 - , 18.0 - , 19.0 - , 20.0 - , 21.0 - , 22.0 - , 23.0 - , 24.0 - , 25.0 - , 26.0 - , 27.0 - , 28.0 - , 29.0 - , 30.0 - , 31.0 - , 32.0 - , 33.0 - , 34.0 - , 35.0 - , 36.0 - , 37.0 - , 38.0 - , 39.0 - , 40.0 - , 41.0 - , 42.0 - , 43.0 - , 44.0 - , 45.0 - , 46.0 - , 47.0 - , 48.0 - , 49.0 - , 50.0 - , 3.14 - , 2.22 - , 8.55 - , 23.3 - , 12.22222235049450945049504950 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - , "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 1}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" - expected - actual - ) - -parseIntsAndDoublesAsDoublesWithManyExamples :: Test -parseIntsAndDoublesAsDoublesWithManyExamples = - let afterParse :: [Double] - afterParse = - [ 1 - , 2 - , 3 - , 4 - , 5 - , 6 - , 7 - , 8 - , 9 - , 10 - , 11 - , 12 - , 13 - , 14 - , 15 - , 16 - , 17 - , 18 - , 19 - , 20 - , 21 - , 22 - , 23 - , 24 - , 25 - , 26 - , 27 - , 28 - , 29 - , 30 - , 31 - , 32 - , 33 - , 34 - , 35 - , 36 - , 37 - , 38 - , 39 - , 40 - , 41 - , 42 - , 43 - , 44 - , 45 - , 46 - , 47 - , 48 - , 49 - , 50 - , 1.0 - , 2.0 - , 3.0 - , 4.0 - , 5.0 - , 6.0 - , 7.0 - , 8.0 - , 9.0 - , 10.0 - , 11.0 - , 12.0 - , 13.0 - , 14.0 - , 15.0 - , 16.0 - , 17.0 - , 18.0 - , 19.0 - , 20.0 - , 21.0 - , 22.0 - , 23.0 - , 24.0 - , 25.0 - , 26.0 - , 27.0 - , 28.0 - , 29.0 - , 30.0 - , 31.0 - , 32.0 - , 33.0 - , 34.0 - , 35.0 - , 36.0 - , 37.0 - , 38.0 - , 39.0 - , 40.0 - , 41.0 - , 42.0 - , 43.0 - , 44.0 - , 45.0 - , 46.0 - , 47.0 - , 48.0 - , 49.0 - , 50.0 - , 3.14 - , 2.22 - , 8.55 - , 23.3 - , 12.22222235049450945049504950 - ] - beforeParse :: [T.Text] - beforeParse = - [ "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - , "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.UnboxedColumn $ VU.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 50}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" - expected - actual - ) - -parseIntsAndDoublesAndEmptyStringsAsDoublesWithOneExampleWithSafeReadOff :: Test -parseIntsAndDoublesAndEmptyStringsAsDoublesWithOneExampleWithSafeReadOff = - let afterParse :: [Maybe Double] - afterParse = - [ Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Just 1.0 - , Just 2.0 - , Just 3.0 - , Just 4.0 - , Just 5.0 - , Just 6.0 - , Just 7.0 - , Just 8.0 - , Just 9.0 - , Just 10.0 - , Just 11.0 - , Just 12.0 - , Just 13.0 - , Just 14.0 - , Just 15.0 - , Just 16.0 - , Just 17.0 - , Just 18.0 - , Just 19.0 - , Just 20.0 - , Just 21.0 - , Just 22.0 - , Just 23.0 - , Just 24.0 - , Just 25.0 - , Just 26.0 - , Just 27.0 - , Just 28.0 - , Just 29.0 - , Just 30.0 - , Just 31.0 - , Just 32.0 - , Just 33.0 - , Just 34.0 - , Just 35.0 - , Just 36.0 - , Just 37.0 - , Just 38.0 - , Just 39.0 - , Just 40.0 - , Just 41.0 - , Just 42.0 - , Just 43.0 - , Just 44.0 - , Just 45.0 - , Just 46.0 - , Just 47.0 - , Just 48.0 - , Just 49.0 - , Just 50.0 - , Just 1.0 - , Just 2.0 - , Just 3.0 - , Just 4.0 - , Just 5.0 - , Just 6.0 - , Just 7.0 - , Just 8.0 - , Just 9.0 - , Just 10.0 - , Just 11.0 - , Just 12.0 - , Just 13.0 - , Just 14.0 - , Just 15.0 - , Just 16.0 - , Just 17.0 - , Just 18.0 - , Just 19.0 - , Just 20.0 - , Just 21.0 - , Just 22.0 - , Just 23.0 - , Just 24.0 - , Just 25.0 - , Just 26.0 - , Just 27.0 - , Just 28.0 - , Just 29.0 - , Just 30.0 - , Just 31.0 - , Just 32.0 - , Just 33.0 - , Just 34.0 - , Just 35.0 - , Just 36.0 - , Just 37.0 - , Just 38.0 - , Just 39.0 - , Just 40.0 - , Just 41.0 - , Just 42.0 - , Just 43.0 - , Just 44.0 - , Just 45.0 - , Just 46.0 - , Just 47.0 - , Just 48.0 - , Just 49.0 - , Just 50.0 - , Just 3.14 - , Just 2.22 - , Just 8.55 - , Just 23.3 - , Just 12.22222235049451 - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - , "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 1, D.parseSafe = False}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" - expected - actual - ) - -parseIntsAndDoublesAndEmptyStringsAsDoublesWithManyExamplesWithSafeReadOff :: - Test -parseIntsAndDoublesAndEmptyStringsAsDoublesWithManyExamplesWithSafeReadOff = - let afterParse :: [Maybe Double] - afterParse = - [ Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Just 1.0 - , Just 2.0 - , Just 3.0 - , Just 4.0 - , Just 5.0 - , Just 6.0 - , Just 7.0 - , Just 8.0 - , Just 9.0 - , Just 10.0 - , Just 11.0 - , Just 12.0 - , Just 13.0 - , Just 14.0 - , Just 15.0 - , Just 16.0 - , Just 17.0 - , Just 18.0 - , Just 19.0 - , Just 20.0 - , Just 21.0 - , Just 22.0 - , Just 23.0 - , Just 24.0 - , Just 25.0 - , Just 26.0 - , Just 27.0 - , Just 28.0 - , Just 29.0 - , Just 30.0 - , Just 31.0 - , Just 32.0 - , Just 33.0 - , Just 34.0 - , Just 35.0 - , Just 36.0 - , Just 37.0 - , Just 38.0 - , Just 39.0 - , Just 40.0 - , Just 41.0 - , Just 42.0 - , Just 43.0 - , Just 44.0 - , Just 45.0 - , Just 46.0 - , Just 47.0 - , Just 48.0 - , Just 49.0 - , Just 50.0 - , Just 1.0 - , Just 2.0 - , Just 3.0 - , Just 4.0 - , Just 5.0 - , Just 6.0 - , Just 7.0 - , Just 8.0 - , Just 9.0 - , Just 10.0 - , Just 11.0 - , Just 12.0 - , Just 13.0 - , Just 14.0 - , Just 15.0 - , Just 16.0 - , Just 17.0 - , Just 18.0 - , Just 19.0 - , Just 20.0 - , Just 21.0 - , Just 22.0 - , Just 23.0 - , Just 24.0 - , Just 25.0 - , Just 26.0 - , Just 27.0 - , Just 28.0 - , Just 29.0 - , Just 30.0 - , Just 31.0 - , Just 32.0 - , Just 33.0 - , Just 34.0 - , Just 35.0 - , Just 36.0 - , Just 37.0 - , Just 38.0 - , Just 39.0 - , Just 40.0 - , Just 41.0 - , Just 42.0 - , Just 43.0 - , Just 44.0 - , Just 45.0 - , Just 46.0 - , Just 47.0 - , Just 48.0 - , Just 49.0 - , Just 50.0 - , Just 3.14 - , Just 2.22 - , Just 8.55 - , Just 23.3 - , Just 12.22222235049451 - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "21" - , "22" - , "23" - , "24" - , "25" - , "26" - , "27" - , "28" - , "29" - , "30" - , "31" - , "32" - , "33" - , "34" - , "35" - , "36" - , "37" - , "38" - , "39" - , "40" - , "41" - , "42" - , "43" - , "44" - , "45" - , "46" - , "47" - , "48" - , "49" - , "50" - , "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "31.0" - , "32.0" - , "33.0" - , "34.0" - , "35.0" - , "36.0" - , "37.0" - , "38.0" - , "39.0" - , "40.0" - , "41.0" - , "42.0" - , "43.0" - , "44.0" - , "45.0" - , "46.0" - , "47.0" - , "48.0" - , "49.0" - , "50.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 30, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" - expected - actual - ) - -parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithOneExampleWithSafeReadOff :: - Test -parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithOneExampleWithSafeReadOff = - let afterParse :: [Maybe T.Text] - afterParse = - [ Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Just "NaN" - , Just "N/A" - , Just "NaN" - , Just "N/A" - , Just "NaN" - , Just "N/A" - , Just "NaN" - , Just "N/A" - , Just "NaN" - , Just "N/A" - , Just "1" - , Just "2" - , Just "3" - , Just "4" - , Just "5" - , Just "6" - , Just "7" - , Just "8" - , Just "9" - , Just "10" - , Just "11" - , Just "12" - , Just "13" - , Just "14" - , Just "15" - , Just "16" - , Just "17" - , Just "18" - , Just "19" - , Just "20" - , Just "1.0" - , Just "2.0" - , Just "3.0" - , Just "4.0" - , Just "5.0" - , Just "6.0" - , Just "7.0" - , Just "8.0" - , Just "9.0" - , Just "10.0" - , Just "11.0" - , Just "12.0" - , Just "13.0" - , Just "14.0" - , Just "15.0" - , Just "16.0" - , Just "17.0" - , Just "18.0" - , Just "19.0" - , Just "20.0" - , Just "21.0" - , Just "22.0" - , Just "23.0" - , Just "24.0" - , Just "25.0" - , Just "26.0" - , Just "27.0" - , Just "28.0" - , Just "29.0" - , Just "30.0" - , Just "3.14" - , Just "2.22" - , Just "8.55" - , Just "23.3" - , Just "12.22222235049451" - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "NaN" - , "N/A" - , "NaN" - , "N/A" - , "NaN" - , "N/A" - , "NaN" - , "N/A" - , "NaN" - , "N/A" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 1, D.parseSafe = False}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with just one example, when safeRead is off" - expected - actual - ) - -parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithManyExamplesWithSafeReadOff :: - Test -parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithManyExamplesWithSafeReadOff = - let afterParse :: [Maybe T.Text] - afterParse = - [ Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Nothing - , Just "NaN" - , Just "N/A" - , Just "NaN" - , Just "N/A" - , Just "NaN" - , Just "N/A" - , Just "NaN" - , Just "N/A" - , Just "NaN" - , Just "N/A" - , Just "1" - , Just "2" - , Just "3" - , Just "4" - , Just "5" - , Just "6" - , Just "7" - , Just "8" - , Just "9" - , Just "10" - , Just "11" - , Just "12" - , Just "13" - , Just "14" - , Just "15" - , Just "16" - , Just "17" - , Just "18" - , Just "19" - , Just "20" - , Just "1.0" - , Just "2.0" - , Just "3.0" - , Just "4.0" - , Just "5.0" - , Just "6.0" - , Just "7.0" - , Just "8.0" - , Just "9.0" - , Just "10.0" - , Just "11.0" - , Just "12.0" - , Just "13.0" - , Just "14.0" - , Just "15.0" - , Just "16.0" - , Just "17.0" - , Just "18.0" - , Just "19.0" - , Just "20.0" - , Just "21.0" - , Just "22.0" - , Just "23.0" - , Just "24.0" - , Just "25.0" - , Just "26.0" - , Just "27.0" - , Just "28.0" - , Just "29.0" - , Just "30.0" - , Just "3.14" - , Just "2.22" - , Just "8.55" - , Just "23.3" - , Just "12.22222235049451" - ] - beforeParse :: [T.Text] - beforeParse = - [ "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "" - , "NaN" - , "N/A" - , "NaN" - , "N/A" - , "NaN" - , "N/A" - , "NaN" - , "N/A" - , "NaN" - , "N/A" - , "1" - , "2" - , "3" - , "4" - , "5" - , "6" - , "7" - , "8" - , "9" - , "10" - , "11" - , "12" - , "13" - , "14" - , "15" - , "16" - , "17" - , "18" - , "19" - , "20" - , "1.0" - , "2.0" - , "3.0" - , "4.0" - , "5.0" - , "6.0" - , "7.0" - , "8.0" - , "9.0" - , "10.0" - , "11.0" - , "12.0" - , "13.0" - , "14.0" - , "15.0" - , "16.0" - , "17.0" - , "18.0" - , "19.0" - , "20.0" - , "21.0" - , "22.0" - , "23.0" - , "24.0" - , "25.0" - , "26.0" - , "27.0" - , "28.0" - , "29.0" - , "30.0" - , "3.14" - , "2.22" - , "8.55" - , "23.3" - , "12.22222235049451" - ] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault - (D.defaultParseOptions{D.sampleSize = 30, D.parseSafe = False}) - $ DI.fromVector - $ V.fromList beforeParse - in TestCase - ( assertEqual - "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with many examples, when safeRead is off" - expected - actual - ) - --- 5. EDGE CASES THAT HAVE TO BE INTERPRETED CORRECTLY - -parseManyNullishAndOneInt :: Test -parseManyNullishAndOneInt = - let afterParse :: [Maybe Int] - afterParse = replicate 100 Nothing ++ [Just 100000] - beforeParse :: [T.Text] - beforeParse = replicate 100 "NaN" ++ ["100000"] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) - -parseManyNullishAndOneDouble :: Test -parseManyNullishAndOneDouble = - let afterParse :: [Maybe Double] - afterParse = replicate 100 Nothing ++ [Just 3.14] - beforeParse :: [T.Text] - beforeParse = replicate 100 "NaN" ++ ["3.14"] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) - -parseManyNullishAndOneDate :: Test -parseManyNullishAndOneDate = - let afterParse :: [Maybe Day] - afterParse = replicate 100 Nothing ++ [Just $ fromGregorian 2024 12 25] - beforeParse :: [T.Text] - beforeParse = replicate 100 "NaN" ++ ["2024-12-25"] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) - -parseManyNullishAndIncorrectDates :: Test -parseManyNullishAndIncorrectDates = - let afterParse :: [Maybe T.Text] - afterParse = replicate 100 Nothing ++ [Just "2024-12-25", Just "2024-12-w6"] - beforeParse :: [T.Text] - beforeParse = replicate 100 "NaN" ++ ["2024-12-25", "2024-12-w6"] - expected = DI.OptionalColumn $ V.fromList afterParse - actual = - D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ - DI.fromVector $ - V.fromList beforeParse - in TestCase - (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) - -parseRepeatedNullish :: Test -parseRepeatedNullish = - let afterParse :: [Maybe T.Text] - afterParse = replicate 100 Nothing - beforeParse :: [T.Text] - beforeParse = replicate 100 "NaN" - expected = DI.OptionalColumn $ V.fromList afterParse +import qualified Data.Text.IO as TIO +import qualified Data.Vector as V +import qualified Data.Vector.Unboxed as VU +import qualified DataFrame as D +import qualified DataFrame.Internal.Column as DI +import qualified DataFrame.Operations.Typing as D + +import Data.Time (Day, fromGregorian) +import Test.HUnit (Test (TestCase, TestLabel), assertEqual) + +testData :: D.DataFrame +testData = + D.fromNamedColumns + [ ("test1", DI.fromList ([1 .. 26] :: [Int])) + , ("test2", DI.fromList ['a' .. 'z']) + ] + +-- Dimensions +correctDimensions :: Test +correctDimensions = TestCase (assertEqual "should be (26, 2)" (26, 2) (D.dimensions testData)) + +emptyDataframeDimensions :: Test +emptyDataframeDimensions = TestCase (assertEqual "should be (0, 0)" (0, 0) (D.dimensions D.empty)) + +dimensionsTest :: [Test] +dimensionsTest = + [ TestLabel "dimensions_correctDimensions" correctDimensions + , TestLabel "dimensions_emptyDataframeDimensions" emptyDataframeDimensions + ] + +-- Shared data directory +typingDataDir :: FilePath +typingDataDir = "./tests/data/typing/" + +-- Shared input/expected data +intsInput :: [T.Text] +intsInput = T.pack . show <$> ([1 .. 50] :: [Int]) + +intsExpected :: [Int] +intsExpected = [1 .. 50] + +doublesSpecial :: [Double] +doublesSpecial = [3.14, 2.22, 8.55, 23.3, 12.22222235049450945049504950] + +doublesSpecialInput :: [T.Text] +doublesSpecialInput = ["3.14", "2.22", "8.55", "23.3", "12.22222235049451"] + +doublesExpected :: [Double] +doublesExpected = [1.0 .. 50.0] ++ doublesSpecial + +doublesInput :: [T.Text] +doublesInput = (T.pack . show <$> ([1.0 .. 50.0] :: [Double])) ++ doublesSpecialInput + +intsAndDoublesExpected :: [Double] +intsAndDoublesExpected = ([1 .. 50] :: [Double]) ++ doublesExpected + +intsAndDoublesInput :: [T.Text] +intsAndDoublesInput = intsInput ++ doublesInput + +datesExpected :: [Day] +datesExpected = [fromGregorian 2020 02 12 .. fromGregorian 2020 02 26] + +datesInput :: [T.Text] +datesInput = T.pack . show <$> datesExpected + +-- PARSING TESTS +------- 1. SIMPLE CASES +parseBools :: Test +parseBools = + let afterParse :: [Bool] + afterParse = [True, True, True] ++ [False, False, False] + beforeParse :: [T.Text] + beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Bools without missing values as OptionalColumn of Maybe Bools" + expected + actual + ) + +parseInts :: Test +parseInts = + let afterParse :: [Int] + afterParse = [1 .. 50] + beforeParse :: [T.Text] + beforeParse = T.pack . show <$> [1 .. 50] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as OptionalColumn of Maybe Ints" + expected + actual + ) + +parseDoubles :: Test +parseDoubles = + let afterParse :: [Double] + afterParse = [1.0 .. 50.0] ++ [3.14, 2.22, 8.55, 23.3, 12.22222235049450945049504950] + beforeParse :: [T.Text] + beforeParse = + T.pack . show + <$> [1.0 .. 50.0] ++ [3.14, 2.22, 8.55, 23.3, 12.22222235049450945049504950] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Doubles without missing values as OptionalColumn of Maybe Doubles" + expected + actual + ) + +parseDates :: Test +parseDates = + let afterParse = datesExpected + beforeParse = datesInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates without missing values as OptionalColumn of Maybe Days" + expected + actual + ) + +parseTexts :: Test +parseTexts = TestCase $ do + texts <- T.lines <$> TIO.readFile (typingDataDir <> "texts.txt") + let expected = DI.fromVector $ V.fromList texts + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList texts + assertEqual + "Correctly parses Text without missing values as OptionalColumn of Maybe Text" + expected + actual + +--- 2. COMBINATION CASES +parseBoolsAndIntsAsTexts :: Test +parseBoolsAndIntsAsTexts = + let afterParse :: [T.Text] + afterParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] ++ ["1", "0", "1"] + beforeParse :: [T.Text] + beforeParse = ["True", "true", "TRUE"] ++ ["False", "false", "FALSE"] ++ ["1", "0", "1"] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses mixture of Bools and Ints as Maybe Text" + expected + actual + ) + +parseIntsAndDoublesAsDoubles :: Test +parseIntsAndDoublesAsDoubles = + let afterParse = intsAndDoublesExpected + beforeParse = intsAndDoublesInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as OptionalColumn of Maybe Doubles" + expected + actual + ) + +parseIntsAndDatesAsTexts :: Test +parseIntsAndDatesAsTexts = + let data30 = T.pack . show <$> ([1 .. 30] :: [Int]) + data9dates = take 9 datesInput + afterParse :: [T.Text] + afterParse = data30 ++ data9dates + beforeParse :: [T.Text] + beforeParse = data30 ++ data9dates + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Dates as OptionalColumn of Maybe Texts" + expected + actual + ) + +parseTextsAndDoublesAsTexts :: Test +parseTextsAndDoublesAsTexts = + let shortTexts = ["To", "protect", "the", "world", "from", "devastation"] + afterParse :: [T.Text] + afterParse = shortTexts ++ doublesInput + beforeParse :: [T.Text] + beforeParse = shortTexts ++ doublesInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Texts and Doubles as OptionalColumn of Maybe Texts" + expected + actual + ) + +parseDatesAndTextsAsTexts :: Test +parseDatesAndTextsAsTexts = + let extra = ["Jessie", "James", "Meowth"] + afterParse :: [T.Text] + afterParse = datesInput ++ extra + beforeParse :: [T.Text] + beforeParse = datesInput ++ extra + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Dates and Texts as OptionalColumn of Maybe Texts" + expected + actual + ) + +-- 3A. PARSING WITH SAFEREAD OFF + +parseBoolsWithoutSafeRead :: Test +parseBoolsWithoutSafeRead = + let afterParse :: [Bool] + afterParse = replicate 10 True ++ replicate 10 False + beforeParse :: [T.Text] + beforeParse = replicate 10 "true" ++ replicate 10 "false" + expected = DI.UnboxedColumn Nothing $ VU.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Bools without missing values as UnboxedColumn of Bools, when safeRead is off" + expected + actual + ) + +parseIntsWithoutSafeRead :: Test +parseIntsWithoutSafeRead = + let afterParse = intsExpected + beforeParse = intsInput + expected = DI.UnboxedColumn Nothing $ VU.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as UnboxedColumn of Ints, when safeRead is off" + expected + actual + ) + +parseDoublesWithoutSafeRead :: Test +parseDoublesWithoutSafeRead = + let afterParse = doublesExpected + beforeParse = doublesInput + expected = DI.UnboxedColumn Nothing $ VU.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Doubles without missing values as UnboxedColumn of Doubles, when safeRead is off" + expected + actual + ) + +parseDatesWithoutSafeRead :: Test +parseDatesWithoutSafeRead = + let afterParse = datesExpected + beforeParse = datesInput + expected = DI.BoxedColumn Nothing $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates without missing values as BoxedColumn of Days" + expected + actual + ) + +parseTextsWithoutSafeRead :: Test +parseTextsWithoutSafeRead = TestCase $ do + texts <- T.lines <$> TIO.readFile (typingDataDir <> "texts.txt") + let expected = DI.BoxedColumn Nothing $ V.fromList texts + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList texts + assertEqual + "Correctly parses Text without missing values as BoxedColumn of Text" + expected + actual + +parseBoolsAndEmptyStringsWithoutSafeRead :: Test +parseBoolsAndEmptyStringsWithoutSafeRead = + let afterParse :: [Maybe Bool] + afterParse = replicate 10 Nothing ++ replicate 10 (Just True) ++ replicate 10 (Just False) + beforeParse :: [T.Text] + beforeParse = replicate 10 "" ++ replicate 10 "true" ++ replicate 10 "false" + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Bools and empty Strings as OptionalColumn of Bools, when safeRead is off" + expected + actual + ) + +parseIntsAndEmptyStringsWithoutSafeRead :: Test +parseIntsAndEmptyStringsWithoutSafeRead = + let beforeParse = replicate 5 "" ++ intsInput + afterParse :: [Maybe Int] + afterParse = replicate 5 Nothing ++ map Just intsExpected + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is off" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsWithoutSafeRead :: Test +parseIntsAndDoublesAndEmptyStringsWithoutSafeRead = + let intGrp = T.pack . show <$> ([1 .. 10] :: [Int]) + dblGrps = + [ T.pack . show <$> ([11.0 .. 20.0] :: [Double]) + , T.pack . show <$> ([21.0 .. 30.0] :: [Double]) + , T.pack . show <$> ([31.0 .. 40.0] :: [Double]) + , T.pack . show <$> ([41.0 .. 50.0] :: [Double]) + ] + beforeParse = intGrp ++ [""] ++ concatMap (\g -> g ++ [""]) dblGrps ++ doublesSpecialInput + afterParse :: [Maybe Double] + afterParse = + map Just ([1.0 .. 10.0] :: [Double]) + ++ [Nothing] + ++ concatMap + (\g -> map Just g ++ [Nothing]) + [[11.0 .. 20.0], [21.0 .. 30.0], [31.0 .. 40.0], [41.0 .. 50.0]] + ++ map Just doublesSpecial + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" + expected + actual + ) + +parseDatesAndEmptyStringsWithoutSafeRead :: Test +parseDatesAndEmptyStringsWithoutSafeRead = + -- Pattern: 3 dates, empty, 3 dates, empty, 3 dates, empty, 3 dates, empty, 3 dates, empty + let groups = chunksOf3 datesExpected + beforeParse = concatMap (\g -> map (T.pack . show) g ++ [""]) groups + afterParse :: [Maybe Day] + afterParse = concatMap (\g -> map Just g ++ [Nothing]) groups + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead off" + expected + actual + ) + where + chunksOf3 [] = [] + chunksOf3 xs = take 3 xs : chunksOf3 (drop 3 xs) + +parseTextsAndEmptyStringsWithoutSafeRead :: Test +parseTextsAndEmptyStringsWithoutSafeRead = TestCase $ do + raw <- T.lines <$> TIO.readFile (typingDataDir <> "texts_with_empties.txt") + let afterParse = map (\t -> if t == "" then Nothing else Just t) raw + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList raw + assertEqual + "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" + expected + actual + +parseBoolsAndNullishStringsWithoutSafeRead :: Test +parseBoolsAndNullishStringsWithoutSafeRead = + let afterParse :: [T.Text] + afterParse = replicate 10 "N/A" ++ replicate 10 "True" ++ replicate 10 "False" + beforeParse :: [T.Text] + beforeParse = replicate 10 "N/A" ++ replicate 10 "True" ++ replicate 10 "False" + expected = DI.BoxedColumn Nothing $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Bools with nullish values as BoxedColumn of Texts, when safeRead is off" + expected + actual + ) + +parseIntsAndNullishStringsWithoutSafeRead :: Test +parseIntsAndNullishStringsWithoutSafeRead = + let beforeParse = replicate 5 "N/A" ++ intsInput + afterParse :: [T.Text] + afterParse = beforeParse + expected = DI.BoxedColumn Nothing $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints with nullish values as BoxedColumn of Texts, when safeRead is off" + expected + actual + ) + +parseIntsAndDoublesAndNullishStringsWithoutSafeRead :: Test +parseIntsAndDoublesAndNullishStringsWithoutSafeRead = + -- Nullish separators are literal text strings (not interpreted as null when safeRead=False) + let nullishSeps = ["Nothing", "N/A", "NULL", "null", "NAN"] + intGrp = T.pack . show <$> ([1 .. 10] :: [Int]) + dblGrps = + [ T.pack . show <$> ([11.0 .. 20.0] :: [Double]) + , T.pack . show <$> ([21.0 .. 30.0] :: [Double]) + , T.pack . show <$> ([31.0 .. 40.0] :: [Double]) + , T.pack . show <$> ([41.0 .. 50.0] :: [Double]) + ] + beforeParse = + intGrp + ++ concat (zipWith (:) nullishSeps dblGrps) + ++ doublesSpecialInput + afterParse :: [T.Text] + afterParse = beforeParse + expected = DI.BoxedColumn Nothing $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" + expected + actual + ) + +parseIntsAndNullishAndEmptyStringsWithoutSafeRead :: Test +parseIntsAndNullishAndEmptyStringsWithoutSafeRead = + -- Pattern: 5x"N/A", then groups of ("" : 10 ints), with trailing "" + let groups = [[1 .. 10], [11 .. 20], [21 .. 30], [31 .. 40], [41 .. 50]] :: [[Int]] + beforeParse = + replicate 5 "N/A" + ++ concatMap (\g -> "" : map (T.pack . show) g) groups + ++ [""] + afterParse :: [Maybe T.Text] + afterParse = + replicate 5 (Just "N/A") + ++ concatMap (\g -> Nothing : map (Just . T.pack . show) g) groups + ++ [Nothing] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Texts, when safeRead is off" + expected + actual + ) + +parseTextsAndEmptyAndNullishStringsWithoutSafeRead :: Test +parseTextsAndEmptyAndNullishStringsWithoutSafeRead = TestCase $ do + raw <- + T.lines <$> TIO.readFile (typingDataDir <> "texts_with_empties_and_nullish.txt") + -- safeRead=False: empty strings -> Nothing, nullish text stays as Just + let afterParse = map (\t -> if t == "" then Nothing else Just t) raw + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 10, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList raw + assertEqual + "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" + expected + actual + +-- 3B. PARSING WITH SAFEREAD ON +parseBoolsAndEmptyStringsWithSafeRead :: Test +parseBoolsAndEmptyStringsWithSafeRead = + let afterParse :: [Maybe Bool] + afterParse = replicate 10 Nothing ++ replicate 10 (Just True) + beforeParse :: [T.Text] + beforeParse = replicate 10 "" ++ replicate 10 "true" + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Bools and empty strings as OptionalColumn of Bools, when safeRead is on" + expected + actual + ) + +parseIntsAndEmptyStringsWithSafeRead :: Test +parseIntsAndEmptyStringsWithSafeRead = + let beforeParse = replicate 5 "" ++ intsInput + afterParse :: [Maybe Int] + afterParse = replicate 5 Nothing ++ map Just intsExpected + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints and empty strings as OptionalColumn of Ints, when safeRead is on" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsWithSafeRead :: Test +parseIntsAndDoublesAndEmptyStringsWithSafeRead = + let intGrp = T.pack . show <$> ([1 .. 10] :: [Int]) + dblGrps = + [ T.pack . show <$> ([11.0 .. 20.0] :: [Double]) + , T.pack . show <$> ([21.0 .. 30.0] :: [Double]) + , T.pack . show <$> ([31.0 .. 40.0] :: [Double]) + , T.pack . show <$> ([41.0 .. 50.0] :: [Double]) + ] + beforeParse = intGrp ++ [""] ++ concatMap (\g -> g ++ [""]) dblGrps ++ doublesSpecialInput + afterParse :: [Maybe Double] + afterParse = + map Just ([1.0 .. 10.0] :: [Double]) + ++ [Nothing] + ++ concatMap + (\g -> map Just g ++ [Nothing]) + [[11.0 .. 20.0], [21.0 .. 30.0], [31.0 .. 40.0], [41.0 .. 50.0]] + ++ map Just doublesSpecial + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is on" + expected + actual + ) + +parseDatesAndEmptyStringsWithSafeRead :: Test +parseDatesAndEmptyStringsWithSafeRead = + let groups = chunksOf3 datesExpected + beforeParse = concatMap (\g -> map (T.pack . show) g ++ [""]) groups + afterParse :: [Maybe Day] + afterParse = concatMap (\g -> map Just g ++ [Nothing]) groups + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates and Empty Strings as OptionalColumn of Dates, with safeRead on" + expected + actual + ) + where + chunksOf3 [] = [] + chunksOf3 xs = take 3 xs : chunksOf3 (drop 3 xs) + +parseTextsAndEmptyStringsWithSafeRead :: Test +parseTextsAndEmptyStringsWithSafeRead = TestCase $ do + raw <- T.lines <$> TIO.readFile (typingDataDir <> "texts_with_empties.txt") + let afterParse = map (\t -> if t == "" then Nothing else Just t) raw + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList raw + assertEqual + "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead on" + expected + actual + +parseIntsAndNullishStringsWithSafeRead :: Test +parseIntsAndNullishStringsWithSafeRead = + let beforeParse = replicate 5 "N/A" ++ intsInput + afterParse :: [Maybe Int] + afterParse = replicate 5 Nothing ++ map Just intsExpected + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints with nullish values as OptionalColumn of Ints, when safeRead is on" + expected + actual + ) + +parseIntsAndDoublesAndNullishStringsWithSafeRead :: Test +parseIntsAndDoublesAndNullishStringsWithSafeRead = + -- Note: this test uses different special doubles and nullish separators than the safeRead=False version + let nullishSeps = ["Nothing", "N/A", "NULL", "null", "NAN"] + intGrp = T.pack . show <$> ([1 .. 10] :: [Int]) + dblGrps = + [ T.pack . show <$> ([11.0 .. 20.0] :: [Double]) + , T.pack . show <$> ([21.0 .. 30.0] :: [Double]) + , T.pack . show <$> ([31.0 .. 40.0] :: [Double]) + , T.pack . show <$> ([41.0 .. 50.0] :: [Double]) + ] + beforeParse = + intGrp + ++ concat (zipWith (:) nullishSeps dblGrps) + ++ [nullishSeps !! 4] + ++ ["3.14", "2.22", "8.55", "23.3", "12.03"] + afterParse :: [Maybe Double] + afterParse = + map Just ([1.0 .. 10.0] :: [Double]) + ++ [Nothing] + ++ concatMap + (\g -> map Just g ++ [Nothing]) + [[11.0 .. 20.0], [21.0 .. 30.0], [31.0 .. 40.0], [41.0 .. 50.0]] + ++ map Just [3.14, 2.22, 8.55, 23.3, 12.03] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses combination of Ints, Doubles and empty strings as OptionalColumn of Doubles, when safeRead is off" + expected + actual + ) + +parseIntsAndNullishAndEmptyStringsWithSafeRead :: Test +parseIntsAndNullishAndEmptyStringsWithSafeRead = + -- safeRead=True: N/A, Nothing -> Nothing; "" -> Nothing + let groups = [[1 .. 10], [11 .. 20], [21 .. 30], [31 .. 40], [41 .. 50]] :: [[Int]] + beforeParse = + ["N/A", "N/A", "N/A", "N/A", "Nothing"] + ++ concatMap (\g -> "" : map (T.pack . show) g) groups + ++ [""] + afterParse :: [Maybe Int] + afterParse = + replicate 5 Nothing + ++ concatMap (\g -> Nothing : map Just g) groups + ++ [Nothing] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints with nullish values AND empty strings as OptionalColumn of Ints, when safeRead is on" + expected + actual + ) + +parseIntsAndDoublesAndNullishAndEmptyStringsWithSafeRead :: Test +parseIntsAndDoublesAndNullishAndEmptyStringsWithSafeRead = + -- Data: 4xN/A, "Nothing", "", then 10 ints (1-10), "", 10 ints (11-20), "", 10 ints (21-30), "", 3.14 + let beforeParse = + ["N/A", "N/A", "N/A", "N/A", "Nothing", ""] + ++ map (T.pack . show) ([1 .. 10] :: [Int]) + ++ [""] + ++ map (T.pack . show) ([11 .. 20] :: [Int]) + ++ [""] + ++ map (T.pack . show) ([21 .. 30] :: [Int]) + ++ ["", "3.14"] + afterParse :: [Maybe Double] + afterParse = + replicate 6 Nothing + ++ map Just ([1 .. 10] :: [Double]) + ++ [Nothing] + ++ map Just ([11 .. 20] :: [Double]) + ++ [Nothing] + ++ map Just ([21 .. 30] :: [Double]) + ++ [Nothing, Just 3.14] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints and Doubles with nullish values AND empty strings as OptionalColumn of Doubles, when safeRead is on" + expected + actual + ) + +parseTextsAndEmptyAndNullishStringsWithSafeRead :: Test +parseTextsAndEmptyAndNullishStringsWithSafeRead = TestCase $ do + raw <- + T.lines <$> TIO.readFile (typingDataDir <> "texts_with_empties_and_nullish.txt") + -- safeRead=True: empty strings and nullish tokens (NaN, Nothing, N/A) -> Nothing + let isNullish t = t `elem` ["NaN", "Nothing", "N/A", "nan", "null", "NULL", "NA", "na", "NAN"] + afterParse = map (\t -> if t == "" || isNullish t then Nothing else Just t) raw + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList raw + assertEqual + "Correctly parses Texts and Empty Strings as OptionalColumn of Text, with safeRead off" + expected + actual + +-- 4. PARSING SHOULD NOT DEPEND ON THE NUMBER OF EXAMPLES. +parseBoolsWithOneExample :: Test +parseBoolsWithOneExample = + let afterParse :: [Bool] + afterParse = False : replicate 50 True + beforeParse :: [T.Text] + beforeParse = "false" : replicate 50 "true" + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 1}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Bools without missing values as OptionalColumn of Maybe Bools with only one example" + expected + actual + ) + +parseBoolsWithManyExamples :: Test +parseBoolsWithManyExamples = + let afterParse :: [Bool] + afterParse = False : replicate 50 True + beforeParse :: [T.Text] + beforeParse = "false" : replicate 50 "true" + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 49}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Bools without missing values as OptionalColumn of Maybe Bools with only one example" + expected + actual + ) + +parseIntsWithOneExample :: Test +parseIntsWithOneExample = + let afterParse = intsExpected + beforeParse = intsInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 1}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as OptionalColumn of Maybe Ints with only one example" + expected + actual + ) + +parseIntsWithTwentyFiveExamples :: Test +parseIntsWithTwentyFiveExamples = + let afterParse = intsExpected + beforeParse = intsInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 25}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as OptionalColumn of Maybe Ints with some examples" + expected + actual + ) + +parseIntsWithFortyNineExamples :: Test +parseIntsWithFortyNineExamples = + let afterParse = intsExpected + beforeParse = intsInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 49}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Ints without missing values as OptionalColumn of Maybe Ints with many examples" + expected + actual + ) + +parseDatesWithOneExample :: Test +parseDatesWithOneExample = + let afterParse = datesExpected + beforeParse = datesInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 1}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates without missing values as OptionalColumn of Maybe Days with only one example" + expected + actual + ) + +parseDatesWithFifteenExamples :: Test +parseDatesWithFifteenExamples = + let afterParse = datesExpected + beforeParse = datesInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 15}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses Dates without missing values as OptionalColumn of Maybe Days with many examples" + expected + actual + ) + +parseIntsAndDoublesAsDoublesWithOneExample :: Test +parseIntsAndDoublesAsDoublesWithOneExample = + let afterParse = intsAndDoublesExpected + beforeParse = intsAndDoublesInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 1}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as OptionalColumn of Maybe Doubles with just one example" + expected + actual + ) + +parseIntsAndDoublesAsDoublesWithManyExamples :: Test +parseIntsAndDoublesAsDoublesWithManyExamples = + let afterParse = intsAndDoublesExpected + beforeParse = intsAndDoublesInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 50}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as OptionalColumn of Maybe Doubles with many examples" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsAsDoublesWithOneExampleWithSafeReadOff :: Test +parseIntsAndDoublesAndEmptyStringsAsDoublesWithOneExampleWithSafeReadOff = + -- Pattern: 10 empties, 50 ints (as int text), 50 doubles (as double text), 5 specials + let beforeParse = + replicate 10 "" + ++ intsInput + ++ (T.pack . show <$> ([1.0 .. 50.0] :: [Double])) + ++ doublesSpecialInput + afterParse :: [Maybe Double] + afterParse = + replicate 10 Nothing + ++ map Just ([1.0 .. 50.0] :: [Double]) + ++ map Just ([1.0 .. 50.0] :: [Double]) + ++ map Just doublesSpecial + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 1, D.parseSafe = False}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsAsDoublesWithManyExamplesWithSafeReadOff :: + Test +parseIntsAndDoublesAndEmptyStringsAsDoublesWithManyExamplesWithSafeReadOff = + let beforeParse = + replicate 10 "" + ++ intsInput + ++ (T.pack . show <$> ([1.0 .. 50.0] :: [Double])) + ++ doublesSpecialInput + afterParse :: [Maybe Double] + afterParse = + replicate 10 Nothing + ++ map Just ([1.0 .. 50.0] :: [Double]) + ++ map Just ([1.0 .. 50.0] :: [Double]) + ++ map Just doublesSpecial + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 30, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints and Doubles as UnboxedColumn of Doubles with just one example" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithOneExampleWithSafeReadOff :: + Test +parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithOneExampleWithSafeReadOff = + -- Pattern: 10 empties, 5x(NaN,N/A), 20 ints (1-20), 30 doubles (1.0-30.0), 5 specials + let nullishPairs = concat (replicate 5 ["NaN", "N/A"]) + beforeParse = + replicate 10 "" + ++ nullishPairs + ++ (T.pack . show <$> ([1 .. 20] :: [Int])) + ++ (T.pack . show <$> ([1.0 .. 30.0] :: [Double])) + ++ doublesSpecialInput + afterParse :: [Maybe T.Text] + afterParse = + replicate 10 Nothing + ++ map Just nullishPairs + ++ map (Just . T.pack . show) ([1 .. 20] :: [Int]) + ++ map (Just . T.pack . show) ([1.0 .. 30.0] :: [Double]) + ++ map Just doublesSpecialInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 1, D.parseSafe = False}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with just one example, when safeRead is off" + expected + actual + ) + +parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithManyExamplesWithSafeReadOff :: + Test +parseIntsAndDoublesAndEmptyStringsAndNullishAsStringssWithManyExamplesWithSafeReadOff = + let nullishPairs = concat (replicate 5 ["NaN", "N/A"]) + beforeParse = + replicate 10 "" + ++ nullishPairs + ++ (T.pack . show <$> ([1 .. 20] :: [Int])) + ++ (T.pack . show <$> ([1.0 .. 30.0] :: [Double])) + ++ doublesSpecialInput + afterParse :: [Maybe T.Text] + afterParse = + replicate 10 Nothing + ++ map Just nullishPairs + ++ map (Just . T.pack . show) ([1 .. 20] :: [Int]) + ++ map (Just . T.pack . show) ([1.0 .. 30.0] :: [Double]) + ++ map Just doublesSpecialInput + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault + (D.defaultParseOptions{D.sampleSize = 30, D.parseSafe = False}) + $ DI.fromVector + $ V.fromList beforeParse + in TestCase + ( assertEqual + "Correctly parses a mixture of Ints, Doubles, empty strings, nullish as OptionalColumn of Text with many examples, when safeRead is off" + expected + actual + ) + +-- 5. EDGE CASES THAT HAVE TO BE INTERPRETED CORRECTLY + +parseManyNullishAndOneInt :: Test +parseManyNullishAndOneInt = + let afterParse :: [Maybe Int] + afterParse = replicate 100 Nothing ++ [Just 100000] + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" ++ ["100000"] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) + +parseManyNullishAndOneDouble :: Test +parseManyNullishAndOneDouble = + let afterParse :: [Maybe Double] + afterParse = replicate 100 Nothing ++ [Just 3.14] + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" ++ ["3.14"] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) + +parseManyNullishAndOneDate :: Test +parseManyNullishAndOneDate = + let afterParse :: [Maybe Day] + afterParse = replicate 100 Nothing ++ [Just $ fromGregorian 2024 12 25] + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" ++ ["2024-12-25"] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) + +parseManyNullishAndIncorrectDates :: Test +parseManyNullishAndIncorrectDates = + let afterParse :: [Maybe T.Text] + afterParse = replicate 100 Nothing ++ [Just "2024-12-25", Just "2024-12-w6"] + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" ++ ["2024-12-25", "2024-12-w6"] + expected = DI.fromVector $ V.fromList afterParse + actual = + D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ + DI.fromVector $ + V.fromList beforeParse + in TestCase + (assertEqual "Correctly parses many Nulls followed by one Int" expected actual) + +parseRepeatedNullish :: Test +parseRepeatedNullish = + let afterParse :: [Maybe T.Text] + afterParse = replicate 100 Nothing + beforeParse :: [T.Text] + beforeParse = replicate 100 "NaN" + expected = DI.fromVector $ V.fromList afterParse actual = D.parseDefault (D.defaultParseOptions{D.sampleSize = 10}) $ DI.fromVector $ diff --git a/tests/Operations/WriteCsv.hs b/tests/Operations/WriteCsv.hs new file mode 100644 --- /dev/null +++ b/tests/Operations/WriteCsv.hs @@ -0,0 +1,88 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +module Operations.WriteCsv where + +import qualified Data.Text as T +import qualified Data.Text.IO as TIO +import qualified DataFrame as D +import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.DataFrame (DataFrame (..), toCsv, toSeparated) +import Test.HUnit + +-- Basic test: Int and Text columns produce correct CSV +toCsvBasic :: Test +toCsvBasic = TestLabel "toCsv_basic" $ TestCase $ do + let df = + D.fromNamedColumns + [ ("name", DI.fromList @T.Text ["Alice", "Bob", "Charlie"]) + , ("age", DI.fromList @Int [30, 25, 35]) + ] + expected = "name,age\nAlice,30\nBob,25\nCharlie,35\n" + assertEqual "basic toCsv" expected (toCsv df) + +-- Empty DataFrame produces empty text +toCsvEmpty :: Test +toCsvEmpty = + TestLabel "toCsv_empty" $ + TestCase $ + assertEqual "empty toCsv" T.empty (toCsv D.empty) + +-- toSeparated with tab produces tab-delimited output +toSeparatedTab :: Test +toSeparatedTab = TestLabel "toSeparated_tab" $ TestCase $ do + let df = + D.fromNamedColumns + [ ("x", DI.fromList @Int [1, 2]) + , ("y", DI.fromList @Int [3, 4]) + ] + expected = "x\ty\n1\t3\n2\t4\n" + assertEqual "tab separated" expected (toSeparated '\t' df) + +-- Double values render correctly +toCsvDouble :: Test +toCsvDouble = TestLabel "toCsv_double" $ TestCase $ do + let df = + D.fromNamedColumns + [ ("value", DI.fromList @Double [1.5, 2.0, 2.5]) + ] + expected = "value\n1.5\n2.0\n2.5\n" + assertEqual "double toCsv" expected (toCsv df) + +-- Single column DataFrame +toCsvSingleColumn :: Test +toCsvSingleColumn = TestLabel "toCsv_single_column" $ TestCase $ do + let df = + D.fromNamedColumns + [ ("id", DI.fromList @Int [10, 20, 30]) + ] + expected = "id\n10\n20\n30\n" + assertEqual "single column toCsv" expected (toCsv df) + +-- Round trip: toCsv then readCsv preserves data +toCsvRoundTrip :: Test +toCsvRoundTrip = TestLabel "toCsv_roundTrip" $ TestCase $ do + let df = + D.fromNamedColumns + [ ("a", DI.fromList @Int [1, 2, 3]) + , ("b", DI.fromList @T.Text ["hello", "world", "test"]) + ] + let csvText = toCsv df + let tmpPath = "/tmp/dataframe_test_toCsv_roundtrip.csv" + TIO.writeFile tmpPath csvText + df' <- D.readCsv tmpPath + assertEqual + "round trip dimensions" + (dataframeDimensions df) + (dataframeDimensions df') + assertEqual "round trip data" df df' + +tests :: [Test] +tests = + [ toCsvBasic + , toCsvEmpty + , toSeparatedTab + , toCsvDouble + , toCsvSingleColumn + , toCsvRoundTrip + ] diff --git a/tests/Parquet.hs b/tests/Parquet.hs --- a/tests/Parquet.hs +++ b/tests/Parquet.hs @@ -7,13 +7,12 @@ import qualified DataFrame as D import qualified DataFrame.Functions as F import qualified DataFrame.IO.Parquet as DP +import ParquetTestData (allTypes, mtCarsDataset, tinyPagesLast10, transactions) import qualified Data.ByteString as BS import Data.Int import qualified Data.Set as S -import Data.Text (Text) import qualified Data.Text as T -import Data.Time import Data.Word import DataFrame.IO.Parquet.Thrift ( columnMetaData, @@ -35,46 +34,6 @@ import GHC.IO (unsafePerformIO) import Test.HUnit -allTypes :: D.DataFrame -allTypes = - D.fromNamedColumns - [ ("id", D.fromList [4 :: Int32, 5, 6, 7, 2, 3, 0, 1]) - , ("bool_col", D.fromList [True, False, True, False, True, False, True, False]) - , ("tinyint_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1]) - , ("smallint_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1]) - , ("int_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1]) - , ("bigint_col", D.fromList [0 :: Int64, 10, 0, 10, 0, 10, 0, 10]) - , ("float_col", D.fromList [0 :: Float, 1.1, 0, 1.1, 0, 1.1, 0, 1.1]) - , ("double_col", D.fromList [0 :: Double, 10.1, 0, 10.1, 0, 10.1, 0, 10.1]) - , - ( "date_string_col" - , D.fromList - [ "03/01/09" :: Text - , "03/01/09" - , "04/01/09" - , "04/01/09" - , "02/01/09" - , "02/01/09" - , "01/01/09" - , "01/01/09" - ] - ) - , ("string_col", D.fromList (take 8 (cycle ["0" :: Text, "1"]))) - , - ( "timestamp_col" - , D.fromList - [ UTCTime{utctDay = fromGregorian 2009 3 1, utctDayTime = secondsToDiffTime 0} - , UTCTime{utctDay = fromGregorian 2009 3 1, utctDayTime = secondsToDiffTime 60} - , UTCTime{utctDay = fromGregorian 2009 4 1, utctDayTime = secondsToDiffTime 0} - , UTCTime{utctDay = fromGregorian 2009 4 1, utctDayTime = secondsToDiffTime 60} - , UTCTime{utctDay = fromGregorian 2009 2 1, utctDayTime = secondsToDiffTime 0} - , UTCTime{utctDay = fromGregorian 2009 2 1, utctDayTime = secondsToDiffTime 60} - , UTCTime{utctDay = fromGregorian 2009 1 1, utctDayTime = secondsToDiffTime 0} - , UTCTime{utctDay = fromGregorian 2009 1 1, utctDayTime = secondsToDiffTime 60} - ] - ) - ] - testBothReadParquetPaths :: ((FilePath -> IO D.DataFrame) -> Test) -> Test testBothReadParquetPaths test = TestList @@ -102,87 +61,6 @@ ) ) -tinyPagesLast10 :: D.DataFrame -tinyPagesLast10 = - D.fromNamedColumns - [ ("id", D.fromList @Int32 (reverse [6174 .. 6183])) - , ("bool_col", D.fromList @Bool (Prelude.take 10 (cycle [False, True]))) - , ("tinyint_col", D.fromList @Int32 [3, 2, 1, 0, 9, 8, 7, 6, 5, 4]) - , ("smallint_col", D.fromList @Int32 [3, 2, 1, 0, 9, 8, 7, 6, 5, 4]) - , ("int_col", D.fromList @Int32 [3, 2, 1, 0, 9, 8, 7, 6, 5, 4]) - , ("bigint_col", D.fromList @Int64 [30, 20, 10, 0, 90, 80, 70, 60, 50, 40]) - , - ( "float_col" - , D.fromList @Float [3.3, 2.2, 1.1, 0, 9.9, 8.8, 7.7, 6.6, 5.5, 4.4] - ) - , - ( "date_string_col" - , D.fromList @Text - [ "09/11/10" - , "09/11/10" - , "09/11/10" - , "09/11/10" - , "09/10/10" - , "09/10/10" - , "09/10/10" - , "09/10/10" - , "09/10/10" - , "09/10/10" - ] - ) - , - ( "string_col" - , D.fromList @Text ["3", "2", "1", "0", "9", "8", "7", "6", "5", "4"] - ) - , - ( "timestamp_col" - , D.fromList @UTCTime - [ UTCTime - { utctDay = fromGregorian 2010 9 10 - , utctDayTime = secondsToDiffTime 85384 - } - , UTCTime - { utctDay = fromGregorian 2010 9 10 - , utctDayTime = secondsToDiffTime 85324 - } - , UTCTime - { utctDay = fromGregorian 2010 9 10 - , utctDayTime = secondsToDiffTime 85264 - } - , UTCTime - { utctDay = fromGregorian 2010 9 10 - , utctDayTime = secondsToDiffTime 85204 - } - , UTCTime - { utctDay = fromGregorian 2010 9 9 - , utctDayTime = secondsToDiffTime 85144 - } - , UTCTime - { utctDay = fromGregorian 2010 9 9 - , utctDayTime = secondsToDiffTime 85084 - } - , UTCTime - { utctDay = fromGregorian 2010 9 9 - , utctDayTime = secondsToDiffTime 85024 - } - , UTCTime - { utctDay = fromGregorian 2010 9 9 - , utctDayTime = secondsToDiffTime 84964 - } - , UTCTime - { utctDay = fromGregorian 2010 9 9 - , utctDayTime = secondsToDiffTime 84904 - } - , UTCTime - { utctDay = fromGregorian 2010 9 9 - , utctDayTime = secondsToDiffTime 84844 - } - ] - ) - , ("year", D.fromList @Int32 (replicate 10 2010)) - , ("month", D.fromList @Int32 (replicate 10 9)) - ] - allTypesTinyPagesLastFew :: Test allTypesTinyPagesLastFew = testBothReadParquetPaths $ \readParquet -> TestCase @@ -327,167 +205,6 @@ ) ) -transactions :: D.DataFrame -transactions = - D.fromNamedColumns - [ ("transaction_id", D.fromList [1 :: Int32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) - , - ( "event_time" - , D.fromList - [ UTCTime - { utctDay = fromGregorian 2024 1 3 - , utctDayTime = secondsToDiffTime 29564 + picosecondsToDiffTime 2311000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 3 - , utctDayTime = secondsToDiffTime 35101 + picosecondsToDiffTime 118900000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 4 - , utctDayTime = secondsToDiffTime 39802 + picosecondsToDiffTime 774512000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 5 - , utctDayTime = secondsToDiffTime 53739 + picosecondsToDiffTime 1000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 6 - , utctDayTime = secondsToDiffTime 8278 + picosecondsToDiffTime 543210000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 6 - , utctDayTime = secondsToDiffTime 8284 + picosecondsToDiffTime 211000000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 7 - , utctDayTime = secondsToDiffTime 63000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 8 - , utctDayTime = secondsToDiffTime 24259 + picosecondsToDiffTime 390000000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 9 - , utctDayTime = secondsToDiffTime 48067 + picosecondsToDiffTime 812345000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 10 - , utctDayTime = secondsToDiffTime 82799 + picosecondsToDiffTime 999999000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 11 - , utctDayTime = secondsToDiffTime 36000 + picosecondsToDiffTime 100000000000 - } - , UTCTime - { utctDay = fromGregorian 2024 1 12 - , utctDayTime = secondsToDiffTime 56028 + picosecondsToDiffTime 667891000000 - } - ] - ) - , - ( "user_email" - , D.fromList - [ "alice@example.com" :: Text - , "bob@example.com" - , "carol@example.com" - , "alice@example.com" - , "dave@example.com" - , "dave@example.com" - , "eve@example.com" - , "frank@example.com" - , "grace@example.com" - , "dave@example.com" - , "alice@example.com" - , "heidi@example.com" - ] - ) - , - ( "transaction_type" - , D.fromList - [ "purchase" :: Text - , "purchase" - , "refund" - , "purchase" - , "purchase" - , "purchase" - , "purchase" - , "withdrawal" - , "purchase" - , "purchase" - , "purchase" - , "refund" - ] - ) - , - ( "amount" - , D.fromList - [ 142.50 :: Double - , 29.99 - , 89.00 - , 2399.00 - , 15.00 - , 15.00 - , 450.75 - , 200.00 - , 55.20 - , 3200.00 - , 74.99 - , 120.00 - ] - ) - , - ( "currency" - , D.fromList - [ "USD" :: Text - , "USD" - , "EUR" - , "USD" - , "GBP" - , "GBP" - , "USD" - , "EUR" - , "CAD" - , "USD" - , "USD" - , "GBP" - ] - ) - , - ( "status" - , D.fromList - [ "approved" :: Text - , "approved" - , "approved" - , "declined" - , "approved" - , "declined" - , "approved" - , "approved" - , "approved" - , "flagged" - , "approved" - , "approved" - ] - ) - , - ( "location" - , D.fromList - [ "New York, US" :: Text - , "London, GB" - , "Berlin, DE" - , "New York, US" - , "Manchester, GB" - , "Lagos, NG" - , "San Francisco, US" - , "Paris, FR" - , "Toronto, CA" - , "New York, US" - , "New York, US" - , "Edinburgh, GB" - ] - ) - ] - transactionsTest :: Test transactionsTest = testBothReadParquetPaths $ \readParquet -> TestCase @@ -496,455 +213,6 @@ transactions (unsafePerformIO (readParquet "./tests/data/transactions.parquet")) ) - -mtCarsDataset :: D.DataFrame -mtCarsDataset = - D.fromNamedColumns - [ - ( "model" - , D.fromList - [ "Mazda RX4" :: Text - , "Mazda RX4 Wag" - , "Datsun 710" - , "Hornet 4 Drive" - , "Hornet Sportabout" - , "Valiant" - , "Duster 360" - , "Merc 240D" - , "Merc 230" - , "Merc 280" - , "Merc 280C" - , "Merc 450SE" - , "Merc 450SL" - , "Merc 450SLC" - , "Cadillac Fleetwood" - , "Lincoln Continental" - , "Chrysler Imperial" - , "Fiat 128" - , "Honda Civic" - , "Toyota Corolla" - , "Toyota Corona" - , "Dodge Challenger" - , "AMC Javelin" - , "Camaro Z28" - , "Pontiac Firebird" - , "Fiat X1-9" - , "Porsche 914-2" - , "Lotus Europa" - , "Ford Pantera L" - , "Ferrari Dino" - , "Maserati Bora" - , "Volvo 142E" - ] - ) - , - ( "mpg" - , D.fromList - [ 21.0 :: Double - , 21.0 - , 22.8 - , 21.4 - , 18.7 - , 18.1 - , 14.3 - , 24.4 - , 22.8 - , 19.2 - , 17.8 - , 16.4 - , 17.3 - , 15.2 - , 10.4 - , 10.4 - , 14.7 - , 32.4 - , 30.4 - , 33.9 - , 21.5 - , 15.5 - , 15.2 - , 13.3 - , 19.2 - , 27.3 - , 26.0 - , 30.4 - , 15.8 - , 19.7 - , 15.0 - , 21.4 - ] - ) - , - ( "cyl" - , D.fromList - [ 6 :: Int32 - , 6 - , 4 - , 6 - , 8 - , 6 - , 8 - , 4 - , 4 - , 6 - , 6 - , 8 - , 8 - , 8 - , 8 - , 8 - , 8 - , 4 - , 4 - , 4 - , 4 - , 8 - , 8 - , 8 - , 8 - , 4 - , 4 - , 4 - , 8 - , 6 - , 8 - , 4 - ] - ) - , - ( "disp" - , D.fromList - [ 160.0 :: Double - , 160.0 - , 108.0 - , 258.0 - , 360.0 - , 225.0 - , 360.0 - , 146.7 - , 140.8 - , 167.6 - , 167.6 - , 275.8 - , 275.8 - , 275.8 - , 472.0 - , 460.0 - , 440.0 - , 78.7 - , 75.7 - , 71.1 - , 120.1 - , 318.0 - , 304.0 - , 350.0 - , 400.0 - , 79.0 - , 120.3 - , 95.1 - , 351.0 - , 145.0 - , 301.0 - , 121.0 - ] - ) - , - ( "hp" - , D.fromList - [ 110 :: Int32 - , 110 - , 93 - , 110 - , 175 - , 105 - , 245 - , 62 - , 95 - , 123 - , 123 - , 180 - , 180 - , 180 - , 205 - , 215 - , 230 - , 66 - , 52 - , 65 - , 97 - , 150 - , 150 - , 245 - , 175 - , 66 - , 91 - , 113 - , 264 - , 175 - , 335 - , 109 - ] - ) - , - ( "drat" - , D.fromList - [ 3.9 :: Double - , 3.9 - , 3.85 - , 3.08 - , 3.15 - , 2.76 - , 3.21 - , 3.69 - , 3.92 - , 3.92 - , 3.92 - , 3.07 - , 3.07 - , 3.07 - , 2.93 - , 3.0 - , 3.23 - , 4.08 - , 4.93 - , 4.22 - , 3.7 - , 2.76 - , 3.15 - , 3.73 - , 3.08 - , 4.08 - , 4.43 - , 3.77 - , 4.22 - , 3.62 - , 3.54 - , 4.11 - ] - ) - , - ( "wt" - , D.fromList - [ 2.62 :: Double - , 2.875 - , 2.32 - , 3.215 - , 3.44 - , 3.46 - , 3.57 - , 3.19 - , 3.15 - , 3.44 - , 3.44 - , 4.07 - , 3.73 - , 3.78 - , 5.25 - , 5.424 - , 5.345 - , 2.2 - , 1.615 - , 1.835 - , 2.465 - , 3.52 - , 3.435 - , 3.84 - , 3.845 - , 1.935 - , 2.14 - , 1.513 - , 3.17 - , 2.77 - , 3.57 - , 2.78 - ] - ) - , - ( "qsec" - , D.fromList - [ 16.46 :: Double - , 17.02 - , 18.61 - , 19.44 - , 17.02 - , 20.22 - , 15.84 - , 20.0 - , 22.9 - , 18.3 - , 18.9 - , 17.4 - , 17.6 - , 18.0 - , 17.98 - , 17.82 - , 17.42 - , 19.47 - , 18.52 - , 19.9 - , 20.01 - , 16.87 - , 17.3 - , 15.41 - , 17.05 - , 18.9 - , 16.7 - , 16.9 - , 14.5 - , 15.5 - , 14.6 - , 18.6 - ] - ) - , - ( "vs" - , D.fromList - [ 0 :: Int32 - , 0 - , 1 - , 1 - , 0 - , 1 - , 0 - , 1 - , 1 - , 1 - , 1 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 1 - , 1 - , 1 - , 1 - , 0 - , 0 - , 0 - , 0 - , 1 - , 0 - , 1 - , 0 - , 0 - , 0 - , 1 - ] - ) - , - ( "am" - , D.fromList - [ 1 :: Int32 - , 1 - , 1 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 0 - , 1 - , 1 - , 1 - , 0 - , 0 - , 0 - , 0 - , 0 - , 1 - , 1 - , 1 - , 1 - , 1 - , 1 - , 1 - ] - ) - , - ( "gear" - , D.fromList - [ 4 :: Int32 - , 4 - , 4 - , 3 - , 3 - , 3 - , 3 - , 4 - , 4 - , 4 - , 4 - , 3 - , 3 - , 3 - , 3 - , 3 - , 3 - , 4 - , 4 - , 4 - , 3 - , 3 - , 3 - , 3 - , 3 - , 4 - , 5 - , 5 - , 5 - , 5 - , 5 - , 4 - ] - ) - , - ( "carb" - , D.fromList - [ 4 :: Int32 - , 4 - , 1 - , 1 - , 2 - , 1 - , 4 - , 2 - , 2 - , 4 - , 4 - , 3 - , 3 - , 3 - , 4 - , 4 - , 4 - , 1 - , 2 - , 1 - , 1 - , 2 - , 2 - , 4 - , 2 - , 1 - , 2 - , 2 - , 4 - , 6 - , 8 - , 2 - ] - ) - ] mtCars :: Test mtCars = diff --git a/tests/ParquetTestData.hs b/tests/ParquetTestData.hs new file mode 100644 --- /dev/null +++ b/tests/ParquetTestData.hs @@ -0,0 +1,727 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} + +{- | Large DataFrame fixtures used by the Parquet test suite. +Kept in a separate module to avoid cluttering Parquet.hs. +-} +module ParquetTestData ( + allTypes, + tinyPagesLast10, + transactions, + mtCarsDataset, +) where + +import Data.Int +import Data.Text (Text) +import Data.Time +import qualified DataFrame as D + +allTypes :: D.DataFrame +allTypes = + D.fromNamedColumns + [ ("id", D.fromList [4 :: Int32, 5, 6, 7, 2, 3, 0, 1]) + , ("bool_col", D.fromList [True, False, True, False, True, False, True, False]) + , ("tinyint_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1]) + , ("smallint_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1]) + , ("int_col", D.fromList [0 :: Int32, 1, 0, 1, 0, 1, 0, 1]) + , ("bigint_col", D.fromList [0 :: Int64, 10, 0, 10, 0, 10, 0, 10]) + , ("float_col", D.fromList [0 :: Float, 1.1, 0, 1.1, 0, 1.1, 0, 1.1]) + , ("double_col", D.fromList [0 :: Double, 10.1, 0, 10.1, 0, 10.1, 0, 10.1]) + , + ( "date_string_col" + , D.fromList + [ "03/01/09" :: Text + , "03/01/09" + , "04/01/09" + , "04/01/09" + , "02/01/09" + , "02/01/09" + , "01/01/09" + , "01/01/09" + ] + ) + , ("string_col", D.fromList (take 8 (cycle ["0" :: Text, "1"]))) + , + ( "timestamp_col" + , D.fromList + [ UTCTime{utctDay = fromGregorian 2009 3 1, utctDayTime = secondsToDiffTime 0} + , UTCTime{utctDay = fromGregorian 2009 3 1, utctDayTime = secondsToDiffTime 60} + , UTCTime{utctDay = fromGregorian 2009 4 1, utctDayTime = secondsToDiffTime 0} + , UTCTime{utctDay = fromGregorian 2009 4 1, utctDayTime = secondsToDiffTime 60} + , UTCTime{utctDay = fromGregorian 2009 2 1, utctDayTime = secondsToDiffTime 0} + , UTCTime{utctDay = fromGregorian 2009 2 1, utctDayTime = secondsToDiffTime 60} + , UTCTime{utctDay = fromGregorian 2009 1 1, utctDayTime = secondsToDiffTime 0} + , UTCTime{utctDay = fromGregorian 2009 1 1, utctDayTime = secondsToDiffTime 60} + ] + ) + ] + +tinyPagesLast10 :: D.DataFrame +tinyPagesLast10 = + D.fromNamedColumns + [ ("id", D.fromList @Int32 (reverse [6174 .. 6183])) + , ("bool_col", D.fromList @Bool (take 10 (cycle [False, True]))) + , ("tinyint_col", D.fromList @Int32 [3, 2, 1, 0, 9, 8, 7, 6, 5, 4]) + , ("smallint_col", D.fromList @Int32 [3, 2, 1, 0, 9, 8, 7, 6, 5, 4]) + , ("int_col", D.fromList @Int32 [3, 2, 1, 0, 9, 8, 7, 6, 5, 4]) + , ("bigint_col", D.fromList @Int64 [30, 20, 10, 0, 90, 80, 70, 60, 50, 40]) + , + ( "float_col" + , D.fromList @Float [3.3, 2.2, 1.1, 0, 9.9, 8.8, 7.7, 6.6, 5.5, 4.4] + ) + , + ( "date_string_col" + , D.fromList @Text + [ "09/11/10" + , "09/11/10" + , "09/11/10" + , "09/11/10" + , "09/10/10" + , "09/10/10" + , "09/10/10" + , "09/10/10" + , "09/10/10" + , "09/10/10" + ] + ) + , + ( "string_col" + , D.fromList @Text ["3", "2", "1", "0", "9", "8", "7", "6", "5", "4"] + ) + , + ( "timestamp_col" + , D.fromList @UTCTime + [ UTCTime + { utctDay = fromGregorian 2010 9 10 + , utctDayTime = secondsToDiffTime 85384 + } + , UTCTime + { utctDay = fromGregorian 2010 9 10 + , utctDayTime = secondsToDiffTime 85324 + } + , UTCTime + { utctDay = fromGregorian 2010 9 10 + , utctDayTime = secondsToDiffTime 85264 + } + , UTCTime + { utctDay = fromGregorian 2010 9 10 + , utctDayTime = secondsToDiffTime 85204 + } + , UTCTime{utctDay = fromGregorian 2010 9 9, utctDayTime = secondsToDiffTime 85144} + , UTCTime{utctDay = fromGregorian 2010 9 9, utctDayTime = secondsToDiffTime 85084} + , UTCTime{utctDay = fromGregorian 2010 9 9, utctDayTime = secondsToDiffTime 85024} + , UTCTime{utctDay = fromGregorian 2010 9 9, utctDayTime = secondsToDiffTime 84964} + , UTCTime{utctDay = fromGregorian 2010 9 9, utctDayTime = secondsToDiffTime 84904} + , UTCTime{utctDay = fromGregorian 2010 9 9, utctDayTime = secondsToDiffTime 84844} + ] + ) + , ("year", D.fromList @Int32 (replicate 10 2010)) + , ("month", D.fromList @Int32 (replicate 10 9)) + ] + +transactions :: D.DataFrame +transactions = + D.fromNamedColumns + [ ("transaction_id", D.fromList [1 :: Int32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) + , + ( "event_time" + , D.fromList + [ UTCTime + { utctDay = fromGregorian 2024 1 3 + , utctDayTime = secondsToDiffTime 29564 + picosecondsToDiffTime 2311000000 + } + , UTCTime + { utctDay = fromGregorian 2024 1 3 + , utctDayTime = secondsToDiffTime 35101 + picosecondsToDiffTime 118900000000 + } + , UTCTime + { utctDay = fromGregorian 2024 1 4 + , utctDayTime = secondsToDiffTime 39802 + picosecondsToDiffTime 774512000000 + } + , UTCTime + { utctDay = fromGregorian 2024 1 5 + , utctDayTime = secondsToDiffTime 53739 + picosecondsToDiffTime 1000000 + } + , UTCTime + { utctDay = fromGregorian 2024 1 6 + , utctDayTime = secondsToDiffTime 8278 + picosecondsToDiffTime 543210000000 + } + , UTCTime + { utctDay = fromGregorian 2024 1 6 + , utctDayTime = secondsToDiffTime 8284 + picosecondsToDiffTime 211000000000 + } + , UTCTime{utctDay = fromGregorian 2024 1 7, utctDayTime = secondsToDiffTime 63000} + , UTCTime + { utctDay = fromGregorian 2024 1 8 + , utctDayTime = secondsToDiffTime 24259 + picosecondsToDiffTime 390000000000 + } + , UTCTime + { utctDay = fromGregorian 2024 1 9 + , utctDayTime = secondsToDiffTime 48067 + picosecondsToDiffTime 812345000000 + } + , UTCTime + { utctDay = fromGregorian 2024 1 10 + , utctDayTime = secondsToDiffTime 82799 + picosecondsToDiffTime 999999000000 + } + , UTCTime + { utctDay = fromGregorian 2024 1 11 + , utctDayTime = secondsToDiffTime 36000 + picosecondsToDiffTime 100000000000 + } + , UTCTime + { utctDay = fromGregorian 2024 1 12 + , utctDayTime = secondsToDiffTime 56028 + picosecondsToDiffTime 667891000000 + } + ] + ) + , + ( "user_email" + , D.fromList + [ "alice@example.com" :: Text + , "bob@example.com" + , "carol@example.com" + , "alice@example.com" + , "dave@example.com" + , "dave@example.com" + , "eve@example.com" + , "frank@example.com" + , "grace@example.com" + , "dave@example.com" + , "alice@example.com" + , "heidi@example.com" + ] + ) + , + ( "transaction_type" + , D.fromList + [ "purchase" :: Text + , "purchase" + , "refund" + , "purchase" + , "purchase" + , "purchase" + , "purchase" + , "withdrawal" + , "purchase" + , "purchase" + , "purchase" + , "refund" + ] + ) + , + ( "amount" + , D.fromList + [ 142.50 :: Double + , 29.99 + , 89.00 + , 2399.00 + , 15.00 + , 15.00 + , 450.75 + , 200.00 + , 55.20 + , 3200.00 + , 74.99 + , 120.00 + ] + ) + , + ( "currency" + , D.fromList + [ "USD" :: Text + , "USD" + , "EUR" + , "USD" + , "GBP" + , "GBP" + , "USD" + , "EUR" + , "CAD" + , "USD" + , "USD" + , "GBP" + ] + ) + , + ( "status" + , D.fromList + [ "approved" :: Text + , "approved" + , "approved" + , "declined" + , "approved" + , "declined" + , "approved" + , "approved" + , "approved" + , "flagged" + , "approved" + , "approved" + ] + ) + , + ( "location" + , D.fromList + [ "New York, US" :: Text + , "London, GB" + , "Berlin, DE" + , "New York, US" + , "Manchester, GB" + , "Lagos, NG" + , "San Francisco, US" + , "Paris, FR" + , "Toronto, CA" + , "New York, US" + , "New York, US" + , "Edinburgh, GB" + ] + ) + ] + +mtCarsDataset :: D.DataFrame +mtCarsDataset = + D.fromNamedColumns + [ + ( "model" + , D.fromList + [ "Mazda RX4" :: Text + , "Mazda RX4 Wag" + , "Datsun 710" + , "Hornet 4 Drive" + , "Hornet Sportabout" + , "Valiant" + , "Duster 360" + , "Merc 240D" + , "Merc 230" + , "Merc 280" + , "Merc 280C" + , "Merc 450SE" + , "Merc 450SL" + , "Merc 450SLC" + , "Cadillac Fleetwood" + , "Lincoln Continental" + , "Chrysler Imperial" + , "Fiat 128" + , "Honda Civic" + , "Toyota Corolla" + , "Toyota Corona" + , "Dodge Challenger" + , "AMC Javelin" + , "Camaro Z28" + , "Pontiac Firebird" + , "Fiat X1-9" + , "Porsche 914-2" + , "Lotus Europa" + , "Ford Pantera L" + , "Ferrari Dino" + , "Maserati Bora" + , "Volvo 142E" + ] + ) + , + ( "mpg" + , D.fromList + [ 21.0 :: Double + , 21.0 + , 22.8 + , 21.4 + , 18.7 + , 18.1 + , 14.3 + , 24.4 + , 22.8 + , 19.2 + , 17.8 + , 16.4 + , 17.3 + , 15.2 + , 10.4 + , 10.4 + , 14.7 + , 32.4 + , 30.4 + , 33.9 + , 21.5 + , 15.5 + , 15.2 + , 13.3 + , 19.2 + , 27.3 + , 26.0 + , 30.4 + , 15.8 + , 19.7 + , 15.0 + , 21.4 + ] + ) + , + ( "cyl" + , D.fromList + [ 6 :: Int32 + , 6 + , 4 + , 6 + , 8 + , 6 + , 8 + , 4 + , 4 + , 6 + , 6 + , 8 + , 8 + , 8 + , 8 + , 8 + , 8 + , 4 + , 4 + , 4 + , 4 + , 8 + , 8 + , 8 + , 8 + , 4 + , 4 + , 4 + , 8 + , 6 + , 8 + , 4 + ] + ) + , + ( "disp" + , D.fromList + [ 160.0 :: Double + , 160.0 + , 108.0 + , 258.0 + , 360.0 + , 225.0 + , 360.0 + , 146.7 + , 140.8 + , 167.6 + , 167.6 + , 275.8 + , 275.8 + , 275.8 + , 472.0 + , 460.0 + , 440.0 + , 78.7 + , 75.7 + , 71.1 + , 120.1 + , 318.0 + , 304.0 + , 350.0 + , 400.0 + , 79.0 + , 120.3 + , 95.1 + , 351.0 + , 145.0 + , 301.0 + , 121.0 + ] + ) + , + ( "hp" + , D.fromList + [ 110 :: Int32 + , 110 + , 93 + , 110 + , 175 + , 105 + , 245 + , 62 + , 95 + , 123 + , 123 + , 180 + , 180 + , 180 + , 205 + , 215 + , 230 + , 66 + , 52 + , 65 + , 97 + , 150 + , 150 + , 245 + , 175 + , 66 + , 91 + , 113 + , 264 + , 175 + , 335 + , 109 + ] + ) + , + ( "drat" + , D.fromList + [ 3.9 :: Double + , 3.9 + , 3.85 + , 3.08 + , 3.15 + , 2.76 + , 3.21 + , 3.69 + , 3.92 + , 3.92 + , 3.92 + , 3.07 + , 3.07 + , 3.07 + , 2.93 + , 3.0 + , 3.23 + , 4.08 + , 4.93 + , 4.22 + , 3.7 + , 2.76 + , 3.15 + , 3.73 + , 3.08 + , 4.08 + , 4.43 + , 3.77 + , 4.22 + , 3.62 + , 3.54 + , 4.11 + ] + ) + , + ( "wt" + , D.fromList + [ 2.62 :: Double + , 2.875 + , 2.32 + , 3.215 + , 3.44 + , 3.46 + , 3.57 + , 3.19 + , 3.15 + , 3.44 + , 3.44 + , 4.07 + , 3.73 + , 3.78 + , 5.25 + , 5.424 + , 5.345 + , 2.2 + , 1.615 + , 1.835 + , 2.465 + , 3.52 + , 3.435 + , 3.84 + , 3.845 + , 1.935 + , 2.14 + , 1.513 + , 3.17 + , 2.77 + , 3.57 + , 2.78 + ] + ) + , + ( "qsec" + , D.fromList + [ 16.46 :: Double + , 17.02 + , 18.61 + , 19.44 + , 17.02 + , 20.22 + , 15.84 + , 20.0 + , 22.9 + , 18.3 + , 18.9 + , 17.4 + , 17.6 + , 18.0 + , 17.98 + , 17.82 + , 17.42 + , 19.47 + , 18.52 + , 19.9 + , 20.01 + , 16.87 + , 17.3 + , 15.41 + , 17.05 + , 18.9 + , 16.7 + , 16.9 + , 14.5 + , 15.5 + , 14.6 + , 18.6 + ] + ) + , + ( "vs" + , D.fromList + [ 0 :: Int32 + , 0 + , 1 + , 1 + , 0 + , 1 + , 0 + , 1 + , 1 + , 1 + , 1 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 1 + , 1 + , 1 + , 1 + , 0 + , 0 + , 0 + , 0 + , 1 + , 0 + , 1 + , 0 + , 0 + , 0 + , 1 + ] + ) + , + ( "am" + , D.fromList + [ 1 :: Int32 + , 1 + , 1 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 0 + , 1 + , 1 + , 1 + , 0 + , 0 + , 0 + , 0 + , 0 + , 1 + , 1 + , 1 + , 1 + , 1 + , 1 + , 1 + ] + ) + , + ( "gear" + , D.fromList + [ 4 :: Int32 + , 4 + , 4 + , 3 + , 3 + , 3 + , 3 + , 4 + , 4 + , 4 + , 4 + , 3 + , 3 + , 3 + , 3 + , 3 + , 3 + , 4 + , 4 + , 4 + , 3 + , 3 + , 3 + , 3 + , 3 + , 4 + , 5 + , 5 + , 5 + , 5 + , 5 + , 4 + ] + ) + , + ( "carb" + , D.fromList + [ 4 :: Int32 + , 4 + , 1 + , 1 + , 2 + , 1 + , 4 + , 2 + , 2 + , 4 + , 4 + , 3 + , 3 + , 3 + , 4 + , 4 + , 4 + , 1 + , 2 + , 1 + , 1 + , 2 + , 2 + , 4 + , 2 + , 1 + , 2 + , 2 + , 4 + , 6 + , 8 + , 2 + ] + ) + ] diff --git a/tests/data/typing/texts.txt b/tests/data/typing/texts.txt new file mode 100644 --- /dev/null +++ b/tests/data/typing/texts.txt @@ -0,0 +1,34 @@ +To +protect +the +world +from +devastation +To +unite +all +people +within +our +nation +To +denounce +the +evils +of +truth +and +love +To +extend +our +reach +to +the +stars +above +JESSIE! +JAMES! +TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT! +Surrender now or prepare to fight! +Meowth, that's right! diff --git a/tests/data/typing/texts_with_empties.txt b/tests/data/typing/texts_with_empties.txt new file mode 100644 --- /dev/null +++ b/tests/data/typing/texts_with_empties.txt @@ -0,0 +1,41 @@ + +To +protect +the +world +from +devastation + +To +unite +all +people +within +our +nation + +To +denounce +the +evils +of +truth +and +love + +To +extend +our +reach +to +the +stars +above + +JESSIE! +JAMES! +TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT! + +Surrender now or prepare to fight! + +Meowth, that's right! diff --git a/tests/data/typing/texts_with_empties_and_nullish.txt b/tests/data/typing/texts_with_empties_and_nullish.txt new file mode 100644 --- /dev/null +++ b/tests/data/typing/texts_with_empties_and_nullish.txt @@ -0,0 +1,44 @@ + +To +protect +the +world +from +devastation + +To +unite +all +people +within +our +nation + +To +denounce +the +evils +of +truth +and +love + +To +extend +our +reach +to +the +stars +above + +JESSIE! +JAMES! +TEAM ROCKET BLASTS OFF AT THE SPEED OF LIGHT! + +Surrender now or prepare to fight! + +Meowth, that's right! +NaN +Nothing +N/A