dataframe 1.0.0.1 → 1.1.0.0

raw patch · 60 files changed

+4784/−8942 lines, 60 filesdep −mmapdep −paralleldep ~dataframePVP ok

version bump matches the API change (PVP)

Dependencies removed: mmap, parallel

Dependency ranges changed: dataframe

API changes (from Hackage documentation)

Files

• CHANGELOG.md +21/−0
• app/Synthesis.hs +94/−82
• benchmark/Main.hs +1/−17
• cbits/process_csv.c +0/−198
• cbits/process_csv.h +0/−33
• dataframe.cabal +13/−18
• ffi/DataFrame/IO/Arrow.hs +82/−125
• ffi/DataFrame/IR.hs +28/−15
• src/DataFrame.hs +2/−7
• src/DataFrame/DecisionTree.hs +21/−13
• src/DataFrame/Display/Terminal/Plot.hs +6/−11
• src/DataFrame/Display/Web/Plot.hs +6/−17
• src/DataFrame/Errors.hs +40/−8
• src/DataFrame/Functions.hs +2/−1
• src/DataFrame/IO/CSV.hs +18/−79
• src/DataFrame/IO/Parquet.hs +23/−22
• src/DataFrame/IO/Parquet/Thrift.hs +2/−3
• src/DataFrame/IO/Unstable/CSV.hs +0/−352
• src/DataFrame/Internal/Column.hs +1694/−1325
• src/DataFrame/Internal/DataFrame.hs +70/−11
• src/DataFrame/Internal/Interpreter.hs +87/−79
• src/DataFrame/Internal/Row.hs +7/−11
• src/DataFrame/Internal/Types.hs +3/−3
• src/DataFrame/Lazy/IO/Binary.hs +37/−40
• src/DataFrame/Lazy/IO/CSV.hs +4/−9
• src/DataFrame/Operations/Aggregation.hs +22/−27
• src/DataFrame/Operations/Core.hs +39/−35
• src/DataFrame/Operations/Join.hs +105/−71
• src/DataFrame/Operations/Permutation.hs +2/−2
• src/DataFrame/Operations/Statistics.hs +5/−8
• src/DataFrame/Operations/Subset.hs +88/−95
• src/DataFrame/Operations/Transformations.hs +9/−7
• src/DataFrame/Operations/Typing.hs +25/−16
• src/DataFrame/Synthesis.hs +1/−1
• src/DataFrame/Typed.hs +8/−1
• src/DataFrame/Typed/Access.hs +3/−3
• src/DataFrame/Typed/Expr.hs +2/−2
• src/DataFrame/Typed/Freeze.hs +14/−2
• src/DataFrame/Typed/Operations.hs +1/−1
• src/DataFrame/Typed/Schema.hs +10/−0
• src/DataFrame/Typed/TH.hs +6/−2
• tests/DecisionTree.hs +12/−12
• tests/GenDataFrame.hs +3/−3
• tests/Main.hs +2/−0
• tests/Operations/Apply.hs +4/−3
• tests/Operations/Derive.hs +1/−0
• tests/Operations/GroupBy.hs +1/−1
• tests/Operations/InsertColumn.hs +8/−6
• tests/Operations/Join.hs +77/−1
• tests/Operations/Nullable.hs +26/−26
• tests/Operations/ReadCsv.hs +10/−284
• tests/Operations/Sort.hs +1/−1
• tests/Operations/Statistics.hs +1/−1
• tests/Operations/Typing.hs +1102/−5119
• tests/Operations/WriteCsv.hs +88/−0
• tests/Parquet.hs +1/−733
• tests/ParquetTestData.hs +727/−0
• tests/data/typing/texts.txt +34/−0
• tests/data/typing/texts_with_empties.txt +41/−0
• tests/data/typing/texts_with_empties_and_nullish.txt +44/−0

@@ -1,5 +1,26 @@ # Revision history for dataframe +## 1.1.0.0+### Breaking changes+* Remove `OptionalColumn` constructor; fold nullability into `BoxedColumn`/`UnboxedColumn` via bit-packed bitmap.+* Remove `NFData` instance from `Columnable` constraint.++### New features+* Add `toCsv` and `toSeparated` for converting a DataFrame to CSV/delimited text without writing to a file.+* `safeRead` now defaults reading columns to `Maybe a`.+* Split SIMD CSV reader into a separate `dataframe-fastcsv` package.++### Bug fixes+* Fix joins for missing key columns (#187).+* Fix single column not found error when using typed dataframe.+* Fix Synthesis to use `SafeLookup` constraint.+* Fix `writeSeparated` ignoring separator parameter (was hardcoded to comma).++### Internal+* Slice groups now use custom backpermute instead of converting unboxed vectors.+* Reuse comparison operators in Subset.+* Refactor `getRowAsText` for readability using pattern guards.+ ## 1.0.0.1 * toMarkdownTable is now toMarkdown (mostly used internally) * Provide toMarkdown' that outputs string

@@ -1,53 +1,51 @@+{-# LANGUAGE DataKinds #-} {-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE TypeApplications #-}+{-# LANGUAGE TypeFamilies #-}+{-# LANGUAGE TypeOperators #-} +import Data.Char import qualified Data.Text as T import qualified DataFrame as D+import DataFrame.DecisionTree import qualified DataFrame.Functions as F+import DataFrame.Operators+import qualified DataFrame.Typed as DT+import System.Random -import Data.Char-import DataFrame.DecisionTree-import DataFrame.Operators hiding (name)+$( DT.deriveSchemaFromCsvFileWith+    D.defaultReadOptions{D.safeRead = True}+    "TrainSchema"+    "./data/titanic/train.csv"+ )+$( DT.deriveSchemaFromCsvFileWith+    D.defaultReadOptions{D.safeRead = True}+    "TestSchema"+    "./data/titanic/test.csv"+ ) -import System.Random+-- Survived is Maybe Int (safeRead = True); prediction is Int (model output).+type RawPredSchema =+    '[DT.Column "Survived" (Maybe Int), DT.Column "prediction" Int] -$(F.declareColumnsFromCsvFile "./data/titanic/train.csv")+prediction :: D.Expr Int+prediction = F.col @Int "prediction"  main :: IO () main = do-    train <- D.readCsv "./data/titanic/train.csv"-    test <- D.readCsv "./data/titanic/test.csv"+    rawTrain <- D.readCsv "./data/titanic/train.csv"+    rawTest <- D.readCsv "./data/titanic/test.csv" -    -- Apply the same transformations to training and test.-    let combined =-            (train <> test)-                |> D.deriveMany-                    [ "Ticket" .= F.lift (T.filter isAlpha) ticket-                    , "Name" .= F.match "\\s*([A-Za-z]+)\\." name-                    , "Cabin" .= F.whenPresent (T.take 1) cabin-                    ]-                |> D.renameMany-                    [ (F.name name, "title")-                    , (F.name cabin, "cabin_prefix")-                    , (F.name pclass, "passenger_class")-                    , (F.name sibsp, "number_of_siblings_and_spouses")-                    , (F.name parch, "number_of_parents_and_children")-                    ]-    print combined+    train <-+        maybe (fail "train.csv schema mismatch") pure (DT.freeze @TrainSchema rawTrain)+    test <-+        maybe (fail "test.csv schema mismatch") pure (DT.freeze @TestSchema rawTest) -    let (train', validation) =-            D.take-                (D.nRows train)-                combined-                |> D.filterJust (F.name survived)-                |> D.randomSplit (mkStdGen 4232) 0.7-        -- Split the test out again.-        test' =-            D.drop-                (D.nRows train)-                combined+    let (trainDf, validDf) =+            D.randomSplit (mkStdGen 4232) 0.7 (DT.thaw (clean train))+        testDf = DT.thaw (clean test)          model =             fitDecisionTree@@ -61,70 +59,84 @@                             { complexityPenalty = 0.1                             , maxExprDepth = 3                             , disallowedCombinations =-                                [ (F.name age, F.name fare)+                                [ ("Age", "Fare")                                 , ("passenger_class", "number_of_siblings_and_spouses")                                 , ("passenger_class", "number_of_parents_and_children")                                 ]                             }                     }                 )-                survived -- Label to predict-                ( train'-                    |> D.exclude [F.name passengerid]-                )+                (F.fromMaybe 0 (F.col @(Maybe Int) "Survived"))+                (trainDf |> D.exclude ["PassengerId"])      print model      putStrLn "Training accuracy: "-    print $-        computeAccuracy-            (train' |> D.derive (F.name prediction) model)+    print $ computeAccuracy (trainDf |> D.derive (F.name prediction) model)      putStrLn "Validation accuracy: "-    print $-        computeAccuracy-            ( validation-                |> D.derive (F.name prediction) model-            )+    print $ computeAccuracy (validDf |> D.derive (F.name prediction) model) -    let predictions = D.derive (F.name survived) model test'     D.writeCsv         "./predictions.csv"-        (predictions |> D.select [F.name passengerid, F.name survived])+        ( testDf+            |> D.derive "Survived" model+            |> D.select ["PassengerId", "Survived"]+        ) -prediction :: D.Expr Int-prediction = F.col @Int "prediction"+clean ::+    ( DT.AssertPresent "Ticket" cols+    , DT.SafeLookup "Ticket" cols ~ Maybe T.Text+    , DT.AssertPresent "Name" cols+    , DT.SafeLookup "Name" cols ~ Maybe T.Text+    , DT.AssertPresent "Cabin" cols+    , DT.SafeLookup "Cabin" cols ~ Maybe T.Text+    ) =>+    DT.TypedDataFrame cols ->+    DT.TypedDataFrame+        ( DT.RenameManyInSchema+            '[ '("Name", "title")+             , '("Cabin", "cabin_prefix")+             , '("Pclass", "passenger_class")+             , '("SibSp", "number_of_siblings_and_spouses")+             , '("Parch", "number_of_parents_and_children")+             ]+            cols+        )+clean tdf =+    tdf+        |> DT.replaceColumn @"Ticket" (DT.nullLift (T.filter isAlpha) (DT.col @"Ticket"))+        |> DT.replaceColumn @"Name" (DT.nullLift extractTitle (DT.col @"Name"))+        |> DT.replaceColumn @"Cabin" (DT.nullLift (T.take 1) (DT.col @"Cabin"))+        |> DT.renameMany+            @'[ '("Name", "title")+              , '("Cabin", "cabin_prefix")+              , '("Pclass", "passenger_class")+              , '("SibSp", "number_of_siblings_and_spouses")+              , '("Parch", "number_of_parents_and_children")+              ] +-- | Extract title (e.g. "Mr", "Mrs") from a full Titanic passenger name.+extractTitle :: T.Text -> T.Text+extractTitle name =+    case filter (T.isSuffixOf ".") (T.words name) of+        (w : _) -> T.dropEnd 1 w+        [] -> ""++{- | Compute binary classification accuracy from a DataFrame containing+  "Survived" and "prediction" columns.+-} computeAccuracy :: D.DataFrame -> Double computeAccuracy df =-    let-        tp =-            fromIntegral $-                D.nRows-                    ( D.filterWhere-                        (survived .== F.lit (1 :: Int) .&& prediction .== F.lit (1 :: Int))-                        df-                    )-        tn =-            fromIntegral $-                D.nRows-                    ( D.filterWhere-                        (survived .== F.lit (0 :: Int) .&& prediction .== F.lit (0 :: Int))-                        df-                    )-        fp =-            fromIntegral $-                D.nRows-                    ( D.filterWhere-                        (survived .== F.lit (0 :: Int) .&& prediction .== F.lit (1 :: Int))-                        df-                    )-        fn =-            fromIntegral $-                D.nRows-                    ( D.filterWhere-                        (survived .== F.lit (1 :: Int) .&& prediction .== F.lit (0 :: Int))-                        df-                    )-     in-        (tp + tn) / (tp + tn + fp + fn)+    let tdf =+            DT.impute @"Survived" 0 $+                DT.unsafeFreeze @RawPredSchema $+                    df |> D.select ["Survived", "prediction"]+        survived = DT.col @"Survived"+        pred = DT.col @"prediction"+        count expr = fromIntegral (DT.nRows (DT.filterWhere expr tdf))+        tp = count ((survived DT..==. DT.lit 1) DT..&&. (pred DT..==. DT.lit 1))+        tn = count ((survived DT..==. DT.lit 0) DT..&&. (pred DT..==. DT.lit 0))+        fp = count ((survived DT..==. DT.lit 0) DT..&&. (pred DT..==. DT.lit 1))+        fn = count ((survived DT..==. DT.lit 1) DT..&&. (pred DT..==. DT.lit 0))+     in (tp + tn) / (tp + tn + fp + fn)

@@ -43,7 +43,7 @@  groupByHaskell :: IO () groupByHaskell = do-    df <- D.fastReadCsvUnstable "./data/housing.csv"+    df <- D.readCsv "./data/housing.csv"     print $         df             |> D.groupBy ["ocean_proximity"]@@ -84,12 +84,6 @@ parseFile :: String -> IO () parseFile = void . D.readCsv -parseFileUnstable :: String -> IO ()-parseFileUnstable = void . D.readCsvUnstable--parseFileUnstableSIMD :: String -> IO ()-parseFileUnstableSIMD = void . D.fastReadCsvUnstable- parseHousingCSV :: IO () parseHousingCSV = parseFile "./data/housing.csv" @@ -215,22 +209,12 @@         , bgroup             "housing.csv (1.4 MB)"             [ bench "Attoparsec" $ nfIO $ parseFile "./data/housing.csv"-            , bench "Native Haskell" $ nfIO $ parseFileUnstable "./data/housing.csv"-            , bench "SIMD" $ nfIO $ parseFileUnstableSIMD "./data/housing.csv"             ]         , bgroup             "effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv (9.1 MB)"             [ bench "Attoparsec" $                 nfIO $                     parseFile-                        "./data/effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv"-            , bench "Native Haskell" $-                nfIO $-                    parseFileUnstable-                        "./data/effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv"-            , bench "SIMD" $-                nfIO $-                    parseFileUnstableSIMD                         "./data/effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv"             ]         , bgroup

@@ -1,198 +0,0 @@-#include "process_csv.h"-#include <stdlib.h>-#include <stdio.h>--/*- * compile with clang -O3- * in cabal use - *   cc-options: -O3- *- * Produce an array of field delimiter indices- * Fields can be delimited by commas and newlines- * TODO: allow the user to provide a custom delimiter- * character to replace commas.- * This should work with UTF-8, so long as the delimiter- * character is a single byte.- *- * Delimiters can be escaped inside of quotes. Quotes- * can also be placed inside quotes by double quoting.- * For the purposes of this parser we can ignore double- * quotes inside quotes, thereby treating the first quote- * as the closing of the string and the next one the- * immediate opening of a new one- * - * We can find the quoted regions by first finding- * the positions of the quotes (cmpeq and then movemask)- * and then using the carryless multiplication operation- * to know the regions that are quoted. We can then simply- * and the inverse of the quotemask to exclude commas and- * newlines inside quotes- *- */--// if the character is found at a particular-// position in the array of bytes, the-// corresponding bit in the returned uint64_t should-// be turned on.-// Example: searching for commas in-// input:  one, two, three-// result: 000100001000000-#ifdef HAS_SIMD_CSV-static uint64_t find_character_in_chunk(uint8_t *in, uint8_t c) {-#ifdef USE_AVX2-  // AVX2 implementation: load two 32-byte chunks-  __m256i v0 = _mm256_loadu_si256((const __m256i *)(in));-  __m256i v1 = _mm256_loadu_si256((const __m256i *)(in + 32));-  __m256i b = _mm256_set1_epi8((char)c);-  __m256i m0 = _mm256_cmpeq_epi8(v0, b);-  __m256i m1 = _mm256_cmpeq_epi8(v1, b);-  uint32_t lo = (uint32_t)_mm256_movemask_epi8(m0);-  uint32_t hi = (uint32_t)_mm256_movemask_epi8(m1);-  return ((uint64_t)hi << 32) | (uint64_t)lo;-#else // USE_NEON-  // ARM NEON implementation: load 64 bytes deinterleaved-  uint8x16x4_t src = vld4q_u8(in);-  uint8x16_t mask = vmovq_n_u8(c);-  uint8x16_t cmp0 = vceqq_u8(src.val[0], mask);-  uint8x16_t cmp1 = vceqq_u8(src.val[1], mask);-  uint8x16_t cmp2 = vceqq_u8(src.val[2], mask);-  uint8x16_t cmp3 = vceqq_u8(src.val[3], mask);--  // For an explanation of how to do movemask in-  // NEON, see: https://branchfree.org/2019/04/01/fitting-my-head-through-the-arm-holes-or-two-sequences-to-substitute-for-the-missing-pmovmskb-instruction-on-arm-neon/-  // The specific implementation below is owed to the -  // user 'aqrit' in a comment on the blog above-  // There's also https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon-  // -  // The input to the move mask must be de-interleaved.-  // That is, for a string after vceqq_u8  we have,-  // cmp0 = aaaaaaaa / eeeeeeee / ...-  // cmp1 = bbbbbbbb / ffffffff / ...-  // cmp2 = cccccccc / gggggggg / ...-  // cmp3 = dddddddd / hhhhhhhh / ...-  // Luckily vld4q_u8 does this for us. Now we want-  // to interleave this into a 64bit integer with bits-  // abcdefgh...-  // cmp0 holds bits for positions-  // 0,4,8,..., cmp1 holds bits for 1,5,9,..., and so on.-  // So to bring together the bits for different positions-  // we right shift and combine with vsriq_n_u8-  //-  // vsriq_n_u8 shifts each byte of the first operand -  // right by n bits, and combines it with the bits of-  // the second operand.-  // example:-  // uint8_t mask = 0xFF >> n; // if n = 1, 0111 1111-  // uint8_t shifted = operand1 >> n // 0bbb, bbbb-  // operand1 = (operand2 & (!mask)) | shifted-  // (aaaa aaaa & 1000 0000) | 0bbb bbbb = abbb bbbb-  // -  // So we first bring together the bits the first two-  // rows and the next two rows (so to speak)-  // t0 = abbbbbbb/efffffff/...-  uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1);-  // t1 = cddddddd/ghhhhhhh/...-  uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1);-  // Now we must combine each of our combined rows-  // so that we get back our column interleaved-  // t2 = abcddddd/efghhhhh/...-  uint8x16_t t2 = vsriq_n_u8(t1, t0, 2);-  // Then to get rid of the repeated bits in the upper half-  // t3 = abcdabcd/efghefgh/...-  uint8x16_t t3 = vsriq_n_u8(t2, t2, 4);-  // and now it's the relatively simple matter of getting-  // rid of half the bits. We combine our 8bit words into 16-  // bit words for this step, and then we shift right by 4-  // and turn the result into an 8 bit word-  // afterreinterpert: abcdabcdefghefgh/...-  // afterrightshift: 0000abcdabcdefgh/...-  // take the lower bits: abcdefgh/...-  uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4);-  // Finally we recombine them into a 64 bit integer-  // (vreinterpret_u64_u8 here does uint8x8 -> uint64x1-  // and vget_lane_u64 does uint64x1 -> uint64) -  return vget_lane_u64(vreinterpret_u64_u8(t4), 0);-#endif-}-#endif--// I owe a debt to https://github.com/geofflangdale/simdcsv-// Let's go ahead and assume `in` will only ever get 64 bytes-// initial_quoted will be either all_ones ~0ULL or all_zeros 0ULL-#ifdef HAS_SIMD_CSV-static uint64_t parse_chunk(uint8_t *in, uint8_t separator, uint64_t *initial_quoted) {-  uint64_t quotebits = find_character_in_chunk(in, QUOTE_CHAR);-  // See https://wunkolo.github.io/post/2020/05/pclmulqdq-tricks/-  // Also, section 3.1.1 of Parsing Gigabytes of JSON per Second,-  // Geoff Langdale, Daniel Lemire, https://arxiv.org/pdf/1902.08318-#ifdef USE_AVX2-  // Use PCLMUL for carryless multiplication on x86-  __m128i a = _mm_set_epi64x(0, (int64_t)ALL_ONES_MASK);-  __m128i b = _mm_set_epi64x(0, (int64_t)quotebits);-  __m128i result = _mm_clmulepi64_si128(a, b, 0);-  uint64_t quotemask = (uint64_t)_mm_cvtsi128_si64(result);-#else // USE_NEON-  // Use vmull_p64 (PMULL) for carryless multiplication on ARM-  // Requires ARM crypto extensions (compile: __ARM_FEATURE_AES, runtime: pmull flag)-  uint64_t quotemask = vmull_p64(ALL_ONES_MASK, quotebits);-#endif-  quotemask ^= (*initial_quoted);-  // Find out if the chunk ends in a quoted region by looking-  // at the last bit-  (*initial_quoted) = (uint64_t)((int64_t)quotemask >> 63);--  uint64_t commabits = find_character_in_chunk(in, separator);-  uint64_t newlinebits = find_character_in_chunk(in, NEWLINE_CHAR);--  uint64_t delimiter_bits = (commabits | newlinebits) & ~quotemask;-  return delimiter_bits;-}-#endif--#ifdef HAS_SIMD_CSV-static size_t find_one_indices(size_t start_index, uint64_t bits, size_t *indices, size_t *base) {-  size_t position = 0;-  uint64_t bitset = bits;-  while (bitset != 0) {-    // temp only has the least significant bit of-    // bitset turned on.-    // In twos complement: 0 - x = ~ x + 1-    uint64_t temp = bitset & -bitset;-    // count trailing zeros-    size_t r = __builtin_ctzll(bitset);-    indices[(*base) + position] = start_index + r;-    position++;--    bitset ^= temp;-  }-  *base += position;-  return position;-}-#endif--size_t get_delimiter_indices(uint8_t *buf, size_t len, uint8_t separator, size_t *indices) {-  // Recall we padded our file with 64 empty bytes.-  // So if, for example, we had a file of 187 bytes-  // We pad it with zeros and so we have 251 bytes-  // The chunks we have are ptr + 0, ptr + 64, and pt-  // (we don't do ptr + 192 since we do len - 64-  // below. This way, in ptr + 128 we have 59 bytes of-  // actual data and 5 bytes of zeros. If we didn't do this-  // we'd be reading past the end of file on the last row-#ifdef HAS_SIMD_CSV-  size_t unpaddedLen = len < 64 ? 0 : len - 64;-  uint64_t initial_quoted = 0ULL;-  size_t base = 0;-  for (size_t i = 0; i < unpaddedLen; i += 64) {-    uint8_t *in = buf + i;-    uint64_t delimiter_bits = parse_chunk(in, separator, &initial_quoted);-    find_one_indices(i, delimiter_bits, indices, &base);-  }-  return base;-#else-  // SIMD not available or carryless multiplication not supported.-  // Signal fallback to Haskell implementation.-  (void)buf; (void)len; (void)indices;-  return (size_t)-1;-#endif-}

@@ -1,33 +0,0 @@-#ifndef PROCESS_CSV-#define PROCESS_CSV--#include <stddef.h>-#include <stdint.h>--// Define feature macros for SIMD support with carryless multiplication-// We need both SIMD instructions AND carryless multiplication for the CSV parser-#if defined(__AVX2__) && defined(__PCLMUL__)-  #define HAS_SIMD_CSV 1-  #define USE_AVX2 1-  #include <immintrin.h>-  #include <wmmintrin.h>-#elif defined(__ARM_NEON) && (defined(__ARM_FEATURE_AES) || defined(__ARM_FEATURE_CRYPTO))-  // Note: __ARM_FEATURE_CRYPTO is deprecated; prefer __ARM_FEATURE_AES-  // We need polynomial multiply (vmull_p64/PMULL) for carryless multiplication-  // Runtime check: 'pmull' flag in /proc/cpuinfo on Linux-  // We support both macros for compatibility with older compilers-  #define HAS_SIMD_CSV 1-  #define USE_NEON 1-  #include <arm_neon.h>-#endif--// CSV parsing constants-#define COMMA_CHAR 0x2C-#define NEWLINE_CHAR 0x0A-#define QUOTE_CHAR 0x22-#define ALL_ONES_MASK ~0ULL-#define ALL_ZEROS_MASK 0ULL--size_t get_delimiter_indices(uint8_t *buf, size_t len, uint8_t separator, size_t* indices);--#endif

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            1.0.0.1+version:            1.1.0.0  synopsis: A fast, safe, and intuitive DataFrame library. @@ -16,10 +16,12 @@ category: Data tested-with: GHC ==9.4.8 || ==9.6.7 || ==9.8.4 || ==9.10.3 || ==9.12.2 extra-doc-files: CHANGELOG.md README.md-extra-source-files: cbits/process_csv.h-                    cbits/arrow_abi.h+extra-source-files: cbits/arrow_abi.h                     cbits/dataframe_arrow.h                     cbits/rts_init.c+                    tests/data/typing/texts.txt+                    tests/data/typing/texts_with_empties.txt+                    tests/data/typing/texts_with_empties_and_nullish.txt                     data/titanic/*.csv                     data/sharded/*.parquet                     tests/data/*.csv@@ -82,7 +84,6 @@                     DataFrame.Display.Terminal.Plot,                     DataFrame.IO.CSV,                     DataFrame.IO.JSON,-                    DataFrame.IO.Unstable.CSV,                     DataFrame.IO.Parquet,                     DataFrame.IO.Parquet.Binary,                     DataFrame.IO.Parquet.Dictionary,@@ -140,8 +141,6 @@                       vector-algorithms ^>= 0.9,                       zlib >= 0.5 && < 1,                       zstd >= 0.1.2.0 && < 0.2,-                      mmap >= 0.5.8 && < 0.6,-                      parallel >= 3.2.2.0 && < 5,                       stm >= 2.5 && < 3,                       filepath >= 1.4 && < 2,                       Glob >= 0.10 && < 1,@@ -149,10 +148,6 @@                       streamly-core >= 0.2.3 && < 0.4,                       streamly-bytestring >= 0.2.0 && < 0.4     hs-source-dirs:   src-    c-sources:        cbits/process_csv.c-    include-dirs:     cbits-    includes:         process_csv.h-    install-includes: process_csv.h     default-language: Haskell2010  foreign-library dataframe-arrow@@ -163,7 +158,7 @@                       DataFrame.IR     build-depends:         base        >= 4   && < 5,-        dataframe   ^>= 1,+        dataframe   ^>= 1.1,         text        >= 2.0 && < 3,         aeson       >= 0.11 && < 3,         bytestring  >= 0.11 && < 0.13,@@ -180,7 +175,7 @@     import: warnings     main-is: Benchmark.hs     build-depends:    base >= 4 && < 5,-                      dataframe ^>= 1,+                      dataframe ^>= 1.1,                       random >= 1 && < 2,                       time >= 1.12 && < 2,                       vector ^>= 0.13,@@ -192,7 +187,7 @@     import: warnings     main-is: Synthesis.hs     build-depends:    base >= 4 && < 5,-                      dataframe ^>= 1,+                      dataframe ^>= 1.1,                       random >= 1 && < 2,                       text >= 2.0 && < 3     hs-source-dirs:   app@@ -219,7 +214,7 @@     build-depends:    base >= 4 && < 5,                       bytestring >= 0.11 && < 0.13,                       containers >= 0.6.7 && < 0.9,-                      dataframe ^>= 1,+                      dataframe ^>= 1.1,                       directory >= 1.3.0.0 && < 2,                       random >= 1 && < 2,                       text >= 2.0 && < 3,@@ -236,7 +231,7 @@     build-depends: base >= 4 && < 5,                    criterion >= 1 && < 2,                    process >= 1.6 && < 2,-                   dataframe ^>= 1,+                   dataframe ^>= 1.1,                    random >= 1 && < 2,     default-language: Haskell2010     ghc-options:@@ -266,6 +261,7 @@                    Operations.Nullable,                    Operations.Provenance,                    Operations.ReadCsv,+                   Operations.WriteCsv,                    Operations.Shuffle,                    Operations.Sort,                    Operations.Subset,@@ -274,13 +270,12 @@                    Operations.Typing,                    LazyParquet,                    Parquet,+                   ParquetTestData,                    Properties,                    Monad     build-depends:  base >= 4 && < 5,                     bytestring >= 0.11 && < 0.13,-                    dataframe ^>= 1,-                    bytestring >= 0.11 && < 0.13,-                    directory >= 1.3.0.0 && < 2,+                    dataframe ^>= 1.1,                     HUnit ^>= 1.6,                     QuickCheck >= 2 && < 3,                     random-shuffle >= 0.0.4 && < 1,

@@ -25,7 +25,7 @@ import qualified DataFrame.Internal.Column as DI  import Control.Monad (foldM_, forM, join, void, when, zipWithM_)-import Data.Bits (setBit, testBit)+import Data.Bits (popCount, setBit, testBit) import Data.Int (Int32, Int64) import Data.Maybe (fromMaybe, isNothing) import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl))@@ -188,25 +188,28 @@     p `at` _arrayPrivateData $ castStablePtrToPtr cleanup     return p -buildBitmap :: V.Vector (Maybe a) -> IO (Ptr Word8, Int)-buildBitmap vec = do-    let n = V.length vec-        bitmapBytes = max 1 ((n + 7) `div` 8)-        nullCount = V.length (V.filter isNothing vec)-    bitmapPtr <- mallocBytes bitmapBytes :: IO (Ptr Word8)-    mapM_ (\i -> pokeElemOff bitmapPtr i (0 :: Word8)) [0 .. bitmapBytes - 1]-    V.iforM_ vec $ \i mv ->-        case mv of-            Nothing -> return ()-            Just _ -> do-                let byteIdx = i `div` 8-                    bitIdx = i `mod` 8-                b <- peekElemOff bitmapPtr byteIdx-                pokeElemOff bitmapPtr byteIdx (setBit b bitIdx)+{- | Allocate an Arrow-format validity bitmap from a 'DI.Bitmap'.+Returns (ptr, nullCount). Caller must 'free' the pointer.+-}+bitmapToPtr :: Int -> DI.Bitmap -> IO (Ptr Word8, Int)+bitmapToPtr n bm = do+    let numBytes = max 1 ((n + 7) `div` 8)+        validCount = VU.foldl' (\acc b -> acc + popCount b) 0 bm+        nullCount = n - validCount+    bitmapPtr <- mallocBytes numBytes :: IO (Ptr Word8)+    VU.imapM_ (pokeElemOff bitmapPtr) bm+    when (VU.length bm < numBytes) $+        mapM_+            (\i -> pokeElemOff bitmapPtr i (0 :: Word8))+            [VU.length bm .. numBytes - 1]     return (bitmapPtr, nullCount) +-- | Read an Arrow validity bitmap into a 'DI.Bitmap'.+readArrowBitmap :: Ptr Word8 -> Int -> IO DI.Bitmap+readArrowBitmap bitmapPtr n = VU.generateM ((n + 7) `div` 8) (peekElemOff bitmapPtr)+ columnToArrow :: T.Text -> Column -> IO (Ptr ArrowSchema, Ptr ArrowArray)-columnToArrow colName (UnboxedColumn (vec :: VU.Vector a))+columnToArrow colName (UnboxedColumn _ (vec :: VU.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = do         let n = VU.length vec         dataPtr <- mallocArray (max 1 n) :: IO (Ptr Int64)@@ -214,7 +217,7 @@         sPtr <- makeLeafSchema "l" colName         aPtr <- makeLeafArray n 0 [nullPtr, castPtr dataPtr] (free dataPtr)         return (sPtr, aPtr)-columnToArrow colName (UnboxedColumn (vec :: VU.Vector a))+columnToArrow colName (UnboxedColumn _ (vec :: VU.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = do         let n = VU.length vec         dataPtr <- mallocArray (max 1 n) :: IO (Ptr Double)@@ -222,7 +225,7 @@         sPtr <- makeLeafSchema "g" colName         aPtr <- makeLeafArray n 0 [nullPtr, castPtr dataPtr] (free dataPtr)         return (sPtr, aPtr)-columnToArrow colName (BoxedColumn (vec :: V.Vector a))+columnToArrow colName (BoxedColumn _ (vec :: V.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @T.Text) = do         let n = V.length vec             bss = map TE.encodeUtf8 (V.toList vec)@@ -250,7 +253,7 @@                 [nullPtr, castPtr offPtr, castPtr charsPtr]                 (free offPtr >> free charsPtr)         return (sPtr, aPtr)-columnToArrow colName (BoxedColumn (vec :: V.Vector a))+columnToArrow colName (BoxedColumn _ (vec :: V.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = do         let n = V.length vec         dataPtr <- mallocArray (max 1 n) :: IO (Ptr Double)@@ -258,7 +261,7 @@         sPtr <- makeLeafSchema "g" colName         aPtr <- makeLeafArray n 0 [nullPtr, castPtr dataPtr] (free dataPtr)         return (sPtr, aPtr)-columnToArrow colName (BoxedColumn (vec :: V.Vector a))+columnToArrow colName (BoxedColumn _ (vec :: V.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = do         let n = V.length vec         dataPtr <- mallocArray (max 1 n) :: IO (Ptr Int64)@@ -266,16 +269,13 @@         sPtr <- makeLeafSchema "l" colName         aPtr <- makeLeafArray n 0 [nullPtr, castPtr dataPtr] (free dataPtr)         return (sPtr, aPtr)-columnToArrow colName (OptionalColumn (vec :: V.Vector (Maybe a)))+-- Nullable Int (UnboxedColumn with bitmap)+columnToArrow colName (UnboxedColumn (Just bm) (vec :: VU.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Int) = do-        let n = V.length vec-        (bitmapPtr, nullCount) <- buildBitmap vec+        let n = VU.length vec+        (bitmapPtr, nullCount) <- bitmapToPtr n bm         dataPtr <- mallocArray (max 1 n) :: IO (Ptr Int64)-        V.imapM_-            ( \i mv ->-                pokeElemOff dataPtr i (fromIntegral (fromMaybe 0 mv) :: Int64)-            )-            vec+        VU.imapM_ (\i v -> pokeElemOff dataPtr i (fromIntegral v :: Int64)) vec         sPtr <- makeLeafSchema "l" colName         aPtr <-             makeLeafArray@@ -284,16 +284,13 @@                 [castPtr bitmapPtr, castPtr dataPtr]                 (free bitmapPtr >> free dataPtr)         return (sPtr, aPtr)-columnToArrow colName (OptionalColumn (vec :: V.Vector (Maybe a)))+-- Nullable Double (UnboxedColumn with bitmap)+columnToArrow colName (UnboxedColumn (Just bm) (vec :: VU.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Double) = do-        let n = V.length vec-        (bitmapPtr, nullCount) <- buildBitmap vec+        let n = VU.length vec+        (bitmapPtr, nullCount) <- bitmapToPtr n bm         dataPtr <- mallocArray (max 1 n) :: IO (Ptr Double)-        V.imapM_-            ( \i mv ->-                pokeElemOff dataPtr i (maybe 0.0 realToFrac mv :: Double)-            )-            vec+        VU.imapM_ (\i v -> pokeElemOff dataPtr i (realToFrac v :: Double)) vec         sPtr <- makeLeafSchema "g" colName         aPtr <-             makeLeafArray@@ -302,14 +299,18 @@                 [castPtr bitmapPtr, castPtr dataPtr]                 (free bitmapPtr >> free dataPtr)         return (sPtr, aPtr)-columnToArrow colName (OptionalColumn (vec :: V.Vector (Maybe a)))+-- Nullable Text (BoxedColumn with bitmap)+columnToArrow colName (BoxedColumn (Just bm) (vec :: V.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @T.Text) = do         let n = V.length vec-            texts = V.toList vec-            bss = map (maybe BS.empty TE.encodeUtf8) texts+            -- For null positions, use empty BS (null placeholder in vec is never evaluated)+            bss =+                map+                    (\i -> if DI.bitmapTestBit bm i then TE.encodeUtf8 (vec V.! i) else BS.empty)+                    [0 .. n - 1]             cumOff = scanl (+) 0 (map BS.length bss)             total = last cumOff-        (bitmapPtr, nullCount) <- buildBitmap vec+        (bitmapPtr, nullCount) <- bitmapToPtr n bm         offPtr <- mallocArray (n + 1) :: IO (Ptr Int32)         zipWithM_             (\i o -> pokeElemOff offPtr i (fromIntegral o :: Int32))@@ -353,7 +354,7 @@      let nRows = case colsInOrder of             [] -> 0-            _ -> DI.columnLength (snd (head colsInOrder))+            (_, col) : _ -> DI.columnLength col     topSchema <- mallocBytes arrowSchemaSize     fmtStr <- newCString "+s"     nameStr <- newCString ""@@ -463,19 +464,12 @@     if nullCnt > 0         then do             let bitmapPtr = castPtr bitmapVoid :: Ptr Word8-            vec <- V.generateM n $ \i -> do-                valid <- bitmapIsSet bitmapPtr i-                if valid-                    then do-                        v <- peekElemOff dataPtr i-                        return (Just (fromIntegral v :: Int))-                    else return Nothing-            return $ OptionalColumn (vec :: V.Vector (Maybe Int))+            bm <- readArrowBitmap bitmapPtr n+            vec <- VU.generateM n $ \i -> fmap fromIntegral (peekElemOff dataPtr i :: IO Int64)+            return $ UnboxedColumn (Just bm) (vec :: VU.Vector Int)         else do-            vec <- VU.generateM n $ \i -> do-                v <- peekElemOff dataPtr i-                return (fromIntegral v :: Int)-            return $ UnboxedColumn (vec :: VU.Vector Int)+            vec <- VU.generateM n $ \i -> fmap fromIntegral (peekElemOff dataPtr i :: IO Int64)+            return $ UnboxedColumn Nothing (vec :: VU.Vector Int)  readInt32Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column readInt32Col n nullCnt bufArr = do@@ -485,19 +479,12 @@     if nullCnt > 0         then do             let bitmapPtr = castPtr bitmapVoid :: Ptr Word8-            vec <- V.generateM n $ \i -> do-                valid <- bitmapIsSet bitmapPtr i-                if valid-                    then do-                        v <- peekElemOff dataPtr i-                        return (Just (fromIntegral v :: Int))-                    else return Nothing-            return $ OptionalColumn (vec :: V.Vector (Maybe Int))+            bm <- readArrowBitmap bitmapPtr n+            vec <- VU.generateM n $ \i -> fmap fromIntegral (peekElemOff dataPtr i :: IO Int32)+            return $ UnboxedColumn (Just bm) (vec :: VU.Vector Int)         else do-            vec <- VU.generateM n $ \i -> do-                v <- peekElemOff dataPtr i-                return (fromIntegral v :: Int)-            return $ UnboxedColumn (vec :: VU.Vector Int)+            vec <- VU.generateM n $ \i -> fmap fromIntegral (peekElemOff dataPtr i :: IO Int32)+            return $ UnboxedColumn Nothing (vec :: VU.Vector Int)  readFloat64Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column readFloat64Col n nullCnt bufArr = do@@ -507,15 +494,12 @@     if nullCnt > 0         then do             let bitmapPtr = castPtr bitmapVoid :: Ptr Word8-            vec <- V.generateM n $ \i -> do-                valid <- bitmapIsSet bitmapPtr i-                if valid-                    then Just <$> (peekElemOff dataPtr i :: IO Double)-                    else return Nothing-            return $ OptionalColumn (vec :: V.Vector (Maybe Double))+            bm <- readArrowBitmap bitmapPtr n+            vec <- VU.generateM n (peekElemOff dataPtr)+            return $ UnboxedColumn (Just bm) (vec :: VU.Vector Double)         else do             vec <- VU.generateM n (peekElemOff dataPtr)-            return $ UnboxedColumn (vec :: VU.Vector Double)+            return $ UnboxedColumn Nothing (vec :: VU.Vector Double)  readFloat32Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column readFloat32Col n nullCnt bufArr = do@@ -525,19 +509,12 @@     if nullCnt > 0         then do             let bitmapPtr = castPtr bitmapVoid :: Ptr Word8-            vec <- V.generateM n $ \i -> do-                valid <- bitmapIsSet bitmapPtr i-                if valid-                    then do-                        v <- peekElemOff dataPtr i-                        return (Just (realToFrac v :: Double))-                    else return Nothing-            return $ OptionalColumn (vec :: V.Vector (Maybe Double))+            bm <- readArrowBitmap bitmapPtr n+            vec <- VU.generateM n $ \i -> fmap (realToFrac :: Float -> Double) (peekElemOff dataPtr i)+            return $ UnboxedColumn (Just bm) (vec :: VU.Vector Double)         else do-            vec <- VU.generateM n $ \i -> do-                v <- peekElemOff dataPtr i-                return (realToFrac v :: Double)-            return $ UnboxedColumn (vec :: VU.Vector Double)+            vec <- VU.generateM n $ \i -> fmap (realToFrac :: Float -> Double) (peekElemOff dataPtr i)+            return $ UnboxedColumn Nothing (vec :: VU.Vector Double)  -- | Read a large_string (format "U") column with int64 offsets. readLargeUtf8Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column@@ -547,30 +524,20 @@     charVoid <- peekElemOff bufArr 2     let offsetPtr = castPtr offsetVoid :: Ptr Int64         charPtr = castPtr charVoid :: Ptr Word8+    let readText i = do+            start <- fromIntegral <$> peekElemOff offsetPtr i+            end <- fromIntegral <$> peekElemOff offsetPtr (i + 1)+            TE.decodeUtf8+                <$> BS.packCStringLen (castPtr (charPtr `plusPtr` start), end - start)     if nullCnt > 0         then do             let bitmapPtr = castPtr bitmapVoid :: Ptr Word8-            vec <- V.generateM n $ \i -> do-                valid <- bitmapIsSet bitmapPtr i-                if valid-                    then do-                        start <- fromIntegral <$> peekElemOff offsetPtr i-                        end <- fromIntegral <$> peekElemOff offsetPtr (i + 1)-                        bs <--                            BS.packCStringLen-                                (castPtr (charPtr `plusPtr` start), end - start)-                        return $ Just (TE.decodeUtf8 bs)-                    else return Nothing-            return $ OptionalColumn vec+            bm <- readArrowBitmap bitmapPtr n+            vec <- V.generateM n readText+            return $ BoxedColumn (Just bm) vec         else do-            vec <- V.generateM n $ \i -> do-                start <- fromIntegral <$> peekElemOff offsetPtr i-                end <- fromIntegral <$> peekElemOff offsetPtr (i + 1)-                bs <--                    BS.packCStringLen-                        (castPtr (charPtr `plusPtr` start), end - start)-                return $ TE.decodeUtf8 bs-            return $ BoxedColumn vec+            vec <- V.generateM n readText+            return $ BoxedColumn Nothing vec  -- | Read a utf8 (format "u") column with int32 offsets. readUtf8Col :: Int -> Int64 -> Ptr (Ptr ()) -> IO Column@@ -580,27 +547,17 @@     charVoid <- peekElemOff bufArr 2     let offsetPtr = castPtr offsetVoid :: Ptr Int32         charPtr = castPtr charVoid :: Ptr Word8+        readText i = do+            start <- fromIntegral <$> peekElemOff offsetPtr i+            end <- fromIntegral <$> peekElemOff offsetPtr (i + 1)+            TE.decodeUtf8+                <$> BS.packCStringLen (castPtr (charPtr `plusPtr` start), end - start)     if nullCnt > 0         then do             let bitmapPtr = castPtr bitmapVoid :: Ptr Word8-            vec <- V.generateM n $ \i -> do-                valid <- bitmapIsSet bitmapPtr i-                if valid-                    then do-                        start <- fromIntegral <$> peekElemOff offsetPtr i-                        end <- fromIntegral <$> peekElemOff offsetPtr (i + 1)-                        bs <--                            BS.packCStringLen-                                (castPtr (charPtr `plusPtr` start), end - start)-                        return $ Just (TE.decodeUtf8 bs)-                    else return Nothing-            return $ OptionalColumn vec+            bm <- readArrowBitmap bitmapPtr n+            vec <- V.generateM n readText+            return $ BoxedColumn (Just bm) vec         else do-            vec <- V.generateM n $ \i -> do-                start <- fromIntegral <$> peekElemOff offsetPtr i-                end <- fromIntegral <$> peekElemOff offsetPtr (i + 1)-                bs <--                    BS.packCStringLen-                        (castPtr (charPtr `plusPtr` start), end - start)-                return $ TE.decodeUtf8 bs-            return $ BoxedColumn vec+            vec <- V.generateM n readText+            return $ BoxedColumn Nothing vec

@@ -113,12 +113,14 @@  -- | Build a SortOrder from a column's runtime type. mkSortOrder :: Bool -> T.Text -> Column -> SortOrder-mkSortOrder True n (UnboxedColumn (_ :: VU.Vector a)) = Asc (Col @a n)-mkSortOrder False n (UnboxedColumn (_ :: VU.Vector a)) = Desc (Col @a n)-mkSortOrder True n (BoxedColumn (_ :: V.Vector a)) = Asc (Col @a n)-mkSortOrder False n (BoxedColumn (_ :: V.Vector a)) = Desc (Col @a n)-mkSortOrder True n (OptionalColumn (_ :: V.Vector (Maybe a))) = Asc (Col @(Maybe a) n)-mkSortOrder False n (OptionalColumn (_ :: V.Vector (Maybe a))) = Desc (Col @(Maybe a) n)+mkSortOrder True n (UnboxedColumn Nothing (_ :: VU.Vector a)) = Asc (Col @a n)+mkSortOrder False n (UnboxedColumn Nothing (_ :: VU.Vector a)) = Desc (Col @a n)+mkSortOrder True n (UnboxedColumn (Just _) (_ :: VU.Vector a)) = Asc (Col @(Maybe a) n)+mkSortOrder False n (UnboxedColumn (Just _) (_ :: VU.Vector a)) = Desc (Col @(Maybe a) n)+mkSortOrder True n (BoxedColumn Nothing (_ :: V.Vector a)) = Asc (Col @a n)+mkSortOrder False n (BoxedColumn Nothing (_ :: V.Vector a)) = Desc (Col @a n)+mkSortOrder True n (BoxedColumn (Just _) (_ :: V.Vector a)) = Asc (Col @(Maybe a) n)+mkSortOrder False n (BoxedColumn (Just _) (_ :: V.Vector a)) = Desc (Col @(Maybe a) n)  -- | Dispatch aggregation by fn name and runtime column type. buildNamedExpr :: DataFrame -> AggSpec -> IO NamedExpr@@ -133,22 +135,28 @@                     "DataFrame.IR: unknown aggregation '" ++ T.unpack other ++ "'"  countExpr :: T.Text -> T.Text -> Column -> IO NamedExpr-countExpr name colName (UnboxedColumn (_ :: VU.Vector a)) = return $ name .= count (Col @a colName)-countExpr name colName (BoxedColumn (_ :: V.Vector a)) = return $ name .= count (Col @a colName)-countExpr name colName (OptionalColumn (_ :: V.Vector (Maybe a))) = return $ name .= count (Col @(Maybe a) colName)+countExpr name colName (UnboxedColumn Nothing (_ :: VU.Vector a)) = return $ name .= count (Col @a colName)+countExpr name colName (UnboxedColumn (Just _) (_ :: VU.Vector a)) = return $ name .= count (Col @(Maybe a) colName)+countExpr name colName (BoxedColumn Nothing (_ :: V.Vector a)) = return $ name .= count (Col @a colName)+countExpr name colName (BoxedColumn (Just _) (_ :: V.Vector a)) = return $ name .= count (Col @(Maybe a) colName)  sumExpr :: T.Text -> T.Text -> Column -> IO NamedExpr-sumExpr name colName (UnboxedColumn (_ :: VU.Vector a))+sumExpr name colName (UnboxedColumn Nothing (_ :: VU.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Int) =         return $ name .= Functions.sum (Col @Int colName)     | Just Refl <- testEquality (typeRep @a) (typeRep @Double) =         return $ name .= Functions.sum (Col @Double colName)-sumExpr name colName (BoxedColumn (_ :: V.Vector a))+sumExpr name colName (UnboxedColumn (Just _) (_ :: VU.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Int) =+        return $ name .= sumMaybe (Col @(Maybe Int) colName)+    | Just Refl <- testEquality (typeRep @a) (typeRep @Double) =+        return $ name .= sumMaybe (Col @(Maybe Double) colName)+sumExpr name colName (BoxedColumn Nothing (_ :: V.Vector a))+    | Just Refl <- testEquality (typeRep @a) (typeRep @Int) =         return $ name .= Functions.sum (Col @Int colName)     | Just Refl <- testEquality (typeRep @a) (typeRep @Double) =         return $ name .= Functions.sum (Col @Double colName)-sumExpr name colName (OptionalColumn (_ :: V.Vector (Maybe a)))+sumExpr name colName (BoxedColumn (Just _) (_ :: V.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Int) =         return $ name .= sumMaybe (Col @(Maybe Int) colName)     | Just Refl <- testEquality (typeRep @a) (typeRep @Double) =@@ -159,15 +167,20 @@             "DataFrame.IR: sum: unsupported column type for '" ++ T.unpack colName ++ "'"  meanExpr :: T.Text -> T.Text -> Column -> IO NamedExpr-meanExpr name colName (UnboxedColumn (_ :: VU.Vector a))+meanExpr name colName (UnboxedColumn Nothing (_ :: VU.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Int) =         return $ name .= mean (Col @Int colName)     | Just Refl <- testEquality (typeRep @a) (typeRep @Double) =         return $ name .= mean (Col @Double colName)-meanExpr name colName (BoxedColumn (_ :: V.Vector a))+meanExpr name colName (UnboxedColumn (Just _) (_ :: VU.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Double) =+        return $ name .= meanMaybe (Col @(Maybe Double) colName)+    | Just Refl <- testEquality (typeRep @a) (typeRep @Int) =+        return $ name .= meanMaybe (Col @(Maybe Int) colName)+meanExpr name colName (BoxedColumn Nothing (_ :: V.Vector a))+    | Just Refl <- testEquality (typeRep @a) (typeRep @Double) =         return $ name .= mean (Col @Double colName)-meanExpr name colName (OptionalColumn (_ :: V.Vector (Maybe a)))+meanExpr name colName (BoxedColumn (Just _) (_ :: V.Vector a))     | Just Refl <- testEquality (typeRep @a) (typeRep @Double) =         return $ name .= meanMaybe (Col @(Maybe Double) colName)     | Just Refl <- testEquality (typeRep @a) (typeRep @Int) =

@@ -216,7 +216,6 @@      -- * I/O     module CSV,-    module UnstableCSV,     module Parquet,      -- * Type conversion@@ -266,12 +265,6 @@     readParquetFilesWithOpts,     readParquetWithOpts,  )-import DataFrame.IO.Unstable.CSV as UnstableCSV (-    fastReadCsvUnstable,-    fastReadTsvUnstable,-    readCsvUnstable,-    readTsvUnstable,- ) import DataFrame.Internal.Column as Column (     Column,     fromList,@@ -288,8 +281,10 @@     GroupedDataFrame,     empty,     null,+    toCsv,     toMarkdown,     toMarkdown',+    toSeparated,  ) import DataFrame.Internal.Expression as Expression (Expr, prettyPrint) import DataFrame.Internal.Row as Row (

@@ -638,18 +638,24 @@ numericCols df = concatMap extract (columnNames df)   where     extract col = case unsafeGetColumn col df of-        UnboxedColumn (_ :: VU.Vector b) ->+        UnboxedColumn Nothing (_ :: VU.Vector b) ->             case testEquality (typeRep @b) (typeRep @Double) of                 Just Refl -> [NDouble (Col col)]                 Nothing -> case sIntegral @b of                     STrue -> [NDouble (F.toDouble (Col @b col))]                     SFalse -> []-        OptionalColumn (_ :: V.Vector (Maybe b)) ->+        BoxedColumn (Just _) (_ :: V.Vector b) ->             case testEquality (typeRep @b) (typeRep @Double) of                 Just Refl -> [NMaybeDouble (Col @(Maybe b) col)]                 Nothing -> case sIntegral @b of                     STrue -> [NMaybeDouble (F.whenPresent (realToFrac @b @Double) (Col @(Maybe b) col))]                     SFalse -> []+        UnboxedColumn (Just _) (_ :: VU.Vector b) ->+            case testEquality (typeRep @b) (typeRep @Double) of+                Just Refl -> [NMaybeDouble (Col @(Maybe b) col)]+                Nothing -> case sIntegral @b of+                    STrue -> [NMaybeDouble (F.whenPresent (realToFrac @b @Double) (Col @(Maybe b) col))]+                    SFalse -> []         _ -> []  numericExprs ::@@ -697,18 +703,18 @@     let         genConds :: T.Text -> [Expr Bool]         genConds colName = case unsafeGetColumn colName df of-            (BoxedColumn (col :: V.Vector a)) ->+            (BoxedColumn Nothing (col :: V.Vector a)) ->                 let ps = map (Lit . (`percentileOrd'` col)) [1, 25, 75, 99]                  in map (F.lift2 (==) (Col @a colName)) ps-            (OptionalColumn (col :: V.Vector (Maybe a))) -> case sFloating @a of+            (BoxedColumn (Just _) (col :: V.Vector a)) -> case sFloating @a of                 STrue -> [] -- handled by numericCols / numericExprs                 SFalse -> case sIntegral @a of                     STrue -> [] -- handled by numericCols / numericExprs                     SFalse ->                         map-                            (F.lift2 (==) (Col @(Maybe a) colName) . Lit . (`percentileOrd'` col))+                            (F.lift2 (==) (Col @(Maybe a) colName) . Lit . Just . (`percentileOrd'` col))                             [1, 25, 75, 99]-            (UnboxedColumn (_ :: VU.Vector a)) -> []+            (UnboxedColumn _ (_ :: VU.Vector a)) -> []          columnConds =             concatMap@@ -724,13 +730,15 @@                 ]           where             colConds (!l, !r) = case (unsafeGetColumn l df, unsafeGetColumn r df) of-                (BoxedColumn (col1 :: V.Vector a), BoxedColumn (_ :: V.Vector b)) ->-                    case testEquality (typeRep @a) (typeRep @b) of-                        Nothing -> []-                        Just Refl -> [F.lift2 (==) (Col @a l) (Col @a r)]-                (UnboxedColumn (_ :: VU.Vector a), UnboxedColumn (_ :: VU.Vector b)) -> []-                ( OptionalColumn (_ :: V.Vector (Maybe a))-                    , OptionalColumn (_ :: V.Vector (Maybe b))+                ( BoxedColumn Nothing (col1 :: V.Vector a)+                    , BoxedColumn Nothing (_ :: V.Vector b)+                    ) ->+                        case testEquality (typeRep @a) (typeRep @b) of+                            Nothing -> []+                            Just Refl -> [F.lift2 (==) (Col @a l) (Col @a r)]+                (UnboxedColumn _ (_ :: VU.Vector a), UnboxedColumn _ (_ :: VU.Vector b)) -> []+                ( BoxedColumn (Just _) (_ :: V.Vector a)+                    , BoxedColumn (Just _) (_ :: V.Vector b)                     ) -> case testEquality (typeRep @a) (typeRep @b) of                         Nothing -> []                         Just Refl -> case testEquality (typeRep @a) (typeRep @T.Text) of

@@ -339,15 +339,12 @@         Just idx ->             let col = columns df V.! idx              in case col of-                    BoxedColumn vec ->+                    BoxedColumn _ vec ->                         let counts = countValues vec                          in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts]-                    UnboxedColumn vec ->+                    UnboxedColumn _ vec ->                         let counts = countValuesUnboxed vec                          in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts]-                    OptionalColumn vec ->-                        let counts = countValues vec-                         in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts]   where     countValues :: (Ord a, Show a) => V.Vector a -> [(a, Int)]     countValues vec = M.toList $ V.foldr' (\x acc -> M.insertWith (+) x 1 acc) M.empty vec@@ -365,9 +362,8 @@         Just idx ->             let col = columns df V.! idx              in case col of-                    BoxedColumn vec -> V.toList $ V.map (T.pack . show) vec-                    UnboxedColumn vec -> V.toList $ VG.map (T.pack . show) (VG.convert vec)-                    OptionalColumn vec -> V.toList $ V.map (T.pack . show) vec+                    BoxedColumn _ vec -> V.toList $ V.map (T.pack . show) vec+                    UnboxedColumn _ vec -> V.toList $ VG.map (T.pack . show) (VG.convert vec)  extractNumericColumn :: (HasCallStack) => T.Text -> DataFrame -> [Double] extractNumericColumn colName df =@@ -376,9 +372,8 @@         Just idx ->             let col = columns df V.! idx              in case col of-                    BoxedColumn vec -> vectorToDoubles vec-                    UnboxedColumn vec -> unboxedVectorToDoubles vec-                    _ -> []+                    BoxedColumn _ vec -> vectorToDoubles vec+                    UnboxedColumn _ vec -> unboxedVectorToDoubles vec  vectorToDoubles :: forall a. (Columnable a, Show a) => V.Vector a -> [Double] vectorToDoubles vec =

@@ -850,13 +850,10 @@         Just idx ->             let col = columns df V.! idx              in case col of-                    BoxedColumn (vec :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of+                    BoxedColumn _ (vec :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of                         Just Refl -> V.toList vec                         Nothing -> V.toList $ V.map (T.pack . show) vec-                    UnboxedColumn vec -> V.toList $ VG.map (T.pack . show) (VG.convert vec)-                    OptionalColumn (vec :: V.Vector (Maybe a)) -> case testEquality (typeRep @a) (typeRep @T.Text) of-                        Nothing -> V.toList $ V.map (T.pack . show) vec-                        Just Refl -> V.toList $ V.map (maybe "Nothing" ("Just " <>)) vec+                    UnboxedColumn _ vec -> V.toList $ VG.map (T.pack . show) (VG.convert vec)  extractNumericColumn :: (HasCallStack) => T.Text -> DataFrame -> [Double] extractNumericColumn colName df =@@ -865,9 +862,8 @@         Just idx ->             let col = columns df V.! idx              in case col of-                    BoxedColumn vec -> vectorToDoubles vec-                    UnboxedColumn vec -> unboxedVectorToDoubles vec-                    _ -> []+                    BoxedColumn _ vec -> vectorToDoubles vec+                    UnboxedColumn _ vec -> unboxedVectorToDoubles vec  vectorToDoubles :: forall a. (Columnable a, Show a) => V.Vector a -> [Double] vectorToDoubles vec =@@ -898,21 +894,14 @@         Just idx ->             let col = columns df V.! idx              in case col of-                    BoxedColumn (vec :: V.Vector a) ->+                    BoxedColumn _ (vec :: V.Vector a) ->                         let counts = countValues vec                          in case testEquality (typeRep @a) (typeRep @T.Text) of                                 Nothing -> Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts]                                 Just Refl -> Just [(k, fromIntegral v) | (k, v) <- counts]-                    UnboxedColumn vec ->+                    UnboxedColumn _ vec ->                         let counts = countValuesUnboxed vec                          in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts]-                    OptionalColumn (vec :: V.Vector (Maybe a)) ->-                        let counts = countValues vec-                         in case testEquality (typeRep @a) (typeRep @T.Text) of-                                Nothing -> Just [((T.pack . show) k, fromIntegral v) | (k, v) <- counts]-                                Just Refl ->-                                    Just-                                        [(maybe "Nothing" ("Just " <>) k, fromIntegral v) | (k, v) <- counts]   where     countValues :: (Ord a, Show a) => V.Vector a -> [(a, Int)]     countValues vec = M.toList $ V.foldr' (\x acc -> M.insertWith (+) x 1 acc) M.empty vec

@@ -11,6 +11,7 @@  import Control.Exception import Data.Array+import qualified Data.List as L import Data.Typeable (Typeable) import DataFrame.Display.Terminal.Colours import Type.Reflection (TypeRep)@@ -29,7 +30,7 @@         TypeErrorContext a b ->         DataFrameException     AggregatedAndNonAggregatedException :: T.Text -> T.Text -> DataFrameException-    ColumnNotFoundException :: T.Text -> T.Text -> [T.Text] -> DataFrameException+    ColumnsNotFoundException :: [T.Text] -> T.Text -> [T.Text] -> DataFrameException     EmptyDataSetException :: T.Text -> DataFrameException     InternalException :: T.Text -> DataFrameException     NonColumnReferenceException :: T.Text -> DataFrameException@@ -51,7 +52,7 @@                 (errorColumnName context)                 (callingFunctionName context)                 errorString-    show (ColumnNotFoundException columnName callPoint availableColumns) = columnNotFound columnName callPoint availableColumns+    show (ColumnsNotFoundException columnNames callPoint availableColumns) = columnsNotFound columnNames callPoint availableColumns     show (EmptyDataSetException callPoint) = emptyDataSetError callPoint     show (WrongQuantileNumberException q) = wrongQuantileNumberError q     show (WrongQuantileIndexException qs q) = wrongQuantileIndexError qs q@@ -65,15 +66,46 @@             ++ T.unpack expr2  columnNotFound :: T.Text -> T.Text -> [T.Text] -> String-columnNotFound name callPoint columns =+columnNotFound missingColumn = columnsNotFound [missingColumn]++columnsNotFound :: [T.Text] -> T.Text -> [T.Text] -> String+columnsNotFound missingColumns callPoint availableColumns =     red "\n\n[ERROR] "-        ++ "Column not found: "-        ++ T.unpack name+        ++ missingColumnsLabel missingColumns+        ++ ": "+        ++ T.unpack (T.intercalate ", " missingColumns)         ++ " for operation "         ++ T.unpack callPoint-        ++ "\n\tDid you mean "-        ++ T.unpack (guessColumnName name columns)-        ++ "?\n\n"+        ++ formatSuggestions missingColumns availableColumns+        ++ "\n\n"+  where+    missingColumnsLabel [_] = "Column not found"+    missingColumnsLabel _ = "Columns not found"++    formatSuggestions [missingColumn] columns =+        case guessColumnName missingColumn columns of+            "" -> ""+            guessed ->+                "\n\tDid you mean "+                    ++ T.unpack guessed+                    ++ "?"+    formatSuggestions names columns =+        case traverse (`suggestColumnName` columns) names of+            Just guessedColumns+                | not (null guessedColumns) ->+                    "\n\tDid you mean "+                        ++ formatColumnSuggestions guessedColumns+                        ++ "?"+            _ -> ""++    suggestColumnName missingColumn columns = case guessColumnName missingColumn columns of+        "" -> Nothing+        guessed -> Just guessed++    formatColumnSuggestions guessedColumns =+        "["+            ++ L.intercalate ", " (map (show . T.unpack) guessedColumns)+            ++ "]"  typeMismatchError :: String -> String -> String typeMismatchError givenType expectedType =

@@ -628,7 +628,8 @@      in         fmap concat $ forM specs $ \(raw, nm, tyStr) -> do             ty <- typeFromString (words tyStr)-            trace (T.unpack (nm <> " :: Expr " <> T.pack tyStr)) pure ()+            let tyDisplay = if ' ' `elem` tyStr then "(" <> T.pack tyStr <> ")" else T.pack tyStr+            trace (T.unpack (nm <> " :: Expr " <> tyDisplay)) pure ()             let n = mkName (T.unpack nm)             sig <- sigD n [t|Expr $(pure ty)|]             val <- valD (varP n) (normalB [|col $(TH.lift raw)|]) []

@@ -39,7 +39,7 @@ import Data.Type.Equality (TestEquality (testEquality)) import Data.Word (Word8) import DataFrame.Internal.Column-import DataFrame.Internal.DataFrame (DataFrame (..))+import DataFrame.Internal.DataFrame (DataFrame (..), toSeparated) import DataFrame.Internal.Parsing import DataFrame.Internal.Schema import DataFrame.Operations.Typing@@ -220,7 +220,7 @@     ReadOptions         { headerSpec = UseFirstRow         , typeSpec = InferFromSample 100-        , safeRead = True+        , safeRead = False         , dateFormat = "%Y-%m-%d"         , columnSeparator = ','         , numColumns = Nothing@@ -389,30 +389,33 @@     vec <- freezePagedUnboxedVector gv     valid <- freezePagedUnboxedVector validRef     if VU.all (== 1) valid-        then return $! UnboxedColumn vec+        then return $! UnboxedColumn Nothing vec         else constructOptional vec valid freezeBuilderColumn (BuilderDouble gv validRef) = do     vec <- freezePagedUnboxedVector gv     valid <- freezePagedUnboxedVector validRef     if VU.all (== 1) valid-        then return $! UnboxedColumn vec+        then return $! UnboxedColumn Nothing vec         else constructOptional vec valid freezeBuilderColumn (BuilderText gv validRef) = do     vec <- freezePagedVector gv     valid <- freezePagedUnboxedVector validRef     if VU.all (== 1) valid-        then return $! BoxedColumn vec+        then return $! BoxedColumn Nothing vec         else constructOptionalBoxed vec valid freezeBuilderColumn (BuilderBS _ _) =     error         "freezeBuilderColumn: BuilderBS must be finalized via finalizeBuilderColumn"  finalizeBuilderColumn :: ReadOptions -> BuilderColumn -> IO Column-finalizeBuilderColumn opts (BuilderBS gv validRef) = do-    vec <- freezePagedVector gv-    valid <- freezePagedUnboxedVector validRef-    return $! inferColumnFromBS opts vec valid-finalizeBuilderColumn _ bc = freezeBuilderColumn bc+finalizeBuilderColumn opts bc = do+    col <- case bc of+        BuilderBS gv validRef -> do+            vec <- freezePagedVector gv+            valid <- freezePagedUnboxedVector validRef+            return $! inferColumnFromBS opts vec valid+        _ -> freezeBuilderColumn bc+    return $! if safeRead opts then ensureOptional col else col  inferColumnFromBS ::     ReadOptions -> V.Vector BS.ByteString -> VU.Vector Word8 -> Column@@ -523,23 +526,13 @@ constructOptional ::     (VU.Unbox a, Columnable a) => VU.Vector a -> VU.Vector Word8 -> IO Column constructOptional vec valid = do-    let size = VU.length vec-    mvec <- VM.new size-    forM_ [0 .. size - 1] $ \i ->-        if (valid VU.! i) == 0-            then VM.write mvec i Nothing-            else VM.write mvec i (Just (vec VU.! i))-    OptionalColumn <$> V.freeze mvec+    let bm = buildBitmapFromValid valid+    pure $ UnboxedColumn (Just bm) vec  constructOptionalBoxed :: V.Vector T.Text -> VU.Vector Word8 -> IO Column constructOptionalBoxed vec valid = do-    let size = V.length vec-    mvec <- VM.new size-    forM_ [0 .. size - 1] $ \i ->-        if (valid VU.! i) == 0-            then VM.write mvec i Nothing-            else VM.write mvec i (Just (vec V.! i))-    OptionalColumn <$> V.freeze mvec+    let bm = buildBitmapFromValid valid+    pure $ BoxedColumn (Just bm) vec  writeCsv :: FilePath -> DataFrame -> IO () writeCsv = writeSeparated ','@@ -554,61 +547,7 @@     FilePath ->     DataFrame ->     IO ()-writeSeparated c filepath df = withFile filepath WriteMode $ \handle -> do-    let (rows, _) = dataframeDimensions df-    let headers = map fst (L.sortBy (compare `on` snd) (M.toList (columnIndices df)))-    TIO.hPutStrLn handle (T.intercalate "," headers)-    forM_ [0 .. (rows - 1)] $ \i -> do-        let row = getRowAsText df i-        TIO.hPutStrLn handle (T.intercalate "," row)--getRowAsText :: DataFrame -> Int -> [T.Text]-getRowAsText df i = V.ifoldr go [] (columns df)-  where-    indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df))-    go k (BoxedColumn (c :: V.Vector a)) acc = case c V.!? i of-        Just e -> textRep : acc-          where-            textRep = case testEquality (typeRep @a) (typeRep @T.Text) of-                Just Refl -> e-                Nothing -> case typeRep @a of-                    App t1 t2 -> case eqTypeRep t1 (typeRep @Maybe) of-                        Just HRefl -> case testEquality t2 (typeRep @T.Text) of-                            Just Refl -> fromMaybe "null" e-                            Nothing -> (fromOptional . T.pack . show) e-                              where-                                fromOptional s-                                    | T.isPrefixOf "Just " s = T.drop (T.length "Just ") s-                                    | otherwise = "null"-                        Nothing -> (T.pack . show) e-                    _ -> (T.pack . show) e-        Nothing ->-            error $-                "Column "-                    ++ T.unpack (indexMap M.! k)-                    ++ " has less items than "-                    ++ "the other columns at index "-                    ++ show i-    go k (UnboxedColumn c) acc = case c VU.!? i of-        Just e -> T.pack (show e) : acc-        Nothing ->-            error $-                "Column "-                    ++ T.unpack (indexMap M.! k)-                    ++ " has less items than "-                    ++ "the other columns at index "-                    ++ show i-    go k (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of-        Just e -> case testEquality (typeRep @a) (typeRep @T.Text) of-            Just Refl -> fromMaybe T.empty e : acc-            Nothing -> maybe T.empty (T.pack . show) e : acc-        Nothing ->-            error $-                "Column "-                    ++ T.unpack (indexMap M.! k)-                    ++ " has less items than "-                    ++ "the other columns at index "-                    ++ show i+writeSeparated c filepath df = TIO.writeFile filepath (toSeparated c df)  stripQuotes :: T.Text -> T.Text stripQuotes txt =

@@ -19,10 +19,11 @@ import Data.Text.Encoding import Data.Time import Data.Time.Clock.POSIX (posixSecondsToUTCTime)-import DataFrame.Errors (DataFrameException (ColumnNotFoundException))+import qualified Data.Vector as V+import DataFrame.Errors (DataFrameException (ColumnsNotFoundException)) import DataFrame.Internal.Binary (littleEndianWord32) import qualified DataFrame.Internal.Column as DI-import DataFrame.Internal.DataFrame (DataFrame)+import DataFrame.Internal.DataFrame (DataFrame, columns) import DataFrame.Internal.Expression (Expr, getColumns) import qualified DataFrame.Operations.Core as DI import DataFrame.Operations.Merge ()@@ -75,6 +76,8 @@     -- ^ Optional row filter expression applied before projection.     , rowRange :: Maybe (Int, Int)     -- ^ Optional row slice @(start, end)@ with start-inclusive/end-exclusive semantics.+    , safeColumns :: Bool+    -- ^ When True, every column is promoted to OptionalColumn after read, regardless of nullability in the schema.     }     deriving (Eq, Show) @@ -87,6 +90,7 @@     { selectedColumns = Nothing     , predicate = Nothing     , rowRange = Nothing+    , safeColumns = False     } @ -}@@ -96,6 +100,7 @@         { selectedColumns = Nothing         , predicate = Nothing         , rowRange = Nothing+        , safeColumns = False         }  -- Public API --------------------------------------------------------------@@ -179,16 +184,15 @@              in unless                     (L.null missing)                     ( throw-                        ( ColumnNotFoundException-                            (T.pack $ show missing)+                        ( ColumnsNotFoundException+                            missing                             "readParquetWithOpts"                             availableSelectedColumns                         )                     ) -    let totalRows = sum (map (fromIntegral . rowGroupNumRows) (rowGroups fileMetadata)) :: Int-    colMutMap <- newIORef (M.empty :: M.Map T.Text DI.MutableColumn)-    colOffMap <- newIORef (M.empty :: M.Map T.Text Int)+    -- Collect per-column chunk lists; concatenate at the end to preserve bitmaps.+    colListMap <- newIORef (M.empty :: M.Map T.Text [DI.Column])     lTypeMap <- newIORef (M.empty :: M.Map T.Text LogicalType)      let schemaElements = schema fileMetadata@@ -261,22 +265,13 @@                         maybeTypeLength                         lType -                mutMapSnap <- readIORef colMutMap-                case M.lookup colFullName mutMapSnap of-                    Nothing -> do-                        mc <- DI.newMutableColumn totalRows column-                        DI.copyIntoMutableColumn mc 0 column-                        modifyIORef' colMutMap (M.insert colFullName mc)-                        modifyIORef' colOffMap (M.insert colFullName (DI.columnLength column))-                    Just mc -> do-                        off <- (M.! colFullName) <$> readIORef colOffMap-                        DI.copyIntoMutableColumn mc off column-                        modifyIORef' colOffMap (M.adjust (+ DI.columnLength column) colFullName)+                modifyIORef' colListMap (M.insertWith (++) colFullName [column])                 modifyIORef' lTypeMap (M.insert colFullName lType) -    finalMutMap <- readIORef colMutMap-    finalColMap <--        M.traverseWithKey (\_ mc -> DI.freezeMutableColumn mc) finalMutMap+    finalListMap <- readIORef colListMap+    -- Reverse the accumulated lists (they were prepended) and concat columns per-name,+    -- preserving bitmaps correctly via concatManyColumns.+    let finalColMap = M.map (DI.concatManyColumns . reverse) finalListMap     finalLTypeMap <- readIORef lTypeMap     let orderedColumns =             map@@ -349,9 +344,15 @@ applyPredicate opts df =     maybe df (`DS.filterWhere` df) (predicate opts) +applySafeRead :: ParquetReadOptions -> DataFrame -> DataFrame+applySafeRead opts df+    | safeColumns opts = df{columns = V.map DI.ensureOptional (columns df)}+    | otherwise = df+ applyReadOptions :: ParquetReadOptions -> DataFrame -> DataFrame applyReadOptions opts =-    applyRowRange opts+    applySafeRead opts+        . applyRowRange opts         . applySelectedColumns opts         . applyPredicate opts 

@@ -68,9 +68,8 @@         let             colType :: TType             colType = case unsafeGetColumn colName df of-                (DI.BoxedColumn (col :: V.Vector a)) -> haskellToTType @a-                (DI.UnboxedColumn (col :: VU.Vector a)) -> haskellToTType @a-                (DI.OptionalColumn (col :: V.Vector (Maybe a))) -> haskellToTType @a+                (DI.BoxedColumn _ (col :: V.Vector a)) -> haskellToTType @a+                (DI.UnboxedColumn _ (col :: VU.Vector a)) -> haskellToTType @a             lType =                 if DI.hasElemType @T.Text (unsafeGetColumn colName df)                     || DI.hasElemType @(Maybe T.Text) (unsafeGetColumn colName df)

@@ -1,352 +0,0 @@-{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE CApiFFI #-}-{-# LANGUAGE ForeignFunctionInterface #-}--module DataFrame.IO.Unstable.CSV (-    fastReadCsvUnstable,-    readCsvUnstable,-    fastReadTsvUnstable,-    readTsvUnstable,-    getDelimiterIndices,-) where--import qualified Data.Vector as Vector-import qualified Data.Vector.Storable as VS-import Data.Vector.Storable.Mutable (-    grow,-    unsafeFromForeignPtr,- )-import qualified Data.Vector.Storable.Mutable as VSM-import System.IO.MMap (-    Mode (WriteCopy),-    mmapFileForeignPtr,- )--import Foreign (-    Ptr,-    castForeignPtr,-    castPtr,-    mallocArray,-    newForeignPtr_,- )-import Foreign.C.Types--import qualified Data.ByteString as BS-import Data.ByteString.Internal (ByteString (PS))-import qualified Data.Map as M-import Data.Text (Text)-import qualified Data.Text as Text-import qualified Data.Text.Encoding as TextEncoding-import Data.Word (Word8)--import Control.Parallel.Strategies (parList, rpar, using)-import Data.Array.IArray (array, (!))-import Data.Array.Unboxed (UArray)-import Data.Ix (range)--import DataFrame.IO.CSV (-    HeaderSpec (..),-    ReadOptions (..),-    defaultReadOptions,-    shouldInferFromSample,-    stripQuotes,-    typeInferenceSampleSize,- )-import DataFrame.Internal.DataFrame (DataFrame (..))-import DataFrame.Operations.Typing (ParseOptions (..), parseFromExamples)--readSeparatedDefaultFast :: Word8 -> FilePath -> IO DataFrame-readSeparatedDefaultFast separator =-    readSeparated-        separator-        defaultReadOptions-        getDelimiterIndices--readSeparatedDefault :: Word8 -> FilePath -> IO DataFrame-readSeparatedDefault separator =-    readSeparated-        separator-        defaultReadOptions-        ( \separator originalLen v -> do-            indices <- mallocArray originalLen-            getDelimiterIndices_ separator originalLen v indices-        )--fastReadCsvUnstable :: FilePath -> IO DataFrame-fastReadCsvUnstable = readSeparatedDefaultFast comma--readCsvUnstable :: FilePath -> IO DataFrame-readCsvUnstable = readSeparatedDefault comma--fastReadTsvUnstable :: FilePath -> IO DataFrame-fastReadTsvUnstable = readSeparatedDefaultFast tab--readTsvUnstable :: FilePath -> IO DataFrame-readTsvUnstable = readSeparatedDefault tab--readSeparated ::-    Word8 ->-    ReadOptions ->-    (Word8 -> Int -> VS.Vector Word8 -> IO (VS.Vector CSize)) ->-    FilePath ->-    IO DataFrame-readSeparated separator opts delimiterIndices filePath = do-    -- We use write copy mode so that we can append-    -- padding to the end of the memory space-    (bufferPtr, offset, len) <--        mmapFileForeignPtr-            filePath-            WriteCopy-            Nothing-    let mutableFile = unsafeFromForeignPtr bufferPtr offset len-    paddedMutableFile <- grow mutableFile 64-    paddedCSVFile <- VS.unsafeFreeze paddedMutableFile-    indices <- delimiterIndices separator len paddedCSVFile-    let numCol = countColumnsInFirstRow paddedCSVFile indices-        totalRows = VS.length indices `div` numCol-        extractField' = extractField paddedCSVFile indices-        (columnNames, dataStartRow) = case headerSpec opts of-            NoHeader ->-                ( Vector.fromList $-                    map (Text.pack . show) [0 .. numCol - 1]-                , 0-                )-            UseFirstRow ->-                ( Vector.fromList $-                    map (stripQuotes . extractField') [0 .. numCol - 1]-                , 1-                )-            ProvideNames ns ->-                (Vector.fromList ns, 0)-        numRow = totalRows - dataStartRow-        parseTypes col =-            let n =-                    if shouldInferFromSample (typeSpec opts)-                        then typeInferenceSampleSize (typeSpec opts)-                        else 0-                parseOpts =-                    ParseOptions-                        { missingValues = missingIndicators opts-                        , sampleSize = n-                        , parseSafe = safeRead opts-                        , parseDateFormat = dateFormat opts-                        }-             in parseFromExamples parseOpts col-        generateColumn col =-            parseTypes $-                Vector.fromListN-                    numRow-                    ( map-                        ( \row ->-                            (stripQuotes . extractField')-                                (row * numCol + col)-                        )-                        [dataStartRow .. totalRows - 1]-                    )-        columns =-            Vector.fromListN-                numCol-                ( map generateColumn [0 .. numCol - 1]-                    `using` parList rpar-                )-        columnIndices =-            M.fromList $-                zip (Vector.toList columnNames) [0 ..]-        dataframeDimensions = (numRow, numCol)-    return $-        DataFrame columns columnIndices dataframeDimensions M.empty--{-# INLINE extractField #-}-extractField ::-    VS.Vector Word8 ->-    VS.Vector CSize ->-    Int ->-    Text-extractField file indices position =-    Text.strip-        . TextEncoding.decodeUtf8Lenient-        . unsafeToByteString-        $ VS.slice-            previous-            (next - previous)-            file-  where-    previous =-        if position == 0-            then 0-            else 1 + fromIntegral (indices VS.! (position - 1))-    next = fromIntegral $ indices VS.! position-    unsafeToByteString :: VS.Vector Word8 -> BS.ByteString-    unsafeToByteString v = PS (castForeignPtr ptr) 0 len-      where-        (ptr, len) = VS.unsafeToForeignPtr0 v--foreign import capi "process_csv.h get_delimiter_indices"-    get_delimiter_indices ::-        Ptr CUChar -> -- input-        CSize -> -- input size-        CUChar -> -- separator character-        Ptr CSize -> -- result array-        IO CSize -- occupancy of result array--{-# INLINE getDelimiterIndices #-}-getDelimiterIndices ::-    Word8 ->-    Int ->-    VS.Vector Word8 ->-    IO (VS.Vector CSize)-getDelimiterIndices separator originalLen csvFile =-    VS.unsafeWith csvFile $ \buffer -> do-        let paddedLen = VS.length csvFile-        -- GC-managed pinned memory: freed automatically, no leak in streaming use.-        resultMV <- VSM.unsafeNew paddedLen-        num_fields <--            VSM.unsafeWith resultMV $ \indicesPtr ->-                get_delimiter_indices-                    (castPtr buffer)-                    (fromIntegral paddedLen)-                    (fromIntegral separator)-                    (castPtr indicesPtr)-        if num_fields == -1-            then do-                -- Haskell state-machine fallback, writing directly into resultMV.-                let trans = stateTransitionTable separator-                    processChar (!state, !idx) i byte =-                        case state of-                            UnEscaped ->-                                if byte == lf || byte == separator-                                    then do-                                        VSM.unsafeWrite resultMV idx (fromIntegral i)-                                        return (toEnum (trans ! (fromEnum state, byte)), idx + 1)-                                    else return (toEnum (trans ! (fromEnum state, byte)), idx)-                            Escaped ->-                                return (toEnum (trans ! (fromEnum state, byte)), idx)-                (_, finalIdx) <- VS.ifoldM' processChar (UnEscaped, 0 :: Int) csvFile-                finalLen <--                    if originalLen > 0 && csvFile VS.! (originalLen - 1) /= lf-                        then do-                            VSM.unsafeWrite resultMV finalIdx (fromIntegral originalLen)-                            return (finalIdx + 1)-                        else return finalIdx-                VS.unsafeFreeze (VSM.slice 0 finalLen resultMV)-            else do-                let n = fromIntegral num_fields-                finalLen <--                    if originalLen > 0 && csvFile VS.! (originalLen - 1) /= lf-                        then do-                            VSM.write resultMV n (fromIntegral originalLen)-                            return (n + 1)-                        else return n-                VS.unsafeFreeze (VSM.slice 0 finalLen resultMV)---- We have a Native version in case the C version--- cannot be used. For example if neither ARM_NEON--- nor AVX2 are available--lf, cr, comma, tab, quote :: Word8-lf = 0x0A-cr = 0x0D-comma = 0x2C-tab = 0x09-quote = 0x22---- We parse using a state machine-data State-    = UnEscaped -- non quoted-    | Escaped -- quoted-    deriving (Enum)--{-# INLINE stateTransitionTable #-}-stateTransitionTable :: Word8 -> UArray (Int, Word8) Int-stateTransitionTable separator = array ((0, 0), (1, 255)) [(i, f i) | i <- range ((0, 0), (1, 255))]-  where-    f (0, character)-        -- Unescaped newline-        | character == 0x0A = fromEnum UnEscaped-        -- Unescaped separator-        | character == separator = fromEnum UnEscaped-        -- Unescaped quote-        | character == 0x22 = fromEnum Escaped-        | otherwise = fromEnum UnEscaped-    -- Escaped quote-    -- escaped quote in fields are dealt as-    -- consecutive quoted sections of a field-    -- example: If we have-    -- field1, "abc""def""ghi, field3-    -- we end up processing abc, def, and ghi-    -- as consecutive quoted strings.-    f (1, 0x22) = fromEnum UnEscaped-    -- Everything else-    f (state, _) = state--{-# INLINE getDelimiterIndices_ #-}-getDelimiterIndices_ ::-    Word8 ->-    Int ->-    VS.Vector Word8 ->-    Ptr CSize ->-    IO (VS.Vector CSize)-getDelimiterIndices_ separator originalLen csvFile resultPtr = do-    resultVector <- resultVectorM-    (_, resultLen) <--        VS.ifoldM'-            (processCharacter resultVector)-            (UnEscaped, 0)-            csvFile-    -- Handle the case where the file doesn't end with a newline-    -- We need to add a final delimiter for the last field-    finalResultLen <--        if originalLen > 0 && csvFile VS.! (originalLen - 1) /= lf-            then do-                VSM.write resultVector resultLen (fromIntegral originalLen)-                return (resultLen + 1)-            else return resultLen-    VS.unsafeFreeze $ VSM.slice 0 finalResultLen resultVector-  where-    paddedLen = VS.length csvFile-    resultVectorM = do-        resultForeignPtr <- newForeignPtr_ resultPtr-        return $ VSM.unsafeFromForeignPtr0 resultForeignPtr paddedLen-    transitionTable = stateTransitionTable separator-    processCharacter ::-        VSM.IOVector CSize ->-        (State, Int) ->-        Int ->-        Word8 ->-        IO (State, Int)-    processCharacter-        resultVector-        (!state, !resultIndex)-        index-        character =-            case state of-                UnEscaped ->-                    if character == lf || character == separator-                        then do-                            VSM.write-                                resultVector-                                resultIndex-                                (fromIntegral index)-                            return (newState, resultIndex + 1)-                        else return (newState, resultIndex)-                Escaped -> return (newState, resultIndex)-          where-            newState =-                toEnum $-                    transitionTable-                        ! (fromEnum state, character)--{-# INLINE countColumnsInFirstRow #-}-countColumnsInFirstRow ::-    VS.Vector Word8 ->-    VS.Vector CSize ->-    Int-countColumnsInFirstRow file indices-    | VS.length indices == 0 = 0-    | otherwise =-        1-            + VS.length-                ( VS.takeWhile-                    (\i -> file VS.! fromIntegral i /= lf)-                    indices-                )

@@ -28,1329 +28,1698 @@  import Control.DeepSeq (NFData (..), rnf) import Control.Exception (throw)-import Control.Monad.ST (runST)-import Data.Kind (Type)-import Data.Maybe-import Data.These-import Data.Type.Equality (TestEquality (..))-import DataFrame.Errors-import DataFrame.Internal.Parsing-import DataFrame.Internal.Types-import System.IO.Unsafe (unsafePerformIO)-import Type.Reflection--{- | Our representation of a column is a GADT that can store data based on the underlying data.--This allows us to pattern match on data kinds and limit some operations to only some-kinds of vectors. E.g. operations for missing data only happen in an OptionalColumn.--}-data Column where-    BoxedColumn :: (Columnable a) => VB.Vector a -> Column-    UnboxedColumn :: (Columnable a, VU.Unbox a) => VU.Vector a -> Column-    OptionalColumn :: (Columnable a) => VB.Vector (Maybe a) -> Column--{- | A mutable companion struct to dataframe columns.--Used mostly as an intermediate structure for I/O.--}-data MutableColumn where-    MBoxedColumn :: (Columnable a) => VBM.IOVector a -> MutableColumn-    MUnboxedColumn :: (Columnable a, VU.Unbox a) => VUM.IOVector a -> MutableColumn-    MOptionalColumn :: (Columnable a) => VBM.IOVector (Maybe a) -> MutableColumn--{- | A TypedColumn is a wrapper around our type-erased column.-It is used to type check expressions on columns.--Note: there is no guarantee that the Phanton type is the-same as the underlying vector type.--}-data TypedColumn a where-    TColumn :: (Columnable a) => Column -> TypedColumn a--instance (Eq a) => Eq (TypedColumn a) where-    (==) :: (Eq a) => TypedColumn a -> TypedColumn a -> Bool-    (==) (TColumn a) (TColumn b) = a == b--instance (Ord a) => Ord (TypedColumn a) where-    compare :: (Ord a) => TypedColumn a -> TypedColumn a -> Ordering-    compare (TColumn a) (TColumn b) = compare a b---- | Gets the underlying value from a TypedColumn.-unwrapTypedColumn :: TypedColumn a -> Column-unwrapTypedColumn (TColumn value) = value---- | Gets the underlying vector from a TypedColumn.-vectorFromTypedColumn :: TypedColumn a -> VB.Vector a-vectorFromTypedColumn (TColumn value) = either throw id (toVector value)---- | Checks if a column contains missing values.-hasMissing :: Column -> Bool-hasMissing (OptionalColumn column) = True-hasMissing _ = False---- | Checks if a column contains only missing values.-allMissing :: Column -> Bool-allMissing (OptionalColumn column) = VB.length (VB.filter isNothing column) == VB.length column-allMissing _ = False---- | Checks if a column contains numeric values.-isNumeric :: Column -> Bool-isNumeric (UnboxedColumn (vec :: VU.Vector a)) = case sNumeric @a of-    STrue -> True-    _ -> False-isNumeric (BoxedColumn (vec :: VB.Vector a)) = case testEquality (typeRep @a) (typeRep @Integer) of-    Nothing -> False-    Just Refl -> True-isNumeric _ = False---- | Checks if a column is of a given type values.-hasElemType :: forall a. (Columnable a) => Column -> Bool-hasElemType = \case-    BoxedColumn (column :: VB.Vector b) -> check (typeRep @b)-    UnboxedColumn (column :: VU.Vector b) -> check (typeRep @b)-    OptionalColumn (column :: VB.Vector b) -> check (typeRep @b)-  where-    check :: forall (b :: Type). TypeRep b -> Bool-    check = isJust . testEquality (typeRep @a)---- | An internal/debugging function to get the column type of a column.-columnVersionString :: Column -> String-columnVersionString column = case column of-    BoxedColumn _ -> "Boxed"-    UnboxedColumn _ -> "Unboxed"-    OptionalColumn _ -> "Optional"--{- | An internal/debugging function to get the type stored in the outermost vector-of a column.--}-columnTypeString :: Column -> String-columnTypeString column = case column of-    BoxedColumn (column :: VB.Vector a) -> show (typeRep @a)-    UnboxedColumn (column :: VU.Vector a) -> show (typeRep @a)-    OptionalColumn (column :: VB.Vector a) -> show (typeRep @a)--instance (Show a) => Show (TypedColumn a) where-    show :: (Show a) => TypedColumn a -> String-    show (TColumn col) = show col--instance NFData Column where-    rnf (BoxedColumn (v :: VB.Vector a)) = VB.foldl' (const (`seq` ())) () v-    rnf (UnboxedColumn v) = v `seq` ()-    rnf (OptionalColumn (v :: VB.Vector (Maybe a))) = VB.foldl' (const (`seq` ())) () v--instance Show Column where-    show :: Column -> String-    show (BoxedColumn column) = show column-    show (UnboxedColumn column) = show column-    show (OptionalColumn column) = show column--instance Eq Column where-    (==) :: Column -> Column -> Bool-    (==) (BoxedColumn (a :: VB.Vector t1)) (BoxedColumn (b :: VB.Vector t2)) =-        case testEquality (typeRep @t1) (typeRep @t2) of-            Nothing -> False-            Just Refl -> a == b-    (==) (OptionalColumn (a :: VB.Vector t1)) (OptionalColumn (b :: VB.Vector t2)) =-        case testEquality (typeRep @t1) (typeRep @t2) of-            Nothing -> False-            Just Refl -> a == b-    (==) (UnboxedColumn (a :: VU.Vector t1)) (UnboxedColumn (b :: VU.Vector t2)) =-        case testEquality (typeRep @t1) (typeRep @t2) of-            Nothing -> False-            Just Refl -> a == b-    (==) _ _ = False---- Generalised LEQ that does reflection.-generalLEQ ::-    forall a b. (Typeable a, Typeable b, Ord a, Ord b) => a -> b -> Bool-generalLEQ x y = case testEquality (typeRep @a) (typeRep @b) of-    Nothing -> False-    Just Refl -> x <= y--instance Ord Column where-    (<=) :: Column -> Column -> Bool-    (<=) (BoxedColumn (a :: VB.Vector t1)) (BoxedColumn (b :: VB.Vector t2)) = generalLEQ a b-    (<=) (OptionalColumn (a :: VB.Vector t1)) (OptionalColumn (b :: VB.Vector t2)) = generalLEQ a b-    (<=) (UnboxedColumn (a :: VU.Vector t1)) (UnboxedColumn (b :: VU.Vector t2)) = generalLEQ a b-    (<=) _ _ = False--{- | A class for converting a vector to a column of the appropriate type.-Given each Rep we tell the `toColumnRep` function which Column type to pick.--}-class ColumnifyRep (r :: Rep) a where-    toColumnRep :: VB.Vector a -> Column---- | Constraint synonym for what we can put into columns.-type Columnable a =-    ( Columnable' a-    , ColumnifyRep (KindOf a) a-    , UnboxIf a-    , IntegralIf a-    , FloatingIf a-    , SBoolI (Unboxable a)-    , SBoolI (Numeric a)-    , SBoolI (IntegralTypes a)-    , SBoolI (FloatingTypes a)-    )--instance-    (Columnable a, VU.Unbox a) =>-    ColumnifyRep 'RUnboxed a-    where-    toColumnRep :: (Columnable a, VUM.Unbox a) => VB.Vector a -> Column-    toColumnRep = UnboxedColumn . VU.convert--instance-    (Columnable a) =>-    ColumnifyRep 'RBoxed a-    where-    toColumnRep :: (Columnable a) => VB.Vector a -> Column-    toColumnRep = BoxedColumn--instance-    (Columnable a) =>-    ColumnifyRep 'ROptional (Maybe a)-    where-    toColumnRep = OptionalColumn--{- | O(n) Convert a vector to a column. Automatically picks the best representation of a vector to store the underlying data in.--__Examples:__--@-> import qualified Data.Vector as V-> fromVector (VB.fromList [(1 :: Int), 2, 3, 4])-[1,2,3,4]-@--}-fromVector ::-    forall a.-    (Columnable a, ColumnifyRep (KindOf a) a) =>-    VB.Vector a -> Column-fromVector = toColumnRep @(KindOf a)--{- | O(n) Convert an unboxed vector to a column. This avoids the extra conversion if you already have the data in an unboxed vector.--__Examples:__--@-> import qualified Data.Vector.Unboxed as V-> fromUnboxedVector (VB.fromList [(1 :: Int), 2, 3, 4])-[1,2,3,4]-@--}-fromUnboxedVector ::-    forall a. (Columnable a, VU.Unbox a) => VU.Vector a -> Column-fromUnboxedVector = UnboxedColumn--{- | O(n) Convert a list to a column. Automatically picks the best representation of a vector to store the underlying data in.--__Examples:__--@-> fromList [(1 :: Int), 2, 3, 4]-[1,2,3,4]-@--}-fromList ::-    forall a.-    (Columnable a, ColumnifyRep (KindOf a) a) =>-    [a] -> Column-fromList = toColumnRep @(KindOf a) . VB.fromList---- An internal helper for type errors-throwTypeMismatch ::-    forall (a :: Type) (b :: Type).-    (Typeable a, Typeable b) => Either DataFrameException Column-throwTypeMismatch =-    Left $-        TypeMismatchException-            MkTypeErrorContext-                { userType = Right (typeRep @b)-                , expectedType = Right (typeRep @a)-                , callingFunctionName = Nothing-                , errorColumnName = Nothing-                }---- | An internal function to map a function over the values of a column.-mapColumn ::-    forall b c.-    (Columnable b, Columnable c) =>-    (b -> c) -> Column -> Either DataFrameException Column-mapColumn f = \case-    BoxedColumn (col :: VB.Vector a) -> run col-    OptionalColumn (col :: VB.Vector (Maybe a)) -> runOpt col-    UnboxedColumn (col :: VU.Vector a) -> runUnboxed col-  where-    run :: forall a. (Typeable a) => VB.Vector a -> Either DataFrameException Column-    run col = case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> Right (fromVector @c (VB.map f col))-        Nothing -> throwTypeMismatch @a @b--    -- For OptionalColumn: first try exact match (b = Maybe a, user maps over Maybe-    -- values directly), then try lenient fmap fallback (b = a, auto-fmap over inner).-    runOpt ::-        forall a.-        (Columnable a) => VB.Vector (Maybe a) -> Either DataFrameException Column-    runOpt col = case testEquality (typeRep @(Maybe a)) (typeRep @b) of-        Just Refl -> Right (fromVector @c (VB.map f col))-        Nothing -> case testEquality (typeRep @a) (typeRep @b) of-            Just Refl -> Right (OptionalColumn (VB.map (fmap f) col))-            Nothing -> throwTypeMismatch @(Maybe a) @b--    runUnboxed ::-        forall a.-        (Typeable a, VU.Unbox a) => VU.Vector a -> Either DataFrameException Column-    runUnboxed col = case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> Right $ case sUnbox @c of-            STrue -> UnboxedColumn (VU.map f col)-            SFalse -> fromVector @c (VB.generate (VU.length col) (f . VU.unsafeIndex col))-        Nothing -> throwTypeMismatch @a @b-{-# INLINEABLE mapColumn #-}---- | Applies a function that returns an unboxed result to an unboxed vector, storing the result in a column.-imapColumn ::-    forall b c.-    (Columnable b, Columnable c) =>-    (Int -> b -> c) -> Column -> Either DataFrameException Column-imapColumn f = \case-    BoxedColumn (col :: VB.Vector a) -> run col-    OptionalColumn (col :: VB.Vector (Maybe a)) -> runOpt col-    UnboxedColumn (col :: VU.Vector a) -> runUnboxed col-  where-    run :: forall a. (Typeable a) => VB.Vector a -> Either DataFrameException Column-    run col = case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> Right (fromVector @c (VB.imap f col))-        Nothing -> throwTypeMismatch @a @b--    runOpt ::-        forall a.-        (Columnable a) => VB.Vector (Maybe a) -> Either DataFrameException Column-    runOpt col = case testEquality (typeRep @(Maybe a)) (typeRep @b) of-        Just Refl -> Right (fromVector @c (VB.imap f col))-        Nothing -> case testEquality (typeRep @a) (typeRep @b) of-            Just Refl -> Right (OptionalColumn (VB.imap (fmap . f) col))-            Nothing -> throwTypeMismatch @(Maybe a) @b--    runUnboxed ::-        forall a.-        (Typeable a, VU.Unbox a) => VU.Vector a -> Either DataFrameException Column-    runUnboxed col = case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> Right $ case sUnbox @c of-            STrue -> UnboxedColumn (VU.imap f col)-            SFalse -> BoxedColumn (VB.imap f (VG.convert col))-        Nothing -> throwTypeMismatch @a @b---- | O(1) Gets the number of elements in the column.-columnLength :: Column -> Int-columnLength (BoxedColumn xs) = VB.length xs-columnLength (UnboxedColumn xs) = VU.length xs-columnLength (OptionalColumn xs) = VB.length xs-{-# INLINE columnLength #-}---- | O(n) Gets the number of elements in the column.-numElements :: Column -> Int-numElements (BoxedColumn xs) = VB.length xs-numElements (UnboxedColumn xs) = VU.length xs-numElements (OptionalColumn xs) = VB.foldl' (\acc x -> acc + fromEnum (isJust x)) 0 xs-{-# INLINE numElements #-}---- | O(n) Takes the first n values of a column.-takeColumn :: Int -> Column -> Column-takeColumn n (BoxedColumn xs) = BoxedColumn $ VG.take n xs-takeColumn n (UnboxedColumn xs) = UnboxedColumn $ VG.take n xs-takeColumn n (OptionalColumn xs) = OptionalColumn $ VG.take n xs-{-# INLINE takeColumn #-}---- | O(n) Takes the last n values of a column.-takeLastColumn :: Int -> Column -> Column-takeLastColumn n column = sliceColumn (columnLength column - n) n column-{-# INLINE takeLastColumn #-}---- | O(n) Takes n values after a given column index.-sliceColumn :: Int -> Int -> Column -> Column-sliceColumn start n (BoxedColumn xs) = BoxedColumn $ VG.slice start n xs-sliceColumn start n (UnboxedColumn xs) = UnboxedColumn $ VG.slice start n xs-sliceColumn start n (OptionalColumn xs) = OptionalColumn $ VG.slice start n xs-{-# INLINE sliceColumn #-}---- | O(n) Selects the elements at a given set of indices. Does not change the order.-atIndicesStable :: VU.Vector Int -> Column -> Column-atIndicesStable indexes (BoxedColumn column) =-    BoxedColumn $-        VB.generate-            (VU.length indexes)-            (\i -> column `VB.unsafeIndex` (indexes `VU.unsafeIndex` i))-atIndicesStable indexes (UnboxedColumn column) = UnboxedColumn $ VU.unsafeBackpermute column indexes-atIndicesStable indexes (OptionalColumn column) =-    OptionalColumn $-        VB.generate-            (VU.length indexes)-            (\i -> column `VB.unsafeIndex` (indexes `VU.unsafeIndex` i))-{-# INLINE atIndicesStable #-}--{- | Like 'atIndicesStable' but treats negative indices as null,-producing an 'OptionalColumn'. Keeps the index vector fully-unboxed (no @VB.Vector (Maybe Int)@).--}-gatherWithSentinel :: VU.Vector Int -> Column -> Column-gatherWithSentinel indices col =-    let !n = VU.length indices-     in case col of-            BoxedColumn v ->-                OptionalColumn $-                    VB.generate n $ \i ->-                        let !idx = indices `VU.unsafeIndex` i-                         in if idx < 0 then Nothing else Just (v `VB.unsafeIndex` idx)-            UnboxedColumn v ->-                OptionalColumn $-                    VB.generate n $ \i ->-                        let !idx = indices `VU.unsafeIndex` i-                         in if idx < 0 then Nothing else Just (v `VU.unsafeIndex` idx)-            OptionalColumn v ->-                OptionalColumn $-                    VB.generate n $ \i ->-                        let !idx = indices `VU.unsafeIndex` i-                         in if idx < 0 then Nothing else v `VB.unsafeIndex` idx-{-# INLINE gatherWithSentinel #-}---- | Internal helper to get indices in a boxed vector.-getIndices :: VU.Vector Int -> VB.Vector a -> VB.Vector a-getIndices indices xs = VB.generate (VU.length indices) (\i -> xs VB.! (indices VU.! i))-{-# INLINE getIndices #-}---- | Internal helper to get indices in an unboxed vector.-getIndicesUnboxed :: (VU.Unbox a) => VU.Vector Int -> VU.Vector a -> VU.Vector a-getIndicesUnboxed indices xs = VU.generate (VU.length indices) (\i -> xs VU.! (indices VU.! i))-{-# INLINE getIndicesUnboxed #-}--findIndices ::-    forall a.-    (Columnable a) =>-    (a -> Bool) ->-    Column ->-    Either DataFrameException (VU.Vector Int)-findIndices pred = \case-    BoxedColumn (v :: VB.Vector b) -> run v VG.convert-    OptionalColumn (v :: VB.Vector b) -> run v VG.convert-    UnboxedColumn (v :: VU.Vector b) -> run v id-  where-    run ::-        forall b v.-        (Typeable b, VG.Vector v b, VG.Vector v Int) =>-        v b ->-        (v Int -> VU.Vector Int) ->-        Either DataFrameException (VU.Vector Int)-    run column finalize = case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> Right . finalize $ VG.findIndices pred column-        Nothing ->-            Left $-                TypeMismatchException-                    MkTypeErrorContext-                        { userType = Right (typeRep @a)-                        , expectedType = Right (typeRep @b)-                        , callingFunctionName = Just "findIndices"-                        , errorColumnName = Nothing-                        }---- | An internal function that returns a vector of how indexes change after a column is sorted.-sortedIndexes :: Bool -> Column -> VU.Vector Int-sortedIndexes asc = \case-    BoxedColumn column -> sortWorker VG.convert column-    UnboxedColumn column -> sortWorker id column-    OptionalColumn column -> sortWorker VG.convert column-  where-    sortWorker ::-        (VG.Vector v a, Ord a, VG.Vector v (Int, a), VG.Vector v Int) =>-        (v Int -> VU.Vector Int) -> v a -> VU.Vector Int-    sortWorker finalize column = runST $ do-        withIndexes <- VG.thaw $ VG.indexed column-        let cmp = if asc then compare else flip compare-        VA.sortBy (\(_, b) (_, b') -> cmp b b') withIndexes-        sorted <- VG.unsafeFreeze withIndexes-        return $ finalize $ VG.map fst sorted-{-# INLINE sortedIndexes #-}---- | Fold (right) column with index.-ifoldrColumn ::-    forall a b.-    (Columnable a, Columnable b) =>-    (Int -> a -> b -> b) -> b -> Column -> Either DataFrameException b-ifoldrColumn f acc = \case-    BoxedColumn column -> foldrWorker column-    UnboxedColumn column -> foldrWorker column-    OptionalColumn column -> foldrWorker column-  where-    foldrWorker ::-        forall c v.-        (Typeable c, VG.Vector v c) =>-        v c ->-        Either DataFrameException b-    foldrWorker vec = case testEquality (typeRep @a) (typeRep @c) of-        Just Refl -> pure $ VG.ifoldr f acc vec-        Nothing ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @a)-                        , expectedType = Right (typeRep @c)-                        , callingFunctionName = Just "ifoldrColumn"-                        , errorColumnName = Nothing-                        }-                    )--foldlColumn ::-    forall a b.-    (Columnable a, Columnable b) =>-    (b -> a -> b) -> b -> Column -> Either DataFrameException b-foldlColumn f acc = \case-    BoxedColumn column -> foldlWorker column-    UnboxedColumn column -> foldlWorker column-    OptionalColumn column -> foldlWorker column-  where-    foldlWorker ::-        forall c v.-        (Typeable c, VG.Vector v c) =>-        v c ->-        Either DataFrameException b-    foldlWorker vec = case testEquality (typeRep @a) (typeRep @c) of-        Just Refl -> pure $ VG.foldl' f acc vec-        Nothing ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @a)-                        , expectedType = Right (typeRep @c)-                        , callingFunctionName = Just "ifoldrColumn"-                        , errorColumnName = Nothing-                        }-                    )--foldl1Column ::-    forall a.-    (Columnable a) =>-    (a -> a -> a) -> Column -> Either DataFrameException a-foldl1Column f = \case-    BoxedColumn column -> foldl1Worker column-    UnboxedColumn column -> foldl1Worker column-    OptionalColumn column -> foldl1Worker column-  where-    foldl1Worker ::-        forall c v.-        (Typeable c, VG.Vector v c) =>-        v c ->-        Either DataFrameException a-    foldl1Worker vec = case testEquality (typeRep @a) (typeRep @c) of-        Just Refl -> pure $ VG.foldl1' f vec-        Nothing ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @a)-                        , expectedType = Right (typeRep @c)-                        , callingFunctionName = Just "foldl1Column"-                        , errorColumnName = Nothing-                        }-                    )--{- | O(n) Seedless fold over groups using the first element of each group as seed.-Like 'foldDirectGroups' but for the case where no initial accumulator is available.--}-foldl1DirectGroups ::-    forall a.-    (Columnable a) =>-    (a -> a -> a) ->-    Column ->-    VU.Vector Int ->-    VU.Vector Int ->-    Either DataFrameException Column-foldl1DirectGroups f col valueIndices offsets-    | VU.length offsets <= 1 = pure $ fromVector @a VB.empty-    | otherwise = case col of-        UnboxedColumn (vec :: VU.Vector d) -> UnboxedColumn <$> foldl1Worker vec-        BoxedColumn (vec :: VB.Vector d) -> BoxedColumn <$> foldl1Worker vec-        OptionalColumn (vec :: VB.Vector (Maybe d)) -> OptionalColumn <$> foldl1Worker vec-  where-    foldl1Worker ::-        forall c v.-        (Typeable c, VG.Vector v c) =>-        v c ->-        Either DataFrameException (v c)-    foldl1Worker vec = case testEquality (typeRep @a) (typeRep @c) of-        Just Refl ->-            Right $-                VG.generate (VU.length offsets - 1) foldGroup-          where-            foldGroup k =-                let !s = VU.unsafeIndex offsets k-                    !e = VU.unsafeIndex offsets (k + 1)-                    !seed = VG.unsafeIndex vec (VU.unsafeIndex valueIndices s)-                 in go (s + 1) e seed-            go !i !e !acc-                | i >= e = acc-                | otherwise =-                    go (i + 1) e $!-                        f acc (VG.unsafeIndex vec (VU.unsafeIndex valueIndices i))-        Nothing ->-            Left $-                TypeMismatchException-                    MkTypeErrorContext-                        { userType = Right (typeRep @a)-                        , expectedType = Right (typeRep @c)-                        , callingFunctionName = Just "foldl1DirectGroups"-                        , errorColumnName = Nothing-                        }-{-# INLINEABLE foldl1DirectGroups #-}--{- | O(n) fold over groups by scanning the column LINEARLY.-rowToGroup[i] = group index for row i.-Avoids random column reads; random writes go to the accumulator array which is-small (nGroups entries) and typically cache-resident.-When @acc@ is unboxable, uses an unboxed mutable vector for the accumulator-array, eliminating pointer indirection on every read/write.--}-foldLinearGroups ::-    forall b acc.-    (Columnable b, Columnable acc) =>-    (acc -> b -> acc) ->-    acc ->-    Column ->-    VU.Vector Int -> -- rowToGroup (length n)-    Int -> -- nGroups-    Either DataFrameException Column-foldLinearGroups f seed col rowToGroup nGroups-    | nGroups == 0 = Right (fromVector @acc VB.empty)-    | otherwise = case col of-        UnboxedColumn (vec :: VU.Vector d) -> foldLinearWorker vec-        BoxedColumn (vec :: VB.Vector d) -> foldLinearWorker vec-        OptionalColumn (vec :: VB.Vector (Maybe d)) -> foldLinearWorker vec-  where-    foldLinearWorker ::-        forall c v.-        (Typeable c, VG.Vector v c) =>-        v c ->-        Either DataFrameException Column-    foldLinearWorker vec = case testEquality (typeRep @b) (typeRep @c) of-        Just Refl ->-            Right $-                unsafePerformIO $-                    runWith-                        ( \readAt writeAt ->-                            VG.iforM_ vec $ \row x -> do-                                let !k = VG.unsafeIndex rowToGroup row-                                cur <- readAt k-                                writeAt k $! f cur x-                        )-        Nothing ->-            Left $-                TypeMismatchException-                    MkTypeErrorContext-                        { userType = Right (typeRep @b)-                        , expectedType = Right (typeRep @c)-                        , callingFunctionName = Just "foldLinearGroups"-                        , errorColumnName = Nothing-                        }--    -- \| Allocate accumulators, run the traversal, return a frozen Column.-    -- When @acc@ is unboxable, uses an unboxed mutable vector (no pointer-    -- indirection per read/write) and returns UnboxedColumn directly —-    -- avoiding a round-trip through VB.Vector.-    runWith :: ((Int -> IO acc) -> (Int -> acc -> IO ()) -> IO ()) -> IO Column-    runWith body = case sUnbox @acc of-        STrue -> do-            accs <- VUM.replicate nGroups seed-            body (VUM.unsafeRead accs) (VUM.unsafeWrite accs)-            UnboxedColumn <$> VU.unsafeFreeze accs-        SFalse -> do-            accs <- VBM.replicate nGroups seed-            body (VBM.unsafeRead accs) (VBM.unsafeWrite accs)-            fromVector @acc <$> VB.unsafeFreeze accs-    {-# INLINE runWith #-}-{-# INLINEABLE foldLinearGroups #-}--headColumn :: forall a. (Columnable a) => Column -> Either DataFrameException a-headColumn = \case-    BoxedColumn col -> headWorker col-    UnboxedColumn col -> headWorker col-    OptionalColumn col -> headWorker col-  where-    headWorker ::-        forall c v.-        (Typeable c, VG.Vector v c) =>-        v c ->-        Either DataFrameException a-    headWorker vec = case testEquality (typeRep @a) (typeRep @c) of-        Just Refl ->-            if VG.null vec-                then Left (EmptyDataSetException "headColumn")-                else pure (VG.head vec)-        Nothing ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @a)-                        , expectedType = Right (typeRep @c)-                        , callingFunctionName = Just "headColumn"-                        , errorColumnName = Nothing-                        }-                    )---- | An internal, column version of zip.-zipColumns :: Column -> Column -> Column-zipColumns (BoxedColumn column) (BoxedColumn other) = BoxedColumn (VG.zip column other)-zipColumns (BoxedColumn column) (UnboxedColumn other) =-    BoxedColumn-        ( VB.generate-            (min (VG.length column) (VG.length other))-            (\i -> (column VG.! i, other VG.! i))-        )-zipColumns (BoxedColumn column) (OptionalColumn optcolumn) = BoxedColumn (VG.zip (VB.convert column) optcolumn)-zipColumns (UnboxedColumn column) (BoxedColumn other) =-    BoxedColumn-        ( VB.generate-            (min (VG.length column) (VG.length other))-            (\i -> (column VG.! i, other VG.! i))-        )-zipColumns (UnboxedColumn column) (UnboxedColumn other) = UnboxedColumn (VG.zip column other)-zipColumns (UnboxedColumn column) (OptionalColumn optcolumn) = BoxedColumn (VG.zip (VB.convert column) optcolumn)-zipColumns (OptionalColumn optcolumn) (BoxedColumn column) = BoxedColumn (VG.zip optcolumn (VB.convert column))-zipColumns (OptionalColumn optcolumn) (UnboxedColumn column) = BoxedColumn (VG.zip optcolumn (VB.convert column))-zipColumns (OptionalColumn optcolumn) (OptionalColumn optother) = BoxedColumn (VG.zip optcolumn optother)-{-# INLINE zipColumns #-}---- | Merge two columns using `These`.-mergeColumns :: Column -> Column -> Column-mergeColumns colA colB = case (colA, colB) of-    (OptionalColumn c1, OptionalColumn c2) ->-        OptionalColumn $ mkVec c1 c2 $ \v1 v2 ->-            case (v1, v2) of-                (Nothing, Nothing) -> Nothing-                (Just x, Nothing) -> Just (This x)-                (Nothing, Just y) -> Just (That y)-                (Just x, Just y) -> Just (These x y)-    (OptionalColumn c1, BoxedColumn c2) -> optReq c1 c2-    (OptionalColumn c1, UnboxedColumn c2) -> optReq c1 c2-    (BoxedColumn c1, OptionalColumn c2) -> reqOpt c1 c2-    (UnboxedColumn c1, OptionalColumn c2) -> reqOpt c1 c2-    (BoxedColumn c1, BoxedColumn c2) -> reqReq c1 c2-    (BoxedColumn c1, UnboxedColumn c2) -> reqReq c1 c2-    (UnboxedColumn c1, BoxedColumn c2) -> reqReq c1 c2-    (UnboxedColumn c1, UnboxedColumn c2) -> reqReq c1 c2-  where-    mkVec c1 c2 combineElements =-        VB.generate-            (min (VG.length c1) (VG.length c2))-            (\i -> combineElements (c1 VG.! i) (c2 VG.! i))-    {-# INLINE mkVec #-}--    reqReq c1 c2 = BoxedColumn $ mkVec c1 c2 These--    reqOpt c1 c2 = BoxedColumn $ mkVec c1 c2 $ \v1 v2 ->-        case v2 of-            Nothing -> This v1-            Just y -> These v1 y--    optReq c1 c2 = BoxedColumn $ mkVec c1 c2 $ \v1 v2 ->-        case v1 of-            Nothing -> That v2-            Just x -> These x v2-{-# INLINE mergeColumns #-}---- | An internal, column version of zipWith.-zipWithColumns ::-    forall a b c.-    (Columnable a, Columnable b, Columnable c) =>-    (a -> b -> c) -> Column -> Column -> Either DataFrameException Column-zipWithColumns f (UnboxedColumn (column :: VU.Vector d)) (UnboxedColumn (other :: VU.Vector e)) = case testEquality (typeRep @a) (typeRep @d) of-    Just Refl -> case testEquality (typeRep @b) (typeRep @e) of-        Just Refl -> pure $ case sUnbox @c of-            STrue -> UnboxedColumn (VU.zipWith f column other)-            SFalse -> fromVector $ VB.zipWith f (VG.convert column) (VG.convert other)-        Nothing ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @b)-                        , expectedType = Right (typeRep @e)-                        , callingFunctionName = Just "zipWithColumns"-                        , errorColumnName = Nothing-                        }-                    )-    Nothing ->-        Left $-            TypeMismatchException-                ( MkTypeErrorContext-                    { userType = Right (typeRep @a)-                    , expectedType = Right (typeRep @d)-                    , callingFunctionName = Just "zipWithColumns"-                    , errorColumnName = Nothing-                    }-                )--- TODO: mchavinda - reuse pattern from interpret where we augment the--- error at the end.-zipWithColumns f left right = case toVector @a left of-    Left (TypeMismatchException context) ->-        Left $-            TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"})-    Left e -> Left e-    Right left' -> case toVector @b right of-        Left (TypeMismatchException context) ->-            Left $-                TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"})-        Left e -> Left e-        Right right' -> pure $ fromVector $ VB.zipWith f left' right'-{-# INLINE zipWithColumns #-}---- Functions for mutable columns (intended for IO).-writeColumn :: Int -> T.Text -> MutableColumn -> IO (Either T.Text Bool)-writeColumn i value (MBoxedColumn (col :: VBM.IOVector a)) =-    let-     in case testEquality (typeRep @a) (typeRep @T.Text) of-            Just Refl ->-                ( if isNullish value-                    then VBM.unsafeWrite col i "" >> return (Left $! value)-                    else VBM.unsafeWrite col i value >> return (Right True)-                )-            Nothing -> return (Left value)-writeColumn i value (MOptionalColumn (col :: VBM.IOVector (Maybe a))) =-    let-     in case testEquality (typeRep @a) (typeRep @T.Text) of-            Just Refl ->-                ( if isNullish value-                    then VBM.unsafeWrite col i Nothing >> return (Left $! value)-                    else VBM.unsafeWrite col i (Just value) >> return (Right True)-                )-            Nothing -> return (Left value)-writeColumn i value (MUnboxedColumn (col :: VUM.IOVector a)) =-    case testEquality (typeRep @a) (typeRep @Int) of-        Just Refl -> case readInt value of-            Just v -> VUM.unsafeWrite col i v >> return (Right True)-            Nothing -> VUM.unsafeWrite col i 0 >> return (Left value)-        Nothing -> case testEquality (typeRep @a) (typeRep @Double) of-            Nothing -> return (Left $! value)-            Just Refl -> case readDouble value of-                Just v -> VUM.unsafeWrite col i v >> return (Right True)-                Nothing -> VUM.unsafeWrite col i 0 >> return (Left $! value)-{-# INLINE writeColumn #-}--freezeColumn' :: [(Int, T.Text)] -> MutableColumn -> IO Column-freezeColumn' nulls (MOptionalColumn col) = OptionalColumn <$> VB.unsafeFreeze col-freezeColumn' nulls (MBoxedColumn col)-    | null nulls = BoxedColumn <$> VB.unsafeFreeze col-    | all (isNullish . snd) nulls =-        OptionalColumn-            . VB.imap (\i v -> if i `elem` map fst nulls then Nothing else Just v)-            <$> VB.unsafeFreeze col-    | otherwise =-        BoxedColumn-            . VB.imap-                ( \i v ->-                    if i `elem` map fst nulls-                        then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls))-                        else Right v-                )-            <$> VB.unsafeFreeze col-freezeColumn' nulls (MUnboxedColumn col)-    | null nulls = UnboxedColumn <$> VU.unsafeFreeze col-    | all (isNullish . snd) nulls =-        VU.unsafeFreeze col >>= \c ->-            return $-                OptionalColumn $-                    VB.generate-                        (VU.length c)-                        (\i -> if i `elem` map fst nulls then Nothing else Just (c VU.! i))-    | otherwise =-        VU.unsafeFreeze col >>= \c ->-            return $-                BoxedColumn $-                    VB.generate-                        (VU.length c)-                        ( \i ->-                            if i `elem` map fst nulls-                                then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls))-                                else Right (c VU.! i)-                        )-{-# INLINE freezeColumn' #-}---- | Fills the end of a column, up to n, with Nothing. Does nothing if column has length greater than n.-expandColumn :: Int -> Column -> Column-expandColumn n (OptionalColumn col) = OptionalColumn $ col <> VB.replicate (n - VG.length col) Nothing-expandColumn n column@(BoxedColumn col)-    | n > VG.length col =-        OptionalColumn $ VB.map Just col <> VB.replicate (n - VG.length col) Nothing-    | otherwise = column-expandColumn n column@(UnboxedColumn col)-    | n > VG.length col =-        OptionalColumn $-            VB.map Just (VU.convert col) <> VB.replicate (n - VG.length col) Nothing-    | otherwise = column---- | Fills the beginning of a column, up to n, with Nothing. Does nothing if column has length greater than n.-leftExpandColumn :: Int -> Column -> Column-leftExpandColumn n column@(OptionalColumn col)-    | n > VG.length col =-        OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> col-    | otherwise = column-leftExpandColumn n column@(BoxedColumn col)-    | n > VG.length col =-        OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> VG.map Just col-    | otherwise = column-leftExpandColumn n column@(UnboxedColumn col)-    | n > VG.length col =-        OptionalColumn $-            VG.replicate (n - VG.length col) Nothing <> VG.map Just (VU.convert col)-    | otherwise = column--{- | Concatenates two columns.-Returns Nothing if the columns are of different types.--}-concatColumns :: Column -> Column -> Either DataFrameException Column-concatColumns left right = case (left, right) of-    (OptionalColumn l, OptionalColumn r) -> case testEquality (typeOf l) (typeOf r) of-        Just Refl -> pure (OptionalColumn $ l <> r)-        Nothing -> Left (mismatchErr (typeOf r) (typeOf l))-    (BoxedColumn l, BoxedColumn r) -> case testEquality (typeOf l) (typeOf r) of-        Just Refl -> pure (BoxedColumn $ l <> r)-        Nothing -> Left (mismatchErr (typeOf r) (typeOf l))-    (UnboxedColumn l, UnboxedColumn r) -> case testEquality (typeOf l) (typeOf r) of-        Just Refl -> pure (UnboxedColumn $ l <> r)-        Nothing -> Left (mismatchErr (typeOf r) (typeOf l))-    _ -> Left (mismatchErr (typeOf right) (typeOf left))-  where-    mismatchErr ::-        forall (x :: Type) (y :: Type). TypeRep x -> TypeRep y -> DataFrameException-    mismatchErr ta tb =-        withTypeable ta $-            withTypeable tb $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right ta-                        , expectedType = Right tb-                        , callingFunctionName = Just "concatColumns"-                        , errorColumnName = Nothing-                        }-                    )--{- | Concatenates two columns.--Works similar to 'concatColumns', but unlike that function, it will also combine columns of different types-by wrapping the values in an Either.--E.g. combining Column containing [1,2] with Column containing ["a","b"]-will result in a Column containing [Left 1, Left 2, Right "a", Right "b"].--}--{- | O(n) Concatenate a list of same-type columns in a single allocation.-All columns must have the same constructor and element type (as they will-within a single Parquet column). Calls 'error' on mismatch.--}-concatManyColumns :: [Column] -> Column-concatManyColumns [] = fromList ([] :: [Maybe Int])-concatManyColumns [c] = c-concatManyColumns (c0 : cs) = case c0 of-    OptionalColumn v0 ->-        let getVec (OptionalColumn v) = case testEquality (typeOf v0) (typeOf v) of-                Just Refl -> v-                Nothing -> error "concatManyColumns: OptionalColumn type mismatch"-            getVec _ = error "concatManyColumns: column constructor mismatch"-         in OptionalColumn (VB.concat (v0 : map getVec cs))-    BoxedColumn v0 ->-        let getVec (BoxedColumn v) = case testEquality (typeOf v0) (typeOf v) of-                Just Refl -> v-                Nothing -> error "concatManyColumns: BoxedColumn type mismatch"-            getVec _ = error "concatManyColumns: column constructor mismatch"-         in BoxedColumn (VB.concat (v0 : map getVec cs))-    UnboxedColumn v0 ->-        let getVec (UnboxedColumn v) = case testEquality (typeOf v0) (typeOf v) of-                Just Refl -> v-                Nothing -> error "concatManyColumns: UnboxedColumn type mismatch"-            getVec _ = error "concatManyColumns: column constructor mismatch"-         in UnboxedColumn (VU.concat (v0 : map getVec cs))--concatColumnsEither :: Column -> Column -> Column-concatColumnsEither (OptionalColumn left) (OptionalColumn right) = case testEquality (typeOf left) (typeOf right) of-    Nothing ->-        OptionalColumn $ fmap (fmap Left) left <> fmap (fmap Right) right-    Just Refl ->-        OptionalColumn $ left <> right-concatColumnsEither (BoxedColumn left) (BoxedColumn right) = case testEquality (typeOf left) (typeOf right) of-    Nothing ->-        BoxedColumn $ fmap Left left <> fmap Right right-    Just Refl ->-        BoxedColumn $ left <> right-concatColumnsEither (UnboxedColumn left) (UnboxedColumn right) = case testEquality (typeOf left) (typeOf right) of-    Nothing ->-        BoxedColumn $ fmap Left (VG.convert left) <> fmap Right (VG.convert right)-    Just Refl ->-        UnboxedColumn $ left <> right-concatColumnsEither (BoxedColumn left) (UnboxedColumn right) =-    BoxedColumn $ fmap Left left <> fmap Right (VG.convert right)-concatColumnsEither (UnboxedColumn left) (BoxedColumn right) =-    BoxedColumn $ fmap Left (VG.convert left) <> fmap Right right-concatColumnsEither (OptionalColumn (left :: VB.Vector (Maybe a))) (BoxedColumn (right :: VB.Vector b)) =-    case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> OptionalColumn $ left <> fmap Just right-        Nothing -> OptionalColumn $ fmap (fmap Left) left <> fmap (Just . Right) right-concatColumnsEither (BoxedColumn (left :: VB.Vector a)) (OptionalColumn (right :: VB.Vector (Maybe b))) =-    case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> OptionalColumn $ fmap Just left <> right-        Nothing -> OptionalColumn $ fmap (Just . Left) left <> fmap (fmap Right) right-concatColumnsEither (OptionalColumn (left :: VB.Vector (Maybe a))) (UnboxedColumn (right :: VU.Vector b)) =-    case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> OptionalColumn $ left <> fmap Just (VG.convert right)-        Nothing ->-            OptionalColumn $ fmap (fmap Left) left <> fmap (Just . Right) (VG.convert right)-concatColumnsEither (UnboxedColumn (left :: VU.Vector a)) (OptionalColumn (right :: VB.Vector (Maybe b))) =-    case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> OptionalColumn $ fmap Just (VG.convert left) <> right-        Nothing ->-            OptionalColumn $ fmap (Just . Left) (VG.convert left) <> fmap (fmap Right) right---- | Allocate a mutable column of size @n@ matching the constructor/type of the given column.-newMutableColumn :: Int -> Column -> IO MutableColumn-newMutableColumn n (OptionalColumn (_ :: VB.Vector (Maybe a))) =-    MOptionalColumn <$> (VBM.new n :: IO (VBM.IOVector (Maybe a)))-newMutableColumn n (BoxedColumn (_ :: VB.Vector a)) =-    MBoxedColumn <$> (VBM.new n :: IO (VBM.IOVector a))-newMutableColumn n (UnboxedColumn (_ :: VU.Vector a)) =-    MUnboxedColumn <$> (VUM.new n :: IO (VUM.IOVector a))---- | Copy a column chunk into a mutable column starting at offset @off@.-copyIntoMutableColumn :: MutableColumn -> Int -> Column -> IO ()-copyIntoMutableColumn (MOptionalColumn (mv :: VBM.IOVector (Maybe b))) off (OptionalColumn (v :: VB.Vector (Maybe a))) =-    case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> VG.imapM_ (\i x -> VBM.unsafeWrite mv (off + i) x) v-        Nothing -> error "copyIntoMutableColumn: Optional type mismatch"-copyIntoMutableColumn (MBoxedColumn (mv :: VBM.IOVector b)) off (BoxedColumn (v :: VB.Vector a)) =-    case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> VG.imapM_ (\i x -> VBM.unsafeWrite mv (off + i) x) v-        Nothing -> error "copyIntoMutableColumn: Boxed type mismatch"-copyIntoMutableColumn (MUnboxedColumn (mv :: VUM.IOVector b)) off (UnboxedColumn (v :: VU.Vector a)) =-    case testEquality (typeRep @a) (typeRep @b) of-        Just Refl -> VG.imapM_ (\i x -> VUM.unsafeWrite mv (off + i) x) v-        Nothing -> error "copyIntoMutableColumn: Unboxed type mismatch"-copyIntoMutableColumn _ _ _ =-    error "copyIntoMutableColumn: constructor mismatch"---- | Freeze a mutable column into an immutable column.-freezeMutableColumn :: MutableColumn -> IO Column-freezeMutableColumn (MOptionalColumn mv) = OptionalColumn <$> VB.unsafeFreeze mv-freezeMutableColumn (MBoxedColumn mv) = BoxedColumn <$> VB.unsafeFreeze mv-freezeMutableColumn (MUnboxedColumn mv) = UnboxedColumn <$> VU.unsafeFreeze mv--{- | O(n) Converts a column to a list. Throws an exception if the wrong type is specified.--__Examples:__--@-> column = fromList [(1 :: Int), 2, 3, 4]-> toList @Int column-[1,2,3,4]-> toList @Double column-exception: ...-@--}-toList :: forall a. (Columnable a) => Column -> [a]-toList xs = case toVector @a xs of-    Left err -> throw err-    Right val -> VB.toList val--{- | Converts a column to a vector of a specific type.--This is a type-safe conversion that requires the column's element type-to exactly match the requested type. You must specify the desired type-via type applications.--==== __Type Parameters__--[@a@] The element type to convert to-[@v@] The vector type (e.g., 'VU.Vector', 'VB.Vector')--==== __Examples__-->>> toVector @Int @VU.Vector column-Right (unboxed vector of Ints)-->>> toVector @Text @VB.Vector column-Right (boxed vector of Text)--==== __Returns__--* 'Right' - The converted vector if types match-* 'Left' 'TypeMismatchException' - If the column's type doesn't match the requested type--==== __See also__--For numeric conversions with automatic type coercion, see 'toDoubleVector',-'toFloatVector', and 'toIntVector'.--}-toVector ::-    forall a v.-    (VG.Vector v a, Columnable a) => Column -> Either DataFrameException (v a)-toVector = \case-    OptionalColumn col -> toVectorWorker col-    BoxedColumn col -> toVectorWorker col-    UnboxedColumn col -> toVectorWorker col-  where-    toVectorWorker ::-        forall c w.-        (Typeable c, VG.Vector w c) =>-        w c ->-        Either DataFrameException (v a)-    toVectorWorker vec = case testEquality (typeRep @a) (typeRep @c) of-        Just Refl -> Right $ VG.convert vec-        Nothing ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @a)-                        , expectedType = Right (typeRep @c)-                        , callingFunctionName = Just "toVector"-                        , errorColumnName = Nothing-                        }-                    )---- Some common types we will use for numerical computing.--{- | Converts a column to an unboxed vector of 'Double' values.--This function performs intelligent type coercion for numeric types:--* If the column is already 'Double', returns it directly-* If the column contains other floating-point types, converts via 'realToFrac'-* If the column contains integral types, converts via 'fromIntegral' (beware of overflow if the type is `Integer`).--==== __Optional column handling__--For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number).-This allows optional numeric data to be represented in the resulting vector.--==== __Returns__--* 'Right' - The converted 'Double' vector-* 'Left' 'TypeMismatchException' - If the column is not numeric--}-toDoubleVector :: Column -> Either DataFrameException (VU.Vector Double)-toDoubleVector column =-    case column of-        UnboxedColumn (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Double) of-            Just Refl -> Right f-            Nothing -> case sFloating @a of-                STrue -> Right (VU.map realToFrac f)-                SFalse -> case sIntegral @a of-                    STrue -> Right (VU.map fromIntegral f)-                    SFalse ->-                        Left $-                            TypeMismatchException-                                ( MkTypeErrorContext-                                    { userType = Right (typeRep @Double)-                                    , expectedType = Right (typeRep @a)-                                    , callingFunctionName = Just "toDoubleVector"-                                    , errorColumnName = Nothing-                                    }-                                )-        OptionalColumn (f :: VB.Vector (Maybe a)) -> case testEquality (typeRep @a) (typeRep @Double) of-            Just Refl -> Right (VB.convert $ VB.map (fromMaybe (read @Double "NaN")) f)-            Nothing -> case sFloating @a of-                STrue ->-                    Right-                        (VB.convert $ VB.map (maybe (read @Double "NaN") realToFrac) f)-                SFalse -> case sIntegral @a of-                    STrue ->-                        Right-                            (VB.convert $ VB.map (maybe (read @Double "NaN") fromIntegral) f)-                    SFalse ->-                        Left $-                            TypeMismatchException-                                ( MkTypeErrorContext-                                    { userType = Right (typeRep @Double)-                                    , expectedType = Right (typeRep @a)-                                    , callingFunctionName = Just "toDoubleVector"-                                    , errorColumnName = Nothing-                                    }-                                )-        BoxedColumn (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of-            Just Refl -> Right (VB.convert $ VB.map fromIntegral f)-            Nothing ->-                Left $-                    TypeMismatchException-                        ( MkTypeErrorContext-                            { userType = Right (typeRep @Double)-                            , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())-                            , callingFunctionName = Just "toDoubleVector"-                            , errorColumnName = Nothing-                            }-                        )--{- | Converts a column to an unboxed vector of 'Float' values.--This function performs intelligent type coercion for numeric types:--* If the column is already 'Float', returns it directly-* If the column contains other floating-point types, converts via 'realToFrac'-* If the column contains integral types, converts via 'fromIntegral'-* If the column is boxed 'Integer', converts via 'fromIntegral' (beware of overflow for 64-bit integers and `Integer`)--==== __Optional column handling__--For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number).-This allows optional numeric data to be represented in the resulting vector.--==== __Returns__--* 'Right' - The converted 'Float' vector-* 'Left' 'TypeMismatchException' - If the column is not numeric--==== __Precision warning__--Converting from 'Double' to 'Float' may result in loss of precision.--}-toFloatVector :: Column -> Either DataFrameException (VU.Vector Float)-toFloatVector column =-    case column of-        UnboxedColumn (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Float) of-            Just Refl -> Right f-            Nothing -> case sFloating @a of-                STrue -> Right (VU.map realToFrac f)-                SFalse -> case sIntegral @a of-                    STrue -> Right (VU.map fromIntegral f)-                    SFalse ->-                        Left $-                            TypeMismatchException-                                ( MkTypeErrorContext-                                    { userType = Right (typeRep @Float)-                                    , expectedType = Right (typeRep @a)-                                    , callingFunctionName = Just "toFloatVector"-                                    , errorColumnName = Nothing-                                    }-                                )-        OptionalColumn (f :: VB.Vector (Maybe a)) -> case testEquality (typeRep @a) (typeRep @Float) of-            Just Refl -> Right (VB.convert $ VB.map (fromMaybe (read @Float "NaN")) f)-            Nothing -> case sFloating @a of-                STrue ->-                    Right-                        (VB.convert $ VB.map (maybe (read @Float "NaN") realToFrac) f)-                SFalse -> case sIntegral @a of-                    STrue ->-                        Right-                            (VB.convert $ VB.map (maybe (read @Float "NaN") fromIntegral) f)-                    SFalse ->-                        Left $-                            TypeMismatchException-                                ( MkTypeErrorContext-                                    { userType = Right (typeRep @Float)-                                    , expectedType = Right (typeRep @a)-                                    , callingFunctionName = Just "toFloatVector"-                                    , errorColumnName = Nothing-                                    }-                                )-        BoxedColumn (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of-            Just Refl -> Right (VB.convert $ VB.map fromIntegral f)-            Nothing ->-                Left $-                    TypeMismatchException-                        ( MkTypeErrorContext-                            { userType = Right (typeRep @Float)-                            , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())-                            , callingFunctionName = Just "toFloatVector"-                            , errorColumnName = Nothing-                            }-                        )--{- | Converts a column to an unboxed vector of 'Int' values.--This function performs intelligent type coercion for numeric types:--* If the column is already 'Int', returns it directly-* If the column contains floating-point types, rounds via 'round' and converts-* If the column contains other integral types, converts via 'fromIntegral'-* If the column is boxed 'Integer', converts via 'fromIntegral'--==== __Returns__--* 'Right' - The converted 'Int' vector-* 'Left' 'TypeMismatchException' - If the column is not numeric--==== __Note__--Unlike 'toDoubleVector' and 'toFloatVector', this function does NOT support-'OptionalColumn'. Optional columns must be handled separately.--==== __Rounding behavior__--Floating-point values are rounded to the nearest integer using 'round'.-For example: 2.5 rounds to 2, 3.5 rounds to 4 (banker's rounding).--}-toIntVector :: Column -> Either DataFrameException (VU.Vector Int)-toIntVector column =-    case column of-        UnboxedColumn (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of-            Just Refl -> Right f-            Nothing -> case sFloating @a of-                STrue -> Right (VU.map (round . realToFrac) f)-                SFalse -> case sIntegral @a of-                    STrue -> Right (VU.map fromIntegral f)-                    SFalse ->-                        Left $-                            TypeMismatchException-                                ( MkTypeErrorContext-                                    { userType = Right (typeRep @Int)-                                    , expectedType = Right (typeRep @a)-                                    , callingFunctionName = Just "toIntVector"-                                    , errorColumnName = Nothing-                                    }-                                )-        BoxedColumn (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of-            Just Refl -> Right (VB.convert $ VB.map fromIntegral f)-            Nothing ->-                Left $-                    TypeMismatchException-                        ( MkTypeErrorContext-                            { userType = Right (typeRep @Int)-                            , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())-                            , callingFunctionName = Just "toIntVector"-                            , errorColumnName = Nothing-                            }-                        )-        _ ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @Int)-                        , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())-                        , callingFunctionName = Just "toIntVector"-                        , errorColumnName = Nothing-                        }-                    )--toUnboxedVector ::-    forall a.-    (Columnable a, VU.Unbox a) => Column -> Either DataFrameException (VU.Vector a)-toUnboxedVector column =-    case column of-        UnboxedColumn (f :: VU.Vector b) -> case testEquality (typeRep @a) (typeRep @b) of-            Just Refl -> Right f-            Nothing ->-                Left $-                    TypeMismatchException-                        ( MkTypeErrorContext-                            { userType = Right (typeRep @Int)-                            , expectedType = Right (typeRep @a)-                            , callingFunctionName = Just "toUnboxedVector"-                            , errorColumnName = Nothing-                            }-                        )-        _ ->-            Left $-                TypeMismatchException-                    ( MkTypeErrorContext-                        { userType = Right (typeRep @a)-                        , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())-                        , callingFunctionName = Just "toUnboxedVector"-                        , errorColumnName = Nothing-                        }-                    )-{-# SPECIALIZE toUnboxedVector ::-    Column -> Either DataFrameException (VU.Vector Double)-    #-}+import Control.Monad (forM_, when)+import Control.Monad.ST (runST)+import Data.Bits (+    complement,+    popCount,+    setBit,+    shiftL,+    shiftR,+    testBit,+    (.&.),+ )+import Data.Kind (Type)+import Data.Maybe+import Data.These+import Data.Type.Equality (TestEquality (..))+import Data.Word (Word8)+import DataFrame.Errors+import DataFrame.Internal.Parsing+import DataFrame.Internal.Types+import System.IO.Unsafe (unsafePerformIO)+import Type.Reflection++-- | A bit-packed validity bitmap. Bit @i@ = 1 means row @i@ is valid (not null).+type Bitmap = VU.Vector Word8++{- | Our representation of a column is a GADT that can store data based on the underlying data.++This allows us to pattern match on data kinds and limit some operations to only some+kinds of vectors. Nullability is represented via an optional bit-packed 'Bitmap':+@Nothing@ = no nulls; @Just bm@ = bit @i@ of @bm@ is 1 iff row @i@ is valid.+-}+data Column where+    BoxedColumn :: (Columnable a) => Maybe Bitmap -> VB.Vector a -> Column+    UnboxedColumn ::+        (Columnable a, VU.Unbox a) => Maybe Bitmap -> VU.Vector a -> Column++{- | A mutable companion struct to dataframe columns.++Used mostly as an intermediate structure for I/O.+-}+data MutableColumn where+    MBoxedColumn :: (Columnable a) => VBM.IOVector a -> MutableColumn+    MUnboxedColumn :: (Columnable a, VU.Unbox a) => VUM.IOVector a -> MutableColumn++-- ---------------------------------------------------------------------------+-- Bitmap helpers+-- ---------------------------------------------------------------------------++-- | Test whether row @i@ is valid (not null) in a bitmap.+bitmapTestBit :: Bitmap -> Int -> Bool+bitmapTestBit bm i = testBit (VU.unsafeIndex bm (i `shiftR` 3)) (i .&. 7)+{-# INLINE bitmapTestBit #-}++-- | Build a fully-valid bitmap for @n@ rows (all bits set).+allValidBitmap :: Int -> Bitmap+allValidBitmap n =+    let bytes = (n + 7) `shiftR` 3+        lastBits = n .&. 7+        full = VU.replicate (bytes - 1) 0xFF+        lastByte = if lastBits == 0 then 0xFF else (1 `shiftL` lastBits) - 1+     in if bytes == 0 then VU.empty else VU.snoc full lastByte+{-# INLINE allValidBitmap #-}++{- | Build a bitmap from a @VU.Vector Word8@ validity vector+(1 = valid, 0 = null), as produced by Arrow / Parquet decoders.+-}+buildBitmapFromValid :: VU.Vector Word8 -> Bitmap+buildBitmapFromValid valid =+    let n = VU.length valid+        bytes = (n + 7) `shiftR` 3+     in VU.generate bytes $ \b ->+            let base = b `shiftL` 3+                setBitIf acc bit =+                    let idx = base + bit+                     in if idx < n && VU.unsafeIndex valid idx /= 0+                            then setBit acc bit+                            else acc+             in foldl setBitIf (0 :: Word8) [0 .. 7]++{- | Build a bitmap from a list of null-row indices.+@nullIdxs@ are the positions that are NULL.+-}+buildBitmapFromNulls :: Int -> [Int] -> Bitmap+buildBitmapFromNulls n nullIdxs =+    let base = allValidBitmap n+     in VU.modify+            ( \mv ->+                forM_ nullIdxs $ \i -> do+                    let byteIdx = i `shiftR` 3+                        bitIdx = i .&. 7+                    v <- VUM.unsafeRead mv byteIdx+                    VUM.unsafeWrite mv byteIdx (clearBit8 v bitIdx)+            )+            base+  where+    clearBit8 :: Word8 -> Int -> Word8+    clearBit8 b bit = b .&. complement (1 `shiftL` bit)++-- | Slice a bitmap for rows @[start .. start+len-1]@.+bitmapSlice :: Int -> Int -> Bitmap -> Bitmap+bitmapSlice start len bm+    | start .&. 7 == 0 =+        -- byte-aligned: simple slice; clamp so we never ask for more bytes than exist+        let startByte = start `shiftR` 3+            bytes = min ((len + 7) `shiftR` 3) (VU.length bm - startByte)+         in VU.slice startByte bytes bm+    | otherwise =+        -- non-aligned: unpack bit-by-bit and repack+        let n = min len (VU.length bm `shiftL` 3 - start)+         in buildBitmapFromValid $+                VU.generate n $+                    \i -> if bitmapTestBit bm (start + i) then 1 else 0++-- | Concatenate two bitmaps covering @n1@ and @n2@ rows respectively.+bitmapConcat :: Int -> Bitmap -> Int -> Bitmap -> Bitmap+bitmapConcat n1 bm1 n2 bm2 =+    buildBitmapFromValid $+        VU.generate (n1 + n2) $ \i ->+            if i < n1+                then if bitmapTestBit bm1 i then 1 else 0+                else if bitmapTestBit bm2 (i - n1) then 1 else 0++-- | Combine two bitmaps with AND (both must be valid for result to be valid).+mergeBitmaps :: Bitmap -> Bitmap -> Bitmap+mergeBitmaps = VU.zipWith (.&.)++{- | Materialize a nullable column from @VB.Vector (Maybe a)@.+When @a@ is unboxable, creates an 'UnboxedColumn' (more compact).+Otherwise creates a 'BoxedColumn'.+Always attaches a bitmap so the column is recognized as nullable even when+no 'Nothing' values are present (preserves the Maybe type marker).+-}+fromMaybeVec :: forall a. (Columnable a) => VB.Vector (Maybe a) -> Column+fromMaybeVec v = case sUnbox @a of+    STrue -> fromMaybeVecUnboxed v+    SFalse ->+        let n = VB.length v+            nullIdxs = [i | i <- [0 .. n - 1], isNothing (VB.unsafeIndex v i)]+            bm = if null nullIdxs then allValidBitmap n else buildBitmapFromNulls n nullIdxs+            dat = VB.map (fromMaybe (errorWithoutStackTrace "fromMaybeVec: Nothing slot")) v+         in BoxedColumn (Just bm) dat++{- | Materialize a nullable 'UnboxedColumn' to @VB.Vector (Maybe a)@ using runST.+Always attaches a bitmap so the column is recognized as nullable even when+no 'Nothing' values are present (preserves the Maybe type marker).+-}+fromMaybeVecUnboxed ::+    forall a. (Columnable a, VU.Unbox a) => VB.Vector (Maybe a) -> Column+fromMaybeVecUnboxed v =+    let n = VB.length v+        nullIdxs = [i | i <- [0 .. n - 1], isNothing (VB.unsafeIndex v i)]+        bm = if null nullIdxs then allValidBitmap n else buildBitmapFromNulls n nullIdxs+        dat = runST $ do+            mv <- VUM.new n+            VG.iforM_ v $ \i mx -> forM_ mx (VUM.unsafeWrite mv i)+            VU.unsafeFreeze mv+     in UnboxedColumn (Just bm) dat++-- | Materialize an element from a column at index @i@, respecting the bitmap.+columnElemIsNull :: Column -> Int -> Bool+columnElemIsNull (BoxedColumn (Just bm) _) i = not (bitmapTestBit bm i)+columnElemIsNull (UnboxedColumn (Just bm) _) i = not (bitmapTestBit bm i)+columnElemIsNull _ _ = False++-- | Return the 'Maybe Bitmap' from a column.+columnBitmap :: Column -> Maybe Bitmap+columnBitmap (BoxedColumn bm _) = bm+columnBitmap (UnboxedColumn bm _) = bm++-- ---------------------------------------------------------------------------+-- End bitmap helpers+-- ---------------------------------------------------------------------------++{- | A TypedColumn is a wrapper around our type-erased column.+It is used to type check expressions on columns.++Note: there is no guarantee that the Phanton type is the+same as the underlying vector type.+-}+data TypedColumn a where+    TColumn :: (Columnable a) => Column -> TypedColumn a++instance (Eq a) => Eq (TypedColumn a) where+    (==) :: (Eq a) => TypedColumn a -> TypedColumn a -> Bool+    (==) (TColumn a) (TColumn b) = a == b++instance (Ord a) => Ord (TypedColumn a) where+    compare :: (Ord a) => TypedColumn a -> TypedColumn a -> Ordering+    compare (TColumn a) (TColumn b) = compare a b++-- | Gets the underlying value from a TypedColumn.+unwrapTypedColumn :: TypedColumn a -> Column+unwrapTypedColumn (TColumn value) = value++-- | Gets the underlying vector from a TypedColumn.+vectorFromTypedColumn :: TypedColumn a -> VB.Vector a+vectorFromTypedColumn (TColumn value) = either throw id (toVector value)++-- | Checks if a column contains missing values (has a bitmap).+hasMissing :: Column -> Bool+hasMissing (BoxedColumn (Just _) _) = True+hasMissing (UnboxedColumn (Just _) _) = True+hasMissing _ = False++-- | Checks if a column contains only missing values.+allMissing :: Column -> Bool+allMissing (BoxedColumn (Just bm) col) = VU.all (== 0) bm && not (VB.null col)+allMissing (UnboxedColumn (Just bm) col) = VU.all (== 0) bm && not (VU.null col)+allMissing _ = False++-- | Checks if a column contains numeric values.+isNumeric :: Column -> Bool+isNumeric (UnboxedColumn _ (vec :: VU.Vector a)) = case sNumeric @a of+    STrue -> True+    _ -> False+isNumeric (BoxedColumn _ (vec :: VB.Vector a)) = case testEquality (typeRep @a) (typeRep @Integer) of+    Nothing -> False+    Just Refl -> True++{- | Checks if a column is of a given type values.+For nullable columns (@BoxedColumn (Just _)@ or @UnboxedColumn (Just _)@),+also returns @True@ when @a = Maybe b@ and the column stores @b@ internally.+-}+hasElemType :: forall a. (Columnable a) => Column -> Bool+hasElemType = \case+    BoxedColumn bm (column :: VB.Vector b) -> checkBoxed bm (typeRep @b)+    UnboxedColumn bm (column :: VU.Vector b) -> checkUnboxed bm (typeRep @b)+  where+    -- Direct type match+    directMatch :: forall (b :: Type). TypeRep b -> Bool+    directMatch = isJust . testEquality (typeRep @a)+    -- For a nullable column (has bitmap), also accept a = Maybe b+    checkMaybe :: forall (b :: Type). TypeRep b -> Bool+    checkMaybe tb = case typeRep @a of+        App tMaybe tInner -> case eqTypeRep tMaybe (typeRep @Maybe) of+            Just HRefl -> isJust (testEquality tInner tb)+            Nothing -> False+        _ -> False+    checkBoxed :: forall (b :: Type). Maybe Bitmap -> TypeRep b -> Bool+    checkBoxed bm tb = directMatch tb || (isJust bm && checkMaybe tb)+    checkUnboxed :: forall (b :: Type). Maybe Bitmap -> TypeRep b -> Bool+    checkUnboxed bm tb = directMatch tb || (isJust bm && checkMaybe tb)++-- | An internal/debugging function to get the column type of a column.+columnVersionString :: Column -> String+columnVersionString column = case column of+    BoxedColumn Nothing _ -> "Boxed"+    BoxedColumn (Just _) _ -> "NullableBoxed"+    UnboxedColumn Nothing _ -> "Unboxed"+    UnboxedColumn (Just _) _ -> "NullableUnboxed"++{- | An internal/debugging function to get the type stored in the outermost vector+of a column.+-}+columnTypeString :: Column -> String+columnTypeString column = case column of+    BoxedColumn Nothing (_ :: VB.Vector a) -> show (typeRep @a)+    BoxedColumn (Just _) (_ :: VB.Vector a) -> showMaybeType @a+    UnboxedColumn Nothing (_ :: VU.Vector a) -> show (typeRep @a)+    UnboxedColumn (Just _) (_ :: VU.Vector a) -> showMaybeType @a+  where+    showMaybeType :: forall a. (Typeable a) => String+    showMaybeType =+        let s = show (typeRep @a)+         in "Maybe " ++ if ' ' `elem` s then "(" ++ s ++ ")" else s++instance (Show a) => Show (TypedColumn a) where+    show :: (Show a) => TypedColumn a -> String+    show (TColumn col) = show col++instance NFData Column where+    rnf (BoxedColumn Nothing (v :: VB.Vector a)) = VB.foldl' (const (`seq` ())) () v+    rnf (BoxedColumn (Just bm) (v :: VB.Vector a)) =+        let n = VB.length v+            go !i+                | i >= n = ()+                | bitmapTestBit bm i = VB.unsafeIndex v i `seq` go (i + 1)+                | otherwise = go (i + 1)+         in go 0+    rnf (UnboxedColumn _ v) = v `seq` ()++instance Show Column where+    show :: Column -> String+    show (BoxedColumn Nothing column) = show column+    show (BoxedColumn (Just bm) column) =+        let n = VB.length column+            elems =+                [ if bitmapTestBit bm i then show (VB.unsafeIndex column i) else "null"+                | i <- [0 .. n - 1]+                ]+         in "[" ++ foldl (\acc e -> if null acc then e else acc ++ "," ++ e) "" elems ++ "]"+    show (UnboxedColumn Nothing column) = show column+    show (UnboxedColumn (Just bm) column) =+        let n = VU.length column+            elems =+                [ if bitmapTestBit bm i then show (VU.unsafeIndex column i) else "null"+                | i <- [0 .. n - 1]+                ]+         in "[" ++ foldl (\acc e -> if null acc then e else acc ++ "," ++ e) "" elems ++ "]"++{- | Compare two nullable boxed columns element by element, skipping null slots.+Uses a manual loop to avoid stream fusion forcing null-slot error thunks.+-}+eqBoxedCols ::+    (Eq a) => Maybe Bitmap -> VB.Vector a -> Maybe Bitmap -> VB.Vector a -> Bool+eqBoxedCols bm1 a bm2 b+    | VB.length a /= VB.length b = False+    | otherwise = go 0+  where+    !n = VB.length a+    go !i+        | i >= n = True+        | nullA || nullB = (nullA == nullB) && go (i + 1)+        | VB.unsafeIndex a i == VB.unsafeIndex b i = go (i + 1)+        | otherwise = False+      where+        nullA = maybe False (\bm -> not (bitmapTestBit bm i)) bm1+        nullB = maybe False (\bm -> not (bitmapTestBit bm i)) bm2+{-# INLINE eqBoxedCols #-}++instance Eq Column where+    (==) :: Column -> Column -> Bool+    (==) (BoxedColumn bm1 (a :: VB.Vector t1)) (BoxedColumn bm2 (b :: VB.Vector t2)) =+        case testEquality (typeRep @t1) (typeRep @t2) of+            Nothing -> False+            Just Refl -> eqBoxedCols bm1 a bm2 b+    (==) (UnboxedColumn bm1 (a :: VU.Vector t1)) (UnboxedColumn bm2 (b :: VU.Vector t2)) =+        case testEquality (typeRep @t1) (typeRep @t2) of+            Nothing -> False+            Just Refl ->+                VU.length a == VU.length b+                    && VU.and+                        ( VU.imap+                            ( \i x ->+                                let nullA = maybe False (\bm -> not (bitmapTestBit bm i)) bm1+                                    nullB = maybe False (\bm -> not (bitmapTestBit bm i)) bm2+                                 in if nullA || nullB then nullA == nullB else x == VU.unsafeIndex b i+                            )+                            a+                        )+    (==) _ _ = False++-- Generalised LEQ that does reflection.+generalLEQ ::+    forall a b. (Typeable a, Typeable b, Ord a, Ord b) => a -> b -> Bool+generalLEQ x y = case testEquality (typeRep @a) (typeRep @b) of+    Nothing -> False+    Just Refl -> x <= y++instance Ord Column where+    (<=) :: Column -> Column -> Bool+    (<=) (BoxedColumn _ (a :: VB.Vector t1)) (BoxedColumn _ (b :: VB.Vector t2)) = generalLEQ a b+    (<=) (UnboxedColumn _ (a :: VU.Vector t1)) (UnboxedColumn _ (b :: VU.Vector t2)) = generalLEQ a b+    (<=) _ _ = False++{- | A class for converting a vector to a column of the appropriate type.+Given each Rep we tell the `toColumnRep` function which Column type to pick.+-}+class ColumnifyRep (r :: Rep) a where+    toColumnRep :: VB.Vector a -> Column++-- | Constraint synonym for what we can put into columns.+type Columnable a =+    ( Columnable' a+    , ColumnifyRep (KindOf a) a+    , UnboxIf a+    , IntegralIf a+    , FloatingIf a+    , SBoolI (Unboxable a)+    , SBoolI (Numeric a)+    , SBoolI (IntegralTypes a)+    , SBoolI (FloatingTypes a)+    )++instance+    (Columnable a, VU.Unbox a) =>+    ColumnifyRep 'RUnboxed a+    where+    toColumnRep :: (Columnable a, VUM.Unbox a) => VB.Vector a -> Column+    toColumnRep v = UnboxedColumn Nothing (VU.convert v)++instance+    (Columnable a) =>+    ColumnifyRep 'RBoxed a+    where+    toColumnRep :: (Columnable a) => VB.Vector a -> Column+    toColumnRep = BoxedColumn Nothing++instance+    (Columnable a) =>+    ColumnifyRep 'RNullableBoxed (Maybe a)+    where+    toColumnRep :: (Columnable a) => VB.Vector (Maybe a) -> Column+    toColumnRep = fromMaybeVec++{- | O(n) Convert a vector to a column. Automatically picks the best representation of a vector to store the underlying data in.++__Examples:__++@+> import qualified Data.Vector as V+> fromVector (VB.fromList [(1 :: Int), 2, 3, 4])+[1,2,3,4]+@+-}+fromVector ::+    forall a.+    (Columnable a, ColumnifyRep (KindOf a) a) =>+    VB.Vector a -> Column+fromVector = toColumnRep @(KindOf a)++{- | O(n) Convert an unboxed vector to a column. This avoids the extra conversion if you already have the data in an unboxed vector.++__Examples:__++@+> import qualified Data.Vector.Unboxed as V+> fromUnboxedVector (VB.fromList [(1 :: Int), 2, 3, 4])+[1,2,3,4]+@+-}+fromUnboxedVector ::+    forall a. (Columnable a, VU.Unbox a) => VU.Vector a -> Column+fromUnboxedVector = UnboxedColumn Nothing++{- | O(n) Convert a list to a column. Automatically picks the best representation of a vector to store the underlying data in.++__Examples:__++@+> fromList [(1 :: Int), 2, 3, 4]+[1,2,3,4]+@+-}+fromList ::+    forall a.+    (Columnable a, ColumnifyRep (KindOf a) a) =>+    [a] -> Column+fromList = toColumnRep @(KindOf a) . VB.fromList++-- An internal helper for type errors+throwTypeMismatch ::+    forall (a :: Type) (b :: Type).+    (Typeable a, Typeable b) => Either DataFrameException Column+throwTypeMismatch =+    Left $+        TypeMismatchException+            MkTypeErrorContext+                { userType = Right (typeRep @b)+                , expectedType = Right (typeRep @a)+                , callingFunctionName = Nothing+                , errorColumnName = Nothing+                }++-- | An internal function to map a function over the values of a column.+mapColumn ::+    forall b c.+    (Columnable b, Columnable c) =>+    (b -> c) -> Column -> Either DataFrameException Column+mapColumn f = \case+    BoxedColumn bm (col :: VB.Vector a) -> runBoxed bm col+    UnboxedColumn bm (col :: VU.Vector a) -> runUnboxed bm col+  where+    runBoxed ::+        forall a.+        (Columnable a) =>+        Maybe Bitmap -> VB.Vector a -> Either DataFrameException Column+    runBoxed bm col = case testEquality (typeRep @b) (typeRep @(Maybe a)) of+        -- user maps over Maybe a (nullable column as Maybe)+        Just Refl ->+            let !n = VB.length col+             in -- Build result directly without intermediate Maybe vector to avoid+                -- fusion forcing null slots via VU.convert.+                Right $ case sUnbox @c of+                    STrue -> UnboxedColumn Nothing $+                        VU.generate n $ \i ->+                            f+                                ( if maybe True (`bitmapTestBit` i) bm+                                    then Just (VB.unsafeIndex col i)+                                    else Nothing+                                )+                    SFalse -> fromVector @c $+                        VB.generate n $ \i ->+                            f+                                ( if maybe True (`bitmapTestBit` i) bm+                                    then Just (VB.unsafeIndex col i)+                                    else Nothing+                                )+        Nothing -> case testEquality (typeRep @a) (typeRep @b) of+            Just Refl ->+                -- user maps over inner type a; preserve bitmap+                Right $ case sUnbox @c of+                    STrue -> UnboxedColumn bm (VU.generate (VB.length col) (f . VB.unsafeIndex col))+                    SFalse -> BoxedColumn bm (VB.map f col)+            Nothing -> throwTypeMismatch @a @b++    runUnboxed ::+        forall a.+        (Columnable a, VU.Unbox a) =>+        Maybe Bitmap -> VU.Vector a -> Either DataFrameException Column+    runUnboxed bm col = case testEquality (typeRep @b) (typeRep @(Maybe a)) of+        Just Refl ->+            let !n = VU.length col+             in Right $ case sUnbox @c of+                    STrue -> UnboxedColumn Nothing $+                        VU.generate n $ \i ->+                            f+                                ( if maybe True (`bitmapTestBit` i) bm+                                    then Just (VU.unsafeIndex col i)+                                    else Nothing+                                )+                    SFalse -> fromVector @c $+                        VB.generate n $ \i ->+                            f+                                ( if maybe True (`bitmapTestBit` i) bm+                                    then Just (VU.unsafeIndex col i)+                                    else Nothing+                                )+        Nothing -> case testEquality (typeRep @a) (typeRep @b) of+            Just Refl -> Right $ case sUnbox @c of+                STrue -> UnboxedColumn bm (VU.map f col)+                SFalse -> BoxedColumn bm (VB.generate (VU.length col) (f . VU.unsafeIndex col))+            Nothing -> throwTypeMismatch @a @b+{-# INLINEABLE mapColumn #-}++-- | Applies a function that returns an unboxed result to an unboxed vector, storing the result in a column.+imapColumn ::+    forall b c.+    (Columnable b, Columnable c) =>+    (Int -> b -> c) -> Column -> Either DataFrameException Column+imapColumn f = \case+    BoxedColumn bm (col :: VB.Vector a) -> runBoxed bm col+    UnboxedColumn bm (col :: VU.Vector a) -> runUnboxed bm col+  where+    runBoxed ::+        forall a.+        (Columnable a) =>+        Maybe Bitmap -> VB.Vector a -> Either DataFrameException Column+    runBoxed bm col = case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> Right $ case sUnbox @c of+            STrue ->+                UnboxedColumn+                    bm+                    (VU.generate (VB.length col) (\i -> f i (VB.unsafeIndex col i)))+            SFalse -> BoxedColumn bm (VB.imap f col)+        Nothing -> throwTypeMismatch @a @b++    runUnboxed ::+        forall a.+        (Columnable a, VU.Unbox a) =>+        Maybe Bitmap -> VU.Vector a -> Either DataFrameException Column+    runUnboxed bm col = case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> Right $ case sUnbox @c of+            STrue -> UnboxedColumn bm (VU.imap f col)+            SFalse -> BoxedColumn bm (VB.imap f (VG.convert col))+        Nothing -> throwTypeMismatch @a @b++-- | O(1) Gets the number of elements in the column.+columnLength :: Column -> Int+columnLength (BoxedColumn _ xs) = VB.length xs+columnLength (UnboxedColumn _ xs) = VU.length xs+{-# INLINE columnLength #-}++-- | O(n) Gets the number of non-null elements in the column.+numElements :: Column -> Int+numElements (BoxedColumn Nothing xs) = VB.length xs+numElements (BoxedColumn (Just bm) xs) = VU.foldl' (\acc b -> acc + popCount b) 0 bm+numElements (UnboxedColumn Nothing xs) = VU.length xs+numElements (UnboxedColumn (Just bm) xs) = VU.foldl' (\acc b -> acc + popCount b) 0 bm+{-# INLINE numElements #-}++-- | O(n) Takes the first n values of a column.+takeColumn :: Int -> Column -> Column+takeColumn n (BoxedColumn bm xs) =+    BoxedColumn (fmap (bitmapSlice 0 n) bm) (VG.take n xs)+takeColumn n (UnboxedColumn bm xs) =+    UnboxedColumn (fmap (bitmapSlice 0 n) bm) (VG.take n xs)+{-# INLINE takeColumn #-}++-- | O(n) Takes the last n values of a column.+takeLastColumn :: Int -> Column -> Column+takeLastColumn n column = sliceColumn (columnLength column - n) n column+{-# INLINE takeLastColumn #-}++-- | O(n) Takes n values after a given column index.+sliceColumn :: Int -> Int -> Column -> Column+sliceColumn start n (BoxedColumn bm xs) =+    BoxedColumn (fmap (bitmapSlice start n) bm) (VG.slice start n xs)+sliceColumn start n (UnboxedColumn bm xs) =+    UnboxedColumn (fmap (bitmapSlice start n) bm) (VG.slice start n xs)+{-# INLINE sliceColumn #-}++-- | O(n) Selects the elements at a given set of indices. Does not change the order.+atIndicesStable :: VU.Vector Int -> Column -> Column+atIndicesStable indexes (BoxedColumn bm column) =+    BoxedColumn+        ( fmap+            ( \bm0 ->+                buildBitmapFromValid $+                    VU.map (\i -> if bitmapTestBit bm0 i then 1 else 0) indexes+            )+            bm+        )+        ( VB.generate+            (VU.length indexes)+            ((column `VB.unsafeIndex`) . (indexes `VU.unsafeIndex`))+        )+atIndicesStable indexes (UnboxedColumn bm column) =+    UnboxedColumn+        ( fmap+            ( \bm0 ->+                buildBitmapFromValid $+                    VU.map (\i -> if bitmapTestBit bm0 i then 1 else 0) indexes+            )+            bm+        )+        (VU.unsafeBackpermute column indexes)+{-# INLINE atIndicesStable #-}++{- | Like 'atIndicesStable' but treats negative indices as null.+Keeps the index vector fully unboxed (no @VB.Vector (Maybe Int)@).+-}+gatherWithSentinel :: VU.Vector Int -> Column -> Column+gatherWithSentinel indices col =+    let !n = VU.length indices+        newBm = buildBitmapFromValid $ VU.generate n $ \i ->+            if VU.unsafeIndex indices i < 0 then 0 else 1+     in case col of+            BoxedColumn srcBm v ->+                let dat = VB.generate n $ \i ->+                        let !idx = VU.unsafeIndex indices i+                         in if idx < 0 then VB.unsafeIndex v 0 else VB.unsafeIndex v idx+                    bm = case srcBm of+                        Nothing -> Just newBm+                        Just sb ->+                            Just+                                ( mergeBitmaps+                                    newBm+                                    ( buildBitmapFromValid $ VU.generate n $ \i ->+                                        let idx = VU.unsafeIndex indices i+                                         in if idx >= 0 && bitmapTestBit sb idx then 1 else 0+                                    )+                                )+                 in BoxedColumn bm dat+            UnboxedColumn srcBm v ->+                let dat = runST $ do+                        mv <- VUM.new n+                        VG.iforM_ indices $ \i idx ->+                            when (idx >= 0) $ VUM.unsafeWrite mv i (VU.unsafeIndex v idx)+                        VU.unsafeFreeze mv+                    bm = case srcBm of+                        Nothing -> Just newBm+                        Just sb ->+                            Just+                                ( mergeBitmaps+                                    newBm+                                    ( buildBitmapFromValid $ VU.generate n $ \i ->+                                        let idx = VU.unsafeIndex indices i+                                         in if idx >= 0 && bitmapTestBit sb idx then 1 else 0+                                    )+                                )+                 in UnboxedColumn bm dat+{-# INLINE gatherWithSentinel #-}++-- | Internal helper to get indices in a boxed vector.+getIndices :: VU.Vector Int -> VB.Vector a -> VB.Vector a+getIndices indices xs = VB.generate (VU.length indices) (\i -> xs VB.! (indices VU.! i))+{-# INLINE getIndices #-}++-- | Internal helper to get indices in an unboxed vector.+getIndicesUnboxed :: (VU.Unbox a) => VU.Vector Int -> VU.Vector a -> VU.Vector a+getIndicesUnboxed indices xs = VU.generate (VU.length indices) (\i -> xs VU.! (indices VU.! i))+{-# INLINE getIndicesUnboxed #-}++findIndices ::+    forall a.+    (Columnable a) =>+    (a -> Bool) ->+    Column ->+    Either DataFrameException (VU.Vector Int)+findIndices pred = \case+    BoxedColumn _ (v :: VB.Vector b) -> run v VG.convert+    UnboxedColumn _ (v :: VU.Vector b) -> run v id+  where+    run ::+        forall b v.+        (Typeable b, VG.Vector v b, VG.Vector v Int) =>+        v b ->+        (v Int -> VU.Vector Int) ->+        Either DataFrameException (VU.Vector Int)+    run column finalize = case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> Right . finalize $ VG.findIndices pred column+        Nothing ->+            Left $+                TypeMismatchException+                    MkTypeErrorContext+                        { userType = Right (typeRep @a)+                        , expectedType = Right (typeRep @b)+                        , callingFunctionName = Just "findIndices"+                        , errorColumnName = Nothing+                        }++-- | An internal function that returns a vector of how indexes change after a column is sorted.+sortedIndexes :: Bool -> Column -> VU.Vector Int+sortedIndexes asc = \case+    BoxedColumn _ column -> sortWorker VG.convert column+    UnboxedColumn _ column -> sortWorker id column+  where+    sortWorker ::+        (VG.Vector v a, Ord a, VG.Vector v (Int, a), VG.Vector v Int) =>+        (v Int -> VU.Vector Int) -> v a -> VU.Vector Int+    sortWorker finalize column = runST $ do+        withIndexes <- VG.thaw $ VG.indexed column+        let cmp = if asc then compare else flip compare+        VA.sortBy (\(_, b) (_, b') -> cmp b b') withIndexes+        sorted <- VG.unsafeFreeze withIndexes+        return $ finalize $ VG.map fst sorted+{-# INLINE sortedIndexes #-}++-- | Fold (right) column with index.+ifoldrColumn ::+    forall a b.+    (Columnable a, Columnable b) =>+    (Int -> a -> b -> b) -> b -> Column -> Either DataFrameException b+ifoldrColumn f acc = \case+    BoxedColumn _ column -> foldrWorker column+    UnboxedColumn _ column -> foldrWorker column+  where+    foldrWorker ::+        forall c v.+        (Typeable c, VG.Vector v c) =>+        v c ->+        Either DataFrameException b+    foldrWorker vec = case testEquality (typeRep @a) (typeRep @c) of+        Just Refl -> pure $ VG.ifoldr f acc vec+        Nothing ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @a)+                        , expectedType = Right (typeRep @c)+                        , callingFunctionName = Just "ifoldrColumn"+                        , errorColumnName = Nothing+                        }+                    )++foldlColumn ::+    forall a b.+    (Columnable a, Columnable b) =>+    (b -> a -> b) -> b -> Column -> Either DataFrameException b+foldlColumn f acc = \case+    BoxedColumn _ column -> foldlWorker column+    UnboxedColumn _ column -> foldlWorker column+  where+    foldlWorker ::+        forall c v.+        (Typeable c, VG.Vector v c) =>+        v c ->+        Either DataFrameException b+    foldlWorker vec = case testEquality (typeRep @a) (typeRep @c) of+        Just Refl -> pure $ VG.foldl' f acc vec+        Nothing ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @a)+                        , expectedType = Right (typeRep @c)+                        , callingFunctionName = Just "ifoldrColumn"+                        , errorColumnName = Nothing+                        }+                    )++foldl1Column ::+    forall a.+    (Columnable a) =>+    (a -> a -> a) -> Column -> Either DataFrameException a+foldl1Column f = \case+    BoxedColumn _ column -> foldl1Worker column+    UnboxedColumn _ column -> foldl1Worker column+  where+    foldl1Worker ::+        forall c v.+        (Typeable c, VG.Vector v c) =>+        v c ->+        Either DataFrameException a+    foldl1Worker vec = case testEquality (typeRep @a) (typeRep @c) of+        Just Refl -> pure $ VG.foldl1' f vec+        Nothing ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @a)+                        , expectedType = Right (typeRep @c)+                        , callingFunctionName = Just "foldl1Column"+                        , errorColumnName = Nothing+                        }+                    )++{- | O(n) Seedless fold over groups using the first element of each group as seed.+Like 'foldDirectGroups' but for the case where no initial accumulator is available.+-}+foldl1DirectGroups ::+    forall a.+    (Columnable a) =>+    (a -> a -> a) ->+    Column ->+    VU.Vector Int ->+    VU.Vector Int ->+    Either DataFrameException Column+foldl1DirectGroups f col valueIndices offsets+    | VU.length offsets <= 1 = pure $ fromVector @a VB.empty+    | otherwise = case col of+        UnboxedColumn _ (vec :: VU.Vector d) -> UnboxedColumn Nothing <$> foldl1Worker vec+        BoxedColumn _ (vec :: VB.Vector d) -> BoxedColumn Nothing <$> foldl1Worker vec+  where+    foldl1Worker ::+        forall c v.+        (Typeable c, VG.Vector v c) =>+        v c ->+        Either DataFrameException (v c)+    foldl1Worker vec = case testEquality (typeRep @a) (typeRep @c) of+        Just Refl ->+            Right $+                VG.generate (VU.length offsets - 1) foldGroup+          where+            foldGroup k =+                let !s = VU.unsafeIndex offsets k+                    !e = VU.unsafeIndex offsets (k + 1)+                    !seed = VG.unsafeIndex vec (VU.unsafeIndex valueIndices s)+                 in go (s + 1) e seed+            go !i !e !acc+                | i >= e = acc+                | otherwise =+                    go (i + 1) e $!+                        f acc (VG.unsafeIndex vec (VU.unsafeIndex valueIndices i))+        Nothing ->+            Left $+                TypeMismatchException+                    MkTypeErrorContext+                        { userType = Right (typeRep @a)+                        , expectedType = Right (typeRep @c)+                        , callingFunctionName = Just "foldl1DirectGroups"+                        , errorColumnName = Nothing+                        }+{-# INLINEABLE foldl1DirectGroups #-}++{- | O(n) fold over groups by scanning the column LINEARLY.+rowToGroup[i] = group index for row i.+Avoids random column reads; random writes go to the accumulator array which is+small (nGroups entries) and typically cache-resident.+When @acc@ is unboxable, uses an unboxed mutable vector for the accumulator+array, eliminating pointer indirection on every read/write.+-}+foldLinearGroups ::+    forall b acc.+    (Columnable b, Columnable acc) =>+    (acc -> b -> acc) ->+    acc ->+    Column ->+    VU.Vector Int -> -- rowToGroup (length n)+    Int -> -- nGroups+    Either DataFrameException Column+foldLinearGroups f seed col rowToGroup nGroups+    | nGroups == 0 = Right (fromVector @acc VB.empty)+    | otherwise = case col of+        UnboxedColumn _ (vec :: VU.Vector d) -> foldLinearWorker vec+        BoxedColumn _ (vec :: VB.Vector d) -> foldLinearWorker vec+  where+    foldLinearWorker ::+        forall c v.+        (Typeable c, VG.Vector v c) =>+        v c ->+        Either DataFrameException Column+    foldLinearWorker vec = case testEquality (typeRep @b) (typeRep @c) of+        Just Refl ->+            Right $+                unsafePerformIO $+                    runWith+                        ( \readAt writeAt ->+                            VG.iforM_ vec $ \row x -> do+                                let !k = VG.unsafeIndex rowToGroup row+                                cur <- readAt k+                                writeAt k $! f cur x+                        )+        Nothing ->+            Left $+                TypeMismatchException+                    MkTypeErrorContext+                        { userType = Right (typeRep @b)+                        , expectedType = Right (typeRep @c)+                        , callingFunctionName = Just "foldLinearGroups"+                        , errorColumnName = Nothing+                        }++    -- \| Allocate accumulators, run the traversal, return a frozen Column.+    -- When @acc@ is unboxable, uses an unboxed mutable vector (no pointer+    -- indirection per read/write) and returns UnboxedColumn directly —+    -- avoiding a round-trip through VB.Vector.+    runWith :: ((Int -> IO acc) -> (Int -> acc -> IO ()) -> IO ()) -> IO Column+    runWith body = case sUnbox @acc of+        STrue -> do+            accs <- VUM.replicate nGroups seed+            body (VUM.unsafeRead accs) (VUM.unsafeWrite accs)+            UnboxedColumn Nothing <$> VU.unsafeFreeze accs+        SFalse -> do+            accs <- VBM.replicate nGroups seed+            body (VBM.unsafeRead accs) (VBM.unsafeWrite accs)+            fromVector @acc <$> VB.unsafeFreeze accs+    {-# INLINE runWith #-}+{-# INLINEABLE foldLinearGroups #-}++headColumn :: forall a. (Columnable a) => Column -> Either DataFrameException a+headColumn = \case+    BoxedColumn _ col -> headWorker col+    UnboxedColumn _ col -> headWorker col+  where+    headWorker ::+        forall c v.+        (Typeable c, VG.Vector v c) =>+        v c ->+        Either DataFrameException a+    headWorker vec = case testEquality (typeRep @a) (typeRep @c) of+        Just Refl ->+            if VG.null vec+                then Left (EmptyDataSetException "headColumn")+                else pure (VG.head vec)+        Nothing ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @a)+                        , expectedType = Right (typeRep @c)+                        , callingFunctionName = Just "headColumn"+                        , errorColumnName = Nothing+                        }+                    )++-- | An internal, column version of zip.+zipColumns :: Column -> Column -> Column+zipColumns (BoxedColumn _ column) (BoxedColumn _ other) = BoxedColumn Nothing (VG.zip column other)+zipColumns (BoxedColumn _ column) (UnboxedColumn _ other) =+    BoxedColumn+        Nothing+        ( VB.generate+            (min (VG.length column) (VG.length other))+            (\i -> (column VG.! i, other VG.! i))+        )+zipColumns (UnboxedColumn _ column) (BoxedColumn _ other) =+    BoxedColumn+        Nothing+        ( VB.generate+            (min (VG.length column) (VG.length other))+            (\i -> (column VG.! i, other VG.! i))+        )+zipColumns (UnboxedColumn _ column) (UnboxedColumn _ other) = UnboxedColumn Nothing (VG.zip column other)+{-# INLINE zipColumns #-}++-- | Merge two columns using `These`.+mergeColumns :: Column -> Column -> Column+mergeColumns colA colB = case (colA, colB) of+    (BoxedColumn bmA c1, BoxedColumn bmB c2) -> case (bmA, bmB) of+        (Just ba, Just bb) ->+            BoxedColumn Nothing $ mkVec c1 c2 $ \i v1 v2 ->+                let nullA = not (bitmapTestBit ba i)+                    nullB = not (bitmapTestBit bb i)+                 in case (nullA, nullB) of+                        (True, True) -> error "mergeColumns: both null"+                        (False, True) -> This v1+                        (True, False) -> That v2+                        (False, False) -> These v1 v2+        (Just ba, Nothing) ->+            BoxedColumn Nothing $ mkVec c1 c2 $ \i v1 v2 ->+                if not (bitmapTestBit ba i) then That v2 else These v1 v2+        (Nothing, Just bb) ->+            BoxedColumn Nothing $ mkVec c1 c2 $ \i v1 v2 ->+                if not (bitmapTestBit bb i) then This v1 else These v1 v2+        (Nothing, Nothing) ->+            BoxedColumn Nothing $ mkVecSimple c1 c2 These+    (BoxedColumn _ c1, UnboxedColumn _ c2) ->+        BoxedColumn Nothing $ mkVecSimple c1 c2 These+    (UnboxedColumn _ c1, BoxedColumn _ c2) ->+        BoxedColumn Nothing $ mkVecSimple c1 c2 These+    (UnboxedColumn _ c1, UnboxedColumn _ c2) ->+        BoxedColumn Nothing $ mkVecSimple c1 c2 These+  where+    mkVec c1 c2 combineElements =+        VB.generate+            (min (VG.length c1) (VG.length c2))+            (\i -> combineElements i (c1 VG.! i) (c2 VG.! i))+    {-# INLINE mkVec #-}++    mkVecSimple c1 c2 f =+        VB.generate+            (min (VG.length c1) (VG.length c2))+            (\i -> f (c1 VG.! i) (c2 VG.! i))+    {-# INLINE mkVecSimple #-}+{-# INLINE mergeColumns #-}++-- | An internal, column version of zipWith.+zipWithColumns ::+    forall a b c.+    (Columnable a, Columnable b, Columnable c) =>+    (a -> b -> c) -> Column -> Column -> Either DataFrameException Column+zipWithColumns f (UnboxedColumn bmL (column :: VU.Vector d)) (UnboxedColumn bmR (other :: VU.Vector e)) = case testEquality (typeRep @a) (typeRep @d) of+    Just Refl -> case testEquality (typeRep @b) (typeRep @e) of+        Just Refl+            -- Fast path: both plain unboxed, no bitmaps involved in the output type+            | isNothing bmL+            , isNothing bmR ->+                pure $ case sUnbox @c of+                    STrue -> UnboxedColumn Nothing (VU.zipWith f column other)+                    SFalse -> fromVector $ VB.zipWith f (VG.convert column) (VG.convert other)+        -- Type mismatch or bitmap involvement: fall through to general toVector path+        _ -> zipWithColumnsGeneral f (UnboxedColumn bmL column) (UnboxedColumn bmR other)+    Nothing -> zipWithColumnsGeneral f (UnboxedColumn bmL column) (UnboxedColumn bmR other)+-- TODO: mchavinda - reuse pattern from interpret where we augment the+-- error at the end.+zipWithColumns f left right = zipWithColumnsGeneral f left right++zipWithColumnsGeneral ::+    forall a b c.+    (Columnable a, Columnable b, Columnable c) =>+    (a -> b -> c) -> Column -> Column -> Either DataFrameException Column+zipWithColumnsGeneral f left right = case toVector @a left of+    Left (TypeMismatchException context) ->+        Left $+            TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"})+    Left e -> Left e+    Right left' -> case toVector @b right of+        Left (TypeMismatchException context) ->+            Left $+                TypeMismatchException (context{callingFunctionName = Just "zipWithColumns"})+        Left e -> Left e+        Right right' -> pure $ fromVector $ VB.zipWith f left' right'+{-# INLINE zipWithColumnsGeneral #-}+{-# INLINE zipWithColumns #-}++-- Functions for mutable columns (intended for IO).+writeColumn :: Int -> T.Text -> MutableColumn -> IO (Either T.Text Bool)+writeColumn i value (MBoxedColumn (col :: VBM.IOVector a)) =+    case testEquality (typeRep @a) (typeRep @T.Text) of+        Just Refl ->+            if isNullish value+                then VBM.unsafeWrite col i "" >> return (Left $! value)+                else VBM.unsafeWrite col i value >> return (Right True)+        Nothing -> return (Left value)+writeColumn i value (MUnboxedColumn (col :: VUM.IOVector a)) =+    case testEquality (typeRep @a) (typeRep @Int) of+        Just Refl -> case readInt value of+            Just v -> VUM.unsafeWrite col i v >> return (Right True)+            Nothing -> VUM.unsafeWrite col i 0 >> return (Left value)+        Nothing -> case testEquality (typeRep @a) (typeRep @Double) of+            Nothing -> return (Left $! value)+            Just Refl -> case readDouble value of+                Just v -> VUM.unsafeWrite col i v >> return (Right True)+                Nothing -> VUM.unsafeWrite col i 0 >> return (Left $! value)+{-# INLINE writeColumn #-}++freezeColumn' :: [(Int, T.Text)] -> MutableColumn -> IO Column+freezeColumn' nulls (MBoxedColumn col)+    | null nulls = BoxedColumn Nothing <$> VB.unsafeFreeze col+    | all (isNullish . snd) nulls = do+        frozen <- VB.unsafeFreeze col+        let n = VB.length frozen+            bm = buildBitmapFromNulls n (map fst nulls)+        return $ BoxedColumn (Just bm) frozen+    | otherwise =+        BoxedColumn Nothing+            . VB.imap+                ( \i v ->+                    if i `elem` map fst nulls+                        then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls))+                        else Right v+                )+            <$> VB.unsafeFreeze col+freezeColumn' nulls (MUnboxedColumn col)+    | null nulls = UnboxedColumn Nothing <$> VU.unsafeFreeze col+    | all (isNullish . snd) nulls = do+        c <- VU.unsafeFreeze col+        let n = VU.length c+            bm = buildBitmapFromNulls n (map fst nulls)+        return $ UnboxedColumn (Just bm) c+    | otherwise = do+        c <- VU.unsafeFreeze col+        return $+            BoxedColumn Nothing $+                VB.generate+                    (VU.length c)+                    ( \i ->+                        if i `elem` map fst nulls+                            then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls))+                            else Right (c VU.! i)+                    )+{-# INLINE freezeColumn' #-}++{- | Promote a non-nullable column to a nullable one (add an all-valid bitmap).+No-op when already nullable.+-}+ensureOptional :: Column -> Column+ensureOptional c@(BoxedColumn (Just _) _) = c+ensureOptional (BoxedColumn Nothing col) =+    BoxedColumn (Just (allValidBitmap (VB.length col))) col+ensureOptional c@(UnboxedColumn (Just _) _) = c+ensureOptional (UnboxedColumn Nothing col) =+    UnboxedColumn (Just (allValidBitmap (VU.length col))) col++-- | Fills the end of a column, up to n, with null rows. Does nothing if column has length >= n.+expandColumn :: Int -> Column -> Column+expandColumn n column@(BoxedColumn bm col)+    | n <= VG.length col = column+    | otherwise =+        let extra = n - VG.length col+            newBm = case bm of+                Nothing -> Just (buildBitmapFromNulls n [VG.length col .. n - 1])+                Just b ->+                    Just+                        (bitmapConcat (VG.length col) b extra (VU.replicate ((extra + 7) `shiftR` 3) 0))+            -- pad data with default (undefined slot, protected by bitmap)+            newCol = col <> VB.replicate extra (errorWithoutStackTrace "expandColumn: null slot")+         in BoxedColumn newBm newCol+expandColumn n column@(UnboxedColumn bm col)+    | n <= VG.length col = column+    | otherwise =+        let extra = n - VG.length col+            newBm = case bm of+                Nothing -> Just (buildBitmapFromNulls n [VG.length col .. n - 1])+                Just b ->+                    Just+                        (bitmapConcat (VG.length col) b extra (VU.replicate ((extra + 7) `shiftR` 3) 0))+            newCol = runST $ do+                mv <- VUM.new n+                VU.imapM_ (VUM.unsafeWrite mv) col+                VU.unsafeFreeze mv+         in UnboxedColumn newBm newCol++-- | Fills the beginning of a column, up to n, with null rows. Does nothing if column has length >= n.+leftExpandColumn :: Int -> Column -> Column+leftExpandColumn n column@(BoxedColumn bm col)+    | n <= VG.length col = column+    | otherwise =+        let extra = n - VG.length col+            origLen = VG.length col+            newBm = case bm of+                Nothing -> Just (buildBitmapFromNulls n [0 .. extra - 1])+                Just b ->+                    let nullPart = VU.replicate ((extra + 7) `shiftR` 3) 0+                     in Just (bitmapConcat extra nullPart origLen b)+            newCol =+                VB.replicate extra (errorWithoutStackTrace "leftExpandColumn: null slot") <> col+         in BoxedColumn newBm newCol+leftExpandColumn n column@(UnboxedColumn bm col)+    | n <= VG.length col = column+    | otherwise =+        let extra = n - VG.length col+            origLen = VG.length col+            newBm = case bm of+                Nothing -> Just (buildBitmapFromNulls n [0 .. extra - 1])+                Just b ->+                    let nullPart = VU.replicate ((extra + 7) `shiftR` 3) 0+                     in Just (bitmapConcat extra nullPart origLen b)+            newCol = runST $ do+                mv <- VUM.new n+                VU.imapM_ (\i x -> VUM.unsafeWrite mv (extra + i) x) col+                VU.unsafeFreeze mv+         in UnboxedColumn newBm newCol++{- | Concatenates two columns.+Returns Nothing if the columns are of different types.+-}+concatColumns :: Column -> Column -> Either DataFrameException Column+concatColumns left right = case (left, right) of+    (BoxedColumn bmL l, BoxedColumn bmR r) -> case testEquality (typeOf l) (typeOf r) of+        Just Refl ->+            let newBm = case (bmL, bmR) of+                    (Nothing, Nothing) -> Nothing+                    (Just bl, Nothing) ->+                        Just+                            (bitmapConcat (VB.length l) bl (VB.length r) (allValidBitmap (VB.length r)))+                    (Nothing, Just br) ->+                        Just+                            (bitmapConcat (VB.length l) (allValidBitmap (VB.length l)) (VB.length r) br)+                    (Just bl, Just br) -> Just (bitmapConcat (VB.length l) bl (VB.length r) br)+             in pure (BoxedColumn newBm (l <> r))+        Nothing -> Left (mismatchErr (typeOf r) (typeOf l))+    (UnboxedColumn bmL l, UnboxedColumn bmR r) -> case testEquality (typeOf l) (typeOf r) of+        Just Refl ->+            let newBm = case (bmL, bmR) of+                    (Nothing, Nothing) -> Nothing+                    (Just bl, Nothing) ->+                        Just+                            (bitmapConcat (VU.length l) bl (VU.length r) (allValidBitmap (VU.length r)))+                    (Nothing, Just br) ->+                        Just+                            (bitmapConcat (VU.length l) (allValidBitmap (VU.length l)) (VU.length r) br)+                    (Just bl, Just br) -> Just (bitmapConcat (VU.length l) bl (VU.length r) br)+             in pure (UnboxedColumn newBm (l <> r))+        Nothing -> Left (mismatchErr (typeOf r) (typeOf l))+    _ -> Left (mismatchErr (typeOf right) (typeOf left))+  where+    mismatchErr ::+        forall (x :: Type) (y :: Type). TypeRep x -> TypeRep y -> DataFrameException+    mismatchErr ta tb =+        withTypeable ta $+            withTypeable tb $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right ta+                        , expectedType = Right tb+                        , callingFunctionName = Just "concatColumns"+                        , errorColumnName = Nothing+                        }+                    )++{- | Concatenates two columns.++Works similar to 'concatColumns', but unlike that function, it will also combine columns of different types+by wrapping the values in an Either.++E.g. combining Column containing [1,2] with Column containing ["a","b"]+will result in a Column containing [Left 1, Left 2, Right "a", Right "b"].+-}++{- | O(n) Concatenate a list of same-type columns in a single allocation.+All columns must have the same constructor and element type (as they will+within a single Parquet column). Calls 'error' on mismatch.+-}+concatManyColumns :: [Column] -> Column+concatManyColumns [] = fromList ([] :: [Maybe Int])+concatManyColumns [c] = c+concatManyColumns (c0 : cs) = case c0 of+    BoxedColumn bm0 v0 ->+        let getCol (BoxedColumn bm v) = case testEquality (typeOf v0) (typeOf v) of+                Just Refl -> (bm, v)+                Nothing -> error "concatManyColumns: BoxedColumn type mismatch"+            getCol _ = error "concatManyColumns: column constructor mismatch"+            rest = map getCol cs+            allVecs = v0 : map snd rest+            allBms = bm0 : map fst rest+            newBm+                | all isNothing allBms = Nothing+                | otherwise =+                    let pairs = zip allVecs allBms+                        expandedBms = map (\(v, mb) -> fromMaybe (allValidBitmap (VB.length v)) mb) pairs+                        go b1 n1 b2 n2 = bitmapConcat n1 b1 n2 b2+                        concatBms [] = VU.empty+                        concatBms [(b, v)] = b+                        concatBms ((b1, v1) : (b2, v2) : rest') =+                            let merged = go b1 (VB.length v1) b2 (VB.length v2)+                             in concatBms ((merged, v1 <> v2) : rest')+                     in Just $ concatBms (zip expandedBms allVecs)+         in BoxedColumn newBm (VB.concat allVecs)+    UnboxedColumn bm0 v0 ->+        let getCol (UnboxedColumn bm v) = case testEquality (typeOf v0) (typeOf v) of+                Just Refl -> (bm, v)+                Nothing -> error "concatManyColumns: UnboxedColumn type mismatch"+            getCol _ = error "concatManyColumns: column constructor mismatch"+            rest = map getCol cs+            allVecs = v0 : map snd rest+            allBms = bm0 : map fst rest+            newBm+                | all isNothing allBms = Nothing+                | otherwise =+                    let pairs = zip allVecs allBms+                        expandedBms = map (\(v, mb) -> fromMaybe (allValidBitmap (VU.length v)) mb) pairs+                        go b1 n1 b2 n2 = bitmapConcat n1 b1 n2 b2+                        concatBms [] = VU.empty+                        concatBms [(b, _)] = b+                        concatBms ((b1, v1) : (b2, v2) : rest') =+                            let merged = go b1 (VU.length v1) b2 (VU.length v2)+                             in concatBms ((merged, v1 <> v2) : rest')+                     in Just $ concatBms (zip expandedBms allVecs)+         in UnboxedColumn newBm (VU.concat allVecs)++concatColumnsEither :: Column -> Column -> Column+concatColumnsEither (BoxedColumn bmL left) (BoxedColumn bmR right) = case testEquality (typeOf left) (typeOf right) of+    Nothing ->+        BoxedColumn Nothing $ fmap Left left <> fmap Right right+    Just Refl ->+        let newBm = case (bmL, bmR) of+                (Nothing, Nothing) -> Nothing+                (Just bl, Nothing) ->+                    Just+                        ( bitmapConcat+                            (VB.length left)+                            bl+                            (VB.length right)+                            (allValidBitmap (VB.length right))+                        )+                (Nothing, Just br) ->+                    Just+                        ( bitmapConcat+                            (VB.length left)+                            (allValidBitmap (VB.length left))+                            (VB.length right)+                            br+                        )+                (Just bl, Just br) -> Just (bitmapConcat (VB.length left) bl (VB.length right) br)+         in BoxedColumn newBm $ left <> right+concatColumnsEither (UnboxedColumn bmL left) (UnboxedColumn bmR right) = case testEquality (typeOf left) (typeOf right) of+    Nothing ->+        BoxedColumn Nothing $+            fmap Left (VG.convert left) <> fmap Right (VG.convert right)+    Just Refl ->+        let newBm = case (bmL, bmR) of+                (Nothing, Nothing) -> Nothing+                (Just bl, Nothing) ->+                    Just+                        ( bitmapConcat+                            (VU.length left)+                            bl+                            (VU.length right)+                            (allValidBitmap (VU.length right))+                        )+                (Nothing, Just br) ->+                    Just+                        ( bitmapConcat+                            (VU.length left)+                            (allValidBitmap (VU.length left))+                            (VU.length right)+                            br+                        )+                (Just bl, Just br) -> Just (bitmapConcat (VU.length left) bl (VU.length right) br)+         in UnboxedColumn newBm $ left <> right+concatColumnsEither (BoxedColumn _ left) (UnboxedColumn _ right) =+    BoxedColumn Nothing $ fmap Left left <> fmap Right (VG.convert right)+concatColumnsEither (UnboxedColumn _ left) (BoxedColumn _ right) =+    BoxedColumn Nothing $ fmap Left (VG.convert left) <> fmap Right right++-- | Allocate a mutable column of size @n@ matching the constructor/type of the given column.+newMutableColumn :: Int -> Column -> IO MutableColumn+newMutableColumn n (BoxedColumn _ (_ :: VB.Vector a)) =+    MBoxedColumn <$> (VBM.new n :: IO (VBM.IOVector a))+newMutableColumn n (UnboxedColumn _ (_ :: VU.Vector a)) =+    MUnboxedColumn <$> (VUM.new n :: IO (VUM.IOVector a))++-- | Copy a column chunk into a mutable column starting at offset @off@.+copyIntoMutableColumn :: MutableColumn -> Int -> Column -> IO ()+copyIntoMutableColumn (MBoxedColumn (mv :: VBM.IOVector b)) off (BoxedColumn _ (v :: VB.Vector a)) =+    case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> VG.imapM_ (\i x -> VBM.unsafeWrite mv (off + i) x) v+        Nothing -> error "copyIntoMutableColumn: Boxed type mismatch"+copyIntoMutableColumn (MUnboxedColumn (mv :: VUM.IOVector b)) off (UnboxedColumn _ (v :: VU.Vector a)) =+    case testEquality (typeRep @a) (typeRep @b) of+        Just Refl -> VG.imapM_ (\i x -> VUM.unsafeWrite mv (off + i) x) v+        Nothing -> error "copyIntoMutableColumn: Unboxed type mismatch"+copyIntoMutableColumn _ _ _ =+    error "copyIntoMutableColumn: constructor mismatch"++-- | Freeze a mutable column into an immutable column.+freezeMutableColumn :: MutableColumn -> IO Column+freezeMutableColumn (MBoxedColumn mv) = BoxedColumn Nothing <$> VB.unsafeFreeze mv+freezeMutableColumn (MUnboxedColumn mv) = UnboxedColumn Nothing <$> VU.unsafeFreeze mv++{- | O(n) Converts a column to a list. Throws an exception if the wrong type is specified.++__Examples:__++@+> column = fromList [(1 :: Int), 2, 3, 4]+> toList @Int column+[1,2,3,4]+> toList @Double column+exception: ...+@+-}+toList :: forall a. (Columnable a) => Column -> [a]+toList xs = case toVector @a xs of+    Left err -> throw err+    Right val -> VB.toList val++{- | Converts a column to a vector of a specific type.++This is a type-safe conversion that requires the column's element type+to exactly match the requested type. You must specify the desired type+via type applications.++==== __Type Parameters__++[@a@] The element type to convert to+[@v@] The vector type (e.g., 'VU.Vector', 'VB.Vector')++==== __Examples__++>>> toVector @Int @VU.Vector column+Right (unboxed vector of Ints)++>>> toVector @Text @VB.Vector column+Right (boxed vector of Text)++==== __Returns__++* 'Right' - The converted vector if types match+* 'Left' 'TypeMismatchException' - If the column's type doesn't match the requested type++==== __See also__++For numeric conversions with automatic type coercion, see 'toDoubleVector',+'toFloatVector', and 'toIntVector'.+-}+toVector ::+    forall a v.+    (VG.Vector v a, Columnable a) => Column -> Either DataFrameException (v a)+toVector col = case col of+    BoxedColumn bm (inner :: VB.Vector c) ->+        -- Check if user wants Maybe c (nullable) or c directly+        case testEquality (typeRep @a) (typeRep @c) of+            Just Refl -> Right $ VG.convert inner+            Nothing ->+                -- Try: a = Maybe c+                case testEquality (typeRep @a) (typeRep @(Maybe c)) of+                    Just Refl ->+                        -- Use VB.generate to avoid fusion forcing null slots+                        let !n = VB.length inner+                            maybeVec = case bm of+                                Nothing -> VB.generate n (Just . VB.unsafeIndex inner)+                                Just bitmap -> VB.generate n $ \i ->+                                    if bitmapTestBit bitmap i then Just (VB.unsafeIndex inner i) else Nothing+                         in Right $ VG.convert maybeVec+                    Nothing ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @a)+                                    , expectedType = Right (typeRep @c)+                                    , callingFunctionName = Just "toVector"+                                    , errorColumnName = Nothing+                                    }+                                )+    UnboxedColumn bm (inner :: VU.Vector c) ->+        case testEquality (typeRep @a) (typeRep @c) of+            Just Refl -> Right $ VG.convert inner+            Nothing ->+                case testEquality (typeRep @a) (typeRep @(Maybe c)) of+                    Just Refl ->+                        let maybeVec = case bm of+                                Nothing -> VB.generate (VU.length inner) (Just . VU.unsafeIndex inner)+                                Just bitmap -> VB.generate (VU.length inner) $ \i ->+                                    if bitmapTestBit bitmap i then Just (VU.unsafeIndex inner i) else Nothing+                         in Right $ VG.convert maybeVec+                    Nothing ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @a)+                                    , expectedType = Right (typeRep @c)+                                    , callingFunctionName = Just "toVector"+                                    , errorColumnName = Nothing+                                    }+                                )++-- Some common types we will use for numerical computing.++{- | Converts a column to an unboxed vector of 'Double' values.++This function performs intelligent type coercion for numeric types:++* If the column is already 'Double', returns it directly+* If the column contains other floating-point types, converts via 'realToFrac'+* If the column contains integral types, converts via 'fromIntegral' (beware of overflow if the type is `Integer`).++==== __Optional column handling__++For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number).+This allows optional numeric data to be represented in the resulting vector.++==== __Returns__++* 'Right' - The converted 'Double' vector+* 'Left' 'TypeMismatchException' - If the column is not numeric+-}+toDoubleVector :: Column -> Either DataFrameException (VU.Vector Double)+toDoubleVector column =+    case column of+        UnboxedColumn bm (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Double) of+            Just Refl -> case bm of+                Nothing -> Right f+                Just bitmap -> Right $ VU.imap (\i x -> if bitmapTestBit bitmap i then x else read "NaN") f+            Nothing -> case sFloating @a of+                STrue ->+                    Right+                        ( VU.imap+                            ( \i x -> case bm of+                                Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN"+                                _ -> realToFrac x+                            )+                            f+                        )+                SFalse -> case sIntegral @a of+                    STrue ->+                        Right+                            ( VU.imap+                                ( \i x -> case bm of+                                    Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN"+                                    _ -> fromIntegral x+                                )+                                f+                            )+                    SFalse ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @Double)+                                    , expectedType = Right (typeRep @a)+                                    , callingFunctionName = Just "toDoubleVector"+                                    , errorColumnName = Nothing+                                    }+                                )+        BoxedColumn bm (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of+            Just Refl ->+                Right+                    ( VB.convert $+                        VB.imap+                            ( \i x -> case bm of+                                Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN"+                                _ -> fromIntegral x+                            )+                            f+                    )+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @Double)+                            , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())+                            , callingFunctionName = Just "toDoubleVector"+                            , errorColumnName = Nothing+                            }+                        )++{- | Converts a column to an unboxed vector of 'Float' values.++This function performs intelligent type coercion for numeric types:++* If the column is already 'Float', returns it directly+* If the column contains other floating-point types, converts via 'realToFrac'+* If the column contains integral types, converts via 'fromIntegral'+* If the column is boxed 'Integer', converts via 'fromIntegral' (beware of overflow for 64-bit integers and `Integer`)++==== __Optional column handling__++For 'OptionalColumn' types, 'Nothing' values are converted to @NaN@ (Not a Number).+This allows optional numeric data to be represented in the resulting vector.++==== __Returns__++* 'Right' - The converted 'Float' vector+* 'Left' 'TypeMismatchException' - If the column is not numeric++==== __Precision warning__++Converting from 'Double' to 'Float' may result in loss of precision.+-}+toFloatVector :: Column -> Either DataFrameException (VU.Vector Float)+toFloatVector column =+    case column of+        UnboxedColumn bm (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Float) of+            Just Refl -> case bm of+                Nothing -> Right f+                Just bitmap -> Right $ VU.imap (\i x -> if bitmapTestBit bitmap i then x else read "NaN") f+            Nothing -> case sFloating @a of+                STrue ->+                    Right+                        ( VU.imap+                            ( \i x -> case bm of+                                Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN"+                                _ -> realToFrac x+                            )+                            f+                        )+                SFalse -> case sIntegral @a of+                    STrue ->+                        Right+                            ( VU.imap+                                ( \i x -> case bm of+                                    Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN"+                                    _ -> fromIntegral x+                                )+                                f+                            )+                    SFalse ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @Float)+                                    , expectedType = Right (typeRep @a)+                                    , callingFunctionName = Just "toFloatVector"+                                    , errorColumnName = Nothing+                                    }+                                )+        BoxedColumn bm (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of+            Just Refl ->+                Right+                    ( VB.convert $+                        VB.imap+                            ( \i x -> case bm of+                                Just bitmap | not (bitmapTestBit bitmap i) -> read "NaN"+                                _ -> fromIntegral x+                            )+                            f+                    )+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @Float)+                            , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())+                            , callingFunctionName = Just "toFloatVector"+                            , errorColumnName = Nothing+                            }+                        )++{- | Converts a column to an unboxed vector of 'Int' values.++This function performs intelligent type coercion for numeric types:++* If the column is already 'Int', returns it directly+* If the column contains floating-point types, rounds via 'round' and converts+* If the column contains other integral types, converts via 'fromIntegral'+* If the column is boxed 'Integer', converts via 'fromIntegral'++==== __Returns__++* 'Right' - The converted 'Int' vector+* 'Left' 'TypeMismatchException' - If the column is not numeric++==== __Note__++Unlike 'toDoubleVector' and 'toFloatVector', this function does NOT support+'OptionalColumn'. Optional columns must be handled separately.++==== __Rounding behavior__++Floating-point values are rounded to the nearest integer using 'round'.+For example: 2.5 rounds to 2, 3.5 rounds to 4 (banker's rounding).+-}+toIntVector :: Column -> Either DataFrameException (VU.Vector Int)+toIntVector column =+    case column of+        UnboxedColumn _ (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of+            Just Refl -> Right f+            Nothing -> case sFloating @a of+                STrue -> Right (VU.map (round . realToFrac) f)+                SFalse -> case sIntegral @a of+                    STrue -> Right (VU.map fromIntegral f)+                    SFalse ->+                        Left $+                            TypeMismatchException+                                ( MkTypeErrorContext+                                    { userType = Right (typeRep @Int)+                                    , expectedType = Right (typeRep @a)+                                    , callingFunctionName = Just "toIntVector"+                                    , errorColumnName = Nothing+                                    }+                                )+        BoxedColumn _ (f :: VB.Vector a) -> case testEquality (typeRep @a) (typeRep @Integer) of+            Just Refl -> Right (VB.convert $ VB.map fromIntegral f)+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @Int)+                            , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())+                            , callingFunctionName = Just "toIntVector"+                            , errorColumnName = Nothing+                            }+                        )++toUnboxedVector ::+    forall a.+    (Columnable a, VU.Unbox a) => Column -> Either DataFrameException (VU.Vector a)+toUnboxedVector column =+    case column of+        UnboxedColumn _ (f :: VU.Vector b) -> case testEquality (typeRep @a) (typeRep @b) of+            Just Refl -> Right f+            Nothing ->+                Left $+                    TypeMismatchException+                        ( MkTypeErrorContext+                            { userType = Right (typeRep @Int)+                            , expectedType = Right (typeRep @a)+                            , callingFunctionName = Just "toUnboxedVector"+                            , errorColumnName = Nothing+                            }+                        )+        _ ->+            Left $+                TypeMismatchException+                    ( MkTypeErrorContext+                        { userType = Right (typeRep @a)+                        , expectedType = Left (columnTypeString column) :: Either String (TypeRep ())+                        , callingFunctionName = Just "toUnboxedVector"+                        , errorColumnName = Nothing+                        }+                    ) {-# INLINE toUnboxedVector #-}

@@ -1,8 +1,10 @@+{-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE InstanceSigs #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE PatternSynonyms #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} @@ -17,13 +19,18 @@ import Control.Exception (throw) import Data.Function (on) import Data.List (sortBy, transpose, (\\))-import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl))+import Data.Maybe (fromMaybe)+import Data.Type.Equality (+    TestEquality (testEquality),+    type (:~:) (Refl),+    type (:~~:) (HRefl),+ ) import DataFrame.Display.Terminal.PrettyPrint import DataFrame.Errors import DataFrame.Internal.Column import DataFrame.Internal.Expression import Text.Printf-import Type.Reflection (typeRep)+import Type.Reflection (Typeable, eqTypeRep, typeRep, pattern App) import Prelude hiding (null)  data DataFrame = DataFrame@@ -110,19 +117,34 @@     let header = map fst (sortBy (compare `on` snd) $ M.toList (columnIndices d))         types = V.toList $ V.filter (/= "") $ V.map getType (columns d)         getType :: Column -> T.Text-        getType (BoxedColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a)-        getType (UnboxedColumn (column :: VU.Vector a)) = T.pack $ show (typeRep @a)-        getType (OptionalColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a)+        showMaybeType :: forall a. (Typeable a) => String+        showMaybeType =+            let s = show (typeRep @a)+             in "Maybe " <> if ' ' `elem` s then "(" <> s <> ")" else s+        getType (BoxedColumn Nothing (_ :: V.Vector a)) = T.pack $ show (typeRep @a)+        getType (BoxedColumn (Just _) (_ :: V.Vector a)) = T.pack $ showMaybeType @a+        getType (UnboxedColumn Nothing (_ :: VU.Vector a)) = T.pack $ show (typeRep @a)+        getType (UnboxedColumn (Just _) (_ :: VU.Vector a)) = T.pack $ showMaybeType @a         -- Separate out cases dynamically so we don't end up making round trip string         -- copies.         get :: Maybe Column -> V.Vector T.Text-        get (Just (BoxedColumn (column :: V.Vector a))) = case testEquality (typeRep @a) (typeRep @T.Text) of+        get (Just (BoxedColumn (Just bm) (column :: V.Vector a))) =+            V.generate (V.length column) $ \i ->+                if bitmapTestBit bm i+                    then T.pack (show (Just (V.unsafeIndex column i)))+                    else "Nothing"+        get (Just (BoxedColumn Nothing (column :: V.Vector a))) = case testEquality (typeRep @a) (typeRep @T.Text) of             Just Refl -> column             Nothing -> case testEquality (typeRep @a) (typeRep @String) of                 Just Refl -> V.map T.pack column                 Nothing -> V.map (T.pack . show) column-        get (Just (UnboxedColumn column)) = V.map (T.pack . show) (V.convert column)-        get (Just (OptionalColumn column)) = V.map (T.pack . show) column+        get (Just (UnboxedColumn (Just bm) column)) =+            let col = V.convert column+             in V.generate (V.length col) $ \i ->+                    if bitmapTestBit bm i+                        then T.pack (show (Just (V.unsafeIndex col i)))+                        else "Nothing"+        get (Just (UnboxedColumn Nothing column)) = V.map (T.pack . show) (V.convert column)         get Nothing = V.empty         getTextColumnFromFrame df (i, name) = get $ (V.!?) (columns d) ((M.!) (columnIndices d) name)         rows =@@ -161,16 +183,16 @@  {- | Retrieves a column by name from the dataframe, throwing an exception if not found. -This is an unsafe version of 'getColumn' that throws 'ColumnNotFoundException'+This is an unsafe version of 'getColumn' that throws 'ColumnsNotFoundException' if the column does not exist. Use this when you are certain the column exists.  ==== __Throws__ -* 'ColumnNotFoundException' - if the column with the given name does not exist+* 'ColumnsNotFoundException' - if the column with the given name does not exist -} unsafeGetColumn :: T.Text -> DataFrame -> Column unsafeGetColumn name df = case getColumn name df of-    Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df)+    Nothing -> throw $ ColumnsNotFoundException [name] "" (M.keys $ columnIndices df)     Just col -> col  {- | Checks if the dataframe is empty (has no columns).@@ -180,3 +202,40 @@ -} null :: DataFrame -> Bool null df = V.null (columns df)++-- | Convert a DataFrame to a CSV (comma-separated) text.+toCsv :: DataFrame -> T.Text+toCsv = toSeparated ','++-- | Convert a DataFrame to a text representation with a custom separator.+toSeparated :: Char -> DataFrame -> T.Text+toSeparated sep df+    | null df = T.empty+    | otherwise =+        let (rows, _) = dataframeDimensions df+            headers = map fst (sortBy (compare `on` snd) (M.toList (columnIndices df)))+            sepText = T.singleton sep+            headerLine = T.intercalate sepText headers+            dataLines = map (T.intercalate sepText . getRowAsText df) [0 .. rows - 1]+         in T.unlines (headerLine : dataLines)++getRowAsText :: DataFrame -> Int -> [T.Text]+getRowAsText df i = map (`showElement` i) (V.toList (columns df))++showElement :: Column -> Int -> T.Text+showElement (BoxedColumn _ (c :: V.Vector a)) i = case c V.!? i of+    Nothing -> error $ "Column index out of bounds at row " ++ show i+    Just e+        | Just Refl <- testEquality (typeRep @a) (typeRep @T.Text) -> e+        | App t1 t2 <- typeRep @a+        , Just HRefl <- eqTypeRep t1 (typeRep @Maybe) ->+            case testEquality t2 (typeRep @T.Text) of+                Just Refl -> fromMaybe "null" e+                Nothing -> stripJust (T.pack (show e))+        | otherwise -> T.pack (show e)+showElement (UnboxedColumn _ c) i = case c VU.!? i of+    Nothing -> error $ "Column index out of bounds at row " ++ show i+    Just e -> T.pack (show e)++stripJust :: T.Text -> T.Text+stripJust = fromMaybe "null" . T.stripPrefix "Just "

@@ -212,18 +212,21 @@ -} sliceGroups :: Column -> VU.Vector Int -> VU.Vector Int -> V.Vector Column sliceGroups col os indices = case col of-    BoxedColumn vec ->-        let !sorted = V.unsafeBackpermute vec (V.convert indices)+    BoxedColumn bm vec ->+        let !sorted =+                V.generate+                    (VU.length indices)+                    ((vec `V.unsafeIndex`) . (indices `VU.unsafeIndex`))          in V.generate nGroups $ \i ->-                BoxedColumn (V.unsafeSlice (start i) (len i) sorted)-    UnboxedColumn vec ->+                BoxedColumn+                    (fmap (bitmapSlice (start i) (len i)) bm)+                    (V.unsafeSlice (start i) (len i) sorted)+    UnboxedColumn bm vec ->         let !sorted = VU.unsafeBackpermute vec indices          in V.generate nGroups $ \i ->-                UnboxedColumn (VU.unsafeSlice (start i) (len i) sorted)-    OptionalColumn vec ->-        let !sorted = V.unsafeBackpermute vec (V.convert indices)-         in V.generate nGroups $ \i ->-                OptionalColumn (V.unsafeSlice (start i) (len i) sorted)+                UnboxedColumn+                    (fmap (bitmapSlice (start i) (len i)) bm)+                    (VU.unsafeSlice (start i) (len i) sorted)   where     !nGroups = VU.length os - 1     start i = os `VU.unsafeIndex` i@@ -272,7 +275,7 @@     (Columnable b) =>     (Either String Double -> b) -> Column -> Either DataFrameException Column promoteToDoubleWith onResult col = case col of-    UnboxedColumn (v :: VU.Vector c) ->+    UnboxedColumn Nothing (v :: VU.Vector c) ->         case sFloating @c of             STrue ->                 Right $@@ -284,35 +287,34 @@                         fromVector @b                             (V.map (onResult . Right . (fromIntegral :: c -> Double)) (VG.convert v))                 SFalse -> castMismatch @c @b-    OptionalColumn (v :: V.Vector (Maybe c)) ->+    UnboxedColumn (Just bm) (v :: VU.Vector c) ->         case sFloating @c of             STrue ->                 Right $                     fromVector @b-                        ( V.map-                            (maybe (onResult (Left "null")) (onResult . Right . (realToFrac :: c -> Double)))-                            v+                        ( V.generate (VU.length v) $ \i ->+                            if bitmapTestBit bm i+                                then onResult (Right (realToFrac (VU.unsafeIndex v i) :: Double))+                                else onResult (Left "null")                         )             SFalse -> case sIntegral @c of                 STrue ->                     Right $                         fromVector @b-                            ( V.map-                                ( maybe-                                    (onResult (Left "null"))-                                    (onResult . Right . (fromIntegral :: c -> Double))-                                )-                                v+                            ( V.generate (VU.length v) $ \i ->+                                if bitmapTestBit bm i+                                    then onResult (Right (fromIntegral (VU.unsafeIndex v i) :: Double))+                                    else onResult (Left "null")                             )-                SFalse -> tryParseWith @Double onResult col-    BoxedColumn _ -> tryParseWith @Double onResult col+                SFalse -> castMismatch @c @b+    BoxedColumn _ _ -> tryParseWith @Double onResult col  promoteToFloatWith ::     forall b.     (Columnable b) =>     (Either String Float -> b) -> Column -> Either DataFrameException Column promoteToFloatWith onResult col = case col of-    UnboxedColumn (v :: VU.Vector c) ->+    UnboxedColumn Nothing (v :: VU.Vector c) ->         case sFloating @c of             STrue ->                 Right $@@ -324,32 +326,34 @@                         fromVector @b                             (V.map (onResult . Right . (fromIntegral :: c -> Float)) (VG.convert v))                 SFalse -> castMismatch @c @b-    OptionalColumn (v :: V.Vector (Maybe c)) ->+    UnboxedColumn (Just bm) (v :: VU.Vector c) ->         case sFloating @c of             STrue ->                 Right $                     fromVector @b-                        ( V.map-                            (maybe (onResult (Left "null")) (onResult . Right . (realToFrac :: c -> Float)))-                            v+                        ( V.generate (VU.length v) $ \i ->+                            if bitmapTestBit bm i+                                then onResult (Right (realToFrac (VU.unsafeIndex v i) :: Float))+                                else onResult (Left "null")                         )             SFalse -> case sIntegral @c of                 STrue ->                     Right $                         fromVector @b-                            ( V.map-                                (maybe (onResult (Left "null")) (onResult . Right . (fromIntegral :: c -> Float)))-                                v+                            ( V.generate (VU.length v) $ \i ->+                                if bitmapTestBit bm i+                                    then onResult (Right (fromIntegral (VU.unsafeIndex v i) :: Float))+                                    else onResult (Left "null")                             )-                SFalse -> tryParseWith @Float onResult col-    BoxedColumn _ -> tryParseWith @Float onResult col+                SFalse -> castMismatch @c @b+    BoxedColumn _ _ -> tryParseWith @Float onResult col  promoteToIntWith ::     forall b.     (Columnable b) =>     (Either String Int -> b) -> Column -> Either DataFrameException Column promoteToIntWith onResult col = case col of-    UnboxedColumn (v :: VU.Vector c) ->+    UnboxedColumn Nothing (v :: VU.Vector c) ->         case sFloating @c of             STrue ->                 Right $@@ -361,28 +365,27 @@                         fromVector @b                             (V.map (onResult . Right . (fromIntegral :: c -> Int)) (VG.convert v))                 SFalse -> castMismatch @c @b-    OptionalColumn (v :: V.Vector (Maybe c)) ->+    UnboxedColumn (Just bm) (v :: VU.Vector c) ->         case sFloating @c of             STrue ->                 Right $                     fromVector @b-                        ( V.map-                            ( maybe-                                (onResult (Left "null"))-                                (onResult . Right . (round . (realToFrac :: c -> Double)))-                            )-                            v+                        ( V.generate (VU.length v) $ \i ->+                            if bitmapTestBit bm i+                                then onResult (Right (round (realToFrac (VU.unsafeIndex v i) :: Double)))+                                else onResult (Left "null")                         )             SFalse -> case sIntegral @c of                 STrue ->                     Right $                         fromVector @b-                            ( V.map-                                (maybe (onResult (Left "null")) (onResult . Right . (fromIntegral :: c -> Int)))-                                v+                            ( V.generate (VU.length v) $ \i ->+                                if bitmapTestBit bm i+                                    then onResult (Right (fromIntegral (VU.unsafeIndex v i) :: Int))+                                    else onResult (Left "null")                             )-                SFalse -> tryParseWith @Int onResult col-    BoxedColumn _ -> tryParseWith @Int onResult col+                SFalse -> castMismatch @c @b+    BoxedColumn _ _ -> tryParseWith @Int onResult col  -- | Single parse primitive: apply @onResult@ to the result of 'reads'. parseWith :: (Read a) => (Either String a -> b) -> String -> b@@ -395,27 +398,34 @@     (Columnable a, Columnable b) =>     (Either String a -> b) -> Column -> Either DataFrameException Column tryParseWith onResult col = case col of-    BoxedColumn (v :: V.Vector c) ->-        case testEquality (typeRep @c) (typeRep @String) of-            Just Refl -> Right $ fromVector @b $ V.map (parseWith onResult) v-            Nothing ->-                case testEquality (typeRep @c) (typeRep @T.Text) of-                    Just Refl -> Right $ fromVector @b $ V.map (parseWith onResult . T.unpack) v-                    Nothing -> castMismatch @c @b-    OptionalColumn (v :: V.Vector (Maybe c)) ->+    BoxedColumn bm (v :: V.Vector c) ->         case testEquality (typeRep @c) (typeRep @String) of-            Just Refl ->-                Right $-                    fromVector @b $-                        V.map (maybe (onResult (Left "null")) (parseWith onResult)) v+            Just Refl -> case bm of+                Nothing -> Right $ fromVector @b $ V.map (parseWith onResult) v+                Just bitmap ->+                    Right $+                        fromVector @b $+                            V.imap+                                ( \i x ->+                                    if bitmapTestBit bitmap i then parseWith onResult x else onResult (Left "null")+                                )+                                v             Nothing ->                 case testEquality (typeRep @c) (typeRep @T.Text) of-                    Just Refl ->-                        Right $-                            fromVector @b $-                                V.map (maybe (onResult (Left "null")) (parseWith onResult . T.unpack)) v+                    Just Refl -> case bm of+                        Nothing -> Right $ fromVector @b $ V.map (parseWith onResult . T.unpack) v+                        Just bitmap ->+                            Right $+                                fromVector @b $+                                    V.imap+                                        ( \i x ->+                                            if bitmapTestBit bitmap i+                                                then parseWith onResult (T.unpack x)+                                                else onResult (Left "null")+                                        )+                                        v                     Nothing -> castMismatch @c @b-    UnboxedColumn (_ :: VU.Vector c) -> castMismatch @c @b+    UnboxedColumn _ (_ :: VU.Vector c) -> castMismatch @c @b  {- | When the output type @b@ is @Maybe c@ (or @Maybe (Maybe c)@) and the column stores plain @c@ values, wrap each element in 'Just'.@@ -430,16 +440,15 @@     (Columnable a, Columnable b) =>     (Either String a -> b) -> Column -> Maybe (Either DataFrameException Column) tryMaybeWrap _onResult col = case col of-    UnboxedColumn (v :: VU.Vector c) ->+    UnboxedColumn Nothing (v :: VU.Vector c) ->         let wrapped = V.map Just (VG.convert v) :: V.Vector (Maybe c)          in case testEquality (typeRep @b) (typeRep @(Maybe c)) of                 Just Refl -> Just $ Right $ fromVector @b wrapped                 Nothing ->-                    -- join: b = Maybe (Maybe c) → produce Maybe c column                     case testEquality (typeRep @b) (typeRep @(Maybe (Maybe c))) of                         Just _ -> Just $ Right $ fromVector @(Maybe c) wrapped                         Nothing -> Nothing-    BoxedColumn (v :: V.Vector c) ->+    BoxedColumn Nothing (v :: V.Vector c) ->         let wrapped = V.map Just v :: V.Vector (Maybe c)          in case testEquality (typeRep @b) (typeRep @(Maybe c)) of                 Just Refl -> Just $ Right $ fromVector @b wrapped@@ -447,7 +456,6 @@                     case testEquality (typeRep @b) (typeRep @(Maybe (Maybe c))) of                         Just _ -> Just $ Right $ fromVector @(Maybe c) wrapped                         Nothing -> Nothing-    -- OptionalColumn and NullableColumn are already handled by the hasElemType guards above.     _ -> Nothing  castMismatch ::@@ -482,7 +490,7 @@ eval (FlatCtx df) (Col name) =     case getColumn name df of         Nothing ->-            Left $ ColumnNotFoundException name "" (M.keys $ columnIndices df)+            Left $ ColumnsNotFoundException [name] "" (M.keys $ columnIndices df)         Just c             | hasElemType @a c -> Right (Flat c)             | otherwise ->@@ -500,8 +508,8 @@     case getColumn name (fullDataframe gdf) of         Nothing ->             Left $-                ColumnNotFoundException-                    name+                ColumnsNotFoundException+                    [name]                     ""                     (M.keys $ columnIndices $ fullDataframe gdf)         Just c@@ -524,14 +532,14 @@     case getColumn name df of         Nothing ->             Left $-                ColumnNotFoundException name "" (M.keys $ columnIndices df)+                ColumnsNotFoundException [name] "" (M.keys $ columnIndices df)         Just c -> Flat <$> promoteColumnWith onResult c eval (GroupCtx gdf) (CastWith name _tag onResult) =     case getColumn name (fullDataframe gdf) of         Nothing ->             Left $-                ColumnNotFoundException-                    name+                ColumnsNotFoundException+                    [name]                     ""                     (M.keys $ columnIndices $ fullDataframe gdf)         Just c -> do@@ -579,8 +587,8 @@         case getColumn name (fullDataframe gdf) of             Nothing ->                 Left $-                    ColumnNotFoundException-                        name+                    ColumnsNotFoundException+                        [name]                         ""                         (M.keys $ columnIndices $ fullDataframe gdf)             Just col ->@@ -599,8 +607,8 @@                 case getColumn name (fullDataframe gdf) of                     Nothing ->                         Left $-                            ColumnNotFoundException-                                name+                            ColumnsNotFoundException+                                [name]                                 ""                                 (M.keys $ columnIndices $ fullDataframe gdf)                     Just col ->@@ -617,8 +625,8 @@             case getColumn name (fullDataframe gdf) of                 Nothing ->                     Left $-                        ColumnNotFoundException-                            name+                        ColumnsNotFoundException+                            [name]                             ""                             (M.keys $ columnIndices $ fullDataframe gdf)                 Just col ->@@ -766,4 +774,4 @@             -- The Column payload is intentionally unused — the only             -- call-site ('aggregate') immediately throws             -- 'UnaggregatedException' on this constructor.-            Right $ UnAggregated $ BoxedColumn @T.Text V.empty+            Right $ UnAggregated $ BoxedColumn @T.Text Nothing V.empty

@@ -169,13 +169,12 @@     get name = case getColumn name df of         Nothing ->             throw $-                ColumnNotFoundException-                    name+                ColumnsNotFoundException+                    [name]                     "[INTERNAL] mkRowFromArgs"                     (M.keys $ columnIndices df)-        Just (BoxedColumn column) -> toAny (column V.! i)-        Just (UnboxedColumn column) -> toAny (column VU.! i)-        Just (OptionalColumn column) -> toAny (column V.! i)+        Just (BoxedColumn _ column) -> toAny (column V.! i)+        Just (UnboxedColumn _ column) -> toAny (column VU.! i)  -- This function will return the items in the order that is specified -- by the user. For example, if the dataframe consists of the columns@@ -193,17 +192,14 @@                 ++ "the other columns at index "                 ++ show i     get name = case getColumn name df of-        Just (BoxedColumn c) -> case c V.!? i of-            Just e -> toAny e-            Nothing -> throwError name-        Just (OptionalColumn c) -> case c V.!? i of+        Just (BoxedColumn _ c) -> case c V.!? i of             Just e -> toAny e             Nothing -> throwError name-        Just (UnboxedColumn c) -> case c VU.!? i of+        Just (UnboxedColumn _ c) -> case c VU.!? i of             Just e -> toAny e             Nothing -> throwError name         Nothing ->-            throw $ ColumnNotFoundException name "mkRowRep" (M.keys $ columnIndices df)+            throw $ ColumnsNotFoundException [name] "mkRowRep" (M.keys $ columnIndices df)  sortedIndexes' :: [Bool] -> V.Vector Row -> VU.Vector Int sortedIndexes' flipCompare rows = runST $ do

@@ -27,7 +27,7 @@ data Rep     = RBoxed     | RUnboxed-    | ROptional+    | RNullableBoxed  -- | Type-level if statement. type family If (cond :: Bool) (yes :: k) (no :: k) :: k where@@ -68,9 +68,9 @@     Numeric Float = 'True     Numeric _ = 'False --- | Compute the column representation tag for any ‘a’.+-- | Compute the column representation tag for any 'a'. type family KindOf a :: Rep where-    KindOf (Maybe a) = 'ROptional+    KindOf (Maybe a) = 'RNullableBoxed     KindOf a = If (Unboxable a) 'RUnboxed 'RBoxed  -- | Type-level boolean for constraint/type comparison.

@@ -51,11 +51,14 @@ import qualified Data.Vector.Unboxed as VU  import Data.Bits (setBit, shiftL, testBit, (.|.))-import Data.Maybe (fromMaybe, isJust) import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) import Data.Word (Word16, Word32, Word64, Word8) import qualified DataFrame.Internal.Binary as Binary-import DataFrame.Internal.Column (Column (..))+import DataFrame.Internal.Column (+    Column (..),+    bitmapTestBit,+    buildBitmapFromValid,+ ) import DataFrame.Internal.DataFrame (DataFrame (..)) import Foreign (ForeignPtr, castForeignPtr, plusForeignPtr, sizeOf) import System.Directory (getTemporaryDirectory, removeFile)@@ -110,48 +113,51 @@     nameLen = fromIntegral (BS.length nameBytes) :: Word16  columnTypeTag :: Column -> Word8-columnTypeTag (UnboxedColumn (_ :: VU.Vector a)) =+columnTypeTag (UnboxedColumn Nothing (_ :: VU.Vector a)) =     case testEquality (typeRep @a) (typeRep @Int) of         Just Refl -> tagInt         Nothing -> case testEquality (typeRep @a) (typeRep @Double) of             Just Refl -> tagDouble             Nothing -> error "spillToDisk: unsupported UnboxedColumn element type"-columnTypeTag (BoxedColumn _) = tagText-columnTypeTag (OptionalColumn (_ :: V.Vector (Maybe a))) =+columnTypeTag (UnboxedColumn (Just _) (_ :: VU.Vector a)) =     case testEquality (typeRep @a) (typeRep @Int) of         Just Refl -> tagMaybeInt         Nothing -> case testEquality (typeRep @a) (typeRep @Double) of             Just Refl -> tagMaybeDouble-            Nothing -> tagMaybeText+            Nothing -> error "spillToDisk: unsupported nullable UnboxedColumn element type"+columnTypeTag (BoxedColumn Nothing _) = tagText+columnTypeTag (BoxedColumn (Just _) _) = tagMaybeText  buildColumnData :: Int -> Column -> BSB.Builder-buildColumnData _ (UnboxedColumn (v :: VU.Vector a)) =+buildColumnData _ (UnboxedColumn Nothing (v :: VU.Vector a)) =     case testEquality (typeRep @a) (typeRep @Int) of         Just Refl -> buildIntVector v         Nothing ->             case testEquality (typeRep @a) (typeRep @Double) of                 Just Refl -> buildDoubleVector v                 Nothing -> error "spillToDisk: unsupported UnboxedColumn element type"-buildColumnData _ (BoxedColumn (v :: V.Vector a)) =-    case testEquality (typeRep @a) (typeRep @T.Text) of-        Just Refl -> buildTextVector v-        Nothing -> error "spillToDisk: unsupported BoxedColumn element type"-buildColumnData _ (OptionalColumn (v :: V.Vector (Maybe a))) =+buildColumnData _ (UnboxedColumn (Just bm) (v :: VU.Vector a)) =     case testEquality (typeRep @a) (typeRep @Int) of         Just Refl ->-            buildNullBitmap (V.map isJust v)-                <> buildIntVector (VU.convert (V.map (fromMaybe 0) v))+            buildNullBitmap (V.generate (VU.length v) (bitmapTestBit bm))+                <> buildIntVector v         Nothing ->             case testEquality (typeRep @a) (typeRep @Double) of                 Just Refl ->-                    buildNullBitmap (V.map isJust v)-                        <> buildDoubleVector (VU.convert (V.map (fromMaybe 0.0) v))-                Nothing ->-                    let showText x = case testEquality (typeRep @a) (typeRep @T.Text) of-                            Just Refl -> x-                            Nothing -> T.pack (show x)-                        texts = V.map (maybe T.empty showText) v-                     in buildNullBitmap (V.map isJust v) <> buildTextVector texts+                    buildNullBitmap (V.generate (VU.length v) (bitmapTestBit bm))+                        <> buildDoubleVector v+                Nothing -> error "spillToDisk: unsupported nullable UnboxedColumn element type"+buildColumnData _ (BoxedColumn Nothing (v :: V.Vector a)) =+    case testEquality (typeRep @a) (typeRep @T.Text) of+        Just Refl -> buildTextVector v+        Nothing -> error "spillToDisk: unsupported BoxedColumn element type"+buildColumnData _ (BoxedColumn (Just bm) (v :: V.Vector a)) =+    let isValidVec = V.generate (V.length v) (bitmapTestBit bm)+        showText x = case testEquality (typeRep @a) (typeRep @T.Text) of+            Just Refl -> x+            Nothing -> T.pack (show x)+        texts = V.imap (\i x -> if bitmapTestBit bm i then showText x else T.empty) v+     in buildNullBitmap isValidVec <> buildTextVector texts  {- | Bulk-encode an Int vector as 8-byte LE values (native layout on LE platforms). hPutBuilder flushes synchronously so the underlying ForeignPtr outlives the Builder.@@ -249,37 +255,28 @@ readColumnData bs off nrows tag     | tag == tagInt = do         (off', v) <- readIntColumn bs off nrows-        return (off', UnboxedColumn v)+        return (off', UnboxedColumn Nothing v)     | tag == tagDouble = do         (off', v) <- readDoubleColumn bs off nrows-        return (off', UnboxedColumn v)+        return (off', UnboxedColumn Nothing v)     | tag == tagText = do         (off', v) <- readTextColumn bs off nrows-        return (off', BoxedColumn v)+        return (off', BoxedColumn Nothing v)     | tag == tagMaybeInt = do         (off1, bitmap) <- readNullBitmap bs off nrows         (off2, v) <- readIntColumn bs off1 nrows-        let maybes =-                V.fromList-                    (zipWith (\valid x -> if valid then Just x else Nothing) bitmap (VU.toList v)) ::-                    V.Vector (Maybe Int)-        return (off2, OptionalColumn maybes)+        let bm = buildBitmapFromValid (VU.fromList (map (\b -> if b then 1 else 0) bitmap))+        return (off2, UnboxedColumn (Just bm) v)     | tag == tagMaybeDouble = do         (off1, bitmap) <- readNullBitmap bs off nrows         (off2, v) <- readDoubleColumn bs off1 nrows-        let maybes =-                V.fromList-                    (zipWith (\valid x -> if valid then Just x else Nothing) bitmap (VU.toList v)) ::-                    V.Vector (Maybe Double)-        return (off2, OptionalColumn maybes)+        let bm = buildBitmapFromValid (VU.fromList (map (\b -> if b then 1 else 0) bitmap))+        return (off2, UnboxedColumn (Just bm) v)     | tag == tagMaybeText = do         (off1, bitmap) <- readNullBitmap bs off nrows         (off2, v) <- readTextColumn bs off1 nrows-        let maybes =-                V.fromList-                    (zipWith (\valid x -> if valid then Just x else Nothing) bitmap (V.toList v)) ::-                    V.Vector (Maybe T.Text)-        return (off2, OptionalColumn maybes)+        let bm = buildBitmapFromValid (VU.fromList (map (\b -> if b then 1 else 0) bitmap))+        return (off2, BoxedColumn (Just bm) v)     | otherwise = Left ("unknown type tag " <> show tag)  {- | Zero-copy Int column read: reuses the ByteString buffer's ForeignPtr.

@@ -30,6 +30,7 @@     Column (..),     MutableColumn (..),     columnLength,+    ensureOptional,     freezeColumn',     writeColumn,  )@@ -60,7 +61,7 @@     ReadOptions         { hasHeader = True         , inferTypes = True-        , safeRead = True+        , safeRead = False         , rowRange = Nothing         , seekPos = Nothing         , totalRows = Nothing@@ -218,7 +219,8 @@     IO Column freezeColumn mutableCols nulls opts colIndex = do     col <- VM.unsafeRead mutableCols colIndex-    freezeColumn' (nulls V.! colIndex) col+    frozen <- freezeColumn' (nulls V.! colIndex) col+    return $! if safeRead opts then ensureOptional frozen else frozen {-# INLINE freezeColumn #-}  -- ---------------------------------------------------------------------------@@ -341,12 +343,6 @@             let val = TextEncoding.decodeUtf8Lenient bs              in VM.unsafeWrite col i val >> return (Right True)         Nothing -> return (Left (TextEncoding.decodeUtf8Lenient bs))-writeColumnBs i bs (MOptionalColumn (col :: VM.IOVector (Maybe a))) =-    case testEquality (typeRep @a) (typeRep @T.Text) of-        Just Refl ->-            let val = TextEncoding.decodeUtf8Lenient bs-             in VM.unsafeWrite col i (Just val) >> return (Right True)-        Nothing -> return (Left (TextEncoding.decodeUtf8Lenient bs)) writeColumnBs i bs (MUnboxedColumn (col :: VUM.IOVector a)) =     case testEquality (typeRep @a) (typeRep @Double) of         Just Refl -> case readByteStringDouble bs of@@ -417,7 +413,6 @@ sliceCol :: Int -> MutableColumn -> MutableColumn sliceCol n (MBoxedColumn col) = MBoxedColumn (VM.take n col) sliceCol n (MUnboxedColumn col) = MUnboxedColumn (VUM.take n col)-sliceCol n (MOptionalColumn col) = MOptionalColumn (VM.take n col)  {- | Finds the index of the next unquoted newline (0x0A). Fast path: uses memchr (SIMD) and falls back to a quote-aware linear scan

@@ -30,6 +30,7 @@     Column (..),     TypedColumn (..),     atIndicesStable,+    bitmapTestBit,  ) import DataFrame.Internal.DataFrame (DataFrame (..), GroupedDataFrame (..)) import DataFrame.Internal.Expression@@ -49,8 +50,8 @@ groupBy names df     | any (`notElem` columnNames df) names =         throw $-            ColumnNotFoundException-                (T.pack $ show $ names L.\\ columnNames df)+            ColumnsNotFoundException+                (names L.\\ columnNames df)                 "groupBy"                 (columnNames df)     | nRows df == 0 =@@ -81,7 +82,7 @@         let selectedCols = map (columns df V.!) indicesToGroup          forM_ selectedCols $ \case-            UnboxedColumn (v :: VU.Vector a) ->+            UnboxedColumn _ (v :: VU.Vector a) ->                 case testEquality (typeRep @a) (typeRep @Int) of                     Just Refl ->                         VU.imapM_@@ -129,31 +130,28 @@                                                         VUM.unsafeWrite mv i (i, hashWithSalt h x)                                                     )                                                     v-            BoxedColumn (v :: V.Vector a) ->+            BoxedColumn bm (v :: V.Vector a) ->                 case testEquality (typeRep @a) (typeRep @T.Text) of                     Just Refl ->                         V.imapM_                             ( \i t -> do                                 (_, !h) <- VUM.unsafeRead mv i-                                VUM.unsafeWrite mv i (i, hashWithSalt h t)+                                let h' = case bm of+                                        Just bm' | not (bitmapTestBit bm' i) -> hashWithSalt h (0 :: Int) -- null sentinel+                                        _ -> hashWithSalt h t+                                VUM.unsafeWrite mv i (i, h')                             )                             v                     Nothing ->                         V.imapM_                             ( \i d -> do-                                let x = hash (show d)                                 (_, !h) <- VUM.unsafeRead mv i-                                VUM.unsafeWrite mv i (i, hashWithSalt h x)+                                let h' = case bm of+                                        Just bm' | not (bitmapTestBit bm' i) -> hashWithSalt h (0 :: Int) -- null sentinel+                                        _ -> hashWithSalt h (hash (show d))+                                VUM.unsafeWrite mv i (i, h')                             )                             v-            OptionalColumn v ->-                V.imapM_-                    ( \i d -> do-                        let x = hash (show d)-                        (_, !h) <- VUM.unsafeRead mv i-                        VUM.unsafeWrite mv i (i, hashWithSalt h x)-                    )-                    v          let numPasses = 4             bucketSize = 65536@@ -197,7 +195,7 @@     let selectedCols = map (columns df V.!) indices      forM_ selectedCols $ \case-        UnboxedColumn (v :: VU.Vector a) ->+        UnboxedColumn _ (v :: VU.Vector a) ->             case testEquality (typeRep @a) (typeRep @Int) of                 Just Refl ->                     VU.imapM_@@ -245,31 +243,28 @@                                                     VUM.unsafeWrite mv i (hashWithSalt h x)                                                 )                                                 v-        BoxedColumn (v :: V.Vector a) ->+        BoxedColumn bm (v :: V.Vector a) ->             case testEquality (typeRep @a) (typeRep @T.Text) of                 Just Refl ->                     V.imapM_                         ( \i (t :: T.Text) -> do                             h <- VUM.unsafeRead mv i-                            VUM.unsafeWrite mv i (hashWithSalt h t)+                            let h' = case bm of+                                    Just bm' | not (bitmapTestBit bm' i) -> hashWithSalt h (0 :: Int)+                                    _ -> hashWithSalt h t+                            VUM.unsafeWrite mv i h'                         )                         v                 Nothing ->                     V.imapM_                         ( \i d -> do-                            let x = hash (show d)+                            let x = case bm of+                                    Just bm' | not (bitmapTestBit bm' i) -> 0 :: Int+                                    _ -> hash (show d)                             h <- VUM.unsafeRead mv i                             VUM.unsafeWrite mv i (hashWithSalt h x)                         )                         v-        OptionalColumn v ->-            V.imapM_-                ( \i d -> do-                    let x = hash (show d)-                    h <- VUM.unsafeRead mv i-                    VUM.unsafeWrite mv i (hashWithSalt h x)-                )-                v      VU.unsafeFreeze mv   where

@@ -17,6 +17,7 @@ import qualified Data.Vector.Unboxed as VU  import Control.Exception (throw)+import Data.Bits (popCount) import Data.Either import qualified Data.Foldable as Fold import Data.Function (on, (&))@@ -306,7 +307,7 @@     -- | DataFrame to add the column to     DataFrame ->     DataFrame-insertUnboxedVector name xs = insertColumn name (UnboxedColumn xs)+insertUnboxedVector name xs = insertColumn name (UnboxedColumn Nothing xs)  {- | /O(n)/ Add a column to the dataframe. @@ -394,7 +395,7 @@     | null df = throw (EmptyDataSetException "cloneColumn")     | otherwise = fromMaybe         ( throw $-            ColumnNotFoundException original "cloneColumn" (M.keys $ columnIndices df)+            ColumnsNotFoundException [original] "cloneColumn" (M.keys $ columnIndices df)         )         $ do             column <- getColumn original df@@ -485,7 +486,7 @@ renameSafe orig new df     | null df = throw (EmptyDataSetException "rename")     | otherwise = fromMaybe-        (Left $ ColumnNotFoundException orig "rename" (M.keys $ columnIndices df))+        (Left $ ColumnsNotFoundException [orig] "rename" (M.keys $ columnIndices df))         $ do             columnIndex <- M.lookup orig (columnIndices df)             let origRemoved = M.delete orig (columnIndices df)@@ -530,11 +531,11 @@             [ColumnInfo]     indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df))     columnName i = M.lookup i indexMap-    go acc i col@(OptionalColumn (c :: V.Vector a)) =+    go acc i col@(BoxedColumn bm (c :: V.Vector a)) =         let             cname = columnName i             countNulls = nulls col-            columnType = T.pack $ show $ typeRep @a+            columnType = T.pack $ columnTypeString col          in             if isNothing cname                 then acc@@ -545,43 +546,35 @@                         countNulls                         columnType                         : acc-    go acc i col@(BoxedColumn (c :: V.Vector a)) =+    go acc i col@(UnboxedColumn bm c) =         let             cname = columnName i-            columnType = T.pack $ show $ typeRep @a+            countNulls = nulls col+            columnType = T.pack $ columnTypeString col          in             if isNothing cname                 then acc                 else                     ColumnInfo                         (fromMaybe "" cname)-                        (columnLength col)-                        0+                        (columnLength col - countNulls)+                        countNulls                         columnType                         : acc-    go acc i col@(UnboxedColumn c) =-        let-            cname = columnName i-            columnType = T.pack $ columnTypeString col-         in-            -- Unboxed columns cannot have nulls since Maybe-            -- is not an instance of Unbox a-            if isNothing cname-                then acc-                else-                    ColumnInfo (fromMaybe "" cname) (columnLength col) 0 columnType : acc  nulls :: Column -> Int-nulls (OptionalColumn xs) = VG.length $ VG.filter isNothing xs-nulls (BoxedColumn (xs :: V.Vector a)) = case testEquality (typeRep @a) (typeRep @T.Text) of+nulls (BoxedColumn (Just bm) xs) =+    -- count null bits in bitmap+    let n = VG.length xs+     in n - VU.foldl' (\acc b -> acc + popCount b) 0 bm+nulls (BoxedColumn Nothing (xs :: V.Vector a)) = case testEquality (typeRep @a) (typeRep @T.Text) of     Just Refl -> VG.length $ VG.filter isNullish xs     Nothing -> case testEquality (typeRep @a) (typeRep @String) of         Just Refl -> VG.length $ VG.filter (isNullish . T.pack) xs-        Nothing -> case typeRep @a of-            App t1 t2 -> case eqTypeRep t1 (typeRep @Maybe) of-                Just HRefl -> VG.length $ VG.filter isNothing xs-                Nothing -> 0-            _ -> 0+        Nothing -> 0+nulls (UnboxedColumn (Just bm) xs) =+    let n = VG.length xs+     in n - VU.foldl' (\acc b -> acc + popCount b) 0 bm nulls _ = 0  {- | Creates a dataframe from a list of tuples with name and column.@@ -859,7 +852,8 @@         (Col name) -> case getColumn name df of             Just col -> toVector col             Nothing ->-                Left $ ColumnNotFoundException name "columnAsVector" (M.keys $ columnIndices df)+                Left $+                    ColumnsNotFoundException [name] "columnAsVector" (M.keys $ columnIndices df)         _ -> case interpret df expr of             Left e -> throw e             Right (TColumn col) -> toVector col@@ -876,7 +870,7 @@     Just col -> toIntVector col     Nothing ->         Left $-            ColumnNotFoundException name "columnAsIntVector" (M.keys $ columnIndices df)+            ColumnsNotFoundException [name] "columnAsIntVector" (M.keys $ columnIndices df) columnAsIntVector expr df = case interpret df expr of     Left e -> throw e     Right (TColumn col) -> toIntVector col@@ -893,7 +887,10 @@     Just col -> toDoubleVector col     Nothing ->         Left $-            ColumnNotFoundException name "columnAsDoubleVector" (M.keys $ columnIndices df)+            ColumnsNotFoundException+                [name]+                "columnAsDoubleVector"+                (M.keys $ columnIndices df) columnAsDoubleVector expr df = case interpret df expr of     Left e -> throw e     Right (TColumn col) -> toDoubleVector col@@ -910,7 +907,10 @@     Just col -> toFloatVector col     Nothing ->         Left $-            ColumnNotFoundException name "columnAsFloatVector" (M.keys $ columnIndices df)+            ColumnsNotFoundException+                [name]+                "columnAsFloatVector"+                (M.keys $ columnIndices df) columnAsFloatVector expr df = case interpret df expr of     Left e -> throw e     Right (TColumn col) -> toFloatVector col@@ -923,7 +923,10 @@     Just col -> toUnboxedVector col     Nothing ->         Left $-            ColumnNotFoundException name "columnAsFloatVector" (M.keys $ columnIndices df)+            ColumnsNotFoundException+                [name]+                "columnAsFloatVector"+                (M.keys $ columnIndices df) columnAsUnboxedVector expr df = case interpret df expr of     Left e -> throw e     Right (TColumn col) -> toUnboxedVector col@@ -967,7 +970,8 @@      in map toNamedExpr names   where     identityUExpr name = case getColumn name df of-        Just (BoxedColumn (_ :: V.Vector a)) -> UExpr (Col @a name)-        Just (UnboxedColumn (_ :: VU.Vector a)) -> UExpr (Col @a name)-        Just (OptionalColumn (_ :: V.Vector (Maybe a))) -> UExpr (Col @(Maybe a) name)+        Just (BoxedColumn (Just _) (_ :: V.Vector a)) -> UExpr (Col @(Maybe a) name)+        Just (BoxedColumn Nothing (_ :: V.Vector a)) -> UExpr (Col @a name)+        Just (UnboxedColumn (Just _) (_ :: VU.Vector a)) -> UExpr (Col @(Maybe a) name)+        Just (UnboxedColumn Nothing (_ :: VU.Vector a)) -> UExpr (Col @a name)         Nothing -> error $ "showDerivedExpressions: column not found: " ++ T.unpack name

@@ -11,6 +11,7 @@ module DataFrame.Operations.Join where  import Control.Applicative ((<|>))+import Control.Exception (throw) import Control.Monad (forM_, when) import Control.Monad.ST (ST, runST) import qualified Data.HashMap.Strict as HM@@ -27,6 +28,9 @@ import qualified Data.Vector.Algorithms.Merge as VA import qualified Data.Vector.Unboxed as VU import qualified Data.Vector.Unboxed.Mutable as VUM+import DataFrame.Errors (+    DataFrameException (ColumnsNotFoundException),+ ) import DataFrame.Internal.Column as D import DataFrame.Internal.DataFrame as D import DataFrame.Operations.Aggregation as D@@ -148,6 +152,15 @@ keyColIndices :: S.Set T.Text -> DataFrame -> [Int] keyColIndices csSet df = M.elems $ M.restrictKeys (D.columnIndices df) csSet +-- | Validate that all requested join keys exist, then return their indices.+validatedKeyColIndices :: T.Text -> S.Set T.Text -> DataFrame -> [Int]+validatedKeyColIndices callPoint csSet df =+    let columnIdxs = D.columnIndices df+        missingKeys = S.toAscList (csSet `S.difference` M.keysSet columnIdxs)+     in case missingKeys of+            [] -> M.elems $ M.restrictKeys columnIdxs csSet+            _ -> throw (ColumnsNotFoundException missingKeys callPoint (M.keys columnIdxs))+ -- ============================================================ -- Inner Join -- ============================================================@@ -175,36 +188,39 @@ innerJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame innerJoin cs left right     | D.null right || D.null left = D.empty-    | otherwise =-        let-            csSet = S.fromList cs-            leftRows = fst (D.dimensions left)-            rightRows = fst (D.dimensions right)+    | otherwise = innerJoinNonEmpty cs left right -            leftKeyIdxs = keyColIndices csSet left-            rightKeyIdxs = keyColIndices csSet right-            leftHashes = D.computeRowHashes leftKeyIdxs left-            rightHashes = D.computeRowHashes rightKeyIdxs right+innerJoinNonEmpty :: [T.Text] -> DataFrame -> DataFrame -> DataFrame+innerJoinNonEmpty cs left right =+    let+        csSet = S.fromList cs+        leftRows = fst (D.dimensions left)+        rightRows = fst (D.dimensions right) -            buildRows = min leftRows rightRows-            (leftIxs, rightIxs)-                | buildRows > joinStrategyThreshold =-                    sortMergeInnerKernel leftHashes rightHashes-                | rightRows <= leftRows =-                    -- Build on right (smaller or equal), probe with left-                    hashInnerKernel leftHashes rightHashes-                | otherwise =-                    -- Build on left (smaller), probe with right, swap result-                    let (!rIxs, !lIxs) = hashInnerKernel rightHashes leftHashes-                     in (lIxs, rIxs)-         in-            assembleInner csSet left right leftIxs rightIxs+        leftKeyIdxs = validatedKeyColIndices "innerJoin" csSet left+        rightKeyIdxs = validatedKeyColIndices "innerJoin" csSet right+        leftHashes = D.computeRowHashes leftKeyIdxs left+        rightHashes = D.computeRowHashes rightKeyIdxs right +        buildRows = min leftRows rightRows+        (leftIxs, rightIxs)+            | buildRows > joinStrategyThreshold =+                sortMergeInnerKernel leftHashes rightHashes+            | rightRows <= leftRows =+                -- Build on right (smaller or equal), probe with left+                hashInnerKernel leftHashes rightHashes+            | otherwise =+                -- Build on left (smaller), probe with right, swap result+                let (!rIxs, !lIxs) = hashInnerKernel rightHashes leftHashes+                 in (lIxs, rIxs)+     in+        assembleInner csSet left right leftIxs rightIxs+ -- | Compute hashes for the given key column names in a DataFrame. buildHashColumn :: [T.Text] -> DataFrame -> VU.Vector Int buildHashColumn keys df =     let csSet = S.fromList keys-        keyIdxs = keyColIndices csSet df+        keyIdxs = validatedKeyColIndices "buildHashColumn" csSet df      in D.computeRowHashes keyIdxs df  {- | Probe one batch of rows against a pre-built 'CompactIndex'.@@ -530,29 +546,36 @@ @ -} leftJoin :: [T.Text] -> DataFrame -> DataFrame -> DataFrame-leftJoin cs left right+leftJoin = leftJoinWithCallPoint "leftJoin"++leftJoinWithCallPoint ::+    T.Text -> [T.Text] -> DataFrame -> DataFrame -> DataFrame+leftJoinWithCallPoint callPoint cs left right     | D.null right || D.nRows right == 0 = left     | D.null left || D.nRows left == 0 = D.empty-    | otherwise =-        let-            csSet = S.fromList cs-            rightRows = fst (D.dimensions right)+    | otherwise = leftJoinNonEmpty callPoint cs left right -            leftKeyIdxs = keyColIndices csSet left-            rightKeyIdxs = keyColIndices csSet right-            leftHashes = D.computeRowHashes leftKeyIdxs left-            rightHashes = D.computeRowHashes rightKeyIdxs right+leftJoinNonEmpty :: T.Text -> [T.Text] -> DataFrame -> DataFrame -> DataFrame+leftJoinNonEmpty callPoint cs left right =+    let+        csSet = S.fromList cs+        rightRows = fst (D.dimensions right) -            -- Right is always the build side for left join-            (leftIxs, rightIxs)-                | rightRows > joinStrategyThreshold =-                    sortMergeLeftKernel leftHashes rightHashes-                | otherwise =-                    hashLeftKernel leftHashes rightHashes-         in-            -- rightIxs uses -1 as sentinel for "no match"-            assembleLeft csSet left right leftIxs rightIxs+        leftKeyIdxs = validatedKeyColIndices callPoint csSet left+        rightKeyIdxs = validatedKeyColIndices callPoint csSet right+        leftHashes = D.computeRowHashes leftKeyIdxs left+        rightHashes = D.computeRowHashes rightKeyIdxs right +        -- Right is always the build side for left join+        (leftIxs, rightIxs)+            | rightRows > joinStrategyThreshold =+                sortMergeLeftKernel leftHashes rightHashes+            | otherwise =+                hashLeftKernel leftHashes rightHashes+     in+        -- rightIxs uses -1 as sentinel for "no match"+        assembleLeft csSet left right leftIxs rightIxs+ {- | Hash-based left join kernel. Returns @(leftExpandedIndices, rightExpandedIndices)@ where right indices use @-1@ as sentinel for unmatched rows.@@ -801,34 +824,37 @@ -} rightJoin ::     [T.Text] -> DataFrame -> DataFrame -> DataFrame-rightJoin cs left right = leftJoin cs right left+rightJoin cs left right = leftJoinWithCallPoint "rightJoin" cs right left  fullOuterJoin ::     [T.Text] -> DataFrame -> DataFrame -> DataFrame fullOuterJoin cs left right     | D.null right || D.nRows right == 0 = left     | D.null left || D.nRows left == 0 = right-    | otherwise =-        let-            csSet = S.fromList cs-            leftRows = fst (D.dimensions left)-            rightRows = fst (D.dimensions right)+    | otherwise = fullOuterJoinNonEmpty cs left right -            leftKeyIdxs = keyColIndices csSet left-            rightKeyIdxs = keyColIndices csSet right-            leftHashes = D.computeRowHashes leftKeyIdxs left-            rightHashes = D.computeRowHashes rightKeyIdxs right+fullOuterJoinNonEmpty :: [T.Text] -> DataFrame -> DataFrame -> DataFrame+fullOuterJoinNonEmpty cs left right =+    let+        csSet = S.fromList cs+        leftRows = fst (D.dimensions left)+        rightRows = fst (D.dimensions right) -            -- Both sides can have nulls in full outer-            (leftIxs, rightIxs)-                | max leftRows rightRows > joinStrategyThreshold =-                    sortMergeFullOuterKernel leftHashes rightHashes-                | otherwise =-                    hashFullOuterKernel leftHashes rightHashes-         in-            -- Both index vectors use -1 as sentinel-            assembleFullOuter csSet left right leftIxs rightIxs+        leftKeyIdxs = validatedKeyColIndices "fullOuterJoin" csSet left+        rightKeyIdxs = validatedKeyColIndices "fullOuterJoin" csSet right+        leftHashes = D.computeRowHashes leftKeyIdxs left+        rightHashes = D.computeRowHashes rightKeyIdxs right +        -- Both sides can have nulls in full outer+        (leftIxs, rightIxs)+            | max leftRows rightRows > joinStrategyThreshold =+                sortMergeFullOuterKernel leftHashes rightHashes+            | otherwise =+                hashFullOuterKernel leftHashes rightHashes+     in+        -- Both index vectors use -1 as sentinel+        assembleFullOuter csSet left right leftIxs rightIxs+ {- | Hash-based full outer join kernel. Builds compact indices on both sides. Returns @(leftExpandedIndices, rightExpandedIndices)@ with @-1@ sentinels.@@ -1031,23 +1057,31 @@         insertIfPresent _ Nothing df = df         insertIfPresent name (Just c) df = D.insertColumn name c df -        -- Coalesce two OptionalColumns: take first non-Nothing per row,+        -- Coalesce two nullable columns: take first non-Nothing per row,         -- producing a non-optional column.         coalesceKeyColumn :: Column -> Column -> Column         coalesceKeyColumn-            (OptionalColumn (lCol :: VB.Vector (Maybe a)))-            (OptionalColumn (rCol :: VB.Vector (Maybe b))) =+            (BoxedColumn lBm (lCol :: VB.Vector a))+            (BoxedColumn rBm (rCol :: VB.Vector b)) =                 case testEquality (typeRep @a) (typeRep @b) of                     Just Refl ->-                        D.fromVector $-                            VB.zipWith-                                ( \l r ->-                                    fromMaybe (error "fullOuterJoin: null on both sides of key column") (l <|> r)-                                )-                                lCol-                                rCol+                        let asMaybe bm =+                                VB.imap+                                    ( \i v -> case bm of+                                        Just bm' -> if bitmapTestBit bm' i then Just v else Nothing+                                        Nothing -> Just v+                                    )+                            lMaybe = asMaybe lBm lCol+                            rMaybe = asMaybe rBm rCol+                         in D.fromVector $+                                VB.zipWith+                                    ( \l r ->+                                        fromMaybe (error "fullOuterJoin: null on both sides of key column") (l <|> r)+                                    )+                                    lMaybe+                                    rMaybe                     Nothing -> error "Cannot join columns of different types"-        coalesceKeyColumn _ _ = error "fullOuterJoin: expected OptionalColumn for key columns"+        coalesceKeyColumn _ _ = error "fullOuterJoin: expected nullable column for key columns"      in D.fold             ( \name df ->                 if S.member name csSet

@@ -53,8 +53,8 @@ sortBy sortOrds df     | any (`notElem` columnNames df) names =         throw $-            ColumnNotFoundException-                (T.pack $ show $ names L.\\ columnNames df)+            ColumnsNotFoundException+                (names L.\\ columnNames df)                 "sortBy"                 (columnNames df)     | otherwise =

@@ -213,7 +213,7 @@  _getColumnAsDouble :: T.Text -> DataFrame -> Maybe (VU.Vector Double) _getColumnAsDouble name df = case getColumn name df of-    Just (UnboxedColumn (f :: VU.Vector a)) -> case testEquality (typeRep @a) (typeRep @Double) of+    Just (UnboxedColumn _ (f :: VU.Vector a)) -> case testEquality (typeRep @a) (typeRep @Double) of         Just Refl -> Just f         Nothing -> case sIntegral @a of             STrue -> Just (VU.map fromIntegral f)@@ -222,7 +222,7 @@                 SFalse -> Nothing     Nothing ->         throw $-            ColumnNotFoundException name "_getColumnAsDouble" (M.keys $ columnIndices df)+            ColumnsNotFoundException [name] "_getColumnAsDouble" (M.keys $ columnIndices df)     _ -> Nothing -- Return a type mismatch error here. {-# INLINE _getColumnAsDouble #-} @@ -237,15 +237,12 @@ sum ::     forall a. (Columnable a, Num a) => Expr a -> DataFrame -> a sum (Col name) df = case getColumn name df of-    Nothing -> throw $ ColumnNotFoundException name "sum" (M.keys $ columnIndices df)-    Just ((UnboxedColumn (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of+    Nothing -> throw $ ColumnsNotFoundException [name] "sum" (M.keys $ columnIndices df)+    Just ((UnboxedColumn _ (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of         Just Refl -> VG.sum column         Nothing -> 0-    Just ((BoxedColumn (column :: V.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of+    Just ((BoxedColumn _ (column :: V.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of         Just Refl -> VG.sum column-        Nothing -> 0-    Just ((OptionalColumn (column :: V.Vector (Maybe a')))) -> case testEquality (typeRep @a') (typeRep @a) of-        Just Refl -> VG.sum (VG.map (fromMaybe 0) column)         Nothing -> 0 sum expr df = case interpret df expr of     Left e -> throw e

@@ -1,4 +1,6 @@ {-# LANGUAGE BangPatterns #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE ConstraintKinds #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-}@@ -44,10 +46,23 @@ import DataFrame.Operations.Core import DataFrame.Operations.Merge () import DataFrame.Operations.Transformations (apply)+import DataFrame.Operators import System.Random import Type.Reflection import Prelude hiding (filter, take) +#if MIN_VERSION_random(1,3,0)+type SplittableGen g = (SplitGen g, RandomGen g)++splitForStratified :: SplittableGen g => g -> (g, g)+splitForStratified = splitGen+#else+type SplittableGen g = RandomGen g++splitForStratified :: SplittableGen g => g -> (g, g)+splitForStratified = split+#endif+ -- | O(k * n) Take the first n rows of a DataFrame. take :: Int -> DataFrame -> DataFrame take n d = d{columns = V.map (takeColumn n') (columns d), dataframeDimensions = (n', c)}@@ -116,10 +131,33 @@ filter (Col filterColumnName) condition df = case getColumn filterColumnName df of     Nothing ->         throw $-            ColumnNotFoundException filterColumnName "filter" (M.keys $ columnIndices df)-    Just (BoxedColumn (column :: V.Vector b)) -> filterByVector filterColumnName column condition df-    Just (OptionalColumn (column :: V.Vector b)) -> filterByVector filterColumnName column condition df-    Just (UnboxedColumn (column :: VU.Vector b)) -> filterByVector filterColumnName column condition df+            ColumnsNotFoundException [filterColumnName] "filter" (M.keys $ columnIndices df)+    Just col@(BoxedColumn bm (column :: V.Vector b)) ->+        -- Check direct type match first, then try Maybe b match for nullable columns+        case testEquality (typeRep @a) (typeRep @b) of+            Just Refl -> filterByVector filterColumnName column condition df+            Nothing -> case (bm, typeRep @a) of+                (Just bm', App tMaybe tInner) -> case eqTypeRep tMaybe (typeRep @Maybe) of+                    Just HRefl -> case testEquality tInner (typeRep @b) of+                        Just Refl ->+                            let maybeVec = V.imap (\i v -> if bitmapTestBit bm' i then Just v else Nothing) column+                             in filterByVector filterColumnName maybeVec condition df+                        Nothing -> filterByVector filterColumnName column condition df+                    Nothing -> filterByVector filterColumnName column condition df+                _ -> filterByVector filterColumnName column condition df+    Just col@(UnboxedColumn bm (column :: VU.Vector b)) ->+        case testEquality (typeRep @a) (typeRep @b) of+            Just Refl -> filterByVector filterColumnName column condition df+            Nothing -> case (bm, typeRep @a) of+                (Just bm', App tMaybe tInner) -> case eqTypeRep tMaybe (typeRep @Maybe) of+                    Just HRefl -> case testEquality tInner (typeRep @b) of+                        Just Refl ->+                            let maybeVec = V.generate (VU.length column) $ \i ->+                                    if bitmapTestBit bm' i then Just (VU.unsafeIndex column i) else Nothing+                             in filterByVector filterColumnName maybeVec condition df+                        Nothing -> filterByVector filterColumnName column condition df+                    Nothing -> filterByVector filterColumnName column condition df+                _ -> filterByVector filterColumnName column condition df filter expr condition df =     let         (TColumn col) = case interpret @a df (normalize expr) of@@ -193,9 +231,12 @@ filterJust :: T.Text -> DataFrame -> DataFrame filterJust name df = case getColumn name df of     Nothing ->-        throw $ ColumnNotFoundException name "filterJust" (M.keys $ columnIndices df)-    Just column@(OptionalColumn (col :: V.Vector (Maybe a))) -> filter (Col @(Maybe a) name) isJust df & apply @(Maybe a) fromJust name-    Just column -> df+        throw $ ColumnsNotFoundException [name] "filterJust" (M.keys $ columnIndices df)+    Just column | hasMissing column -> case column of+        BoxedColumn (Just _) (col :: V.Vector a) -> filter (Col @(Maybe a) name) isJust df & apply @(Maybe a) fromJust name+        UnboxedColumn (Just _) (col :: VU.Vector a) -> filter (Col @(Maybe a) name) isJust df & apply @(Maybe a) fromJust name+        _ -> df+    Just _ -> df  {- | O(k) returns all rows with `Nothing` in a give column. @@ -204,8 +245,12 @@ filterNothing :: T.Text -> DataFrame -> DataFrame filterNothing name df = case getColumn name df of     Nothing ->-        throw $ ColumnNotFoundException name "filterNothing" (M.keys $ columnIndices df)-    Just (OptionalColumn (col :: V.Vector (Maybe a))) -> filter (Col @(Maybe a) name) isNothing df+        throw $+            ColumnsNotFoundException [name] "filterNothing" (M.keys $ columnIndices df)+    Just column | hasMissing column -> case column of+        BoxedColumn (Just _) (col :: V.Vector a) -> filter (Col @(Maybe a) name) isNothing df+        UnboxedColumn (Just _) (col :: VU.Vector a) -> filter (Col @(Maybe a) name) isNothing df+        _ -> df     _ -> df  {- | O(n * k) removes all rows with `Nothing` from the dataframe.@@ -242,8 +287,8 @@     | L.null cs = empty     | any (`notElem` columnNames df) cs =         throw $-            ColumnNotFoundException-                (T.pack $ show $ cs L.\\ columnNames df)+            ColumnsNotFoundException+                (cs L.\\ columnNames df)                 "select"                 (columnNames df)     | otherwise =@@ -345,23 +390,12 @@ sample pureGen p df =     let         rand = generateRandomVector pureGen (fst (dataframeDimensions df))+        cRand = col @Double "__rand__"      in         df-            & insertUnboxedVector "__rand__" rand-            & filterWhere-                ( Binary-                    ( MkBinaryOp-                        { binaryFn = (>=)-                        , binaryName = "geq"-                        , binarySymbol = Just ">="-                        , binaryCommutative = False-                        , binaryPrecedence = 1-                        }-                    )-                    (Col @Double "__rand__")-                    (Lit (1 - p))-                )-            & exclude ["__rand__"]+            & insertUnboxedVector (name cRand) rand+            & filterWhere (cRand .>=. Lit (1 - p))+            & exclude [name cRand]  {- | Split a dataset into two. The first in the tuple gets a sample of p (0 <= p <= 1) and the second gets (1 - p). This is useful for creating test and train splits. @@ -377,38 +411,16 @@ randomSplit pureGen p df =     let         rand = generateRandomVector pureGen (fst (dataframeDimensions df))-        withRand = df & insertUnboxedVector "__rand__" rand+        cRand = col @Double "__rand__"+        withRand = df & insertUnboxedVector (name cRand) rand      in         ( withRand-            & filterWhere-                ( Binary-                    ( MkBinaryOp-                        { binaryFn = (<=)-                        , binaryName = "leq"-                        , binarySymbol = Just "<="-                        , binaryCommutative = False-                        , binaryPrecedence = 1-                        }-                    )-                    (Col @Double "__rand__")-                    (Lit p)-                )-            & exclude ["__rand__"]+            & filterWhere (cRand .<=. Lit p)+            & exclude [name cRand]         , withRand             & filterWhere-                ( Binary-                    ( MkBinaryOp-                        { binaryFn = (>)-                        , binaryName = "gt"-                        , binarySymbol = Just ">"-                        , binaryCommutative = False-                        , binaryPrecedence = 1-                        }-                    )-                    (Col @Double "__rand__")-                    (Lit p)-                )-            & exclude ["__rand__"]+                (cRand .>. Lit p)+            & exclude [name cRand]         )  {- | Creates n folds of a dataframe.@@ -424,46 +436,20 @@ kFolds pureGen folds df =     let         rand = generateRandomVector pureGen (fst (dataframeDimensions df))-        withRand = df & insertUnboxedVector "__rand__" rand+        cRand = col @Double "__rand__"+        withRand = df & insertUnboxedVector (name cRand) rand         partitionSize = 1 / fromIntegral folds         singleFold n d =-            d-                & filterWhere-                    ( Binary-                        ( MkBinaryOp-                            { binaryFn = (>=)-                            , binaryName = "geq"-                            , binarySymbol = Just ">="-                            , binaryCommutative = False-                            , binaryPrecedence = 1-                            }-                        )-                        (Col @Double "__rand__")-                        (Lit (fromIntegral n * partitionSize))-                    )+            d & filterWhere (cRand .>=. Lit (fromIntegral n * partitionSize))         go (-1) _ = []         go n d =             let                 d' = singleFold n d-                d'' =-                    d-                        & filterWhere-                            ( Binary-                                ( MkBinaryOp-                                    { binaryFn = (<)-                                    , binaryName = "lt"-                                    , binarySymbol = Just "<"-                                    , binaryCommutative = False-                                    , binaryPrecedence = 1-                                    }-                                )-                                (Col @Double "__rand__")-                                (Lit (fromIntegral n * partitionSize))-                            )+                d'' = d & filterWhere (cRand .<. Lit (fromIntegral n * partitionSize))              in                 d' : go (n - 1) d''      in-        map (exclude ["__rand__"]) (go (folds - 1) withRand)+        map (exclude [name cRand]) (go (folds - 1) withRand)  generateRandomVector :: (RandomGen g) => g -> Int -> VU.Vector Double generateRandomVector pureGen k = VU.fromList $ go pureGen k@@ -477,12 +463,19 @@  -- | Convert any Column to a vector of Text labels (one per row). columnToTextVec :: Column -> V.Vector T.Text-columnToTextVec (BoxedColumn (col :: V.Vector a)) =-    case testEquality (typeRep @a) (typeRep @T.Text) of-        Just Refl -> col-        Nothing -> V.map (T.pack . show) col-columnToTextVec (UnboxedColumn col) = V.map (T.pack . show) (V.convert col)-columnToTextVec (OptionalColumn col) = V.map (T.pack . show) col+columnToTextVec (BoxedColumn bm (col :: V.Vector a)) =+    case bm of+        Nothing -> case testEquality (typeRep @a) (typeRep @T.Text) of+            Just Refl -> col+            Nothing -> V.map (T.pack . show) col+        Just bitmap ->+            V.imap (\i x -> if bitmapTestBit bitmap i then T.pack (show x) else "null") col+columnToTextVec (UnboxedColumn bm col) =+    case bm of+        Nothing -> V.map (T.pack . show) (V.convert col)+        Just bitmap ->+            V.generate (VU.length col) $ \i ->+                if bitmapTestBit bitmap i then T.pack (show (col VU.! i)) else "null"  -- | Build a map from stringified label to row indices. groupByIndices :: Column -> M.Map T.Text (VU.Vector Int)@@ -513,7 +506,7 @@ -} stratifiedSample ::     forall a g.-    (SplitGen g, RandomGen g, Columnable a) =>+    (SplittableGen g, Columnable a) =>     g -> Double -> Expr a -> DataFrame -> DataFrame stratifiedSample gen p strataCol df =     let col = case strataCol of@@ -523,7 +516,7 @@         go _ [] = mempty         go g (ixs : rest) =             let stratum = rowsAtIndices ixs df-                (g1, g2) = splitGen g+                (g1, g2) = splitForStratified g              in sample g1 p stratum <> go g2 rest      in go gen groups @@ -537,7 +530,7 @@ -} stratifiedSplit ::     forall a g.-    (SplitGen g, RandomGen g, Columnable a) =>+    (SplittableGen g, Columnable a) =>     g -> Double -> Expr a -> DataFrame -> (DataFrame, DataFrame) stratifiedSplit gen p strataCol df =     let col = case strataCol of@@ -547,7 +540,7 @@         go _ [] = (mempty, mempty)         go g (ixs : rest) =             let stratum = rowsAtIndices ixs df-                (g1, g2) = splitGen g+                (g1, g2) = splitForStratified g                 (tr, va) = randomSplit g1 p stratum                 (trAcc, vaAcc) = go g2 rest              in (tr <> trAcc, va <> vaAcc)

@@ -21,9 +21,9 @@ import Data.Maybe import DataFrame.Errors (DataFrameException (..), TypeErrorContext (..)) import DataFrame.Internal.Column (-    Column (..),     Columnable,     TypedColumn (..),+    hasMissing,     ifoldrColumn,     imapColumn,     mapColumn,@@ -63,7 +63,8 @@     DataFrame ->     Either DataFrameException DataFrame safeApply f columnName d = case getColumn columnName d of-    Nothing -> Left $ ColumnNotFoundException columnName "apply" (M.keys $ columnIndices d)+    Nothing ->+        Left $ ColumnsNotFoundException [columnName] "apply" (M.keys $ columnIndices d)     Just column -> do         column' <- mapColumn f column         pure $ insertColumn columnName column' d@@ -163,8 +164,8 @@ applyWhere condition filterColumnName f columnName df = case getColumn filterColumnName df of     Nothing ->         throw $-            ColumnNotFoundException-                filterColumnName+            ColumnsNotFoundException+                [filterColumnName]                 "applyWhere"                 (M.keys $ columnIndices df)     Just column -> case ifoldrColumn@@ -193,7 +194,7 @@ applyAtIndex i f columnName df = case getColumn columnName df of     Nothing ->         throw $-            ColumnNotFoundException columnName "applyAtIndex" (M.keys $ columnIndices df)+            ColumnsNotFoundException [columnName] "applyAtIndex" (M.keys $ columnIndices df)     Just column -> case imapColumn (\index value -> if index == i then f value else value) column of         Left e -> throw e         Right column' -> insertColumn columnName column' df@@ -208,8 +209,9 @@     DataFrame imputeCore (Col columnName) value df = case getColumn columnName df of     Nothing ->-        throw $ ColumnNotFoundException columnName "impute" (M.keys $ columnIndices df)-    Just (OptionalColumn _) -> case safeApply (fromMaybe value) columnName df of+        throw $+            ColumnsNotFoundException [columnName] "impute" (M.keys $ columnIndices df)+    Just col | hasMissing col -> case safeApply (fromMaybe value) columnName df of         Left (TypeMismatchException context) -> throw $ TypeMismatchException (context{callingFunctionName = Just "impute"})         Left exception -> throw exception         Right res -> res

@@ -11,11 +11,15 @@  import Control.Applicative (asum) import Control.Monad (join)-import Data.Maybe (fromMaybe) import qualified Data.Proxy as P import Data.Time import Data.Type.Equality (TestEquality (..))-import DataFrame.Internal.Column (Column (..), fromVector)+import DataFrame.Internal.Column (+    Column (..),+    bitmapTestBit,+    ensureOptional,+    fromVector,+ ) import DataFrame.Internal.DataFrame (DataFrame (..), unsafeGetColumn) import DataFrame.Internal.Parsing import DataFrame.Internal.Schema@@ -55,19 +59,22 @@ parseDefaults opts df = df{columns = V.map (parseDefault opts) (columns df)}  parseDefault :: ParseOptions -> Column -> Column-parseDefault opts (BoxedColumn (c :: V.Vector a)) =+parseDefault opts (BoxedColumn Nothing (c :: V.Vector a)) =     case (typeRep @a) `testEquality` (typeRep @T.Text) of         Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of             Just Refl -> parseFromExamples opts (V.map T.pack c)-            Nothing -> BoxedColumn c+            Nothing -> BoxedColumn Nothing c         Just Refl -> parseFromExamples opts c-parseDefault opts (OptionalColumn (c :: V.Vector (Maybe a))) =+parseDefault opts (BoxedColumn (Just bm) (c :: V.Vector a)) =     case (typeRep @a) `testEquality` (typeRep @T.Text) of         Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of             Just Refl ->-                parseFromExamples opts (V.map (T.pack . fromMaybe "") c)-            Nothing -> BoxedColumn c-        Just Refl -> parseFromExamples opts (V.map (fromMaybe "") c)+                parseFromExamples+                    opts+                    (V.imap (\i x -> if bitmapTestBit bm i then T.pack x else "") c)+            Nothing -> BoxedColumn (Just bm) c+        Just Refl ->+            parseFromExamples opts (V.imap (\i x -> if bitmapTestBit bm i then x else "") c) parseDefault _ column = column  parseFromExamples :: ParseOptions -> V.Vector T.Text -> Column@@ -78,14 +85,16 @@         examples = V.map converter (V.take (sampleSize opts) cols)         asMaybeText = V.map converter cols         dfmt = parseDateFormat opts+        result =+            case makeParsingAssumption dfmt examples of+                BoolAssumption -> handleBoolAssumption asMaybeText+                IntAssumption -> handleIntAssumption asMaybeText+                DoubleAssumption -> handleDoubleAssumption asMaybeText+                TextAssumption -> handleTextAssumption asMaybeText+                DateAssumption -> handleDateAssumption dfmt asMaybeText+                NoAssumption -> handleNoAssumption dfmt asMaybeText      in-        case makeParsingAssumption dfmt examples of-            BoolAssumption -> handleBoolAssumption asMaybeText-            IntAssumption -> handleIntAssumption asMaybeText-            DoubleAssumption -> handleDoubleAssumption asMaybeText-            TextAssumption -> handleTextAssumption asMaybeText-            DateAssumption -> handleDateAssumption dfmt asMaybeText-            NoAssumption -> handleNoAssumption dfmt asMaybeText+        if parseSafe opts then ensureOptional result else result  handleBoolAssumption :: V.Vector (Maybe T.Text) -> Column handleBoolAssumption asMaybeText@@ -232,7 +241,7 @@             ts   where     asType :: SchemaType -> Column -> Column-    asType (SType (_ :: P.Proxy a)) c@(BoxedColumn (col :: V.Vector b)) = case typeRep @a of+    asType (SType (_ :: P.Proxy a)) c@(BoxedColumn _ (col :: V.Vector b)) = case typeRep @a of         App t1 t2 -> case eqTypeRep t1 (typeRep @Maybe) of             Just HRefl -> case testEquality (typeRep @a) (typeRep @b) of                 Just Refl -> c

@@ -177,7 +177,7 @@             Left e -> throw e             Right v -> v         hasInvalid = case res of-            (TColumn (UnboxedColumn (col :: VU.Vector b))) -> case testEquality (typeRep @Double) (typeRep @b) of+            (TColumn (UnboxedColumn _ (col :: VU.Vector b))) -> case testEquality (typeRep @Double) (typeRep @b) of                 Just Refl -> VU.any (\n -> isNaN n || isInfinite n) col                 Nothing -> False             _ -> False

@@ -183,13 +183,16 @@     -- * Template Haskell     deriveSchema,     deriveSchemaFromCsvFile,+    deriveSchemaFromCsvFileWith,      -- * Schema type families (for advanced use)     Lookup,+    SafeLookup,     HasName,     SubsetSchema,     ExcludeSchema,     RenameInSchema,+    RenameManyInSchema,     RemoveColumn,     Impute,     Append,@@ -225,7 +228,11 @@ import DataFrame.Typed.Join (fullOuterJoin, innerJoin, leftJoin, rightJoin) import DataFrame.Typed.Operations import DataFrame.Typed.Schema-import DataFrame.Typed.TH (deriveSchema, deriveSchemaFromCsvFile)+import DataFrame.Typed.TH (+    deriveSchema,+    deriveSchemaFromCsvFile,+    deriveSchemaFromCsvFileWith,+ ) import DataFrame.Typed.Types (     Column,     TSortOrder (..),

@@ -21,7 +21,7 @@ import DataFrame.Internal.Column (Columnable) import DataFrame.Internal.Expression (Expr (Col)) import qualified DataFrame.Operations.Core as D-import DataFrame.Typed.Schema (AssertPresent, Lookup)+import DataFrame.Typed.Schema (AssertPresent, SafeLookup) import DataFrame.Typed.Types (TypedDataFrame (..))  {- | Retrieve a column as a boxed 'Vector', with the type determined by@@ -30,7 +30,7 @@ columnAsVector ::     forall name cols a.     ( KnownSymbol name-    , a ~ Lookup name cols+    , a ~ SafeLookup name cols     , Columnable a     , AssertPresent name cols     ) =>@@ -44,7 +44,7 @@ columnAsList ::     forall name cols a.     ( KnownSymbol name-    , a ~ Lookup name cols+    , a ~ SafeLookup name cols     , Columnable a     , AssertPresent name cols     ) =>

@@ -155,7 +155,7 @@  ) import DataFrame.Internal.Types (Promote, PromoteDiv) -import DataFrame.Typed.Schema (AssertPresent, Lookup)+import DataFrame.Typed.Schema (AssertPresent, SafeLookup) import DataFrame.Typed.Types (TExpr (..), TSortOrder (..)) import Prelude hiding (maximum, minimum, sum) @@ -172,7 +172,7 @@ col ::     forall (name :: Symbol) cols a.     ( KnownSymbol name-    , a ~ Lookup name cols+    , a ~ SafeLookup name cols     , Columnable a     , AssertPresent name cols     ) =>

@@ -18,6 +18,7 @@ import qualified Data.Text as T import Type.Reflection (SomeTypeRep) +import Data.List (stripPrefix) import qualified DataFrame.Internal.Column as C import qualified DataFrame.Internal.DataFrame as D import DataFrame.Operations.Core (columnNames)@@ -81,6 +82,17 @@                             <> ", got "                             <> T.pack (C.columnTypeString col) --- | Check if a Column's element type matches the expected SomeTypeRep.+{- | Check if a Column's element type matches the expected SomeTypeRep.+For nullable columns (those with a bitmap), @Maybe a@ in the schema matches+a column whose inner type is @a@, since we store nullable data as+@BoxedColumn (Just bm) a@ or @UnboxedColumn (Just bm) a@ rather than+@Column (Maybe a)@.+-} matchesType :: SomeTypeRep -> C.Column -> Bool-matchesType expected col = T.pack (show expected) == T.pack (C.columnTypeString col)+matchesType expected col =+    let expectedStr = show expected+        colTypeStr = C.columnTypeString col+     in expectedStr == colTypeStr+            || ( C.hasMissing col -- nullable column: schema says "Maybe X", column stores "X" with a bitmap+                    && Just colTypeStr == stripPrefix "Maybe " expectedStr+               )

@@ -328,7 +328,7 @@     forall name a cols.     ( KnownSymbol name     , Columnable a-    , a ~ Lookup name cols+    , a ~ SafeLookup name cols     , AssertPresent name cols     ) =>     TExpr cols a -> TypedDataFrame cols -> TypedDataFrame cols

@@ -16,6 +16,7 @@ module DataFrame.Typed.Schema (     -- * Type families for schema manipulation     Lookup,+    SafeLookup,     HasName,     RemoveColumn,     Impute,@@ -75,6 +76,15 @@     Lookup name '[] =         TypeError             ('Text "Column '" ':<>: 'Text name ':<>: 'Text "' not found in schema")++{- | Like 'Lookup', but returns a harmless fallback ('Int') instead of+'TypeError' when the column is not found.  Use together with+'AssertPresent' so the error fires exactly once.+-}+type family SafeLookup (name :: Symbol) (cols :: [Type]) :: Type where+    SafeLookup name (Column name a ': _) = a+    SafeLookup name (Column _ _ ': rest) = SafeLookup name rest+    SafeLookup name '[] = Int  -- | Unwrap a Maybe from a type after we impute values. type family Impute (name :: Symbol) (cols :: [Type]) :: [Type] where

@@ -8,6 +8,7 @@     -- * Schema inference     deriveSchema,     deriveSchemaFromCsvFile,+    deriveSchemaFromCsvFileWith,      -- * Re-export for TH splices     TypedDataFrame,@@ -49,8 +50,11 @@     pure [TySynD synName [] schemaType]  deriveSchemaFromCsvFile :: String -> String -> DecsQ-deriveSchemaFromCsvFile typeName path = do-    df <- liftIO (D.readCsv path)+deriveSchemaFromCsvFile = deriveSchemaFromCsvFileWith D.defaultReadOptions++deriveSchemaFromCsvFileWith :: D.ReadOptions -> String -> String -> DecsQ+deriveSchemaFromCsvFileWith opts typeName path = do+    df <- liftIO (D.readSeparated opts path)     deriveSchema typeName df  getSchemaInfo :: D.DataFrame -> [(T.Text, String)]

@@ -439,7 +439,7 @@         [ ("label", DI.fromList (replicate 6 "pos" ++ replicate 6 "neg" :: [T.Text]))         ,             ( "x"-            , DI.OptionalColumn+            , DI.fromVector                 ( V.fromList $                     map (Just . fromIntegral) ([1 .. 6] :: [Int])                         ++ map (Just . fromIntegral) ([7 .. 12] :: [Int]) ::@@ -455,7 +455,7 @@         [ ("label", DI.fromList (["pos", "pos", "pos", "neg", "neg", "neg"] :: [T.Text]))         ,             ( "x"-            , DI.OptionalColumn+            , DI.fromVector                 ( V.fromList                     [Just 1.0, Nothing, Just 3.0, Just 7.0, Nothing, Just 9.0] ::                     V.Vector (Maybe Double)@@ -463,16 +463,16 @@             )         ] --- numericCols picks up OptionalColumn (Maybe Double) as NMaybeDouble.+-- numericCols picks up DI.fromVector (Maybe Double) as NMaybeDouble. numericColsNullableDoubleTest :: Test numericColsNullableDoubleTest = TestCase $ do     let exprs = numericCols nullableSepDF         hasMD = any (\case NMaybeDouble _ -> True; _ -> False) exprs     assertBool-        "numericCols finds NMaybeDouble for OptionalColumn (Maybe Double)"+        "numericCols finds NMaybeDouble for DI.fromVector (Maybe Double)"         hasMD --- numericCols picks up OptionalColumn (Maybe Int) as NMaybeDouble (via whenPresent).+-- numericCols picks up DI.fromVector (Maybe Int) as NMaybeDouble (via whenPresent). numericColsNullableIntTest :: Test numericColsNullableIntTest = TestCase $ do     let df =@@ -480,18 +480,18 @@                 [ ("label", DI.fromList (["pos", "neg"] :: [T.Text]))                 ,                     ( "n"-                    , DI.OptionalColumn (V.fromList [Just (1 :: Int), Just 2] :: V.Vector (Maybe Int))+                    , DI.fromVector (V.fromList [Just (1 :: Int), Just 2] :: V.Vector (Maybe Int))                     )                 ]         hasMD = any (\case NMaybeDouble _ -> True; _ -> False) (numericCols df)-    assertBool "numericCols finds NMaybeDouble for OptionalColumn (Maybe Int)" hasMD+    assertBool "numericCols finds NMaybeDouble for DI.fromVector (Maybe Int)" hasMD --- generateNumericConds is non-empty for a DF with an OptionalColumn (Maybe Double).+-- generateNumericConds is non-empty for a DF with an DI.fromVector (Maybe Double). numericCondsNullableNonEmptyTest :: Test numericCondsNullableNonEmptyTest =     TestCase $         assertBool-            "generateNumericConds non-empty for OptionalColumn (Maybe Double)"+            "generateNumericConds non-empty for DI.fromVector (Maybe Double)"             (not (null (generateNumericConds defaultTreeConfig nullableSepDF)))  -- Null values evaluate to False for threshold conditions (null rows route right).@@ -502,7 +502,7 @@                 [ ("label", DI.fromList (["A", "B"] :: [T.Text]))                 ,                     ( "x"-                    , DI.OptionalColumn+                    , DI.fromVector                         (V.fromList [Nothing, Just (5.0 :: Double)] :: V.Vector (Maybe Double))                     )                 ]@@ -541,14 +541,14 @@         (loss >= 0.0 && loss <= 1.0)  -- numericExprsWithTerms produces cross-column combinations when one col is--- OptionalColumn (Maybe Double) and another is a plain UnboxedColumn Double.+-- DI.fromVector (Maybe Double) and another is a plain UnboxedColumn Double. numericExprsWithTermsMixedTest :: Test numericExprsWithTermsMixedTest = TestCase $ do     let df =             D.fromNamedColumns                 [                     ( "x"-                    , DI.OptionalColumn+                    , DI.fromVector                         (V.fromList [Just 1.0, Just 2.0, Just 3.0] :: V.Vector (Maybe Double))                     )                 , ("y", DI.fromList ([4.0, 5.0, 6.0] :: [Double]))

@@ -14,9 +14,9 @@ genColumn :: Int -> Gen Column genColumn len =     oneof-        [ BoxedColumn . V.fromList <$> vectorOf len (arbitrary @Int)-        , UnboxedColumn . VU.fromList <$> vectorOf len (arbitrary @Double)-        , OptionalColumn . V.fromList <$> vectorOf len (arbitrary @(Maybe Int))+        [ BoxedColumn Nothing . V.fromList <$> vectorOf len (arbitrary @Int)+        , UnboxedColumn Nothing . VU.fromList <$> vectorOf len (arbitrary @Double)+        , fromVector . V.fromList <$> vectorOf len (arbitrary @(Maybe Int))         ]  genDataFrame :: Gen DataFrame

@@ -33,6 +33,7 @@ import qualified Operations.Subset import qualified Operations.Take import qualified Operations.Typing+import qualified Operations.WriteCsv import qualified Parquet import qualified Properties @@ -53,6 +54,7 @@             ++ Operations.Nullable.tests             ++ Operations.Provenance.tests             ++ Operations.ReadCsv.tests+            ++ Operations.WriteCsv.tests             ++ Operations.Shuffle.tests             ++ Operations.Sort.tests             ++ Operations.Statistics.tests

@@ -40,7 +40,7 @@     TestCase         ( assertEqual             "Boxed apply unboxed when result is unboxed"-            (Just $ DI.UnboxedColumn (VU.fromList (replicate 26 (1 :: Int))))+            (Just $ DI.UnboxedColumn Nothing (VU.fromList (replicate 26 (1 :: Int))))             (DI.getColumn "test2" $ D.apply @String (const (1 :: Int)) "test2" testData)         ) @@ -49,7 +49,7 @@     TestCase         ( assertEqual             "Boxed apply remains in boxed vector"-            (Just $ DI.BoxedColumn (V.fromList (replicate 26 (1 :: Integer))))+            (Just $ DI.BoxedColumn Nothing (V.fromList (replicate 26 (1 :: Integer))))             (DI.getColumn "test2" $ D.apply @String (const (1 :: Integer)) "test2" testData)         ) @@ -204,6 +204,7 @@             "applyWhere works as intended"             ( Just $                 DI.UnboxedColumn+                    Nothing                     (VU.fromList (zipWith ($) (cycle [id, (+ 1)]) [(1 :: Int) .. 26]))             )             ( D.getColumn "test5" $@@ -223,7 +224,7 @@     TestCase         ( assertEqual             "impute fills Nothing with the given value"-            (Just $ DI.UnboxedColumn (VU.fromList [1 :: Int, 0, 3]))+            (Just $ DI.UnboxedColumn Nothing (VU.fromList [1 :: Int, 0, 3]))             (DI.getColumn "opt" $ impute (F.col @(Maybe Int) "opt") 0 imputeData)         ) 

@@ -32,6 +32,7 @@             "derive works with column expression"             ( Just $                 DI.BoxedColumn+                    Nothing                     (V.fromList (zipWith (\n c -> show n ++ [c]) [1 .. 26] ['a' .. 'z']))             )             ( DI.getColumn "test4" $

@@ -47,7 +47,7 @@     TestCase         ( assertExpectException             "[Error Case]"-            (D.columnNotFound "[\"test0\"]" "groupBy" (D.columnNames testData))+            (D.columnsNotFound ["test0"] "groupBy" (D.columnNames testData))             (print $ D.groupBy ["test0"] testData)         ) 

@@ -31,7 +31,9 @@     TestCase         ( assertEqual             "Two columns should be equal"-            (Just $ DI.BoxedColumn (V.fromList ["Thuba" :: T.Text, "Zodwa", "Themba"]))+            ( Just $+                DI.BoxedColumn Nothing (V.fromList ["Thuba" :: T.Text, "Zodwa", "Themba"])+            )             ( DI.getColumn "new" $                 D.insertVector "new" (V.fromList ["Thuba" :: T.Text, "Zodwa", "Themba"]) D.empty             )@@ -57,7 +59,7 @@     TestCase         ( assertEqual             "Value should be boxed"-            (Just $ DI.UnboxedColumn (VU.fromList [1 :: Int, 2, 3]))+            (Just $ DI.UnboxedColumn Nothing (VU.fromList [1 :: Int, 2, 3]))             (DI.getColumn "new" $ D.insertVector "new" (V.fromList [1 :: Int, 2, 3]) D.empty)         ) @@ -77,7 +79,7 @@         ( assertEqual             "Missing values should be replaced with Nothing"             ( Just $-                DI.OptionalColumn+                DI.fromVector                     (V.fromList [Just "a" :: Maybe T.Text, Just "b", Just "c", Nothing, Nothing])             )             ( DI.getColumn "newer" $@@ -92,7 +94,7 @@         ( assertEqual             "Missing values should be replaced with Nothing"             ( Just $-                DI.OptionalColumn+                DI.fromVector                     (V.fromList [Just 1 :: Maybe Int, Just 2, Just 3, Nothing, Nothing])             )             ( DI.getColumn "newer" $@@ -106,7 +108,7 @@     TestCase         ( assertEqual             "Missing values should be replaced with Nothing"-            (Just $ DI.UnboxedColumn (VU.fromList [1 :: Int, 2, 3, 0, 0]))+            (Just $ DI.UnboxedColumn Nothing (VU.fromList [1 :: Int, 2, 3, 0, 0]))             ( DI.getColumn "newer" $                 D.insertVectorWithDefault 0 "newer" (V.fromList [1 :: Int, 2, 3]) $                     D.insertVector "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty@@ -118,7 +120,7 @@     TestCase         ( assertEqual             "Lists should be the same size"-            (Just $ DI.UnboxedColumn (VU.fromList [(6 :: Int) .. 10]))+            (Just $ DI.UnboxedColumn Nothing (VU.fromList [(6 :: Int) .. 10]))             ( DI.getColumn "newer" $                 D.insertVectorWithDefault 0 "newer" (V.fromList [(6 :: Int) .. 10]) $                     D.insertVector "new" (V.fromList [1 :: Int, 2, 3, 4, 5]) D.empty

@@ -4,7 +4,9 @@  module Operations.Join where -import Data.Text (Text)+import Assertions (assertExpectException)+import Control.Exception (evaluate)+import Data.Text (Text, unpack) import Data.These import qualified DataFrame as D import qualified DataFrame.Functions as F@@ -26,6 +28,21 @@         , ("B", D.fromList ["B0" :: Text, "B1", "B2"])         ] +assertMissingJoinColumns :: String -> [Text] -> D.DataFrame -> Assertion+assertMissingJoinColumns preface missingKeys result = do+    assertExpectException+        preface+        (if length missingKeys == 1 then "Column not found" else "Columns not found")+        (evaluate (D.nRows result))+    mapM_+        ( \missingKey ->+            assertExpectException preface (unpack missingKey) (evaluate (D.nRows result))+        )+        missingKeys++assertMissingJoinColumn :: String -> Text -> D.DataFrame -> Assertion+assertMissingJoinColumn preface missingKey = assertMissingJoinColumns preface [missingKey]+ testInnerJoin :: Test testInnerJoin =     TestCase@@ -253,6 +270,59 @@             (D.sortBy [D.Asc (F.col @Text "key")] (fullOuterJoin ["key"] dfL dfR))         ) +testInnerJoinMissingKey :: Test+testInnerJoinMissingKey =+    TestCase $+        assertMissingJoinColumn+            "Inner join should fail when the join key is missing"+            "Cats"+            (innerJoin ["Cats"] df1 df2)++testLeftJoinMissingKey :: Test+testLeftJoinMissingKey =+    TestCase $+        assertMissingJoinColumn+            "Left join should fail when the join key is missing"+            "Cats"+            (leftJoin ["Cats"] df1 df2)++testRightJoinMissingKey :: Test+testRightJoinMissingKey =+    TestCase $+        assertMissingJoinColumn+            "Right join should fail when the join key is missing"+            "Animals"+            (rightJoin ["Animals"] df1 df2)++testFullOuterJoinMissingKey :: Test+testFullOuterJoinMissingKey =+    TestCase $+        assertMissingJoinColumn+            "Full outer join should fail when the join key is missing"+            "Cats"+            (fullOuterJoin ["Cats"] df1 df2)++testInnerJoinMissingKeys :: Test+testInnerJoinMissingKeys =+    TestCase $+        assertMissingJoinColumns+            "Inner join should report every missing join key"+            ["Animals", "Cats"]+            (innerJoin ["Animals", "Cats"] df1 df2)++testInnerJoinMissingKeysSuggestion :: Test+testInnerJoinMissingKeysSuggestion =+    TestCase $+        let typoDf =+                D.fromNamedColumns+                    [ ("hello", D.fromList ["H" :: Text])+                    , ("world", D.fromList ["W" :: Text])+                    ]+         in assertExpectException+                "Inner join should report consolidated suggestions for missing join keys"+                "Did you mean [\"hello\", \"world\"]?"+                (evaluate (D.nRows (innerJoin ["helo", "wrld"] typoDf typoDf)))+ -- Empty DataFrame fixtures: same schema as df1/df2 but zero rows. emptyDf1 :: D.DataFrame emptyDf1 =@@ -353,6 +423,12 @@     , TestLabel "leftJoinWithCollisions" testLeftJoinWithCollisions     , TestLabel "rightJoinWithCollisions" testRightJoinWithCollisions     , TestLabel "outerJoinWithCollisions" testOuterJoinWithCollisions+    , TestLabel "innerJoinMissingKey" testInnerJoinMissingKey+    , TestLabel "leftJoinMissingKey" testLeftJoinMissingKey+    , TestLabel "rightJoinMissingKey" testRightJoinMissingKey+    , TestLabel "fullOuterJoinMissingKey" testFullOuterJoinMissingKey+    , TestLabel "innerJoinMissingKeys" testInnerJoinMissingKeys+    , TestLabel "innerJoinMissingKeysSuggestion" testInnerJoinMissingKeysSuggestion     , TestLabel "innerJoinBothEmpty" testInnerJoinBothEmpty     , TestLabel "innerJoinLeftEmpty" testInnerJoinLeftEmpty     , TestLabel "innerJoinRightEmpty" testInnerJoinRightEmpty

@@ -27,7 +27,7 @@ testData =     D.fromNamedColumns         [ ("x", DI.fromList ([1, 2, 3] :: [Int]))-        , ("y", DI.OptionalColumn (V.fromList [Just 10, Nothing, Just 30 :: Maybe Int]))+        , ("y", DI.fromVector (V.fromList [Just 10, Nothing, Just 30 :: Maybe Int]))         ]  -- | col @Int .+ col @(Maybe Int)  should give Maybe Int column@@ -36,7 +36,7 @@     TestCase         ( assertEqual             "Int .+ Maybe Int = Maybe Int"-            (Just $ DI.OptionalColumn (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int]))+            (Just $ DI.fromVector (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int]))             ( DI.getColumn "result" $                 D.derive                     "result"@@ -51,7 +51,7 @@     TestCase         ( assertEqual             "Maybe Int .+ Int = Maybe Int"-            (Just $ DI.OptionalColumn (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int]))+            (Just $ DI.fromVector (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int]))             ( DI.getColumn "result" $                 D.derive                     "result"@@ -81,7 +81,7 @@     TestCase         ( assertEqual             "Maybe Int .+ Maybe Int = Maybe Int"-            (Just $ DI.OptionalColumn (V.fromList [Just 20, Nothing, Just 60 :: Maybe Int]))+            (Just $ DI.fromVector (V.fromList [Just 20, Nothing, Just 60 :: Maybe Int]))             ( DI.getColumn "result" $                 D.derive                     "result"@@ -97,7 +97,7 @@         ( assertEqual             "Int .- Maybe Int = Maybe Int"             ( Just $-                DI.OptionalColumn (V.fromList [Just (-9), Nothing, Just (-27) :: Maybe Int])+                DI.fromVector (V.fromList [Just (-9), Nothing, Just (-27) :: Maybe Int])             )             ( DI.getColumn "result" $                 D.derive@@ -114,7 +114,7 @@         ( assertEqual             "Int .== Maybe Int = Maybe Bool"             ( Just $-                DI.OptionalColumn (V.fromList [Just False, Nothing, Just False :: Maybe Bool])+                DI.fromVector (V.fromList [Just False, Nothing, Just False :: Maybe Bool])             )             ( DI.getColumn "result" $                 D.derive@@ -150,7 +150,7 @@         ( assertEqual             "nullLift negate (Maybe Int) propagates Nothing"             ( Just $-                DI.OptionalColumn (V.fromList [Just (-10), Nothing, Just (-30) :: Maybe Int])+                DI.fromVector (V.fromList [Just (-10), Nothing, Just (-30) :: Maybe Int])             )             ( DI.getColumn "result" $                 D.derive@@ -181,7 +181,7 @@     TestCase         ( assertEqual             "nullLift2 (+) Int (Maybe Int) = Maybe Int"-            (Just $ DI.OptionalColumn (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int]))+            (Just $ DI.fromVector (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int]))             ( DI.getColumn "result" $                 D.derive                     "result"@@ -196,7 +196,7 @@     TestCase         ( assertEqual             "nullLift2 (+) (Maybe Int) Int = Maybe Int"-            (Just $ DI.OptionalColumn (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int]))+            (Just $ DI.fromVector (V.fromList [Just 11, Nothing, Just 33 :: Maybe Int]))             ( DI.getColumn "result" $                 D.derive                     "result"@@ -229,11 +229,11 @@ crossData =     D.fromNamedColumns         [ ("x", DI.fromList ([1, 2, 3] :: [Int]))-        , ("y", DI.OptionalColumn (V.fromList [Just 10, Nothing, Just 30 :: Maybe Int]))+        , ("y", DI.fromVector (V.fromList [Just 10, Nothing, Just 30 :: Maybe Int]))         , ("d", DI.fromList ([1.5, 2.5, 3.5] :: [Double]))         ,             ( "md"-            , DI.OptionalColumn (V.fromList [Just 10.5, Nothing, Just 30.5 :: Maybe Double])+            , DI.fromVector (V.fromList [Just 10.5, Nothing, Just 30.5 :: Maybe Double])             )         ] @@ -268,7 +268,7 @@         ( assertEqual             "Maybe Int .+ Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 11.5, Nothing, Just 33.5 :: Maybe Double])+                DI.fromVector (V.fromList [Just 11.5, Nothing, Just 33.5 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive "result" (F.col @(Maybe Int) "y" .+ F.col @Double "d") crossData@@ -282,7 +282,7 @@         ( assertEqual             "Int .+ Maybe Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 11.5, Nothing, Just 33.5 :: Maybe Double])+                DI.fromVector (V.fromList [Just 11.5, Nothing, Just 33.5 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive "result" (F.col @Int "x" .+ F.col @(Maybe Double) "md") crossData@@ -296,7 +296,7 @@         ( assertEqual             "Maybe Int .+ Maybe Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 20.5, Nothing, Just 60.5 :: Maybe Double])+                DI.fromVector (V.fromList [Just 20.5, Nothing, Just 60.5 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive@@ -349,7 +349,7 @@         ( assertEqual             "Maybe Int .- Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 8.5, Nothing, Just 26.5 :: Maybe Double])+                DI.fromVector (V.fromList [Just 8.5, Nothing, Just 26.5 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive "result" (F.col @(Maybe Int) "y" .- F.col @Double "d") crossData@@ -363,7 +363,7 @@         ( assertEqual             "Int .- Maybe Double = Maybe Double"             ( Just $-                DI.OptionalColumn+                DI.fromVector                     (V.fromList [Just (-9.5), Nothing, Just (-27.5) :: Maybe Double])             )             ( DI.getColumn "result" $@@ -378,7 +378,7 @@         ( assertEqual             "Maybe Int .- Maybe Double = Maybe Double"             ( Just $-                DI.OptionalColumn+                DI.fromVector                     (V.fromList [Just (-0.5), Nothing, Just (-0.5) :: Maybe Double])             )             ( DI.getColumn "result" $@@ -408,7 +408,7 @@         ( assertEqual             "Maybe Int .* Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 15.0, Nothing, Just 105.0 :: Maybe Double])+                DI.fromVector (V.fromList [Just 15.0, Nothing, Just 105.0 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive "result" (F.col @(Maybe Int) "y" .* F.col @Double "d") crossData@@ -422,7 +422,7 @@         ( assertEqual             "Int .* Maybe Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 10.5, Nothing, Just 91.5 :: Maybe Double])+                DI.fromVector (V.fromList [Just 10.5, Nothing, Just 91.5 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive "result" (F.col @Int "x" .* F.col @(Maybe Double) "md") crossData@@ -436,7 +436,7 @@         ( assertEqual             "Maybe Int .* Maybe Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 105.0, Nothing, Just 915.0 :: Maybe Double])+                DI.fromVector (V.fromList [Just 105.0, Nothing, Just 915.0 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive@@ -451,11 +451,11 @@ divData =     D.fromNamedColumns         [ ("x", DI.fromList ([2, 4, 6] :: [Int]))-        , ("y", DI.OptionalColumn (V.fromList [Just 4, Nothing, Just 6 :: Maybe Int]))+        , ("y", DI.fromVector (V.fromList [Just 4, Nothing, Just 6 :: Maybe Int]))         , ("d", DI.fromList ([1.0, 2.0, 3.0] :: [Double]))         ,             ( "md"-            , DI.OptionalColumn (V.fromList [Just 1.0, Nothing, Just 3.0 :: Maybe Double])+            , DI.fromVector (V.fromList [Just 1.0, Nothing, Just 3.0 :: Maybe Double])             )         ] @@ -490,7 +490,7 @@         ( assertEqual             "Maybe Int ./ Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 4.0, Nothing, Just 2.0 :: Maybe Double])+                DI.fromVector (V.fromList [Just 4.0, Nothing, Just 2.0 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive "result" (F.col @(Maybe Int) "y" ./ F.col @Double "d") divData@@ -504,7 +504,7 @@         ( assertEqual             "Int ./ Maybe Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 2.0, Nothing, Just 2.0 :: Maybe Double])+                DI.fromVector (V.fromList [Just 2.0, Nothing, Just 2.0 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive "result" (F.col @Int "x" ./ F.col @(Maybe Double) "md") divData@@ -518,7 +518,7 @@         ( assertEqual             "Maybe Int ./ Maybe Double = Maybe Double"             ( Just $-                DI.OptionalColumn (V.fromList [Just 4.0, Nothing, Just 2.0 :: Maybe Double])+                DI.fromVector (V.fromList [Just 4.0, Nothing, Just 2.0 :: Maybe Double])             )             ( DI.getColumn "result" $                 D.derive "result" (F.col @(Maybe Int) "y" ./ F.col @(Maybe Double) "md") divData@@ -705,7 +705,7 @@     TestCase         ( assertEqual             "apply negate to Maybe Int column fmaps"-            (Just $ DI.OptionalColumn (V.fromList [Just (-10), Nothing, Just (-30 :: Int)]))+            (Just $ DI.fromVector (V.fromList [Just (-10), Nothing, Just (-30 :: Int)]))             ( DI.getColumn "y" $                 D.apply @Int negate "y" testData             )

@@ -3,156 +3,29 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} --- Test fixtures inspired by csv-spectrum (https://github.com/max-mapper/csv-spectrum)- module Operations.ReadCsv where -import qualified Data.List as L-import qualified Data.Map as M import qualified Data.Text as T-import qualified Data.Text.IO as TIO-import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D -import Data.Function (on)-import Data.Maybe (fromMaybe)-import Data.Type.Equality (testEquality, (:~:) (Refl)) import DataFrame.Internal.Column (Column (..), columnTypeString)-import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (     DataFrame (..),-    columnIndices,-    columns,     dataframeDimensions,     getColumn,  )-import System.Directory (removeFile)-import System.IO (IOMode (..), withFile) import Test.HUnit import Type.Reflection (typeRep) -fixtureDir :: FilePath-fixtureDir = "./tests/data/unstable_csv/"--tempDir :: FilePath-tempDir = "./tests/data/unstable_csv/"+arbuthnotPath :: FilePath+arbuthnotPath = "./tests/data/arbuthnot.csv"  readCsvNoInfer :: FilePath -> IO DataFrame readCsvNoInfer =     D.readCsvWithOpts         D.defaultReadOptions{D.typeSpec = D.NoInference} ------------------------------------------------------------------------------------ Pretty-printer-----------------------------------------------------------------------------------prettyPrintCsv :: FilePath -> DataFrame -> IO ()-prettyPrintCsv = prettyPrintSeparated ','--prettyPrintTsv :: FilePath -> DataFrame -> IO ()-prettyPrintTsv = prettyPrintSeparated '\t'--prettyPrintSeparated :: Char -> FilePath -> DataFrame -> IO ()-prettyPrintSeparated sep filepath df = withFile filepath WriteMode $ \handle -> do-    let (rows, _) = dataframeDimensions df-    let headers = map fst (L.sortBy (compare `on` snd) (M.toList (columnIndices df)))-    TIO.hPutStrLn-        handle-        (T.intercalate (T.singleton sep) (map (escapeField sep) headers))-    -- Write data rows-    mapM_-        (TIO.hPutStrLn handle . T.intercalate (T.singleton sep) . getRowEscaped sep df)-        [0 .. rows - 1]---- Note: The unstable parser does not unescape doubled quotes (""  -> "),--- so we must not double-escape them here. We only wrap in quotes when needed.-escapeField :: Char -> T.Text -> T.Text-escapeField sep field-    | needsQuoting = T.concat ["\"", field, "\""]-    | otherwise = field-  where-    needsQuoting =-        T.any (\c -> c == sep || c == '\n' || c == '\r' || c == '"') field---- | Get a row from the DataFrame with all fields escaped-getRowEscaped :: Char -> DataFrame -> Int -> [T.Text]-getRowEscaped sep df i = V.ifoldr go [] (columns df)-  where-    go :: Int -> Column -> [T.Text] -> [T.Text]-    go _ (BoxedColumn (c :: V.Vector a)) acc = case c V.!? i of-        Just e -> escapeField sep textRep : acc-          where-            textRep = case testEquality (typeRep @a) (typeRep @T.Text) of-                Just Refl -> e-                Nothing -> T.pack (show e)-        Nothing -> acc-    go _ (UnboxedColumn c) acc = case c VU.!? i of-        Just e -> escapeField sep (T.pack (show e)) : acc-        Nothing -> acc-    go _ (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of-        Just e -> escapeField sep textRep : acc-          where-            textRep = case testEquality (typeRep @a) (typeRep @T.Text) of-                Just Refl -> fromMaybe "" e-                Nothing -> case e of-                    Just val -> T.pack (show val)-                    Nothing -> ""-        Nothing -> acc--testFastCsv :: String -> FilePath -> Test-testFastCsv name csvPath = TestLabel ("fast_roundtrip_" <> name) $ TestCase $ do-    dfOriginal <- D.fastReadCsvUnstable csvPath-    let tempPath = tempDir <> "temp_fast_" <> name <> ".csv"-    prettyPrintCsv tempPath dfOriginal-    dfRoundtrip <- D.fastReadCsvUnstable tempPath-    assertEqual-        ("Fast round-trip should produce equivalent DataFrame for " <> name)-        dfOriginal-        dfRoundtrip-    removeFile tempPath--testTsv :: String -> FilePath -> Test-testTsv name tsvPath = TestLabel ("roundtrip_tsv_" <> name) $ TestCase $ do-    dfOriginal <- D.readTsvUnstable tsvPath-    let tempPath = tempDir <> "temp_" <> name <> ".tsv"-    prettyPrintTsv tempPath dfOriginal-    dfRoundtrip <- D.readTsvUnstable tempPath-    assertEqual-        ("TSV round-trip should produce equivalent DataFrame for " <> name)-        dfOriginal-        dfRoundtrip-    removeFile tempPath---- Individual round-trip test cases for each fixture--testSimpleFast :: Test-testSimpleFast = testFastCsv "simple" (fixtureDir <> "simple.csv")--testCommaInQuotesFast :: Test-testCommaInQuotesFast = testFastCsv "comma_in_quotes" (fixtureDir <> "comma_in_quotes.csv")--testEscapedQuotesFast :: Test-testEscapedQuotesFast = testFastCsv "escaped_quotes" (fixtureDir <> "escaped_quotes.csv")--testNewlinesFast :: Test-testNewlinesFast = testFastCsv "newlines" (fixtureDir <> "newlines.csv")--testUtf8Fast :: Test-testUtf8Fast = testFastCsv "utf8" (fixtureDir <> "utf8.csv")--testQuotesAndNewlinesFast :: Test-testQuotesAndNewlinesFast = testFastCsv "quotes_and_newlines" (fixtureDir <> "quotes_and_newlines.csv")--testEmptyValuesFast :: Test-testEmptyValuesFast = testFastCsv "empty_values" (fixtureDir <> "empty_values.csv")--testJsonDataFast :: Test-testJsonDataFast = testFastCsv "json_data" (fixtureDir <> "json_data.csv")--arbuthnotPath :: FilePath-arbuthnotPath = "./tests/data/arbuthnot.csv"- -- SpecifyTypes with NoInference fallback: named column is typed, rest stay Text specifyTypesNoInferenceFallback :: Test specifyTypesNoInferenceFallback =@@ -168,11 +41,11 @@                 arbuthnotPath         -- "year" must be Int         case getColumn "year" df of-            Just col@(UnboxedColumn _) -> assertEqual "year should be Int" "Int" (columnTypeString col)+            Just col@(UnboxedColumn _ _) -> assertEqual "year should be Int" "Int" (columnTypeString col)             _ -> assertFailure "expected UnboxedColumn for 'year'"         -- "boys" unspecified + NoInference → stays Text         case getColumn "boys" df of-            Just col@(BoxedColumn _) -> assertEqual "boys should be Text" "Text" (columnTypeString col)+            Just col@(BoxedColumn _ _) -> assertEqual "boys should be Text" "Text" (columnTypeString col)             _ -> assertFailure "expected BoxedColumn for 'boys' with NoInference fallback"  -- SpecifyTypes with InferFromSample fallback: named column typed, rest inferred@@ -190,11 +63,11 @@                 arbuthnotPath         -- "year" must be Int (explicitly specified)         case getColumn "year" df of-            Just col@(UnboxedColumn _) -> assertEqual "year should be Int" "Int" (columnTypeString col)+            Just col@(UnboxedColumn _ _) -> assertEqual "year should be Int" "Int" (columnTypeString col)             _ -> assertFailure "expected UnboxedColumn for 'year'"         -- "boys" unspecified + InferFromSample → inferred as Int         case getColumn "boys" df of-            Just col@(UnboxedColumn _) -> assertEqual "boys should be Int" "Int" (columnTypeString col)+            Just col@(UnboxedColumn _ _) -> assertEqual "boys should be Int" "Int" (columnTypeString col)             _ ->                 assertFailure "expected UnboxedColumn for 'boys' with InferFromSample fallback" @@ -213,128 +86,17 @@                     }                 arbuthnotPath         case getColumn "girls" df of-            Just col@(UnboxedColumn _) -> assertEqual "girls should be Int" "Int" (columnTypeString col)+            Just col@(UnboxedColumn _ _) -> assertEqual "girls should be Int" "Int" (columnTypeString col)             _ ->                 assertFailure                     "expected UnboxedColumn for 'girls' via fallback InferFromSample 10" -testCrlfCsv :: Test-testCrlfCsv = TestLabel "malformed_crlf_csv" $ TestCase $ do-    df <- D.readCsvUnstable (fixtureDir <> "crlf.csv")-    let (rows, cols) = dataframeDimensions df-    assertEqual "crlf.csv: 2 data rows" 2 rows-    assertEqual "crlf.csv: 2 columns" 2 cols-    case getColumn "name" df of-        Nothing -> assertFailure "crlf.csv: column 'name' missing"-        Just col ->-            assertEqual-                "crlf.csv: name has no \\r"-                (DI.fromList @T.Text ["Alice", "Bob"])-                col--testCrlfTsv :: Test-testCrlfTsv = TestLabel "malformed_crlf_tsv" $ TestCase $ do-    df <- D.readTsvUnstable (fixtureDir <> "crlf.tsv")-    let (rows, _) = dataframeDimensions df-    assertEqual "crlf.tsv: 1 data row" 1 rows-    case getColumn "name" df of-        Nothing -> assertFailure "crlf.tsv: column 'name' missing"-        Just col ->-            assertEqual-                "crlf.tsv: name has no \\r"-                (DI.fromList @T.Text ["Alice"])-                col--testHeaderOnly :: Test-testHeaderOnly = TestLabel "malformed_header_only" $ TestCase $ do-    df <- D.readCsvUnstable (fixtureDir <> "header_only.csv")-    let (rows, cols) = dataframeDimensions df-    assertEqual "header_only.csv: 0 data rows" 0 rows-    assertEqual "header_only.csv: 3 columns" 3 cols-    let names = map fst . L.sortBy (compare `on` snd) . M.toList $ columnIndices df-    assertEqual "header_only.csv: column names" ["first", "second", "third"] names--testTrailingBlankLine :: Test-testTrailingBlankLine = TestLabel "malformed_trailing_blank_line" $ TestCase $ do-    df <- D.readCsvUnstable (fixtureDir <> "trailing_blank_line.csv")-    -- blank line contributes 1 extra delimiter; (3+3+1) div 3 = 2, numRow=1-    assertEqual-        "trailing_blank_line.csv: 1 data row visible"-        1-        (fst (dataframeDimensions df))--testAllEmptyRow :: Test-testAllEmptyRow = TestLabel "malformed_all_empty_row" $ TestCase $ do-    df <- D.readCsvUnstable (fixtureDir <> "all_empty_row.csv")-    assertEqual "all_empty_row.csv: 1 data row" 1 (fst (dataframeDimensions df))-    let checkEmpty colName =-            case getColumn colName df of-                Nothing -> assertFailure ("column '" <> T.unpack colName <> "' missing")-                Just col ->-                    assertEqual-                        (T.unpack colName <> " is Nothing (empty field → null)")-                        (DI.fromList @(Maybe T.Text) [Nothing])-                        col-    mapM_ checkEmpty ["a", "b", "c"]--testSingleCol :: Test-testSingleCol = TestLabel "malformed_single_col" $ TestCase $ do-    df <- D.readCsvUnstable (fixtureDir <> "single_col.csv")-    let (rows, cols) = dataframeDimensions df-    assertEqual "single_col.csv: 3 data rows" 3 rows-    assertEqual "single_col.csv: 1 column" 1 cols-    case getColumn "name" df of-        Nothing -> assertFailure "single_col.csv: column 'name' missing"-        Just col ->-            assertEqual-                "single_col.csv: correct values"-                (DI.fromList @T.Text ["Alice", "Bob", "Carol"])-                col--testWhitespaceFields :: Test-testWhitespaceFields = TestLabel "malformed_whitespace_fields" $ TestCase $ do-    df <- D.readCsvUnstable (fixtureDir <> "whitespace_fields.csv")-    assertEqual-        "whitespace_fields.csv: 2 data rows"-        2-        (fst (dataframeDimensions df))-    case getColumn "name" df of-        Nothing -> assertFailure "whitespace_fields.csv: 'name' missing"-        Just col -> assertEqual "name stripped" (DI.fromList @T.Text ["Alice", "Bob"]) col-    case getColumn "city" df of-        Nothing -> assertFailure "whitespace_fields.csv: 'city' missing"-        Just col ->-            assertEqual-                "city stripped"-                (DI.fromList @T.Text ["New York", "Los Angeles"])-                col---- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full)--- Total delimiters: 3 (header) + 2 (short) + 3 (full) = 8--- numCol=3, totalRows=8 div 3=2, numRow=1--- Row-1 stride offsets 3,4,5 → fields "1","2","X"  (X bleeds in from next row)-testMissingFields :: Test-testMissingFields = TestLabel "malformed_missing_fields" $ TestCase $ do-    df <- D.readCsvUnstable (fixtureDir <> "missing_fields.csv")-    assertEqual-        "missing_fields.csv: integer-division gives 1 visible row"-        1-        (fst (dataframeDimensions df))-    case getColumn "c" df of-        Nothing -> assertFailure "missing_fields.csv: column 'c' missing"-        Just col ->-            -- "X" bleeds from the start of the next row into the missing slot-            assertEqual-                "missing_fields.csv: col 'c' bleeds 'X' from next row"-                (DI.fromList @T.Text ["X"])-                col- -- File: a,b,c header; row "1,2,3,EXTRA" -- Total delimiters: 3 + 4 = 7; 7 div 3 = 2, numRow=1 -- Row-1 strides 3,4,5 → "1","2","3" — EXTRA (stride 6) is never accessed testExtraFields :: Test testExtraFields = TestLabel "malformed_extra_fields" $ TestCase $ do-    df <- readCsvNoInfer (fixtureDir <> "extra_fields.csv")+    df <- readCsvNoInfer ("./tests/data/unstable_csv/" <> "extra_fields.csv")     assertEqual "extra_fields.csv: 1 data row" 1 (fst (dataframeDimensions df))     assertEqual "extra_fields.csv: 3 columns" 3 (snd (dataframeDimensions df))     case getColumn "c" df of@@ -342,49 +104,13 @@         Just col ->             assertEqual                 "extra_fields.csv: 'c' = '3' (EXTRA ignored)"-                (DI.fromList @T.Text ["3"])-                col--testNoTrailingNewline :: Test-testNoTrailingNewline = TestLabel "malformed_no_trailing_newline" $ TestCase $ do-    df <- D.readCsvUnstable (fixtureDir <> "no_trailing_newline.csv")-    assertEqual-        "no_trailing_newline.csv: 1 data row"-        1-        (fst (dataframeDimensions df))-    case getColumn "name" df of-        Nothing -> assertFailure "no_trailing_newline.csv: 'name' missing"-        Just col -> assertEqual "name = Alice" (DI.fromList @T.Text ["Alice"]) col-    case getColumn "city" df of-        Nothing -> assertFailure "no_trailing_newline.csv: 'city' missing"-        Just col ->-            assertEqual-                "city = London (synthetic delimiter worked)"-                (DI.fromList @T.Text ["London"])+                (D.fromList @T.Text ["3"])                 col  tests :: [Test] tests =-    [ testSimpleFast-    , testCommaInQuotesFast-    , testQuotesAndNewlinesFast-    , testEscapedQuotesFast-    , testNewlinesFast-    , testUtf8Fast-    , testQuotesAndNewlinesFast-    , testEmptyValuesFast-    , testJsonDataFast-    , specifyTypesNoInferenceFallback+    [ specifyTypesNoInferenceFallback     , specifyTypesInferFallback     , specifyTypesSampleSize-    , testCrlfCsv-    , testCrlfTsv-    , testHeaderOnly-    , testTrailingBlankLine-    , testAllEmptyRow-    , testSingleCol-    , testWhitespaceFields-    , testMissingFields     , testExtraFields-    , testNoTrailingNewline     ]