packages feed

dataframe-fastcsv 1.4.1.1 → 1.4.1.2

raw patch · 15 files changed

+127/−544 lines, 15 filesdep ~dataframe-coredep ~dataframe-operations

Dependency ranges changed: dataframe-core, dataframe-operations

Files

dataframe-fastcsv.cabal view
@@ -1,6 +1,6 @@ cabal-version:      3.4 name:               dataframe-fastcsv-version:            1.4.1.1+version:            1.4.1.2 synopsis: SIMD-accelerated CSV reader for the dataframe library.  description: A fast, SIMD-accelerated CSV/TSV reader using memory-mapped I/O@@ -56,9 +56,9 @@     build-depends:    base >= 4 && < 5,                       bytestring >= 0.11 && < 0.14,                       containers >= 0.6.7 && < 0.10,-                      dataframe-core >= 2.5 && < 2.6,+                      dataframe-core >= 2.6 && < 2.7,                       dataframe-csv >= 2.3 && < 2.4,-                      dataframe-operations >= 2.5 && < 2.6,+                      dataframe-operations >= 2.6 && < 2.7,                       dataframe-parsing >= 2.2 && < 2.3,                       dataframe-parsing >= 2.2 && < 2.3,                       mmap >= 0.5.8 && < 0.7,@@ -90,10 +90,10 @@                    Properties.Csv     build-depends:  base >= 4 && < 5,                     containers >= 0.6.7 && < 0.10,-                    dataframe-core >= 2.5 && < 2.6,+                    dataframe-core >= 2.6 && < 2.7,                     dataframe-csv >= 2.3 && < 2.4,                     dataframe-fastcsv,-                    dataframe-operations >= 2.5 && < 2.6,+                    dataframe-operations >= 2.6 && < 2.7,                     dataframe-parsing >= 2.2 && < 2.3,                     directory >= 1.3.0.0 && < 2,                     HUnit >= 1.6 && < 1.8,
src/DataFrame/IO/CSV/Fast.hs view
@@ -1,7 +1,4 @@-{- | SIMD-accelerated CSV reader: a C scanner finds delimiter positions,-then each column is parsed directly from the mmap'd byte slices into-typed column builders. Chunk-parallel with @-threaded@ + @+RTS -N@.--}+-- | SIMD-accelerated, chunk-parallel CSV reader. module DataFrame.IO.CSV.Fast (     fastReadCsv,     readCsvFast,@@ -50,87 +47,31 @@ separatorByte :: ReadOptions -> Word8 separatorByte = fromIntegral . ord . columnSeparator -{- | Read a CSV file with the default options.--==== __Example__-@-ghci> D.fastReadCsv ".\/data\/taxi.csv"--@--}+-- | Read a CSV file with the default options. fastReadCsv :: FilePath -> IO DataFrame fastReadCsv = fastReadCsvWithOpts defaultReadOptions -{- | Alias for 'fastReadCsv'.--==== __Example__-@-ghci> D.readCsvFast ".\/data\/taxi.csv"--@--}+-- | Alias for 'fastReadCsv'. readCsvFast :: FilePath -> IO DataFrame readCsvFast = fastReadCsv -{- | Read a TSV file with the default options.--==== __Example__-@-ghci> D.fastReadTsv ".\/data\/taxi.tsv"--@--}+-- | Read a TSV file with the default options. fastReadTsv :: FilePath -> IO DataFrame fastReadTsv = fastReadTsvWithOpts defaultReadOptions -{- | Alias for 'fastReadTsv'.--==== __Example__-@-ghci> D.readTsvFast ".\/data\/taxi.tsv"--@--}+-- | Alias for 'fastReadTsv'. readTsvFast :: FilePath -> IO DataFrame readTsvFast = fastReadTsv -{- | Like 'fastReadCsv' but takes a 'ReadOptions' record.  Use this when-you need to tune ragged-row handling, unclosed-quote handling, the-whitespace-trimming knob, or the column selection; otherwise stick with-'fastReadCsv'.  The separator comes from 'columnSeparator', so this is a-'CsvReader' the lazy scan can drive.--==== __Example__-@-ghci> D.fastReadCsvWithOpts D.defaultReadOptions{D.readColumns = Just ["id", "name"]} ".\/data\/customers.csv"--@--}+-- | Like 'fastReadCsv' but takes a 'ReadOptions' record. fastReadCsvWithOpts :: CsvReader fastReadCsvWithOpts opts = readSeparated (separatorByte opts) opts -{- | TSV counterpart to 'fastReadCsvWithOpts'; pins 'columnSeparator' to tab.--==== __Example__-@-ghci> D.fastReadTsvWithOpts D.defaultReadOptions{D.safeRead = D.MaybeRead} ".\/data\/taxi.tsv"--@--}+-- | TSV counterpart to 'fastReadCsvWithOpts'. fastReadTsvWithOpts :: ReadOptions -> FilePath -> IO DataFrame fastReadTsvWithOpts opts = fastReadCsvWithOpts opts{columnSeparator = '\t'} -{- | Read a CSV and coerce each column to the type declared in the-supplied 'Schema'. Schema columns bypass inference: parsed straight from-file bytes as the declared type, avoiding row-1 misclassification.--==== __Example__-@-ghci> schema = D.makeSchema [("id", D.schemaType \@Int)]-ghci> D.fastReadCsvWithSchema schema ".\/data\/customers.csv"--@--}+-- | Read a CSV and coerce each column to the declared 'Schema' type. fastReadCsvWithSchema :: Schema -> FilePath -> IO DataFrame fastReadCsvWithSchema schema =     readSeparated@@ -140,26 +81,11 @@                 SpecifyTypes (M.toList (elements schema)) (typeSpec defaultReadOptions)             } -{- | In-memory version of 'fastReadCsv'.--==== __Example__-@-ghci> D.fastReadCsvFromBytes "id,name\\n1,Ada\\n"--@--}+-- | In-memory version of 'fastReadCsv'. fastReadCsvFromBytes :: BS.ByteString -> IO DataFrame fastReadCsvFromBytes = readSeparatedFromBytes comma defaultReadOptions -{- | In-memory version of 'fastReadCsvWithSchema'.--==== __Example__-@-ghci> schema = D.makeSchema [("id", D.schemaType \@Int)]-ghci> D.fastReadCsvFromBytesWithSchema schema "id,name\\n1,Ada\\n"--@--}+-- | In-memory version of 'fastReadCsvWithSchema'. fastReadCsvFromBytesWithSchema :: Schema -> BS.ByteString -> IO DataFrame fastReadCsvFromBytesWithSchema schema =     readSeparatedFromBytes@@ -169,32 +95,12 @@                 SpecifyTypes (M.toList (elements schema)) (typeSpec defaultReadOptions)             } -{- | Read a CSV and return only the named columns, in the order given.-Shorthand for 'readColumns'. The SIMD scan and delimiter-classification still run over the whole file, but unnamed columns are-never extracted, parsed or allocated. Naming a column the file does not-have raises a 'ColumnsNotFoundException'.--==== __Example__-@-ghci> D.fastReadCsvProj ["id", "name"] ".\/data\/customers.csv"--@--}+-- | Read a CSV and return only the named columns, in the order given. fastReadCsvProj :: [Text] -> FilePath -> IO DataFrame fastReadCsvProj projection =     readSeparated comma defaultReadOptions{readColumns = Just projection} -{- | Reads via a private ('WriteCopy') mmap so the buffer is never copied.-The @separator@ byte and 'columnSeparator' in @opts@ must agree; use-'fastReadCsvWithOpts' unless you need to pass a delimiter byte directly.--==== __Example__-@-ghci> D.readSeparated 44 D.defaultReadOptions ".\/data\/taxi.csv"  -- 44 = ','--@--}+-- | Read a delimited file via a private mmap, without copying. readSeparated ::     Word8 ->     ReadOptions ->@@ -214,16 +120,7 @@         opts         (VS.unsafeFromForeignPtr bufferPtr offset len) -{- | Identical to 'readSeparated' but reads from an already-loaded byte-buffer (e.g. a slice of a memory-mapped file). Zero-copy: the bytes are-only read, never modified or grown.--==== __Example__-@-ghci> D.readSeparatedFromBytes 44 D.defaultReadOptions "id,name\\n1,Ada\\n"  -- 44 = ','--@--}+-- | Like 'readSeparated' but reads from an already-loaded byte buffer. readSeparatedFromBytes ::     Word8 ->     ReadOptions ->@@ -233,16 +130,7 @@     let (fp, off, len) = BSI.toForeignPtr bs     readSeparatedCore Nothing separator opts (VS.unsafeFromForeignPtr fp off len) -{- | 'readSeparatedFromBytes' with a forced chunk count, bypassing the-capability/size gate. Intended for tests and benchmarks that need to pin-the parallel split; @1@ forces the sequential path.--==== __Example__-@-ghci> D.readSeparatedFromBytesChunks 1 44 D.defaultReadOptions "id,name\\n1,Ada\\n"  -- 44 = ','--@--}+-- | 'readSeparatedFromBytes' with a forced chunk count. readSeparatedFromBytesChunks ::     Int ->     Word8 ->
src/DataFrame/IO/CSV/Fast/Columns.hs view
@@ -3,10 +3,7 @@ {-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeApplications #-} -{- | Column-level dispatch for the fast CSV reader: resolve each column's-target type up front into a 'ColumnPlan' (schema hit, inference chain, or-legacy Text pipeline). Plans are chunk-runnable for the parallel reader.--}+-- | Resolves each column's target type up front into a 'ColumnPlan'. module DataFrame.IO.CSV.Fast.Columns (     ColumnEnv (..),     ColumnPlan (..),@@ -103,21 +100,14 @@     | isJust (testEquality (typeRep @a) (typeRep @Day)) = Just TDate     | otherwise = Nothing -{- | How to build one column, resolved once per read (before any chunk-fan-out). 'PlanLegacy' carries the \"re-apply 'parseWithTypes'\" flag.--}+-- | How to build one column, resolved once per read. data ColumnPlan-    = {- | Inference: fallback chain; the 'Bool' asks for the all-null-      pre-check ('NoAssumption' only).-      -}+    = -- | Inference: fallback chain, optionally with an all-null pre-check.       PlanChain !SafeReadMode !NullSpec ![Step] !Bool     | PlanSchema !SafeReadMode !NullSpec !TargetType     | PlanLegacy !SafeReadMode !Bool -{- | Resolve a column's plan. For inferred columns the sample is always-classified against the global (full-range) env, so chunked and sequential-reads share one assumption.--}+-- | Resolve a column's plan against the global (full-range) env. planColumn :: ColumnEnv -> T.Text -> Int -> ColumnPlan planColumn env name col = case mode of     EitherRead -> PlanLegacy mode (isJust schemaEntry)@@ -154,10 +144,7 @@         NoAssumption -> True         _ -> False -{- | What a chain plan produced for one row range: every cell null (only-reported when the all-null pre-check is on), or the index of the first-chain step whose pass succeeded plus its column.--}+-- | What a chain plan produced for one row range. data ChunkOutcome = AllNullChunk | Resolved !Int !PassCol  -- | Run a fallback chain over the env's row range.@@ -174,9 +161,7 @@             Left _ -> go (i + 1) rest     go _ [] = error "fastcsv: type-fallback chain exhausted (StepText must succeed)" -{- | Run a schema-typed pass over the env's row range. @rowOff@ maps a-chunk-local failure index to the global data-row index.--}+-- | Run a schema-typed pass over the env's row range. runSchemaChunk ::     ColumnEnv ->     NullSpec ->@@ -209,10 +194,7 @@ finishMode MaybeRead = ensureOptional finishMode _ = id -{- | Build one column sequentially (single chunk over the full row range).-Returns the column plus a flag telling the caller to re-apply the legacy-'parseWithTypes' conversion afterwards (schema types the passes can't serve).--}+-- | Build one column sequentially over the full row range. buildColumn :: ColumnEnv -> T.Text -> Int -> IO (Column, Bool) buildColumn env name col = case planColumn env name col of     PlanLegacy mode pwt -> (,pwt) <$> legacyColumn env mode col@@ -227,9 +209,7 @@                 Resolved _ c' -> passColumn c'         pure (finishMode mode (dictCompactColumn c), False) -{- | The original Text-materializing pipeline, kept for 'EitherRead' and-exotic schema types (cold paths).--}+-- | The original Text-materializing pipeline, kept for cold paths. legacyColumn :: ColumnEnv -> SafeReadMode -> Int -> IO Column legacyColumn env mode col = do     let opts = ceOpts env
src/DataFrame/IO/CSV/Fast/Core.hs view
@@ -1,11 +1,6 @@ {-# LANGUAGE OverloadedStrings #-} -{- | The shared read pipeline behind every "DataFrame.IO.CSV.Fast" entry-point: scan for delimiters, classify rows, then build every column with-the typed extraction passes — chunk-parallel when the input is large and-capabilities allow ('autoChunkCount'), unless the caller pins a chunk-count. The result is fully forced before it is returned.--}+-- | The shared read pipeline behind every fast CSV entry point. module DataFrame.IO.CSV.Fast.Core (     readSeparatedCore,     checkSeparatorAgrees,@@ -59,9 +54,7 @@ import DataFrame.Internal.DataFrame (DataFrame (..), forceDataFrame) import DataFrame.Operations.Typing (effectiveSafeRead, parseWithTypes) -{- | @readSeparatedCore chunkSpec separator opts buffer@; @chunkSpec@-pins the chunk count ('Nothing' = capability\/size-gated automatic).--}+-- | Run the read pipeline; @chunkSpec@ pins the chunk count if given. readSeparatedCore ::     Maybe Int ->     Word8 ->@@ -74,8 +67,7 @@     let (file, _bomLen) = stripBom fileRaw         contentLen = VS.length file     nChunks <- maybe (autoChunkCount contentLen) pure chunkSpec-    -- Parallel reads scan in parallel too; sequential reads (or builds-    -- without SIMD) keep the original scan + row classification.+    -- Parallel reads scan in parallel too.     mPar <-         if nChunks > 1             then getDelimiterIndicesPar nChunks separator contentLen file@@ -192,19 +184,14 @@                                                 legacySchema                                             )                                             df-                        -- The parallel merge already forced every column-                        -- payload, so only the column thunks themselves-                        -- need forcing; the deep walk would re-chase 6M+-                        -- pointers per boxed column for nothing.+                        -- Parallel merge already forced the payloads.                         if nChunks > 1 && S.null legacySchema                             then do                                 Vector.foldl' (\() c -> c `seq` ()) () cols `seq`                                     return df                             else return $! forceDataFrame df' -{- | The scanner is handed its separator as a byte, so a 'columnSeparator'-saying anything else would be silently ignored. Reject that up front.--}+-- | Reject a 'columnSeparator' that disagrees with the separator byte. checkSeparatorAgrees :: Word8 -> ReadOptions -> IO () checkSeparatorAgrees separator opts =     unless (separator == fromIntegral (ord asked)) $@@ -217,9 +204,6 @@   where     asked = columnSeparator opts -{- | An empty 'DataFrame' — returned when the input has no delimiters-(empty file or single line with no separator and no newline). Guards-against a divide-by-zero in the row-stride math when 'numCol == 0'.--}+-- | An empty 'DataFrame', for input with no delimiters. emptyDataFrame :: DataFrame emptyDataFrame = DataFrame Vector.empty M.empty (0, 0) M.empty
src/DataFrame/IO/CSV/Fast/Index.hs view
@@ -2,12 +2,7 @@ {-# LANGUAGE CApiFFI #-} {-# LANGUAGE ForeignFunctionInterface #-} -{- | Delimiter-index construction for the fast CSV reader. The index buffer-is sized from an exact byte count (not 8 bytes per input byte), the SIMD-scan runs over the 64-byte-aligned prefix of the unpadded mmap, and a-scalar Haskell tail finishes the final partial chunk — so the input is-never copied or grown.--}+-- | Delimiter-index construction for the fast CSV reader. module DataFrame.IO.CSV.Fast.Index (     CsvParseError (..),     getDelimiterIndices,@@ -41,22 +36,11 @@  import DataFrame.IO.CSV (UnclosedQuotePolicy (..)) -{- | Exceptions raised by the fast CSV parser.  Catchable with-'Control.Exception.catch' or 'Control.Exception.try'.--}+-- | Exceptions raised by the fast CSV parser. data CsvParseError-    = {- | The input ends with a quoted field that was never closed.  The-      PCLMUL quote-parity chain in the SIMD scanner treats an unmatched-      @\"@ as if the rest of the file were inside quotes, so we refuse-      to return a silently corrupted 'DataFrame.Internal.DataFrame.DataFrame'-      and raise instead.-      -}+    = -- | The input ends with a quoted field that was never closed.       CsvUnclosedQuote-    | {- | A row has a different number of fields from the header.  Only-      raised when 'DataFrame.IO.CSV.fastCsvOnRaggedRow' is set to-      'RaiseOnRagged'.  Carries the 0-based row index, the expected field-      count, and the actual field count.-      -}+    | -- | A row has a different number of fields from the header.       CsvRaggedRow !Int !Int !Int     deriving (Eq, Show) @@ -69,17 +53,12 @@ tab = 0x09 quote = 0x22 --- Status codes reported via the out-parameter of 'get_delimiter_indices'.--- Must stay in sync with @GDI_*@ in @cbits/process_csv.h@.+-- Status codes of 'get_delimiter_indices', mirroring @GDI_*@ in the C header. gdiOk, gdiUnclosedQuote :: CInt gdiOk = 0 gdiUnclosedQuote = 1 -{- | Return value that the C helper uses to tell Haskell \"SIMD isn't-available on this build / CPU, run the pure-Haskell state machine.\"-Mirrors @GDI_SIMD_UNAVAILABLE@ in @cbits/process_csv.h@.  'CSize' is-unsigned so this has the same bit pattern as 'maxBound'.--}+-- | Return value meaning SIMD is unavailable and Haskell must scan. simdUnavailable :: CSize simdUnavailable = maxBound @@ -97,11 +76,7 @@ byteStringView n v = BSI.fromForeignPtr (fst (VS.unsafeToForeignPtr0 v)) 0 n {-# INLINE byteStringView #-} -{- | Locate delimiter byte positions in the first @originalLen@ bytes of-@csvFile@.  Treats an unclosed quoted field at EOF as a hard error; callers-that want to suppress the exception can use 'getDelimiterIndicesPolicy'-with 'BestEffort'.--}+-- | Locate delimiter byte positions in the first @originalLen@ bytes. {-# INLINE getDelimiterIndices #-} getDelimiterIndices ::     Word8 ->@@ -117,16 +92,11 @@     VS.Vector Word8 ->     IO (VS.Vector CSize) getDelimiterIndicesPolicy policy separator contentLen csvFile = do-    -- Exact upper bound on the index count: every output index is an-    -- unquoted separator or newline byte, so two memchr-speed counts-    -- (plus one synthetic EOF slot) right-size the buffer (audit F4).+    -- Every index is an unquoted separator or newline, plus one EOF slot.     let contentBS = byteStringView contentLen csvFile         capacity = BS.count separator contentBS + BS.count lf contentBS + 1     resultMV <- VSM.unsafeNew capacity-    -- The C scanner reads whole 64-byte chunks; cap it at the largest-    -- chunk boundary inside the content so no padding is needed and the-    -- input never has to be grown or copied (audit F6). The loop in C-    -- runs while i < len - 64, hence the + 64.+    -- The C scanner reads whole 64-byte chunks, so stop at the last one.     let simdLen = (contentLen `div` 64) * 64     (numFound, status) <- VS.unsafeWith csvFile $ \buffer ->         alloca $ \statusPtr -> do@@ -140,8 +110,7 @@                     statusPtr             s <- peek statusPtr             pure (n, s)-    -- The status reports the quote parity at the end of the SIMD region,-    -- which is exactly the starting state for the scalar tail.+    -- The status gives the quote parity the scalar tail starts from.     (found, inQuote) <-         if numFound == simdUnavailable             then scalarScan separator csvFile resultMV 0 0 contentLen False@@ -164,11 +133,7 @@             else pure found     VS.unsafeFreeze (VSM.slice 0 finalLen resultMV) -{- | RFC 4180 quote-parity state machine over bytes @[from, to)@, appending-delimiter indices at @writeIdx@.  Returns the next write index and whether-the scan ended inside a quoted region. Serves as both the SIMD tail and the-full fallback when SIMD is unavailable.--}+-- | RFC 4180 quote-parity state machine over a byte range. scalarScan ::     Word8 ->     VS.Vector Word8 ->@@ -196,10 +161,7 @@                                         go (idx + 1) (i + 1) False                                     else go idx (i + 1) False -{- | Walk every data row and throw 'CsvRaggedRow' on the first one whose-field count differs from the header.  Only called when the user opts-into 'RaiseOnRagged' via 'DataFrame.IO.CSV.fastCsvOnRaggedRow'.--}+-- | Throw 'CsvRaggedRow' on the first row whose field count differs. checkNoRaggedRows ::     VS.Vector Int ->     VS.Vector Int ->@@ -214,18 +176,7 @@         )         dataRows -{- | Classify each entry in the flat delimiter vector as either a row-terminator or a field terminator, and return the indices-into-the-vector-of every row terminator.  A position counts as a row break if either--  * the byte at that position is @\\n@, or-  * the position is beyond the original content length, which only-    happens for the synthetic end-of-file delimiter written when a file-    does not end in a newline.--Field terminators (commas / tabs / the configured separator) are left-implicit: anything between consecutive row terminators is a field.--}+-- | Find the indices of every row terminator in the delimiter vector. {-# INLINE classifyRowEnds #-} classifyRowEnds :: VS.Vector Word8 -> Int -> VS.Vector CSize -> VS.Vector Int classifyRowEnds file contentLen =@@ -235,9 +186,7 @@         let p = fromIntegral pos :: Int          in p >= contentLen || VS.unsafeIndex file p == lf -{- | Number of fields that row @r@ contains, derived directly from the-gap between consecutive entries in @rowEnds@.--}+-- | Number of fields that row @r@ contains. {-# INLINE fieldsInRow #-} fieldsInRow :: VS.Vector Int -> Int -> Int fieldsInRow rowEnds r =@@ -245,12 +194,7 @@         startIdx = if r == 0 then 0 else VS.unsafeIndex rowEnds (r - 1) + 1      in endIdx - startIdx + 1 -{- | Does row @r@ contain exactly one, empty field?  That is the case for-a bare @\\n@ or a @\\r\\n@ \x2014 blank lines, which we skip by default to-match pandas / polars @skip_blank_lines@ semantics.  We also treat a-row whose only byte is @\\r@ as blank, so CRLF-terminated blank lines-don't leak through when only the @\\n@ counts as a row break.--}+-- | Does row @r@ contain exactly one empty field (a blank line)? {-# INLINE isBlankRow #-} isBlankRow ::     VS.Vector Word8 ->@@ -280,9 +224,7 @@                             && VS.unsafeIndex file fieldStart == cr                        ) -{- | Select the row indices that contain actual data: skip @[0 .. skip - 1]@-and drop any blank rows from the remainder.--}+-- | Select the row indices that contain actual data. {-# INLINE collectDataRows #-} collectDataRows ::     VS.Vector Word8 ->
src/DataFrame/IO/CSV/Fast/IndexPar.hs view
@@ -1,15 +1,7 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE CApiFFI #-} -{- | Parallel delimiter-index construction (Round-2 WS-E2): the SIMD region-is split into 64-byte-aligned ranges, one per worker. Quote parity at each-range start comes from a prefix-xor of per-range quote counts (exact: every-quote byte toggles the scanner's PCLMUL parity chain), a parallel counting-pass right-sizes the flat output arrays and gives every worker an exact-write offset, and a parallel emit pass writes delimiter positions plus-row-end ordinals directly (no per-delimiter file-byte re-read). The scalar-tail and the unclosed-quote policy are unchanged from the sequential path.--}+-- | Parallel delimiter-index construction over aligned SIMD ranges. module DataFrame.IO.CSV.Fast.IndexPar (     ParIndex (..),     getDelimiterIndicesPar,@@ -52,18 +44,14 @@ simdUnavailable :: CSize simdUnavailable = maxBound -{- | Result of the parallel scan: the same @(delimiters, rowEnds)@ pair the-sequential 'DataFrame.IO.CSV.Fast.Index.classifyRowEnds' path produces.--}+-- | Result of the parallel scan: the @(delimiters, rowEnds)@ pair. data ParIndex = ParIndex     { piDelims :: !(VS.Vector CSize)     , piRowEnds :: !(VS.Vector Int)     , piEndsInQuote :: !Bool     } -{- | Scan with @nWorkers@ parallel range scans. Returns 'Nothing' when the-build\/CPU lacks SIMD support (callers fall back to the sequential scan).--}+-- | Scan with @nWorkers@ ranges, or 'Nothing' when SIMD is missing. getDelimiterIndicesPar ::     Int -> Word8 -> Int -> VS.Vector Word8 -> IO (Maybe ParIndex) getDelimiterIndicesPar nWorkers separator contentLen file =@@ -156,9 +144,7 @@     rowEnds <- VS.unsafeFreeze (VSM.slice 0 nlFinal rowEndsMV)     pure (ParIndex delims rowEnds inQuote) -{- | Classify the scalar tail's delimiters: append the ordinal of every-newline entry in @[from, to)@ of the delimiter vector.--}+-- | Append the ordinal of every newline entry in the scalar tail. appendTailRowEnds ::     VS.Vector Word8 ->     VSM.IOVector Int ->
src/DataFrame/IO/CSV/Fast/Parallel.hs view
@@ -1,18 +1,6 @@ {-# LANGUAGE TupleSections #-} -{- | Chunk-parallel extraction (Round-2 WS-E2): the data-row range is split-into contiguous, newline-aligned chunks (alignment is by construction —-chunks are row ranges over the quote-resolved delimiter index), each-worker runs every column's 'ColumnPlan' over its chunk with right-sized-builders, and the merger splices the results ('concatColumns' /-'mergeTextChunksPar'). Inference classifies its sample once, globally,-before fan-out; when chunks resolve different types the merger re-parses-only the narrower-typed chunks.--Parallelism requires the executable to be linked with @-threaded@ and run-with @+RTS -N@; otherwise ('getNumCapabilities' == 1) the sequential path-runs. Inputs under 4 MB are always read sequentially.--}+-- | Chunk-parallel column extraction over newline-aligned row ranges. module DataFrame.IO.CSV.Fast.Parallel (     buildAllColumns,     autoChunkCount,@@ -41,11 +29,7 @@ parallelThresholdBytes :: Int parallelThresholdBytes = 4 * 1024 * 1024 -{- | Chunk count for an automatic read: four chunks per capability (the-worker pool bounds concurrency at the capability count; the extra chunks-absorb straggler imbalance), or the sequential path (1) on a single-capability or a small input.--}+-- | Chunk count for an automatic read, 1 when sequential. autoChunkCount :: Int -> IO Int autoChunkCount contentLen = do     caps <- getNumCapabilities@@ -68,11 +52,7 @@     mergeTextChunksPar width [tc | PassText tc <- ps] mergePassCols _ ps = pure $! concatColumns [c | PassFull c <- ps] -{- | Build every column, fanning the row range out over @nChunks@ chunks-on a capability-wide worker pool (sequentially when @nChunks <= 1@). Same-result as the sequential path by construction; see the chunk-determinism-tests and property.--}+-- | Build every column, fanning the row range out over @nChunks@. buildAllColumns ::     Int -> ColumnEnv -> [(T.Text, Int)] -> IO [(Column, Bool)] buildAllColumns nChunks env wanted@@ -91,16 +71,9 @@         traceMarkerIO "fastcsv:join-done"         let byCol = L.transpose perChunk         mapM_ (mapM_ (\cc -> cc `seq` pure ())) byCol-        -- Reset the major-GC trigger here, where the copyable live set is-        -- small (chunk payloads are large objects). Otherwise the heap-        -- doubling crosses its threshold mid-merge and a ~0.5s gen-1-        -- collection lands at the worst point of the read.+        -- Reset the major-GC trigger while the copyable live set is small.         performMajorGC-        -- Merge in parallel too (chunk-level inside each column, pooled-        -- across columns), forcing each spliced column here so no memcpy-        -- is deferred to the final forceDataFrame walk. 'evaluate' resolves-        -- the plan dispatch now, so a schema task is a closure over its own-        -- column's data only — never 'env'.+        -- Merge in parallel too, forcing each spliced column here.         tasks <-             mapM                 (\(wp, cps) -> evaluate (mergeTask width env ranges wp cps))@@ -177,9 +150,7 @@         PlanChain _ nspec steps checkAllNull ->             CCChain <$> runChainChunk cenv nspec steps checkAllNull col -{- | Splice one column's chunk results back together, resolving chunk-level-type promotion. Mirrors 'buildColumn' for the single-chunk case.--}+-- | Splice one column's chunk results back together. mergeColumn ::     Int ->     ColumnEnv ->@@ -195,12 +166,7 @@         c <- mergeChain width env nspec steps col [(r, oc) | (r, CCChain oc) <- parts]         pure (finishMode mode (dictCompactColumn c), False) -{- | Merge chain-plan chunks. The candidate type starts at the widest chunk-resolution; any chunk that resolved narrower is re-parsed at the candidate-step, and a re-parse failure escalates the candidate further down the-chain (exactly the rows the sequential pass would have failed on). The-final 'StepText' never fails, so this terminates.--}+-- | Merge chain-plan chunks, re-parsing narrower chunks as needed. mergeChain ::     Int ->     ColumnEnv ->
src/DataFrame/IO/CSV/Fast/Passes.hs view
@@ -3,11 +3,7 @@ {-# LANGUAGE MagicHash #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | The typed column passes (Round-2 WS-E1): each pass walks one column's-delimiter slices and appends parsed values straight into a WS-C builder —-no per-field 'Data.Text.Text' and no deferred re-parse. Text bytes are-memcpy'd into a shared buffer, only for Text-typed columns.--}+-- | Typed column passes that append parsed values straight into builders. module DataFrame.IO.CSV.Fast.Passes (     ColumnEnv (..),     NullSpec (..),@@ -111,9 +107,7 @@ isNullText NullEmpty t = T.null t isNullText (NullMissing _ toks) t = isNullishOrMissing toks t -{- | What a pass produced: a finished 'Column', or a raw text chunk that the-merger splices at the byte level (no per-chunk 'T.Text' materialization).--}+-- | What a pass produced: a finished 'Column' or a raw text chunk. data PassCol = PassFull !Column | PassText !TextChunk  -- | Finish one 'PassCol' on its own (the single-chunk \/ sequential case).@@ -128,9 +122,7 @@ rowAt env = VS.unsafeIndex (ceRows env) {-# INLINE rowAt #-} -{- | One pass appending unboxed values; @nullOnFail@ turns parse failures-into nulls (schema + 'MaybeRead') instead of aborting (inference demotion).--}+-- | One pass appending unboxed values, nulling parse failures if asked. {-# INLINE unboxedPass #-} unboxedPass ::     (Columnable a, VU.Unbox a) =>@@ -146,8 +138,7 @@         bs = fcBS ctx         !(I# n) = ceNumRow env     b <- stToIO (newNumBuilder nullVal (ceNumRow env))-    -- Loop counter is Int#: join points are not worker/wrapper'd, so a-    -- boxed counter would allocate on every iteration.+    -- Int# counter: a boxed one would allocate on every iteration.     let go :: Int# -> IO PassResult         go i             | isTrue# (i >=# n) =@@ -176,11 +167,7 @@ boolPass env nspec col nullOnFail =     unboxedPass env nspec col nullOnFail False parseBoolFieldSlice -{- | Text pass (never fails): raw bytes are memcpy'd into the shared-builder buffer; only quoted fields containing a doubled quote, and the-legacy trim knob, take the decode path. Returns the raw chunk; 'T.Text'-values materialize once, when the merger splices the chunks.--}+-- | Text pass: copies raw bytes into the shared builder buffer. textPass :: ColumnEnv -> NullSpec -> Int -> IO TextChunk textPass env nspec col = do     let ctx = ceCtx env
src/DataFrame/IO/CSV/Fast/Slice.hs view
@@ -1,10 +1,6 @@ {-# LANGUAGE BangPatterns #-} -{- | Field-slice resolution for the fast CSV reader: maps @(row, col)@ to a-byte range of the input (quote-stripped, CR-stripped, optionally trimmed)-without materializing any intermediate value. 'extractField' is the legacy-Text materialization used for headers, inference samples and cold paths.--}+-- | Maps @(row, col)@ to a byte range of the input without copying. module DataFrame.IO.CSV.Fast.Slice (     FieldCtx (..),     FieldLayout (..),@@ -34,14 +30,9 @@  -- | How @(row, col)@ resolves to delimiter byte positions. data FieldLayout-    = {- | Flat delimiter positions + row-end ordinals: the shape the scanner-      emits. Handles ragged and blank rows; 8 bytes per delimiter.-      -}+    = -- | Flat delimiter positions plus row-end ordinals.       LayoutFlat !(VS.Vector CSize) !(VS.Vector Int)-    | {- | Uniform-stride rows: byte start per row plus 'Word16' deltas of-      each field-end from its row start (@deltas ! (r*stride + col)@).-      Built by 'compactRows'; ~3.5x smaller than the flat index.-      -}+    | -- | Uniform-stride rows with 'Word16' field-end deltas.       LayoutRows !(VS.Vector Int) !(VS.Vector Word16) !Int  -- | Everything needed to resolve a field slice, shared by all columns.@@ -100,12 +91,7 @@   where     totalRows = VS.length rowEnds -{- | Resolve field @col@ of row @r@ and continue with @k start end quoted@.-Quoted fields yield the bytes between the outer quotes (embedded @\"\"@ is-NOT unescaped here); a trailing @\\r@ is stripped first; a column beyond the-row's field count (ragged short row) yields the empty slice. CPS so the hot-loops never box the bounds.--}+-- | Resolve field @col@ of row @r@ and continue with its byte range. {-# INLINE withFieldSlice #-} withFieldSlice :: FieldCtx -> Int -> Int -> (Int -> Int -> Bool -> r) -> r withFieldSlice ctx r col k = case fcLayout ctx of@@ -159,10 +145,7 @@                 then k (fieldStart + 1) (fieldEnd - 1) True                 else k fieldStart fieldEnd False -{- | 'withFieldSlice' with the trim knob applied: when 'fcTrim' is set,-unquoted slices have ASCII whitespace stripped from both ends before the-continuation runs. This is the slice the typed parsers and null tests see.--}+-- | 'withFieldSlice' with the trim knob applied. {-# INLINE withParseSlice #-} withParseSlice :: FieldCtx -> Int -> Int -> (Int -> Int -> r) -> r withParseSlice ctx r col k =@@ -205,12 +188,7 @@ decodeSlice :: BS.ByteString -> Int -> Int -> Text decodeSlice bs s e = TextEncoding.decodeUtf8Lenient (sliceBS bs s e) -{- | Extract field @col@ of row @r@ as a 'Text' with the original fastcsv-semantics: quotes stripped and @\"\"@ unescaped for quoted fields, optional-'Text.strip' (full Unicode) for unquoted fields when 'fcTrim' is set.-Used for header names, inference samples and the legacy (EitherRead /-unsupported-schema) column paths.--}+-- | Extract field @col@ of row @r@ as a 'Text'. {-# INLINE extractField #-} extractField :: FieldCtx -> Int -> Int -> Text extractField ctx r col =@@ -221,10 +199,7 @@                 (if fcTrim ctx then Text.strip else id)                     (decodeSlice (fcBS ctx) s e) -{- | RFC 4180 inner-quote unescape: @\"\"@ → @\"@.  'Text.replace' on a-two-char needle does a single linear pass and is allocation-free when-no doubled quote is present (short-circuits at the first miss).--}+-- | RFC 4180 inner-quote unescape. {-# INLINE unescapeDoubledQuotes #-} unescapeDoubledQuotes :: Text -> Text unescapeDoubledQuotes t@@ -234,9 +209,7 @@     doubledQuote = Text.pack "\"\""     singleQuote = Text.singleton '"' -{- | Strip a leading UTF-8 BOM (EF BB BF) if present. Returns the trimmed-vector and the number of bytes removed (0 or 3).--}+-- | Strip a leading UTF-8 BOM, returning the vector and bytes removed. {-# INLINE stripBom #-} stripBom :: VS.Vector Word8 -> (VS.Vector Word8, Int) stripBom v
src/DataFrame/IO/CSV/Fast/TextMerge.hs view
@@ -1,11 +1,6 @@ {-# LANGUAGE BangPatterns #-} -{- | Parallel byte-level merge of per-chunk text columns: the byte copies-and offset rebase fan out over the chunk list, so a large text column does-not serialize the merge phase. Wraps the merged shared buffer + offsets as-'PackedText' (no 'Data.Text.Text' spine, no eager UTF-8 validation), exactly-as the pure 'mergeTextChunks' now produces.--}+-- | Parallel byte-level merge of per-chunk text columns. module DataFrame.IO.CSV.Fast.TextMerge (     mergeTextChunksPar, ) where@@ -32,10 +27,7 @@     mkPackedContiguous32,  ) -{- | Merge text chunks with @width@-way parallel byte copies + offset-rebase, then wrap the shared buffer as 'PackedText'. Single chunks take-the pure (zero-copy) path.--}+-- | Merge text chunks with parallel byte copies into a 'PackedText'. mergeTextChunksPar :: Int -> [TextChunk] -> IO Column mergeTextChunksPar _ [c] = pure $! mergeTextChunks [c] mergeTextChunksPar width cs = do@@ -59,8 +51,7 @@                     fill 1                 | (c, bOff, rOff) <- zip3 cs byteOffs rowOffs                 ]-    -- The final width is known before allocation: Int32 offsets whenever-    -- the merged buffer stays under 2^31 bytes (the common case).+    -- Use Int32 offsets whenever the merged buffer stays under 2^31 bytes.     packed <-         if totalBytes <= fromIntegral (maxBound :: Int32)             then do
tests/Operations/ChunkParallel.hs view
@@ -1,10 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -{- | Behavior pins for chunk-parallel extraction: a chunked read must be-indistinguishable from the sequential one — same column types (promotion-resolved across chunks), same values, and same errors.--}+-- | Chunked reads must match the sequential read. module Operations.ChunkParallel (tests) where  import qualified Data.Map as M@@ -55,8 +52,7 @@     Nothing -> assertFailure (what <> ": column missing")     Just col -> assertEqual what expected col --- A 200-row mixed dataset: nullable ints, doubles, quoted text with--- embedded separators/newlines/doubled quotes, bools, nullable dates.+-- A 200-row mixed dataset with quoted text, bools, nulls and dates. mixedBody :: T.Text mixedBody = "i,d,t,b,dt\n" <> T.concat (map row [0 .. 199 :: Int])   where@@ -84,8 +80,7 @@     TestLabel "par_matches_seq_mixed" . TestCase $         assertChunkInvariant "mixed nulls+quoted" defaultReadOptions mixedBody --- A Double cell far past the sample, in the last chunk: every chunk that--- resolved Int must be re-parsed as Double by the merger.+-- A Double cell in the last chunk forces Int chunks to be re-parsed. testPromotionAcrossChunks :: Test testPromotionAcrossChunks = TestLabel "par_promotion_int_double" . TestCase $ do     let body =@@ -100,8 +95,7 @@         df         (DI.fromList @Double ([fromIntegral k | k <- [0 .. 189 :: Int]] <> [190.5])) --- A Bool cell in the last chunk of an Int column: candidate escalates past--- Double (the bool cell fails it) all the way to Text.+-- A Bool cell in the last chunk of an Int column escalates to Text. testDemotionToTextAcrossChunks :: Test testDemotionToTextAcrossChunks = TestLabel "par_demotion_text" . TestCase $ do     let body =@@ -129,8 +123,7 @@         df         (DI.fromList @(Maybe T.Text) (replicate 200 Nothing)) --- All-null sample (NoAssumption), data only in the last chunk: all-null--- chunks must be re-parsed at the type the data chunk resolved.+-- Data only in the last chunk: all-null chunks must be re-parsed. testLateDataAfterAllNullSample :: Test testLateDataAfterAllNullSample = TestLabel "par_late_bool" . TestCase $ do     let cell k@@ -152,8 +145,7 @@             ]         ) --- Schema + NoSafeRead failure in a late chunk: same error, same row index--- as the sequential read.+-- A late schema failure reports the same row as the sequential read. testSchemaErrorSameRowAsSequential :: Test testSchemaErrorSameRowAsSequential = TestLabel "par_schema_error" . TestCase $ do     let body =@@ -238,8 +230,7 @@         Left D.CsvUnclosedQuote -> pure ()         other -> assertFailure ("expected CsvUnclosedQuote, got " <> show other) --- The parallel byte-level merge wraps the merged buffer as 'PackedText'--- (no Text spine), an RSS win; values stay identical to the sequential read.+-- The parallel merge wraps the buffer as 'PackedText', values unchanged. testParallelTextFreezesPacked :: Test testParallelTextFreezesPacked = TestLabel "par_text_freezes_packed" . TestCase $ do     let body =@@ -254,7 +245,7 @@         Just col -> do             assertBool "parallel text column is PackedText" (DI.isPackedText col)             assertEqual-                "parallel text values byte-identical"+                "parallel text values equal"                 (DI.fromList @T.Text ["t" <> T.pack (show k) | k <- [0 .. 199 :: Int]])                 col
tests/Operations/Projection.hs view
@@ -1,11 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | 'readColumns' semantics for the SIMD reader, pinned against the-pure reader. Every case asserts both readers agree: the two disagreed-over 'ProvideNames' for a long time, and a projection only one of them-honours is worse than no projection at all.--}+-- | 'readColumns' semantics, pinned against the pure reader. module Operations.Projection (tests) where  import qualified Data.Map as M@@ -33,9 +29,7 @@     removeFile path     pure r -{- | Read @contents@ through both readers with @opts@, check they agree,-then hand the frame to the assertion.--}+-- | Read through both readers, check they agree, then assert. bothReaders ::     String -> CSV.ReadOptions -> T.Text -> (DataFrame -> Assertion) -> Test bothReaders label opts contents check = TestLabel label $
tests/Operations/ReadCsv.hs view
@@ -3,7 +3,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} --- Test fixtures inspired by csv-spectrum (https://github.com/max-mapper/csv-spectrum)+-- Test fixtures inspired by csv-spectrum.  module Operations.ReadCsv where @@ -55,10 +55,6 @@ tempDir :: FilePath tempDir = "./tests/data/unstable_csv/" ------------------------------------------------------------------------------------ Pretty-printer---------------------------------------------------------------------------------- prettyPrintCsv :: FilePath -> DataFrame -> IO () prettyPrintCsv = prettyPrintSeparated ',' @@ -78,9 +74,7 @@         (TIO.hPutStrLn handle . T.intercalate (T.singleton sep) . getRowEscaped sep df)         [0 .. rows - 1] --- RFC 4180-compliant CSV field escaping: wrap in quotes if the field--- contains the separator, a line break, or a quote; double any embedded--- quotes so the parser can reverse the encoding.+-- RFC 4180 field escaping: quote when needed, double embedded quotes. escapeField :: Char -> T.Text -> T.Text escapeField sep field     | needsQuoting = T.concat ["\"", T.replace "\"" "\"\"" field, "\""]@@ -236,9 +230,7 @@                 (DI.fromList @T.Text ["Alice", "Bob", "Carol"])                 col --- After the RFC 4180 alignment, whitespace inside unquoted fields is--- preserved verbatim.  Users who want leading/trailing whitespace--- stripped can opt in via the fastCsvTrimUnquoted knob (Step 7).+-- Whitespace inside unquoted fields is preserved verbatim. testWhitespaceFields :: Test testWhitespaceFields = TestLabel "malformed_whitespace_fields" $ TestCase $ do     df <- D.readCsvFast (fixtureDir <> "whitespace_fields.csv")@@ -261,9 +253,7 @@                 (DI.fromList @T.Text ["  New York", "  Los Angeles"])                 col --- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full). Each line is--- classified individually: the short row survives with a null-padded 'c';--- the full row keeps its value.+-- A short row survives with a null-padded 'c'; the full row keeps its value. testMissingFields :: Test testMissingFields = TestLabel "malformed_missing_fields" $ TestCase $ do     df <- D.readCsvFast (fixtureDir <> "missing_fields.csv")@@ -286,9 +276,7 @@                 (DI.fromList @(Maybe T.Text) [Nothing, Just "Z"])                 col --- File: a,b,c header; row "1,2,3,EXTRA" (over-long). Under the default--- ragged-row policy we read columns 0..numCol-1 and silently drop the--- EXTRA field.+-- An over-long row drops the extra field under the default ragged policy. testExtraFieldsTruncate :: Test testExtraFieldsTruncate =     TestLabel "malformed_extra_fields_truncate" $ TestCase $ do@@ -327,18 +315,14 @@                 (DI.fromList @T.Text ["London"])                 col --- Regression: a zero-byte CSV used to divide by zero in the row-stride math--- (`VS.length indices `div` numCol` with numCol == 0). Now it returns an--- empty DataFrame cleanly.+-- Regression: a zero-byte CSV used to divide by zero; now it is empty. testEmptyFile :: Test testEmptyFile = TestLabel "malformed_empty_file" $ TestCase $ do     df <- D.readCsvFast (fixtureDir <> "empty_file.csv")     assertEqual "empty_file.csv: 0 rows" 0 (fst (dataframeDimensions df))     assertEqual "empty_file.csv: 0 columns" 0 (snd (dataframeDimensions df)) --- RFC 4180 quote unescaping: `""` decodes to a single `"`.  The escaped--- quote fixture contains the two canonical shapes: an embedded doubled--- quote and a plain quoted number.+-- RFC 4180 unescaping: a doubled quote decodes to a single quote. testRfc4180EscapedQuote :: Test testRfc4180EscapedQuote =     TestLabel "rfc4180_escaped_quote" $ TestCase $ do@@ -355,8 +339,7 @@                     (DI.fromList @T.Text ["ha \"ha\" ha", "4"])                     col --- Whitespace inside double quotes is data, not padding, so a quoted--- "  padded  " survives the parser unchanged.+-- Whitespace inside double quotes is data and survives unchanged. testQuotedWhitespacePreserved :: Test testQuotedWhitespacePreserved =     TestLabel "quoted_whitespace_preserved" $ TestCase $ do@@ -372,9 +355,7 @@                     (DI.fromList @T.Text ["  padded  "])                     col --- With `fastCsvOnRaggedRow = RaiseOnRagged`, the short row in--- missing_fields.csv must be rejected with a CsvRaggedRow carrying the--- row index and the expected/actual field counts.+-- RaiseOnRagged rejects the short row with a CsvRaggedRow. testRaiseOnRagged :: Test testRaiseOnRagged = TestLabel "raise_on_ragged" $ TestCase $ do     let opts = defaultReadOptions{fastCsvOnRaggedRow = RaiseOnRagged}@@ -392,8 +373,7 @@             assertFailure                 "fastReadCsvWithOpts RaiseOnRagged should have thrown" --- With `fastCsvTrimUnquoted = True` we recover the legacy behaviour of--- stripping leading/trailing whitespace from unquoted fields.+-- fastCsvTrimUnquoted = True strips whitespace from unquoted fields. testTrimUnquotedOpt :: Test testTrimUnquotedOpt = TestLabel "trim_unquoted_opt" $ TestCase $ do     let opts = defaultReadOptions{fastCsvTrimUnquoted = True}@@ -406,9 +386,7 @@                 (DI.fromList @T.Text ["Alice", "Bob"])                 col --- A declared 'Schema' overrides type inference: the 'a' column is--- forced to 'Int' and 'b' to 'Double', regardless of what the first few--- rows look like.+-- A declared 'Schema' overrides type inference. testSchemaPushdown :: Test testSchemaPushdown = TestLabel "schema_pushdown" $ TestCase $ do     let path = fixtureDir <> "schema_pushdown.csv"@@ -428,8 +406,7 @@         Nothing -> assertFailure "schema_pushdown: 'b' missing"         Just col -> assertEqual "b is Double" (DI.fromList @Double [1.5, 2.5]) col --- `fastReadCsvProj` returns only the requested columns, in the order--- requested; unreferenced columns never appear in the result.+-- fastReadCsvProj returns only the requested columns, in order. testProjection :: Test testProjection = TestLabel "projection" $ TestCase $ do     let path = fixtureDir <> "projection.csv"@@ -443,9 +420,7 @@             map fst . L.sortBy (compare `on` snd) . M.toList $ columnIndices df     assertEqual "projection preserves order" ["c", "a"] names --- An unmatched `"` used to silently swallow the rest of the file (the SIMD--- quote-parity chain never resets). We now raise 'CsvUnclosedQuote' rather--- than returning a corrupted DataFrame.+-- An unmatched quote now raises 'CsvUnclosedQuote' instead of corrupting. testUnclosedQuote :: Test testUnclosedQuote = TestLabel "malformed_unclosed_quote" $ TestCase $ do     let path = fixtureDir <> "unclosed_quote.csv"@@ -461,9 +436,7 @@             assertFailure                 "readCsvFast should have thrown CsvUnclosedQuote on input with a stray quote" --- UTF-8 BOM (EF BB BF) must be stripped before the first header is parsed.--- Without the strip, `getColumn "name"` fails because the column is keyed--- under the three-byte-prefixed "\xEF\xBB\xBFname".+-- The UTF-8 BOM must be stripped before the first header is parsed. testUtf8Bom :: Test testUtf8Bom = TestLabel "malformed_utf8_bom" $ TestCase $ do     df <- D.readCsvFast (fixtureDir <> "utf8_bom.csv")
tests/Operations/TypedExtraction.hs view
@@ -2,10 +2,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Behavior pins for the typed-extraction fastcsv path: schema bypass,-parser semantics, byte-level missing tests, and the chunk-boundary scalar-tail that replaced the full-file copy.--}+-- | Behavior pins for the typed-extraction fastcsv path. module Operations.TypedExtraction (tests) where  import qualified Data.Map as M@@ -13,9 +10,7 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as TE --- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the--- handle's text mode, which on Windows turns \n into \r\n and--- corrupts inputs meant for byte-level parsers.+-- UTF-8 byte-mode IO: the plain writer turns \n into \r\n on Windows. import qualified Data.Text.IO.Utf8 as TIO  import Control.Exception (ErrorCall, evaluate, try)@@ -50,8 +45,7 @@     Nothing -> assertFailure (what <> ": column missing")     Just col -> assertEqual what expected col --- Divergence #1 (pinned new behavior): Int overflow rejects instead of--- wrapping, so an overflowing cell demotes the column to Double.+-- Int overflow rejects instead of wrapping, demoting the column to Double. testIntOverflowDemotesToDouble :: Test testIntOverflowDemotesToDouble = TestLabel "typed_int_overflow_demotes" $     TestCase $@@ -63,9 +57,7 @@                 df                 (DI.fromList @Double [1.0, 9.223372036854776e18]) --- Divergence #2: doubles parse with strip-equivalent semantics, so a padded--- double past the 100-row inference sample no longer demotes the column to--- Text. (Inside the sample the classifier still rejects padding.)+-- A padded double past the inference sample no longer demotes to Text. testPaddedDoubleStaysDouble :: Test testPaddedDoubleStaysDouble = TestLabel "typed_padded_double" $     TestCase $ do@@ -79,8 +71,7 @@                 df                 (DI.fromList @Double (clean <> [3.5])) --- Schema bypasses inference: a numeric-looking column declared Text in the--- schema must come back as Text (old code silently kept the inferred Int).+-- A numeric-looking column declared Text in the schema comes back as Text. testSchemaTextBypassesInference :: Test testSchemaTextBypassesInference = TestLabel "typed_schema_text" $     TestCase $@@ -93,8 +84,7 @@                 df                 (DI.fromList @T.Text ["1", "2"]) --- Schema + NoSafeRead: a cell that cannot parse as the declared type--- raises during the read (strict read), instead of hiding an error thunk.+-- A cell failing the declared type raises during the read. testSchemaStrictFailureThrows :: Test testSchemaStrictFailureThrows = TestLabel "typed_schema_strict_throws" $     TestCase $@@ -133,8 +123,7 @@                 df                 (DI.fromList @(Maybe Int) [Just 1, Nothing, Just 2]) --- Preserved: NoSafeRead treats only empty cells as null, so "NA" still--- demotes a numeric column to Text.+-- NoSafeRead treats only empty cells as null, so "NA" demotes to Text. testNoSafeReadNADemotesToText :: Test testNoSafeReadNADemotesToText = TestLabel "typed_nosaferead_na" $     TestCase $@@ -177,8 +166,7 @@                 df                 (DI.fromList @(Maybe Int) [Just 1, Nothing, Just 2]) --- Preserved: quoted numerics parse as numbers (quote stripping happens--- before the typed parse).+-- Quoted numerics parse as numbers; quotes are stripped first. testQuotedNumbersParse :: Test testQuotedNumbersParse = TestLabel "typed_quoted_numbers" $     TestCase $@@ -217,9 +205,7 @@                     ]                 ) --- The scalar tail (last <64 bytes after the SIMD chunks) must honour the--- quote state from the SIMD region: a quoted field opening before the--- boundary and closing in the tail may contain separators and newlines.+-- A quote opening before the SIMD boundary and closing in the tail is kept. testQuoteSpansChunkBoundary :: Test testQuoteSpansChunkBoundary = TestLabel "typed_quote_spans_boundary" $     TestCase $ do@@ -247,8 +233,7 @@                 other ->                     assertFailure ("expected CsvUnclosedQuote, got " <> show other) --- S2 pin: inference must land on exactly the same columns (types and--- values) as an explicit schema declaring the true types.+-- Inference must land on the same columns as an explicit true schema. testInferredMatchesSchema :: Test testInferredMatchesSchema = TestLabel "typed_inferred_matches_schema" $     TestCase $ do@@ -280,8 +265,7 @@             explicit <- D.fastReadCsvWithSchema schema path             assertEqual "inferred == explicit schema" explicit inferred --- Ingest pin: an inferred string column freezes to a shared-buffer--- 'PackedText' (no per-row boxed Text header), values byte-identical.+-- An inferred string column freezes to a 'PackedText', values identical. testTextFreezesPacked :: Test testTextFreezesPacked = TestLabel "typed_text_freezes_packed" $     TestCase $@@ -291,7 +275,7 @@                 Nothing -> assertFailure "missing text column"                 Just col -> do                     assertBool "text column is PackedText" (DI.isPackedText col)-                    assertEqual "values byte-identical" (DI.fromList @T.Text ["x", "y"]) col+                    assertEqual "values equal" (DI.fromList @T.Text ["x", "y"]) col  -- The in-memory entry point must agree with the file-based one. testFromBytesMatchesFile :: Test
tests/Properties/Csv.hs view
@@ -3,16 +3,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | QuickCheck properties for the SIMD CSV parser.--Each property builds a fresh input on disk under @/tmp@, exercises-'DataFrame.IO.CSV.Fast.fastReadCsv' (or a variant), and asserts a-property-level invariant.  Temp files are cleaned up by the caller.--The properties deliberately avoid depending on type-inference: every-test cell starts with a letter so 'parseFromExamples' always lands on-@Text@, which makes roundtrip comparison unambiguous.--}+-- | QuickCheck properties for the SIMD CSV parser. module Properties.Csv (tests) where  import Control.Exception (try)@@ -41,10 +32,7 @@ import Test.QuickCheck.Monadic (PropertyM, assert, monadicIO, run) import Type.Reflection (typeRep) -{- | A generator for a single CSV cell.  Each cell starts with an ASCII-letter (so type inference always picks 'Text') and may contain a small-menagerie of special characters the parser must escape correctly.--}+-- | A generator for a single CSV cell. newtype Cell = Cell {unCell :: T.Text}     deriving (Eq, Show) @@ -57,11 +45,7 @@         | T.null t || T.length t == 1 = []         | otherwise = [Cell (T.take (T.length t - 1) t)] -{- | Like 'Cell' but guaranteed to contain neither @\\n@ nor @\\r@.  Used-by properties that diff across line-ending dialects, where embedded-line breaks become part of the data under CRLF transform and break-the otherwise-clean invariant.--}+-- | Like 'Cell' but without line breaks. newtype SingleLineCell = SingleLineCell {unSingleLineCell :: T.Text}     deriving (Eq, Show) @@ -86,10 +70,7 @@         | length ns <= 1 = []         | otherwise = [ColumnNames (init ns)] -{- | RFC 4180 cell encoding: wrap a field in @"@ and double embedded-quotes when the cell contains the separator, a line break, or a-quote.--}+-- | RFC 4180 cell encoding. encodeCell :: Char -> T.Text -> T.Text encodeCell sep t     | T.any needsQuote t = T.concat ["\"", T.replace "\"" "\"\"" t, "\""]@@ -97,22 +78,14 @@   where     needsQuote c = c == sep || c == '\n' || c == '\r' || c == '"' -{- | Encode a grid @[[cell]]@ (rows of cells) as an RFC 4180 CSV.  The-first row is written as the header, followed by one line per data-row.  Always uses @\\n@ as the line terminator; callers that need-CRLF can transform the output themselves.--}+-- | Encode a grid of cells as an RFC 4180 CSV. encodeCsv :: Char -> [T.Text] -> [[T.Text]] -> T.Text encodeCsv sep header rows =     T.unlines (encodeRow header : map encodeRow rows)   where     encodeRow = T.intercalate (T.singleton sep) . map (encodeCell sep) -{- | Pull the raw 'Text' values of the named column out of a DataFrame.-Fails the property if the column is missing or is not a 'Text'-column (which would signal a type-inference accident — the cell-generator is designed to avoid that).--}+-- | Pull the raw 'Text' values of the named column out of a DataFrame. columnAsText :: T.Text -> DataFrame -> Maybe [T.Text] columnAsText name df = do     idx <- M.lookup name (columnIndices df)@@ -126,9 +99,7 @@         PackedText{} -> Nothing         MergedColumn{} -> Nothing -{- | Run a property in @IO@ against a generated CSV text, cleaning up-the temp file afterwards no matter what.--}+-- | Run a property in @IO@ against a generated CSV text. withCsvFile :: String -> T.Text -> (FilePath -> IO a) -> IO a withCsvFile label body action = do     let path = "./tests/data/unstable_csv/fastcsv_prop_" <> label <> ".csv"@@ -137,12 +108,9 @@     removeFile path     pure r --------------------------------------------------------------------------------- -- Properties -{- | Any CSV parses the same whether or not a UTF-8 BOM (EF BB BF) is-prepended.  Excel / PowerShell CSV exports all come with a BOM.--}+-- | A UTF-8 BOM does not change how a CSV parses. prop_bom_invariant :: ColumnNames -> [[Cell]] -> Property prop_bom_invariant (ColumnNames header) rows = monadicIO $ do     let cellRows = map (map unCell) rows@@ -153,12 +121,7 @@     assert (plain == withBom)     assert (dataframeDimensions plain == dataframeDimensions withBom) -{- | A CSV with no embedded newlines inside quoted fields parses the-same whether record separators are @\\n@ or @\\r\\n@.  When quoted-cells themselves contain @\\n@, a CRLF-encoded file contains a literal-@\\r\\n@ inside the quotes (that's data, not line ending), so the-invariant correctly no longer holds; we filter such inputs out.--}+-- | A CSV parses the same with @\n@ or @\r\n@ record separators. prop_crlf_invariant :: ColumnNames -> [[SingleLineCell]] -> Property prop_crlf_invariant (ColumnNames header) rows = monadicIO $ do     let cellRows = map (map unSingleLineCell) rows@@ -168,11 +131,7 @@     dfCrlf <- run $ withCsvFile "crlf_crlf" crlfCsv D.fastReadCsv     assert (dfLf == dfCrlf) -{- | A DataFrame produced by the RFC 4180 encoder round-trips through-the fast reader: column names survive, row count survives, and every-cell comes back bit-for-bit.  Restricted to ASCII-letter-prefixed-cells so that type inference always stays on 'Text'.--}+-- | A DataFrame encoded as RFC 4180 round-trips through the fast reader. prop_roundtrip_ascii :: Property prop_roundtrip_ascii = forAll (chooseInt (0, 12)) $ \nRows ->     forAll (chooseInt (1, 4)) $ \nCols ->@@ -198,11 +157,7 @@         )         (zip header expected) -{- | A file that ends with an unmatched @"@ must raise 'CsvUnclosedQuote'-under the default policy.  The generator deliberately builds a valid-prefix so the property is checking the error path, not a random-parse failure.--}+-- | A file ending in an unmatched quote raises 'CsvUnclosedQuote'. prop_unclosed_quote_throws :: Property prop_unclosed_quote_throws = forAll (listOf1 arbitrary) $ \(cells :: [Cell]) ->     monadicIO $ do@@ -219,35 +174,24 @@             Left CsvUnclosedQuote -> assert True             _ -> assert False -{- | Files whose length sits exactly at a SIMD chunk boundary must parse-without an off-by-one.  We generate inputs at the key critical sizes-(64, 127, 128, 129, 192 bytes after the 'v\n' header) by padding a-single column with ASCII letters.--}+-- | Files whose length sits at a SIMD chunk boundary parse correctly. prop_boundary_sizes :: Property prop_boundary_sizes =     forAll (elements [64, 127, 128, 129, 192]) $ \targetBytes ->         let header = "v\n"             bodyLen = targetBytes - T.length header-            -- Build a "aaaa\nbbbb\n..." body of exactly `bodyLen` bytes-            -- ending in a newline.+            -- Build a body of exactly bodyLen bytes ending in a newline.             rows = take bodyLen $ cycle (map T.singleton "abcdefghij")             body = T.intercalate "\n" rows <> "\n"             csv = header <> T.take bodyLen body          in counterexample (show csv) $ monadicIO $ do                 df <- run $ withCsvFile "boundary" csv D.fastReadCsv                 let (rs, cs) = dataframeDimensions df-                -- Property: reader terminates and returns one column; the-                -- row count may vary depending on how the padding lines up,-                -- but the column metadata must be consistent.+                -- Property: one column, consistent metadata, any row count.                 assert (cs == 1)                 assert (rs >= 0) -{- | WS-E2 determinism: a chunk-parallel read returns exactly the same-DataFrame as the sequential read, for any chunk count.  The generated grid-mixes Text cells (quotes, embedded separators and newlines) with a-nullable numeric column so chunk-level type promotion is exercised too.--}+-- | A chunk-parallel read equals the sequential read. prop_parallel_equals_sequential :: ColumnNames -> [[Cell]] -> Property prop_parallel_equals_sequential (ColumnNames header) rows =     forAll (chooseInt (2, 7)) $ \nChunks ->