diff --git a/dataframe-fastcsv.cabal b/dataframe-fastcsv.cabal --- a/dataframe-fastcsv.cabal +++ b/dataframe-fastcsv.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: dataframe-fastcsv -version: 1.4.1.1 +version: 1.4.1.2 synopsis: SIMD-accelerated CSV reader for the dataframe library. description: A fast, SIMD-accelerated CSV/TSV reader using memory-mapped I/O @@ -56,9 +56,9 @@ build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.14, containers >= 0.6.7 && < 0.10, - dataframe-core >= 2.5 && < 2.6, + dataframe-core >= 2.6 && < 2.7, dataframe-csv >= 2.3 && < 2.4, - dataframe-operations >= 2.5 && < 2.6, + dataframe-operations >= 2.6 && < 2.7, dataframe-parsing >= 2.2 && < 2.3, dataframe-parsing >= 2.2 && < 2.3, mmap >= 0.5.8 && < 0.7, @@ -90,10 +90,10 @@ Properties.Csv build-depends: base >= 4 && < 5, containers >= 0.6.7 && < 0.10, - dataframe-core >= 2.5 && < 2.6, + dataframe-core >= 2.6 && < 2.7, dataframe-csv >= 2.3 && < 2.4, dataframe-fastcsv, - dataframe-operations >= 2.5 && < 2.6, + dataframe-operations >= 2.6 && < 2.7, dataframe-parsing >= 2.2 && < 2.3, directory >= 1.3.0.0 && < 2, HUnit >= 1.6 && < 1.8, diff --git a/src/DataFrame/IO/CSV/Fast.hs b/src/DataFrame/IO/CSV/Fast.hs --- a/src/DataFrame/IO/CSV/Fast.hs +++ b/src/DataFrame/IO/CSV/Fast.hs @@ -1,7 +1,4 @@ -{- | SIMD-accelerated CSV reader: a C scanner finds delimiter positions, -then each column is parsed directly from the mmap'd byte slices into -typed column builders. Chunk-parallel with @-threaded@ + @+RTS -N@. --} +-- | SIMD-accelerated, chunk-parallel CSV reader. module DataFrame.IO.CSV.Fast ( fastReadCsv, readCsvFast, @@ -50,87 +47,31 @@ separatorByte :: ReadOptions -> Word8 separatorByte = fromIntegral . ord . columnSeparator -{- | Read a CSV file with the default options. - -==== __Example__ -@ -ghci> D.fastReadCsv ".\/data\/taxi.csv" - -@ --} +-- | Read a CSV file with the default options. fastReadCsv :: FilePath -> IO DataFrame fastReadCsv = fastReadCsvWithOpts defaultReadOptions -{- | Alias for 'fastReadCsv'. - -==== __Example__ -@ -ghci> D.readCsvFast ".\/data\/taxi.csv" - -@ --} +-- | Alias for 'fastReadCsv'. readCsvFast :: FilePath -> IO DataFrame readCsvFast = fastReadCsv -{- | Read a TSV file with the default options. - -==== __Example__ -@ -ghci> D.fastReadTsv ".\/data\/taxi.tsv" - -@ --} +-- | Read a TSV file with the default options. fastReadTsv :: FilePath -> IO DataFrame fastReadTsv = fastReadTsvWithOpts defaultReadOptions -{- | Alias for 'fastReadTsv'. - -==== __Example__ -@ -ghci> D.readTsvFast ".\/data\/taxi.tsv" - -@ --} +-- | Alias for 'fastReadTsv'. readTsvFast :: FilePath -> IO DataFrame readTsvFast = fastReadTsv -{- | Like 'fastReadCsv' but takes a 'ReadOptions' record. Use this when -you need to tune ragged-row handling, unclosed-quote handling, the -whitespace-trimming knob, or the column selection; otherwise stick with -'fastReadCsv'. The separator comes from 'columnSeparator', so this is a -'CsvReader' the lazy scan can drive. - -==== __Example__ -@ -ghci> D.fastReadCsvWithOpts D.defaultReadOptions{D.readColumns = Just ["id", "name"]} ".\/data\/customers.csv" - -@ --} +-- | Like 'fastReadCsv' but takes a 'ReadOptions' record. fastReadCsvWithOpts :: CsvReader fastReadCsvWithOpts opts = readSeparated (separatorByte opts) opts -{- | TSV counterpart to 'fastReadCsvWithOpts'; pins 'columnSeparator' to tab. - -==== __Example__ -@ -ghci> D.fastReadTsvWithOpts D.defaultReadOptions{D.safeRead = D.MaybeRead} ".\/data\/taxi.tsv" - -@ --} +-- | TSV counterpart to 'fastReadCsvWithOpts'. fastReadTsvWithOpts :: ReadOptions -> FilePath -> IO DataFrame fastReadTsvWithOpts opts = fastReadCsvWithOpts opts{columnSeparator = '\t'} -{- | Read a CSV and coerce each column to the type declared in the -supplied 'Schema'. Schema columns bypass inference: parsed straight from -file bytes as the declared type, avoiding row-1 misclassification. - -==== __Example__ -@ -ghci> schema = D.makeSchema [("id", D.schemaType \@Int)] -ghci> D.fastReadCsvWithSchema schema ".\/data\/customers.csv" - -@ --} +-- | Read a CSV and coerce each column to the declared 'Schema' type. fastReadCsvWithSchema :: Schema -> FilePath -> IO DataFrame fastReadCsvWithSchema schema = readSeparated @@ -140,26 +81,11 @@ SpecifyTypes (M.toList (elements schema)) (typeSpec defaultReadOptions) } -{- | In-memory version of 'fastReadCsv'. - -==== __Example__ -@ -ghci> D.fastReadCsvFromBytes "id,name\\n1,Ada\\n" - -@ --} +-- | In-memory version of 'fastReadCsv'. fastReadCsvFromBytes :: BS.ByteString -> IO DataFrame fastReadCsvFromBytes = readSeparatedFromBytes comma defaultReadOptions -{- | In-memory version of 'fastReadCsvWithSchema'. - -==== __Example__ -@ -ghci> schema = D.makeSchema [("id", D.schemaType \@Int)] -ghci> D.fastReadCsvFromBytesWithSchema schema "id,name\\n1,Ada\\n" - -@ --} +-- | In-memory version of 'fastReadCsvWithSchema'. fastReadCsvFromBytesWithSchema :: Schema -> BS.ByteString -> IO DataFrame fastReadCsvFromBytesWithSchema schema = readSeparatedFromBytes @@ -169,32 +95,12 @@ SpecifyTypes (M.toList (elements schema)) (typeSpec defaultReadOptions) } -{- | Read a CSV and return only the named columns, in the order given. -Shorthand for 'readColumns'. The SIMD scan and delimiter -classification still run over the whole file, but unnamed columns are -never extracted, parsed or allocated. Naming a column the file does not -have raises a 'ColumnsNotFoundException'. - -==== __Example__ -@ -ghci> D.fastReadCsvProj ["id", "name"] ".\/data\/customers.csv" - -@ --} +-- | Read a CSV and return only the named columns, in the order given. fastReadCsvProj :: [Text] -> FilePath -> IO DataFrame fastReadCsvProj projection = readSeparated comma defaultReadOptions{readColumns = Just projection} -{- | Reads via a private ('WriteCopy') mmap so the buffer is never copied. -The @separator@ byte and 'columnSeparator' in @opts@ must agree; use -'fastReadCsvWithOpts' unless you need to pass a delimiter byte directly. - -==== __Example__ -@ -ghci> D.readSeparated 44 D.defaultReadOptions ".\/data\/taxi.csv" -- 44 = ',' - -@ --} +-- | Read a delimited file via a private mmap, without copying. readSeparated :: Word8 -> ReadOptions -> @@ -214,16 +120,7 @@ opts (VS.unsafeFromForeignPtr bufferPtr offset len) -{- | Identical to 'readSeparated' but reads from an already-loaded byte -buffer (e.g. a slice of a memory-mapped file). Zero-copy: the bytes are -only read, never modified or grown. - -==== __Example__ -@ -ghci> D.readSeparatedFromBytes 44 D.defaultReadOptions "id,name\\n1,Ada\\n" -- 44 = ',' - -@ --} +-- | Like 'readSeparated' but reads from an already-loaded byte buffer. readSeparatedFromBytes :: Word8 -> ReadOptions -> @@ -233,16 +130,7 @@ let (fp, off, len) = BSI.toForeignPtr bs readSeparatedCore Nothing separator opts (VS.unsafeFromForeignPtr fp off len) -{- | 'readSeparatedFromBytes' with a forced chunk count, bypassing the -capability/size gate. Intended for tests and benchmarks that need to pin -the parallel split; @1@ forces the sequential path. - -==== __Example__ -@ -ghci> D.readSeparatedFromBytesChunks 1 44 D.defaultReadOptions "id,name\\n1,Ada\\n" -- 44 = ',' - -@ --} +-- | 'readSeparatedFromBytes' with a forced chunk count. readSeparatedFromBytesChunks :: Int -> Word8 -> diff --git a/src/DataFrame/IO/CSV/Fast/Columns.hs b/src/DataFrame/IO/CSV/Fast/Columns.hs --- a/src/DataFrame/IO/CSV/Fast/Columns.hs +++ b/src/DataFrame/IO/CSV/Fast/Columns.hs @@ -3,10 +3,7 @@ {-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeApplications #-} -{- | Column-level dispatch for the fast CSV reader: resolve each column's -target type up front into a 'ColumnPlan' (schema hit, inference chain, or -legacy Text pipeline). Plans are chunk-runnable for the parallel reader. --} +-- | Resolves each column's target type up front into a 'ColumnPlan'. module DataFrame.IO.CSV.Fast.Columns ( ColumnEnv (..), ColumnPlan (..), @@ -103,21 +100,14 @@ | isJust (testEquality (typeRep @a) (typeRep @Day)) = Just TDate | otherwise = Nothing -{- | How to build one column, resolved once per read (before any chunk -fan-out). 'PlanLegacy' carries the \"re-apply 'parseWithTypes'\" flag. --} +-- | How to build one column, resolved once per read. data ColumnPlan - = {- | Inference: fallback chain; the 'Bool' asks for the all-null - pre-check ('NoAssumption' only). - -} + = -- | Inference: fallback chain, optionally with an all-null pre-check. PlanChain !SafeReadMode !NullSpec ![Step] !Bool | PlanSchema !SafeReadMode !NullSpec !TargetType | PlanLegacy !SafeReadMode !Bool -{- | Resolve a column's plan. For inferred columns the sample is always -classified against the global (full-range) env, so chunked and sequential -reads share one assumption. --} +-- | Resolve a column's plan against the global (full-range) env. planColumn :: ColumnEnv -> T.Text -> Int -> ColumnPlan planColumn env name col = case mode of EitherRead -> PlanLegacy mode (isJust schemaEntry) @@ -154,10 +144,7 @@ NoAssumption -> True _ -> False -{- | What a chain plan produced for one row range: every cell null (only -reported when the all-null pre-check is on), or the index of the first -chain step whose pass succeeded plus its column. --} +-- | What a chain plan produced for one row range. data ChunkOutcome = AllNullChunk | Resolved !Int !PassCol -- | Run a fallback chain over the env's row range. @@ -174,9 +161,7 @@ Left _ -> go (i + 1) rest go _ [] = error "fastcsv: type-fallback chain exhausted (StepText must succeed)" -{- | Run a schema-typed pass over the env's row range. @rowOff@ maps a -chunk-local failure index to the global data-row index. --} +-- | Run a schema-typed pass over the env's row range. runSchemaChunk :: ColumnEnv -> NullSpec -> @@ -209,10 +194,7 @@ finishMode MaybeRead = ensureOptional finishMode _ = id -{- | Build one column sequentially (single chunk over the full row range). -Returns the column plus a flag telling the caller to re-apply the legacy -'parseWithTypes' conversion afterwards (schema types the passes can't serve). --} +-- | Build one column sequentially over the full row range. buildColumn :: ColumnEnv -> T.Text -> Int -> IO (Column, Bool) buildColumn env name col = case planColumn env name col of PlanLegacy mode pwt -> (,pwt) <$> legacyColumn env mode col @@ -227,9 +209,7 @@ Resolved _ c' -> passColumn c' pure (finishMode mode (dictCompactColumn c), False) -{- | The original Text-materializing pipeline, kept for 'EitherRead' and -exotic schema types (cold paths). --} +-- | The original Text-materializing pipeline, kept for cold paths. legacyColumn :: ColumnEnv -> SafeReadMode -> Int -> IO Column legacyColumn env mode col = do let opts = ceOpts env diff --git a/src/DataFrame/IO/CSV/Fast/Core.hs b/src/DataFrame/IO/CSV/Fast/Core.hs --- a/src/DataFrame/IO/CSV/Fast/Core.hs +++ b/src/DataFrame/IO/CSV/Fast/Core.hs @@ -1,11 +1,6 @@ {-# LANGUAGE OverloadedStrings #-} -{- | The shared read pipeline behind every "DataFrame.IO.CSV.Fast" entry -point: scan for delimiters, classify rows, then build every column with -the typed extraction passes — chunk-parallel when the input is large and -capabilities allow ('autoChunkCount'), unless the caller pins a chunk -count. The result is fully forced before it is returned. --} +-- | The shared read pipeline behind every fast CSV entry point. module DataFrame.IO.CSV.Fast.Core ( readSeparatedCore, checkSeparatorAgrees, @@ -59,9 +54,7 @@ import DataFrame.Internal.DataFrame (DataFrame (..), forceDataFrame) import DataFrame.Operations.Typing (effectiveSafeRead, parseWithTypes) -{- | @readSeparatedCore chunkSpec separator opts buffer@; @chunkSpec@ -pins the chunk count ('Nothing' = capability\/size-gated automatic). --} +-- | Run the read pipeline; @chunkSpec@ pins the chunk count if given. readSeparatedCore :: Maybe Int -> Word8 -> @@ -74,8 +67,7 @@ let (file, _bomLen) = stripBom fileRaw contentLen = VS.length file nChunks <- maybe (autoChunkCount contentLen) pure chunkSpec - -- Parallel reads scan in parallel too; sequential reads (or builds - -- without SIMD) keep the original scan + row classification. + -- Parallel reads scan in parallel too. mPar <- if nChunks > 1 then getDelimiterIndicesPar nChunks separator contentLen file @@ -192,19 +184,14 @@ legacySchema ) df - -- The parallel merge already forced every column - -- payload, so only the column thunks themselves - -- need forcing; the deep walk would re-chase 6M+ - -- pointers per boxed column for nothing. + -- Parallel merge already forced the payloads. if nChunks > 1 && S.null legacySchema then do Vector.foldl' (\() c -> c `seq` ()) () cols `seq` return df else return $! forceDataFrame df' -{- | The scanner is handed its separator as a byte, so a 'columnSeparator' -saying anything else would be silently ignored. Reject that up front. --} +-- | Reject a 'columnSeparator' that disagrees with the separator byte. checkSeparatorAgrees :: Word8 -> ReadOptions -> IO () checkSeparatorAgrees separator opts = unless (separator == fromIntegral (ord asked)) $ @@ -217,9 +204,6 @@ where asked = columnSeparator opts -{- | An empty 'DataFrame' — returned when the input has no delimiters -(empty file or single line with no separator and no newline). Guards -against a divide-by-zero in the row-stride math when 'numCol == 0'. --} +-- | An empty 'DataFrame', for input with no delimiters. emptyDataFrame :: DataFrame emptyDataFrame = DataFrame Vector.empty M.empty (0, 0) M.empty diff --git a/src/DataFrame/IO/CSV/Fast/Index.hs b/src/DataFrame/IO/CSV/Fast/Index.hs --- a/src/DataFrame/IO/CSV/Fast/Index.hs +++ b/src/DataFrame/IO/CSV/Fast/Index.hs @@ -2,12 +2,7 @@ {-# LANGUAGE CApiFFI #-} {-# LANGUAGE ForeignFunctionInterface #-} -{- | Delimiter-index construction for the fast CSV reader. The index buffer -is sized from an exact byte count (not 8 bytes per input byte), the SIMD -scan runs over the 64-byte-aligned prefix of the unpadded mmap, and a -scalar Haskell tail finishes the final partial chunk — so the input is -never copied or grown. --} +-- | Delimiter-index construction for the fast CSV reader. module DataFrame.IO.CSV.Fast.Index ( CsvParseError (..), getDelimiterIndices, @@ -41,22 +36,11 @@ import DataFrame.IO.CSV (UnclosedQuotePolicy (..)) -{- | Exceptions raised by the fast CSV parser. Catchable with -'Control.Exception.catch' or 'Control.Exception.try'. --} +-- | Exceptions raised by the fast CSV parser. data CsvParseError - = {- | The input ends with a quoted field that was never closed. The - PCLMUL quote-parity chain in the SIMD scanner treats an unmatched - @\"@ as if the rest of the file were inside quotes, so we refuse - to return a silently corrupted 'DataFrame.Internal.DataFrame.DataFrame' - and raise instead. - -} + = -- | The input ends with a quoted field that was never closed. CsvUnclosedQuote - | {- | A row has a different number of fields from the header. Only - raised when 'DataFrame.IO.CSV.fastCsvOnRaggedRow' is set to - 'RaiseOnRagged'. Carries the 0-based row index, the expected field - count, and the actual field count. - -} + | -- | A row has a different number of fields from the header. CsvRaggedRow !Int !Int !Int deriving (Eq, Show) @@ -69,17 +53,12 @@ tab = 0x09 quote = 0x22 --- Status codes reported via the out-parameter of 'get_delimiter_indices'. --- Must stay in sync with @GDI_*@ in @cbits/process_csv.h@. +-- Status codes of 'get_delimiter_indices', mirroring @GDI_*@ in the C header. gdiOk, gdiUnclosedQuote :: CInt gdiOk = 0 gdiUnclosedQuote = 1 -{- | Return value that the C helper uses to tell Haskell \"SIMD isn't -available on this build / CPU, run the pure-Haskell state machine.\" -Mirrors @GDI_SIMD_UNAVAILABLE@ in @cbits/process_csv.h@. 'CSize' is -unsigned so this has the same bit pattern as 'maxBound'. --} +-- | Return value meaning SIMD is unavailable and Haskell must scan. simdUnavailable :: CSize simdUnavailable = maxBound @@ -97,11 +76,7 @@ byteStringView n v = BSI.fromForeignPtr (fst (VS.unsafeToForeignPtr0 v)) 0 n {-# INLINE byteStringView #-} -{- | Locate delimiter byte positions in the first @originalLen@ bytes of -@csvFile@. Treats an unclosed quoted field at EOF as a hard error; callers -that want to suppress the exception can use 'getDelimiterIndicesPolicy' -with 'BestEffort'. --} +-- | Locate delimiter byte positions in the first @originalLen@ bytes. {-# INLINE getDelimiterIndices #-} getDelimiterIndices :: Word8 -> @@ -117,16 +92,11 @@ VS.Vector Word8 -> IO (VS.Vector CSize) getDelimiterIndicesPolicy policy separator contentLen csvFile = do - -- Exact upper bound on the index count: every output index is an - -- unquoted separator or newline byte, so two memchr-speed counts - -- (plus one synthetic EOF slot) right-size the buffer (audit F4). + -- Every index is an unquoted separator or newline, plus one EOF slot. let contentBS = byteStringView contentLen csvFile capacity = BS.count separator contentBS + BS.count lf contentBS + 1 resultMV <- VSM.unsafeNew capacity - -- The C scanner reads whole 64-byte chunks; cap it at the largest - -- chunk boundary inside the content so no padding is needed and the - -- input never has to be grown or copied (audit F6). The loop in C - -- runs while i < len - 64, hence the + 64. + -- The C scanner reads whole 64-byte chunks, so stop at the last one. let simdLen = (contentLen `div` 64) * 64 (numFound, status) <- VS.unsafeWith csvFile $ \buffer -> alloca $ \statusPtr -> do @@ -140,8 +110,7 @@ statusPtr s <- peek statusPtr pure (n, s) - -- The status reports the quote parity at the end of the SIMD region, - -- which is exactly the starting state for the scalar tail. + -- The status gives the quote parity the scalar tail starts from. (found, inQuote) <- if numFound == simdUnavailable then scalarScan separator csvFile resultMV 0 0 contentLen False @@ -164,11 +133,7 @@ else pure found VS.unsafeFreeze (VSM.slice 0 finalLen resultMV) -{- | RFC 4180 quote-parity state machine over bytes @[from, to)@, appending -delimiter indices at @writeIdx@. Returns the next write index and whether -the scan ended inside a quoted region. Serves as both the SIMD tail and the -full fallback when SIMD is unavailable. --} +-- | RFC 4180 quote-parity state machine over a byte range. scalarScan :: Word8 -> VS.Vector Word8 -> @@ -196,10 +161,7 @@ go (idx + 1) (i + 1) False else go idx (i + 1) False -{- | Walk every data row and throw 'CsvRaggedRow' on the first one whose -field count differs from the header. Only called when the user opts -into 'RaiseOnRagged' via 'DataFrame.IO.CSV.fastCsvOnRaggedRow'. --} +-- | Throw 'CsvRaggedRow' on the first row whose field count differs. checkNoRaggedRows :: VS.Vector Int -> VS.Vector Int -> @@ -214,18 +176,7 @@ ) dataRows -{- | Classify each entry in the flat delimiter vector as either a row -terminator or a field terminator, and return the indices-into-the-vector -of every row terminator. A position counts as a row break if either - - * the byte at that position is @\\n@, or - * the position is beyond the original content length, which only - happens for the synthetic end-of-file delimiter written when a file - does not end in a newline. - -Field terminators (commas / tabs / the configured separator) are left -implicit: anything between consecutive row terminators is a field. --} +-- | Find the indices of every row terminator in the delimiter vector. {-# INLINE classifyRowEnds #-} classifyRowEnds :: VS.Vector Word8 -> Int -> VS.Vector CSize -> VS.Vector Int classifyRowEnds file contentLen = @@ -235,9 +186,7 @@ let p = fromIntegral pos :: Int in p >= contentLen || VS.unsafeIndex file p == lf -{- | Number of fields that row @r@ contains, derived directly from the -gap between consecutive entries in @rowEnds@. --} +-- | Number of fields that row @r@ contains. {-# INLINE fieldsInRow #-} fieldsInRow :: VS.Vector Int -> Int -> Int fieldsInRow rowEnds r = @@ -245,12 +194,7 @@ startIdx = if r == 0 then 0 else VS.unsafeIndex rowEnds (r - 1) + 1 in endIdx - startIdx + 1 -{- | Does row @r@ contain exactly one, empty field? That is the case for -a bare @\\n@ or a @\\r\\n@ \x2014 blank lines, which we skip by default to -match pandas / polars @skip_blank_lines@ semantics. We also treat a -row whose only byte is @\\r@ as blank, so CRLF-terminated blank lines -don't leak through when only the @\\n@ counts as a row break. --} +-- | Does row @r@ contain exactly one empty field (a blank line)? {-# INLINE isBlankRow #-} isBlankRow :: VS.Vector Word8 -> @@ -280,9 +224,7 @@ && VS.unsafeIndex file fieldStart == cr ) -{- | Select the row indices that contain actual data: skip @[0 .. skip - 1]@ -and drop any blank rows from the remainder. --} +-- | Select the row indices that contain actual data. {-# INLINE collectDataRows #-} collectDataRows :: VS.Vector Word8 -> diff --git a/src/DataFrame/IO/CSV/Fast/IndexPar.hs b/src/DataFrame/IO/CSV/Fast/IndexPar.hs --- a/src/DataFrame/IO/CSV/Fast/IndexPar.hs +++ b/src/DataFrame/IO/CSV/Fast/IndexPar.hs @@ -1,15 +1,7 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE CApiFFI #-} -{- | Parallel delimiter-index construction (Round-2 WS-E2): the SIMD region -is split into 64-byte-aligned ranges, one per worker. Quote parity at each -range start comes from a prefix-xor of per-range quote counts (exact: every -quote byte toggles the scanner's PCLMUL parity chain), a parallel counting -pass right-sizes the flat output arrays and gives every worker an exact -write offset, and a parallel emit pass writes delimiter positions plus -row-end ordinals directly (no per-delimiter file-byte re-read). The scalar -tail and the unclosed-quote policy are unchanged from the sequential path. --} +-- | Parallel delimiter-index construction over aligned SIMD ranges. module DataFrame.IO.CSV.Fast.IndexPar ( ParIndex (..), getDelimiterIndicesPar, @@ -52,18 +44,14 @@ simdUnavailable :: CSize simdUnavailable = maxBound -{- | Result of the parallel scan: the same @(delimiters, rowEnds)@ pair the -sequential 'DataFrame.IO.CSV.Fast.Index.classifyRowEnds' path produces. --} +-- | Result of the parallel scan: the @(delimiters, rowEnds)@ pair. data ParIndex = ParIndex { piDelims :: !(VS.Vector CSize) , piRowEnds :: !(VS.Vector Int) , piEndsInQuote :: !Bool } -{- | Scan with @nWorkers@ parallel range scans. Returns 'Nothing' when the -build\/CPU lacks SIMD support (callers fall back to the sequential scan). --} +-- | Scan with @nWorkers@ ranges, or 'Nothing' when SIMD is missing. getDelimiterIndicesPar :: Int -> Word8 -> Int -> VS.Vector Word8 -> IO (Maybe ParIndex) getDelimiterIndicesPar nWorkers separator contentLen file = @@ -156,9 +144,7 @@ rowEnds <- VS.unsafeFreeze (VSM.slice 0 nlFinal rowEndsMV) pure (ParIndex delims rowEnds inQuote) -{- | Classify the scalar tail's delimiters: append the ordinal of every -newline entry in @[from, to)@ of the delimiter vector. --} +-- | Append the ordinal of every newline entry in the scalar tail. appendTailRowEnds :: VS.Vector Word8 -> VSM.IOVector Int -> diff --git a/src/DataFrame/IO/CSV/Fast/Parallel.hs b/src/DataFrame/IO/CSV/Fast/Parallel.hs --- a/src/DataFrame/IO/CSV/Fast/Parallel.hs +++ b/src/DataFrame/IO/CSV/Fast/Parallel.hs @@ -1,18 +1,6 @@ {-# LANGUAGE TupleSections #-} -{- | Chunk-parallel extraction (Round-2 WS-E2): the data-row range is split -into contiguous, newline-aligned chunks (alignment is by construction — -chunks are row ranges over the quote-resolved delimiter index), each -worker runs every column's 'ColumnPlan' over its chunk with right-sized -builders, and the merger splices the results ('concatColumns' / -'mergeTextChunksPar'). Inference classifies its sample once, globally, -before fan-out; when chunks resolve different types the merger re-parses -only the narrower-typed chunks. - -Parallelism requires the executable to be linked with @-threaded@ and run -with @+RTS -N@; otherwise ('getNumCapabilities' == 1) the sequential path -runs. Inputs under 4 MB are always read sequentially. --} +-- | Chunk-parallel column extraction over newline-aligned row ranges. module DataFrame.IO.CSV.Fast.Parallel ( buildAllColumns, autoChunkCount, @@ -41,11 +29,7 @@ parallelThresholdBytes :: Int parallelThresholdBytes = 4 * 1024 * 1024 -{- | Chunk count for an automatic read: four chunks per capability (the -worker pool bounds concurrency at the capability count; the extra chunks -absorb straggler imbalance), or the sequential path (1) on a single -capability or a small input. --} +-- | Chunk count for an automatic read, 1 when sequential. autoChunkCount :: Int -> IO Int autoChunkCount contentLen = do caps <- getNumCapabilities @@ -68,11 +52,7 @@ mergeTextChunksPar width [tc | PassText tc <- ps] mergePassCols _ ps = pure $! concatColumns [c | PassFull c <- ps] -{- | Build every column, fanning the row range out over @nChunks@ chunks -on a capability-wide worker pool (sequentially when @nChunks <= 1@). Same -result as the sequential path by construction; see the chunk-determinism -tests and property. --} +-- | Build every column, fanning the row range out over @nChunks@. buildAllColumns :: Int -> ColumnEnv -> [(T.Text, Int)] -> IO [(Column, Bool)] buildAllColumns nChunks env wanted @@ -91,16 +71,9 @@ traceMarkerIO "fastcsv:join-done" let byCol = L.transpose perChunk mapM_ (mapM_ (\cc -> cc `seq` pure ())) byCol - -- Reset the major-GC trigger here, where the copyable live set is - -- small (chunk payloads are large objects). Otherwise the heap - -- doubling crosses its threshold mid-merge and a ~0.5s gen-1 - -- collection lands at the worst point of the read. + -- Reset the major-GC trigger while the copyable live set is small. performMajorGC - -- Merge in parallel too (chunk-level inside each column, pooled - -- across columns), forcing each spliced column here so no memcpy - -- is deferred to the final forceDataFrame walk. 'evaluate' resolves - -- the plan dispatch now, so a schema task is a closure over its own - -- column's data only — never 'env'. + -- Merge in parallel too, forcing each spliced column here. tasks <- mapM (\(wp, cps) -> evaluate (mergeTask width env ranges wp cps)) @@ -177,9 +150,7 @@ PlanChain _ nspec steps checkAllNull -> CCChain <$> runChainChunk cenv nspec steps checkAllNull col -{- | Splice one column's chunk results back together, resolving chunk-level -type promotion. Mirrors 'buildColumn' for the single-chunk case. --} +-- | Splice one column's chunk results back together. mergeColumn :: Int -> ColumnEnv -> @@ -195,12 +166,7 @@ c <- mergeChain width env nspec steps col [(r, oc) | (r, CCChain oc) <- parts] pure (finishMode mode (dictCompactColumn c), False) -{- | Merge chain-plan chunks. The candidate type starts at the widest chunk -resolution; any chunk that resolved narrower is re-parsed at the candidate -step, and a re-parse failure escalates the candidate further down the -chain (exactly the rows the sequential pass would have failed on). The -final 'StepText' never fails, so this terminates. --} +-- | Merge chain-plan chunks, re-parsing narrower chunks as needed. mergeChain :: Int -> ColumnEnv -> diff --git a/src/DataFrame/IO/CSV/Fast/Passes.hs b/src/DataFrame/IO/CSV/Fast/Passes.hs --- a/src/DataFrame/IO/CSV/Fast/Passes.hs +++ b/src/DataFrame/IO/CSV/Fast/Passes.hs @@ -3,11 +3,7 @@ {-# LANGUAGE MagicHash #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | The typed column passes (Round-2 WS-E1): each pass walks one column's -delimiter slices and appends parsed values straight into a WS-C builder — -no per-field 'Data.Text.Text' and no deferred re-parse. Text bytes are -memcpy'd into a shared buffer, only for Text-typed columns. --} +-- | Typed column passes that append parsed values straight into builders. module DataFrame.IO.CSV.Fast.Passes ( ColumnEnv (..), NullSpec (..), @@ -111,9 +107,7 @@ isNullText NullEmpty t = T.null t isNullText (NullMissing _ toks) t = isNullishOrMissing toks t -{- | What a pass produced: a finished 'Column', or a raw text chunk that the -merger splices at the byte level (no per-chunk 'T.Text' materialization). --} +-- | What a pass produced: a finished 'Column' or a raw text chunk. data PassCol = PassFull !Column | PassText !TextChunk -- | Finish one 'PassCol' on its own (the single-chunk \/ sequential case). @@ -128,9 +122,7 @@ rowAt env = VS.unsafeIndex (ceRows env) {-# INLINE rowAt #-} -{- | One pass appending unboxed values; @nullOnFail@ turns parse failures -into nulls (schema + 'MaybeRead') instead of aborting (inference demotion). --} +-- | One pass appending unboxed values, nulling parse failures if asked. {-# INLINE unboxedPass #-} unboxedPass :: (Columnable a, VU.Unbox a) => @@ -146,8 +138,7 @@ bs = fcBS ctx !(I# n) = ceNumRow env b <- stToIO (newNumBuilder nullVal (ceNumRow env)) - -- Loop counter is Int#: join points are not worker/wrapper'd, so a - -- boxed counter would allocate on every iteration. + -- Int# counter: a boxed one would allocate on every iteration. let go :: Int# -> IO PassResult go i | isTrue# (i >=# n) = @@ -176,11 +167,7 @@ boolPass env nspec col nullOnFail = unboxedPass env nspec col nullOnFail False parseBoolFieldSlice -{- | Text pass (never fails): raw bytes are memcpy'd into the shared -builder buffer; only quoted fields containing a doubled quote, and the -legacy trim knob, take the decode path. Returns the raw chunk; 'T.Text' -values materialize once, when the merger splices the chunks. --} +-- | Text pass: copies raw bytes into the shared builder buffer. textPass :: ColumnEnv -> NullSpec -> Int -> IO TextChunk textPass env nspec col = do let ctx = ceCtx env diff --git a/src/DataFrame/IO/CSV/Fast/Slice.hs b/src/DataFrame/IO/CSV/Fast/Slice.hs --- a/src/DataFrame/IO/CSV/Fast/Slice.hs +++ b/src/DataFrame/IO/CSV/Fast/Slice.hs @@ -1,10 +1,6 @@ {-# LANGUAGE BangPatterns #-} -{- | Field-slice resolution for the fast CSV reader: maps @(row, col)@ to a -byte range of the input (quote-stripped, CR-stripped, optionally trimmed) -without materializing any intermediate value. 'extractField' is the legacy -Text materialization used for headers, inference samples and cold paths. --} +-- | Maps @(row, col)@ to a byte range of the input without copying. module DataFrame.IO.CSV.Fast.Slice ( FieldCtx (..), FieldLayout (..), @@ -34,14 +30,9 @@ -- | How @(row, col)@ resolves to delimiter byte positions. data FieldLayout - = {- | Flat delimiter positions + row-end ordinals: the shape the scanner - emits. Handles ragged and blank rows; 8 bytes per delimiter. - -} + = -- | Flat delimiter positions plus row-end ordinals. LayoutFlat !(VS.Vector CSize) !(VS.Vector Int) - | {- | Uniform-stride rows: byte start per row plus 'Word16' deltas of - each field-end from its row start (@deltas ! (r*stride + col)@). - Built by 'compactRows'; ~3.5x smaller than the flat index. - -} + | -- | Uniform-stride rows with 'Word16' field-end deltas. LayoutRows !(VS.Vector Int) !(VS.Vector Word16) !Int -- | Everything needed to resolve a field slice, shared by all columns. @@ -100,12 +91,7 @@ where totalRows = VS.length rowEnds -{- | Resolve field @col@ of row @r@ and continue with @k start end quoted@. -Quoted fields yield the bytes between the outer quotes (embedded @\"\"@ is -NOT unescaped here); a trailing @\\r@ is stripped first; a column beyond the -row's field count (ragged short row) yields the empty slice. CPS so the hot -loops never box the bounds. --} +-- | Resolve field @col@ of row @r@ and continue with its byte range. {-# INLINE withFieldSlice #-} withFieldSlice :: FieldCtx -> Int -> Int -> (Int -> Int -> Bool -> r) -> r withFieldSlice ctx r col k = case fcLayout ctx of @@ -159,10 +145,7 @@ then k (fieldStart + 1) (fieldEnd - 1) True else k fieldStart fieldEnd False -{- | 'withFieldSlice' with the trim knob applied: when 'fcTrim' is set, -unquoted slices have ASCII whitespace stripped from both ends before the -continuation runs. This is the slice the typed parsers and null tests see. --} +-- | 'withFieldSlice' with the trim knob applied. {-# INLINE withParseSlice #-} withParseSlice :: FieldCtx -> Int -> Int -> (Int -> Int -> r) -> r withParseSlice ctx r col k = @@ -205,12 +188,7 @@ decodeSlice :: BS.ByteString -> Int -> Int -> Text decodeSlice bs s e = TextEncoding.decodeUtf8Lenient (sliceBS bs s e) -{- | Extract field @col@ of row @r@ as a 'Text' with the original fastcsv -semantics: quotes stripped and @\"\"@ unescaped for quoted fields, optional -'Text.strip' (full Unicode) for unquoted fields when 'fcTrim' is set. -Used for header names, inference samples and the legacy (EitherRead / -unsupported-schema) column paths. --} +-- | Extract field @col@ of row @r@ as a 'Text'. {-# INLINE extractField #-} extractField :: FieldCtx -> Int -> Int -> Text extractField ctx r col = @@ -221,10 +199,7 @@ (if fcTrim ctx then Text.strip else id) (decodeSlice (fcBS ctx) s e) -{- | RFC 4180 inner-quote unescape: @\"\"@ → @\"@. 'Text.replace' on a -two-char needle does a single linear pass and is allocation-free when -no doubled quote is present (short-circuits at the first miss). --} +-- | RFC 4180 inner-quote unescape. {-# INLINE unescapeDoubledQuotes #-} unescapeDoubledQuotes :: Text -> Text unescapeDoubledQuotes t @@ -234,9 +209,7 @@ doubledQuote = Text.pack "\"\"" singleQuote = Text.singleton '"' -{- | Strip a leading UTF-8 BOM (EF BB BF) if present. Returns the trimmed -vector and the number of bytes removed (0 or 3). --} +-- | Strip a leading UTF-8 BOM, returning the vector and bytes removed. {-# INLINE stripBom #-} stripBom :: VS.Vector Word8 -> (VS.Vector Word8, Int) stripBom v diff --git a/src/DataFrame/IO/CSV/Fast/TextMerge.hs b/src/DataFrame/IO/CSV/Fast/TextMerge.hs --- a/src/DataFrame/IO/CSV/Fast/TextMerge.hs +++ b/src/DataFrame/IO/CSV/Fast/TextMerge.hs @@ -1,11 +1,6 @@ {-# LANGUAGE BangPatterns #-} -{- | Parallel byte-level merge of per-chunk text columns: the byte copies -and offset rebase fan out over the chunk list, so a large text column does -not serialize the merge phase. Wraps the merged shared buffer + offsets as -'PackedText' (no 'Data.Text.Text' spine, no eager UTF-8 validation), exactly -as the pure 'mergeTextChunks' now produces. --} +-- | Parallel byte-level merge of per-chunk text columns. module DataFrame.IO.CSV.Fast.TextMerge ( mergeTextChunksPar, ) where @@ -32,10 +27,7 @@ mkPackedContiguous32, ) -{- | Merge text chunks with @width@-way parallel byte copies + offset -rebase, then wrap the shared buffer as 'PackedText'. Single chunks take -the pure (zero-copy) path. --} +-- | Merge text chunks with parallel byte copies into a 'PackedText'. mergeTextChunksPar :: Int -> [TextChunk] -> IO Column mergeTextChunksPar _ [c] = pure $! mergeTextChunks [c] mergeTextChunksPar width cs = do @@ -59,8 +51,7 @@ fill 1 | (c, bOff, rOff) <- zip3 cs byteOffs rowOffs ] - -- The final width is known before allocation: Int32 offsets whenever - -- the merged buffer stays under 2^31 bytes (the common case). + -- Use Int32 offsets whenever the merged buffer stays under 2^31 bytes. packed <- if totalBytes <= fromIntegral (maxBound :: Int32) then do diff --git a/tests/Operations/ChunkParallel.hs b/tests/Operations/ChunkParallel.hs --- a/tests/Operations/ChunkParallel.hs +++ b/tests/Operations/ChunkParallel.hs @@ -1,10 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -{- | Behavior pins for chunk-parallel extraction: a chunked read must be -indistinguishable from the sequential one — same column types (promotion -resolved across chunks), same values, and same errors. --} +-- | Chunked reads must match the sequential read. module Operations.ChunkParallel (tests) where import qualified Data.Map as M @@ -55,8 +52,7 @@ Nothing -> assertFailure (what <> ": column missing") Just col -> assertEqual what expected col --- A 200-row mixed dataset: nullable ints, doubles, quoted text with --- embedded separators/newlines/doubled quotes, bools, nullable dates. +-- A 200-row mixed dataset with quoted text, bools, nulls and dates. mixedBody :: T.Text mixedBody = "i,d,t,b,dt\n" <> T.concat (map row [0 .. 199 :: Int]) where @@ -84,8 +80,7 @@ TestLabel "par_matches_seq_mixed" . TestCase $ assertChunkInvariant "mixed nulls+quoted" defaultReadOptions mixedBody --- A Double cell far past the sample, in the last chunk: every chunk that --- resolved Int must be re-parsed as Double by the merger. +-- A Double cell in the last chunk forces Int chunks to be re-parsed. testPromotionAcrossChunks :: Test testPromotionAcrossChunks = TestLabel "par_promotion_int_double" . TestCase $ do let body = @@ -100,8 +95,7 @@ df (DI.fromList @Double ([fromIntegral k | k <- [0 .. 189 :: Int]] <> [190.5])) --- A Bool cell in the last chunk of an Int column: candidate escalates past --- Double (the bool cell fails it) all the way to Text. +-- A Bool cell in the last chunk of an Int column escalates to Text. testDemotionToTextAcrossChunks :: Test testDemotionToTextAcrossChunks = TestLabel "par_demotion_text" . TestCase $ do let body = @@ -129,8 +123,7 @@ df (DI.fromList @(Maybe T.Text) (replicate 200 Nothing)) --- All-null sample (NoAssumption), data only in the last chunk: all-null --- chunks must be re-parsed at the type the data chunk resolved. +-- Data only in the last chunk: all-null chunks must be re-parsed. testLateDataAfterAllNullSample :: Test testLateDataAfterAllNullSample = TestLabel "par_late_bool" . TestCase $ do let cell k @@ -152,8 +145,7 @@ ] ) --- Schema + NoSafeRead failure in a late chunk: same error, same row index --- as the sequential read. +-- A late schema failure reports the same row as the sequential read. testSchemaErrorSameRowAsSequential :: Test testSchemaErrorSameRowAsSequential = TestLabel "par_schema_error" . TestCase $ do let body = @@ -238,8 +230,7 @@ Left D.CsvUnclosedQuote -> pure () other -> assertFailure ("expected CsvUnclosedQuote, got " <> show other) --- The parallel byte-level merge wraps the merged buffer as 'PackedText' --- (no Text spine), an RSS win; values stay identical to the sequential read. +-- The parallel merge wraps the buffer as 'PackedText', values unchanged. testParallelTextFreezesPacked :: Test testParallelTextFreezesPacked = TestLabel "par_text_freezes_packed" . TestCase $ do let body = @@ -254,7 +245,7 @@ Just col -> do assertBool "parallel text column is PackedText" (DI.isPackedText col) assertEqual - "parallel text values byte-identical" + "parallel text values equal" (DI.fromList @T.Text ["t" <> T.pack (show k) | k <- [0 .. 199 :: Int]]) col diff --git a/tests/Operations/Projection.hs b/tests/Operations/Projection.hs --- a/tests/Operations/Projection.hs +++ b/tests/Operations/Projection.hs @@ -1,11 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | 'readColumns' semantics for the SIMD reader, pinned against the -pure reader. Every case asserts both readers agree: the two disagreed -over 'ProvideNames' for a long time, and a projection only one of them -honours is worse than no projection at all. --} +-- | 'readColumns' semantics, pinned against the pure reader. module Operations.Projection (tests) where import qualified Data.Map as M @@ -33,9 +29,7 @@ removeFile path pure r -{- | Read @contents@ through both readers with @opts@, check they agree, -then hand the frame to the assertion. --} +-- | Read through both readers, check they agree, then assert. bothReaders :: String -> CSV.ReadOptions -> T.Text -> (DataFrame -> Assertion) -> Test bothReaders label opts contents check = TestLabel label $ diff --git a/tests/Operations/ReadCsv.hs b/tests/Operations/ReadCsv.hs --- a/tests/Operations/ReadCsv.hs +++ b/tests/Operations/ReadCsv.hs @@ -3,7 +3,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} --- Test fixtures inspired by csv-spectrum (https://github.com/max-mapper/csv-spectrum) +-- Test fixtures inspired by csv-spectrum. module Operations.ReadCsv where @@ -55,10 +55,6 @@ tempDir :: FilePath tempDir = "./tests/data/unstable_csv/" --------------------------------------------------------------------------------- --- Pretty-printer --------------------------------------------------------------------------------- - prettyPrintCsv :: FilePath -> DataFrame -> IO () prettyPrintCsv = prettyPrintSeparated ',' @@ -78,9 +74,7 @@ (TIO.hPutStrLn handle . T.intercalate (T.singleton sep) . getRowEscaped sep df) [0 .. rows - 1] --- RFC 4180-compliant CSV field escaping: wrap in quotes if the field --- contains the separator, a line break, or a quote; double any embedded --- quotes so the parser can reverse the encoding. +-- RFC 4180 field escaping: quote when needed, double embedded quotes. escapeField :: Char -> T.Text -> T.Text escapeField sep field | needsQuoting = T.concat ["\"", T.replace "\"" "\"\"" field, "\""] @@ -236,9 +230,7 @@ (DI.fromList @T.Text ["Alice", "Bob", "Carol"]) col --- After the RFC 4180 alignment, whitespace inside unquoted fields is --- preserved verbatim. Users who want leading/trailing whitespace --- stripped can opt in via the fastCsvTrimUnquoted knob (Step 7). +-- Whitespace inside unquoted fields is preserved verbatim. testWhitespaceFields :: Test testWhitespaceFields = TestLabel "malformed_whitespace_fields" $ TestCase $ do df <- D.readCsvFast (fixtureDir <> "whitespace_fields.csv") @@ -261,9 +253,7 @@ (DI.fromList @T.Text [" New York", " Los Angeles"]) col --- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full). Each line is --- classified individually: the short row survives with a null-padded 'c'; --- the full row keeps its value. +-- A short row survives with a null-padded 'c'; the full row keeps its value. testMissingFields :: Test testMissingFields = TestLabel "malformed_missing_fields" $ TestCase $ do df <- D.readCsvFast (fixtureDir <> "missing_fields.csv") @@ -286,9 +276,7 @@ (DI.fromList @(Maybe T.Text) [Nothing, Just "Z"]) col --- File: a,b,c header; row "1,2,3,EXTRA" (over-long). Under the default --- ragged-row policy we read columns 0..numCol-1 and silently drop the --- EXTRA field. +-- An over-long row drops the extra field under the default ragged policy. testExtraFieldsTruncate :: Test testExtraFieldsTruncate = TestLabel "malformed_extra_fields_truncate" $ TestCase $ do @@ -327,18 +315,14 @@ (DI.fromList @T.Text ["London"]) col --- Regression: a zero-byte CSV used to divide by zero in the row-stride math --- (`VS.length indices `div` numCol` with numCol == 0). Now it returns an --- empty DataFrame cleanly. +-- Regression: a zero-byte CSV used to divide by zero; now it is empty. testEmptyFile :: Test testEmptyFile = TestLabel "malformed_empty_file" $ TestCase $ do df <- D.readCsvFast (fixtureDir <> "empty_file.csv") assertEqual "empty_file.csv: 0 rows" 0 (fst (dataframeDimensions df)) assertEqual "empty_file.csv: 0 columns" 0 (snd (dataframeDimensions df)) --- RFC 4180 quote unescaping: `""` decodes to a single `"`. The escaped --- quote fixture contains the two canonical shapes: an embedded doubled --- quote and a plain quoted number. +-- RFC 4180 unescaping: a doubled quote decodes to a single quote. testRfc4180EscapedQuote :: Test testRfc4180EscapedQuote = TestLabel "rfc4180_escaped_quote" $ TestCase $ do @@ -355,8 +339,7 @@ (DI.fromList @T.Text ["ha \"ha\" ha", "4"]) col --- Whitespace inside double quotes is data, not padding, so a quoted --- " padded " survives the parser unchanged. +-- Whitespace inside double quotes is data and survives unchanged. testQuotedWhitespacePreserved :: Test testQuotedWhitespacePreserved = TestLabel "quoted_whitespace_preserved" $ TestCase $ do @@ -372,9 +355,7 @@ (DI.fromList @T.Text [" padded "]) col --- With `fastCsvOnRaggedRow = RaiseOnRagged`, the short row in --- missing_fields.csv must be rejected with a CsvRaggedRow carrying the --- row index and the expected/actual field counts. +-- RaiseOnRagged rejects the short row with a CsvRaggedRow. testRaiseOnRagged :: Test testRaiseOnRagged = TestLabel "raise_on_ragged" $ TestCase $ do let opts = defaultReadOptions{fastCsvOnRaggedRow = RaiseOnRagged} @@ -392,8 +373,7 @@ assertFailure "fastReadCsvWithOpts RaiseOnRagged should have thrown" --- With `fastCsvTrimUnquoted = True` we recover the legacy behaviour of --- stripping leading/trailing whitespace from unquoted fields. +-- fastCsvTrimUnquoted = True strips whitespace from unquoted fields. testTrimUnquotedOpt :: Test testTrimUnquotedOpt = TestLabel "trim_unquoted_opt" $ TestCase $ do let opts = defaultReadOptions{fastCsvTrimUnquoted = True} @@ -406,9 +386,7 @@ (DI.fromList @T.Text ["Alice", "Bob"]) col --- A declared 'Schema' overrides type inference: the 'a' column is --- forced to 'Int' and 'b' to 'Double', regardless of what the first few --- rows look like. +-- A declared 'Schema' overrides type inference. testSchemaPushdown :: Test testSchemaPushdown = TestLabel "schema_pushdown" $ TestCase $ do let path = fixtureDir <> "schema_pushdown.csv" @@ -428,8 +406,7 @@ Nothing -> assertFailure "schema_pushdown: 'b' missing" Just col -> assertEqual "b is Double" (DI.fromList @Double [1.5, 2.5]) col --- `fastReadCsvProj` returns only the requested columns, in the order --- requested; unreferenced columns never appear in the result. +-- fastReadCsvProj returns only the requested columns, in order. testProjection :: Test testProjection = TestLabel "projection" $ TestCase $ do let path = fixtureDir <> "projection.csv" @@ -443,9 +420,7 @@ map fst . L.sortBy (compare `on` snd) . M.toList $ columnIndices df assertEqual "projection preserves order" ["c", "a"] names --- An unmatched `"` used to silently swallow the rest of the file (the SIMD --- quote-parity chain never resets). We now raise 'CsvUnclosedQuote' rather --- than returning a corrupted DataFrame. +-- An unmatched quote now raises 'CsvUnclosedQuote' instead of corrupting. testUnclosedQuote :: Test testUnclosedQuote = TestLabel "malformed_unclosed_quote" $ TestCase $ do let path = fixtureDir <> "unclosed_quote.csv" @@ -461,9 +436,7 @@ assertFailure "readCsvFast should have thrown CsvUnclosedQuote on input with a stray quote" --- UTF-8 BOM (EF BB BF) must be stripped before the first header is parsed. --- Without the strip, `getColumn "name"` fails because the column is keyed --- under the three-byte-prefixed "\xEF\xBB\xBFname". +-- The UTF-8 BOM must be stripped before the first header is parsed. testUtf8Bom :: Test testUtf8Bom = TestLabel "malformed_utf8_bom" $ TestCase $ do df <- D.readCsvFast (fixtureDir <> "utf8_bom.csv") diff --git a/tests/Operations/TypedExtraction.hs b/tests/Operations/TypedExtraction.hs --- a/tests/Operations/TypedExtraction.hs +++ b/tests/Operations/TypedExtraction.hs @@ -2,10 +2,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Behavior pins for the typed-extraction fastcsv path: schema bypass, -parser semantics, byte-level missing tests, and the chunk-boundary scalar -tail that replaced the full-file copy. --} +-- | Behavior pins for the typed-extraction fastcsv path. module Operations.TypedExtraction (tests) where import qualified Data.Map as M @@ -13,9 +10,7 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as TE --- UTF-8 byte-mode IO: the plain Data.Text.IO writer honours the --- handle's text mode, which on Windows turns \n into \r\n and --- corrupts inputs meant for byte-level parsers. +-- UTF-8 byte-mode IO: the plain writer turns \n into \r\n on Windows. import qualified Data.Text.IO.Utf8 as TIO import Control.Exception (ErrorCall, evaluate, try) @@ -50,8 +45,7 @@ Nothing -> assertFailure (what <> ": column missing") Just col -> assertEqual what expected col --- Divergence #1 (pinned new behavior): Int overflow rejects instead of --- wrapping, so an overflowing cell demotes the column to Double. +-- Int overflow rejects instead of wrapping, demoting the column to Double. testIntOverflowDemotesToDouble :: Test testIntOverflowDemotesToDouble = TestLabel "typed_int_overflow_demotes" $ TestCase $ @@ -63,9 +57,7 @@ df (DI.fromList @Double [1.0, 9.223372036854776e18]) --- Divergence #2: doubles parse with strip-equivalent semantics, so a padded --- double past the 100-row inference sample no longer demotes the column to --- Text. (Inside the sample the classifier still rejects padding.) +-- A padded double past the inference sample no longer demotes to Text. testPaddedDoubleStaysDouble :: Test testPaddedDoubleStaysDouble = TestLabel "typed_padded_double" $ TestCase $ do @@ -79,8 +71,7 @@ df (DI.fromList @Double (clean <> [3.5])) --- Schema bypasses inference: a numeric-looking column declared Text in the --- schema must come back as Text (old code silently kept the inferred Int). +-- A numeric-looking column declared Text in the schema comes back as Text. testSchemaTextBypassesInference :: Test testSchemaTextBypassesInference = TestLabel "typed_schema_text" $ TestCase $ @@ -93,8 +84,7 @@ df (DI.fromList @T.Text ["1", "2"]) --- Schema + NoSafeRead: a cell that cannot parse as the declared type --- raises during the read (strict read), instead of hiding an error thunk. +-- A cell failing the declared type raises during the read. testSchemaStrictFailureThrows :: Test testSchemaStrictFailureThrows = TestLabel "typed_schema_strict_throws" $ TestCase $ @@ -133,8 +123,7 @@ df (DI.fromList @(Maybe Int) [Just 1, Nothing, Just 2]) --- Preserved: NoSafeRead treats only empty cells as null, so "NA" still --- demotes a numeric column to Text. +-- NoSafeRead treats only empty cells as null, so "NA" demotes to Text. testNoSafeReadNADemotesToText :: Test testNoSafeReadNADemotesToText = TestLabel "typed_nosaferead_na" $ TestCase $ @@ -177,8 +166,7 @@ df (DI.fromList @(Maybe Int) [Just 1, Nothing, Just 2]) --- Preserved: quoted numerics parse as numbers (quote stripping happens --- before the typed parse). +-- Quoted numerics parse as numbers; quotes are stripped first. testQuotedNumbersParse :: Test testQuotedNumbersParse = TestLabel "typed_quoted_numbers" $ TestCase $ @@ -217,9 +205,7 @@ ] ) --- The scalar tail (last <64 bytes after the SIMD chunks) must honour the --- quote state from the SIMD region: a quoted field opening before the --- boundary and closing in the tail may contain separators and newlines. +-- A quote opening before the SIMD boundary and closing in the tail is kept. testQuoteSpansChunkBoundary :: Test testQuoteSpansChunkBoundary = TestLabel "typed_quote_spans_boundary" $ TestCase $ do @@ -247,8 +233,7 @@ other -> assertFailure ("expected CsvUnclosedQuote, got " <> show other) --- S2 pin: inference must land on exactly the same columns (types and --- values) as an explicit schema declaring the true types. +-- Inference must land on the same columns as an explicit true schema. testInferredMatchesSchema :: Test testInferredMatchesSchema = TestLabel "typed_inferred_matches_schema" $ TestCase $ do @@ -280,8 +265,7 @@ explicit <- D.fastReadCsvWithSchema schema path assertEqual "inferred == explicit schema" explicit inferred --- Ingest pin: an inferred string column freezes to a shared-buffer --- 'PackedText' (no per-row boxed Text header), values byte-identical. +-- An inferred string column freezes to a 'PackedText', values identical. testTextFreezesPacked :: Test testTextFreezesPacked = TestLabel "typed_text_freezes_packed" $ TestCase $ @@ -291,7 +275,7 @@ Nothing -> assertFailure "missing text column" Just col -> do assertBool "text column is PackedText" (DI.isPackedText col) - assertEqual "values byte-identical" (DI.fromList @T.Text ["x", "y"]) col + assertEqual "values equal" (DI.fromList @T.Text ["x", "y"]) col -- The in-memory entry point must agree with the file-based one. testFromBytesMatchesFile :: Test diff --git a/tests/Properties/Csv.hs b/tests/Properties/Csv.hs --- a/tests/Properties/Csv.hs +++ b/tests/Properties/Csv.hs @@ -3,16 +3,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | QuickCheck properties for the SIMD CSV parser. - -Each property builds a fresh input on disk under @/tmp@, exercises -'DataFrame.IO.CSV.Fast.fastReadCsv' (or a variant), and asserts a -property-level invariant. Temp files are cleaned up by the caller. - -The properties deliberately avoid depending on type-inference: every -test cell starts with a letter so 'parseFromExamples' always lands on -@Text@, which makes roundtrip comparison unambiguous. --} +-- | QuickCheck properties for the SIMD CSV parser. module Properties.Csv (tests) where import Control.Exception (try) @@ -41,10 +32,7 @@ import Test.QuickCheck.Monadic (PropertyM, assert, monadicIO, run) import Type.Reflection (typeRep) -{- | A generator for a single CSV cell. Each cell starts with an ASCII -letter (so type inference always picks 'Text') and may contain a small -menagerie of special characters the parser must escape correctly. --} +-- | A generator for a single CSV cell. newtype Cell = Cell {unCell :: T.Text} deriving (Eq, Show) @@ -57,11 +45,7 @@ | T.null t || T.length t == 1 = [] | otherwise = [Cell (T.take (T.length t - 1) t)] -{- | Like 'Cell' but guaranteed to contain neither @\\n@ nor @\\r@. Used -by properties that diff across line-ending dialects, where embedded -line breaks become part of the data under CRLF transform and break -the otherwise-clean invariant. --} +-- | Like 'Cell' but without line breaks. newtype SingleLineCell = SingleLineCell {unSingleLineCell :: T.Text} deriving (Eq, Show) @@ -86,10 +70,7 @@ | length ns <= 1 = [] | otherwise = [ColumnNames (init ns)] -{- | RFC 4180 cell encoding: wrap a field in @"@ and double embedded -quotes when the cell contains the separator, a line break, or a -quote. --} +-- | RFC 4180 cell encoding. encodeCell :: Char -> T.Text -> T.Text encodeCell sep t | T.any needsQuote t = T.concat ["\"", T.replace "\"" "\"\"" t, "\""] @@ -97,22 +78,14 @@ where needsQuote c = c == sep || c == '\n' || c == '\r' || c == '"' -{- | Encode a grid @[[cell]]@ (rows of cells) as an RFC 4180 CSV. The -first row is written as the header, followed by one line per data -row. Always uses @\\n@ as the line terminator; callers that need -CRLF can transform the output themselves. --} +-- | Encode a grid of cells as an RFC 4180 CSV. encodeCsv :: Char -> [T.Text] -> [[T.Text]] -> T.Text encodeCsv sep header rows = T.unlines (encodeRow header : map encodeRow rows) where encodeRow = T.intercalate (T.singleton sep) . map (encodeCell sep) -{- | Pull the raw 'Text' values of the named column out of a DataFrame. -Fails the property if the column is missing or is not a 'Text' -column (which would signal a type-inference accident — the cell -generator is designed to avoid that). --} +-- | Pull the raw 'Text' values of the named column out of a DataFrame. columnAsText :: T.Text -> DataFrame -> Maybe [T.Text] columnAsText name df = do idx <- M.lookup name (columnIndices df) @@ -126,9 +99,7 @@ PackedText{} -> Nothing MergedColumn{} -> Nothing -{- | Run a property in @IO@ against a generated CSV text, cleaning up -the temp file afterwards no matter what. --} +-- | Run a property in @IO@ against a generated CSV text. withCsvFile :: String -> T.Text -> (FilePath -> IO a) -> IO a withCsvFile label body action = do let path = "./tests/data/unstable_csv/fastcsv_prop_" <> label <> ".csv" @@ -137,12 +108,9 @@ removeFile path pure r --------------------------------------------------------------------------------- -- Properties -{- | Any CSV parses the same whether or not a UTF-8 BOM (EF BB BF) is -prepended. Excel / PowerShell CSV exports all come with a BOM. --} +-- | A UTF-8 BOM does not change how a CSV parses. prop_bom_invariant :: ColumnNames -> [[Cell]] -> Property prop_bom_invariant (ColumnNames header) rows = monadicIO $ do let cellRows = map (map unCell) rows @@ -153,12 +121,7 @@ assert (plain == withBom) assert (dataframeDimensions plain == dataframeDimensions withBom) -{- | A CSV with no embedded newlines inside quoted fields parses the -same whether record separators are @\\n@ or @\\r\\n@. When quoted -cells themselves contain @\\n@, a CRLF-encoded file contains a literal -@\\r\\n@ inside the quotes (that's data, not line ending), so the -invariant correctly no longer holds; we filter such inputs out. --} +-- | A CSV parses the same with @\n@ or @\r\n@ record separators. prop_crlf_invariant :: ColumnNames -> [[SingleLineCell]] -> Property prop_crlf_invariant (ColumnNames header) rows = monadicIO $ do let cellRows = map (map unSingleLineCell) rows @@ -168,11 +131,7 @@ dfCrlf <- run $ withCsvFile "crlf_crlf" crlfCsv D.fastReadCsv assert (dfLf == dfCrlf) -{- | A DataFrame produced by the RFC 4180 encoder round-trips through -the fast reader: column names survive, row count survives, and every -cell comes back bit-for-bit. Restricted to ASCII-letter-prefixed -cells so that type inference always stays on 'Text'. --} +-- | A DataFrame encoded as RFC 4180 round-trips through the fast reader. prop_roundtrip_ascii :: Property prop_roundtrip_ascii = forAll (chooseInt (0, 12)) $ \nRows -> forAll (chooseInt (1, 4)) $ \nCols -> @@ -198,11 +157,7 @@ ) (zip header expected) -{- | A file that ends with an unmatched @"@ must raise 'CsvUnclosedQuote' -under the default policy. The generator deliberately builds a valid -prefix so the property is checking the error path, not a random -parse failure. --} +-- | A file ending in an unmatched quote raises 'CsvUnclosedQuote'. prop_unclosed_quote_throws :: Property prop_unclosed_quote_throws = forAll (listOf1 arbitrary) $ \(cells :: [Cell]) -> monadicIO $ do @@ -219,35 +174,24 @@ Left CsvUnclosedQuote -> assert True _ -> assert False -{- | Files whose length sits exactly at a SIMD chunk boundary must parse -without an off-by-one. We generate inputs at the key critical sizes -(64, 127, 128, 129, 192 bytes after the 'v\n' header) by padding a -single column with ASCII letters. --} +-- | Files whose length sits at a SIMD chunk boundary parse correctly. prop_boundary_sizes :: Property prop_boundary_sizes = forAll (elements [64, 127, 128, 129, 192]) $ \targetBytes -> let header = "v\n" bodyLen = targetBytes - T.length header - -- Build a "aaaa\nbbbb\n..." body of exactly `bodyLen` bytes - -- ending in a newline. + -- Build a body of exactly bodyLen bytes ending in a newline. rows = take bodyLen $ cycle (map T.singleton "abcdefghij") body = T.intercalate "\n" rows <> "\n" csv = header <> T.take bodyLen body in counterexample (show csv) $ monadicIO $ do df <- run $ withCsvFile "boundary" csv D.fastReadCsv let (rs, cs) = dataframeDimensions df - -- Property: reader terminates and returns one column; the - -- row count may vary depending on how the padding lines up, - -- but the column metadata must be consistent. + -- Property: one column, consistent metadata, any row count. assert (cs == 1) assert (rs >= 0) -{- | WS-E2 determinism: a chunk-parallel read returns exactly the same -DataFrame as the sequential read, for any chunk count. The generated grid -mixes Text cells (quotes, embedded separators and newlines) with a -nullable numeric column so chunk-level type promotion is exercised too. --} +-- | A chunk-parallel read equals the sequential read. prop_parallel_equals_sequential :: ColumnNames -> [[Cell]] -> Property prop_parallel_equals_sequential (ColumnNames header) rows = forAll (chooseInt (2, 7)) $ \nChunks ->