diff --git a/dataframe-fastcsv.cabal b/dataframe-fastcsv.cabal --- a/dataframe-fastcsv.cabal +++ b/dataframe-fastcsv.cabal @@ -1,6 +1,6 @@ -cabal-version: 2.4 +cabal-version: 3.4 name: dataframe-fastcsv -version: 1.1.1.0 +version: 1.3.0.0 synopsis: SIMD-accelerated CSV reader for the dataframe library. description: A fast, SIMD-accelerated CSV/TSV reader using memory-mapped I/O @@ -55,16 +55,17 @@ DataFrame.IO.CSV.Fast.TextMerge DataFrame.IO.CSV.Fast.Workers build-depends: base >= 4 && < 5, - bytestring >= 0.11 && < 0.13, - containers >= 0.6.7 && < 0.9, - dataframe-core ^>= 1.1, - dataframe-csv ^>= 1.0.2, - dataframe-operations ^>= 1.1.1, - dataframe-parsing ^>= 1.0.2, - mmap >= 0.5.8 && < 0.6, - text >= 2.0 && < 3, + bytestring >= 0.11 && < 0.14, + containers >= 0.6.7 && < 0.10, + dataframe-core >= 2.1 && < 2.2, + dataframe-csv >= 2.1 && < 2.2, + dataframe-operations >= 2.1 && < 2.2, + dataframe-parsing >= 2.1 && < 2.2, + dataframe-parsing >= 2.1 && < 2.2, + mmap >= 0.5.8 && < 0.7, + text >= 2.1 && < 3, time >= 1.12 && < 2, - vector ^>= 0.13 + vector >= 0.13 && < 0.15 hs-source-dirs: src c-sources: cbits/process_csv.c include-dirs: cbits @@ -88,19 +89,18 @@ Operations.TypedExtraction Properties.Csv build-depends: base >= 4 && < 5, - containers >= 0.6.7 && < 0.9, - dataframe >= 1 && < 3, - dataframe-core ^>= 1.1, - dataframe-csv ^>= 1.0.2, + containers >= 0.6.7 && < 0.10, + dataframe-core >= 2.1 && < 2.2, + dataframe-csv >= 2.1 && < 2.2, dataframe-fastcsv, - dataframe-operations ^>= 1.1.1, - dataframe-parsing ^>= 1.0.2, + dataframe-operations >= 2.1 && < 2.2, + dataframe-parsing >= 2.1 && < 2.2, directory >= 1.3.0.0 && < 2, - HUnit ^>= 1.6, + HUnit >= 1.6 && < 1.8, QuickCheck >= 2 && < 3, - text >= 2.0 && < 3, + text >= 2.1 && < 3, time >= 1.12 && < 2, - vector ^>= 0.13 + vector >= 0.13 && < 0.15 hs-source-dirs: tests default-language: Haskell2010 if flag(asan) diff --git a/src/DataFrame/IO/CSV/Fast.hs b/src/DataFrame/IO/CSV/Fast.hs --- a/src/DataFrame/IO/CSV/Fast.hs +++ b/src/DataFrame/IO/CSV/Fast.hs @@ -1,13 +1,6 @@ {- | SIMD-accelerated CSV reader: a C scanner finds delimiter positions, then each column is parsed directly from the mmap'd byte slices into -unboxed column builders (Round-2 typed extraction). Reads are strict: the -returned 'DataFrame' is fully built, with no deferred parse work. - -Extraction is chunk-parallel: inputs of 4 MB and up are split into one -row chunk per capability and parsed by 'Control.Concurrent.forkIO' -workers. The library never chooses RTS settings — to actually get -parallel reads, link the executable with @-threaded@ and run with -@+RTS -N@ (otherwise the reader falls back to the sequential path). +typed column builders. Chunk-parallel with @-threaded@ + @+RTS -N@. -} module DataFrame.IO.CSV.Fast ( fastReadCsv, @@ -50,7 +43,7 @@ tab, ) import DataFrame.Internal.DataFrame (DataFrame (..)) -import DataFrame.Internal.Schema (Schema (..)) +import DataFrame.Schema (Schema (..)) readSeparatedDefault :: Word8 -> FilePath -> IO DataFrame readSeparatedDefault separator = @@ -80,10 +73,8 @@ fastReadTsvWithOpts = readSeparated tab {- | Read a CSV and coerce each column to the type declared in the -supplied 'Schema'. Schema columns bypass inference entirely: they are -parsed straight from the file bytes as the declared type, which is both -faster than inference and guards against the row-1 \"looks like Int\" -misclassification trap. +supplied 'Schema'. Schema columns bypass inference: parsed straight from +file bytes as the declared type, avoiding row-1 misclassification. -} fastReadCsvWithSchema :: Schema -> FilePath -> IO DataFrame fastReadCsvWithSchema schema = @@ -117,9 +108,8 @@ pure (projectColumns projection df) {- | Filter a DataFrame's columns down to (and in the order of) the -supplied names. Missing names are silently dropped rather than -raised; callers that want strict semantics can check the resulting -'columnIndices' map. +supplied names. Missing names are silently dropped; check the result's +'columnIndices' map for strict semantics. -} projectColumns :: [Text] -> DataFrame -> DataFrame projectColumns names df = @@ -129,14 +119,13 @@ (rows, _) = dataframeDimensions df in DataFrame newCols newIndices (rows, length idxs) M.empty +-- | Reads via a private ('WriteCopy') mmap so the buffer is never copied. readSeparated :: Word8 -> ReadOptions -> FilePath -> IO DataFrame readSeparated separator opts filePath = do - -- WriteCopy keeps the mapping private; the scanner never needs to - -- grow or pad the buffer, so the file is no longer copied (audit F6). (bufferPtr, offset, len) <- mmapFileForeignPtr filePath diff --git a/src/DataFrame/IO/CSV/Fast/Columns.hs b/src/DataFrame/IO/CSV/Fast/Columns.hs --- a/src/DataFrame/IO/CSV/Fast/Columns.hs +++ b/src/DataFrame/IO/CSV/Fast/Columns.hs @@ -4,10 +4,8 @@ {-# LANGUAGE TypeApplications #-} {- | Column-level dispatch for the fast CSV reader: resolve each column's -target type up front into a 'ColumnPlan' (schema hit -> typed pass directly; -miss -> sample classification + typed fallback chain; EitherRead \/ exotic -schema types -> the legacy Text pipeline). Plans are chunk-runnable so the -parallel reader (WS-E2) can execute them over row sub-ranges. +target type up front into a 'ColumnPlan' (schema hit, inference chain, or +legacy Text pipeline). Plans are chunk-runnable for the parallel reader. -} module DataFrame.IO.CSV.Fast.Columns ( ColumnEnv (..), @@ -46,7 +44,6 @@ import DataFrame.IO.CSV.Fast.Passes import DataFrame.IO.CSV.Fast.Slice (extractField) import DataFrame.Internal.Column (Column, ensureOptional, fromVector) -import DataFrame.Internal.Schema (SchemaType (..)) import DataFrame.Operations.Typing ( ParseOptions (..), ParsingAssumption (..), @@ -56,6 +53,7 @@ makeParsingAssumption, parseFromExamples, ) +import DataFrame.Schema (SchemaType (..)) -- | One step of a type-fallback chain (always ends in 'StepText'). data Step = StepBool | StepInt | StepDouble | StepDate | StepText @@ -212,8 +210,7 @@ {- | Build one column sequentially (single chunk over the full row range). Returns the column plus a flag telling the caller to re-apply the legacy -schema conversion ('parseWithTypes') afterwards — only set for schema'd -columns the typed passes cannot serve. +'parseWithTypes' conversion afterwards (schema types the passes can't serve). -} buildColumn :: ColumnEnv -> T.Text -> Int -> IO (Column, Bool) buildColumn env name col = case planColumn env name col of diff --git a/tests/Operations/ChunkParallel.hs b/tests/Operations/ChunkParallel.hs --- a/tests/Operations/ChunkParallel.hs +++ b/tests/Operations/ChunkParallel.hs @@ -1,9 +1,9 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -{- | Round-2 (WS-E2) behavior pins for chunk-parallel extraction: a chunked -read must be indistinguishable from the sequential one — same column types -(promotion resolved across chunks), same values, same errors. +{- | Behavior pins for chunk-parallel extraction: a chunked read must be +indistinguishable from the sequential one — same column types (promotion +resolved across chunks), same values, and same errors. -} module Operations.ChunkParallel (tests) where @@ -23,8 +23,8 @@ import qualified DataFrame.IO.CSV.Fast as D import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (DataFrame, getColumn) -import DataFrame.Internal.Schema (Schema (..), SchemaType (..), elements) import DataFrame.Operations.Typing (SafeReadMode (..)) +import DataFrame.Schema (Schema (..), SchemaType (..), elements) import Test.HUnit comma :: Word8 @@ -205,8 +205,6 @@ -- Date column with nulls: boxed chunk columns must splice correctly. testDateNullsAcrossChunks :: Test testDateNullsAcrossChunks = TestLabel "par_date_nulls" . TestCase $ do - -- A second column keeps null rows from looking like blank lines - -- (blank lines are skipped by documented behavior). let cell k = if k `mod` 17 == 9 then "" @@ -240,9 +238,8 @@ Left D.CsvUnclosedQuote -> pure () other -> assertFailure ("expected CsvUnclosedQuote, got " <> show other) --- WS-E2 ingest pin: the parallel byte-level merge wraps the merged shared --- buffer as 'PackedText' (no Text spine), so the parallel path is also an --- RSS win. Values stay identical to the sequential read. +-- The parallel byte-level merge wraps the merged buffer as 'PackedText' +-- (no Text spine), an RSS win; values stay identical to the sequential read. testParallelTextFreezesPacked :: Test testParallelTextFreezesPacked = TestLabel "par_text_freezes_packed" . TestCase $ do let body = diff --git a/tests/Operations/ReadCsv.hs b/tests/Operations/ReadCsv.hs --- a/tests/Operations/ReadCsv.hs +++ b/tests/Operations/ReadCsv.hs @@ -35,7 +35,7 @@ dataframeDimensions, getColumn, ) -import DataFrame.Internal.Schema (Schema (..), SchemaType (..)) +import DataFrame.Schema (Schema (..), SchemaType (..)) import System.Directory (removeFile) import System.IO (IOMode (..), withFile) import Test.HUnit @@ -64,7 +64,6 @@ TIO.hPutStrLn handle (T.intercalate (T.singleton sep) (map (escapeField sep) headers)) - -- Write data rows mapM_ (TIO.hPutStrLn handle . T.intercalate (T.singleton sep) . getRowEscaped sep df) [0 .. rows - 1] @@ -193,7 +192,6 @@ testTrailingBlankLine :: Test testTrailingBlankLine = TestLabel "malformed_trailing_blank_line" $ TestCase $ do df <- D.readCsvFast (fixtureDir <> "trailing_blank_line.csv") - -- blank line contributes 1 extra delimiter; (3+3+1) div 3 = 2, numRow=1 assertEqual "trailing_blank_line.csv: 1 data row visible" 1 @@ -252,11 +250,9 @@ (DI.fromList @T.Text [" New York", " Los Angeles"]) col --- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full). --- After the row-index refactor each line is classified individually: --- the short row survives with a null-padded column 'c'; the full row --- keeps its value. This replaces the earlier "X bleeds from next row" --- behaviour, which was data corruption masquerading as a passing test. +-- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full). Each line is +-- classified individually: the short row survives with a null-padded 'c'; +-- the full row keeps its value. testMissingFields :: Test testMissingFields = TestLabel "malformed_missing_fields" $ TestCase $ do df <- D.readCsvFast (fixtureDir <> "missing_fields.csv") @@ -279,10 +275,9 @@ (DI.fromList @(Maybe T.Text) [Nothing, Just "Z"]) col --- File: a,b,c header; row "1,2,3,EXTRA" (over-long). --- Under the default ragged-row policy we read columns 0..numCol-1 and --- silently drop the EXTRA field. A stricter `RaggedRowPolicy = Error` --- will come with Step 7's ReadOptions knob. +-- File: a,b,c header; row "1,2,3,EXTRA" (over-long). Under the default +-- ragged-row policy we read columns 0..numCol-1 and silently drop the +-- EXTRA field. testExtraFieldsTruncate :: Test testExtraFieldsTruncate = TestLabel "malformed_extra_fields_truncate" $ TestCase $ do @@ -437,10 +432,9 @@ map fst . L.sortBy (compare `on` snd) . M.toList $ columnIndices df assertEqual "projection preserves order" ["c", "a"] names --- An unmatched `"` used to silently swallow the rest of the file: the --- SIMD scanner's PCLMUL quote-parity chain never resets, so every byte --- after the stray quote becomes "inside quotes." We now raise --- 'CsvUnclosedQuote' rather than returning a corrupted DataFrame. +-- An unmatched `"` used to silently swallow the rest of the file (the SIMD +-- quote-parity chain never resets). We now raise 'CsvUnclosedQuote' rather +-- than returning a corrupted DataFrame. testUnclosedQuote :: Test testUnclosedQuote = TestLabel "malformed_unclosed_quote" $ TestCase $ do let path = fixtureDir <> "unclosed_quote.csv" diff --git a/tests/Operations/TypedExtraction.hs b/tests/Operations/TypedExtraction.hs --- a/tests/Operations/TypedExtraction.hs +++ b/tests/Operations/TypedExtraction.hs @@ -2,9 +2,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Round-2 (WS-E1) behavior pins for the typed-extraction fastcsv path: -schema bypass, unified WS-B parser semantics, byte-level missing tests, -and the chunk-boundary scalar tail that replaced the full-file copy. +{- | Behavior pins for the typed-extraction fastcsv path: schema bypass, +parser semantics, byte-level missing tests, and the chunk-boundary scalar +tail that replaced the full-file copy. -} module Operations.TypedExtraction (tests) where @@ -25,8 +25,8 @@ import DataFrame.Internal.Column (Column) import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (DataFrame, dataframeDimensions, getColumn) -import DataFrame.Internal.Schema (Schema (..), SchemaType (..), elements) import DataFrame.Operations.Typing (SafeReadMode (..)) +import DataFrame.Schema (Schema (..), SchemaType (..), elements) import System.Directory (removeFile) import Test.HUnit @@ -59,10 +59,9 @@ df (DI.fromList @Double [1.0, 9.223372036854776e18]) --- Divergence #2 (pinned new behavior): doubles parse with strip-equivalent --- semantics, so a padded double past the 100-row inference sample no longer --- demotes the whole column to Text. (Inside the sample the Text-level --- classifier still rejects padding, by design.) +-- Divergence #2: doubles parse with strip-equivalent semantics, so a padded +-- double past the 100-row inference sample no longer demotes the column to +-- Text. (Inside the sample the classifier still rejects padding.) testPaddedDoubleStaysDouble :: Test testPaddedDoubleStaysDouble = TestLabel "typed_padded_double" $ TestCase $ do @@ -215,9 +214,8 @@ ) -- The scalar tail (last <64 bytes after the SIMD chunks) must honour the --- quote state carried out of the SIMD region: a quoted field that opens --- before the 64-byte boundary and closes inside the tail may contain --- separators and newlines. +-- quote state from the SIMD region: a quoted field opening before the +-- boundary and closing in the tail may contain separators and newlines. testQuoteSpansChunkBoundary :: Test testQuoteSpansChunkBoundary = TestLabel "typed_quote_spans_boundary" $ TestCase $ do