packages feed

dataframe-fastcsv 1.1.1.0 → 1.3.0.0

raw patch · 6 files changed

+56/−81 lines, 6 filesdep −dataframedep ~HUnitdep ~bytestringdep ~containersPVP ok

version bump matches the API change (PVP)

Dependencies removed: dataframe

Dependency ranges changed: HUnit, bytestring, containers, dataframe-core, dataframe-csv, dataframe-operations, dataframe-parsing, mmap, vector

API changes (from Hackage documentation)

Files

dataframe-fastcsv.cabal view
@@ -1,6 +1,6 @@-cabal-version:      2.4+cabal-version:      3.4 name:               dataframe-fastcsv-version:            1.1.1.0+version:            1.3.0.0 synopsis: SIMD-accelerated CSV reader for the dataframe library.  description: A fast, SIMD-accelerated CSV/TSV reader using memory-mapped I/O@@ -55,16 +55,17 @@                      DataFrame.IO.CSV.Fast.TextMerge                      DataFrame.IO.CSV.Fast.Workers     build-depends:    base >= 4 && < 5,-                      bytestring >= 0.11 && < 0.13,-                      containers >= 0.6.7 && < 0.9,-                      dataframe-core ^>= 1.1,-                      dataframe-csv ^>= 1.0.2,-                      dataframe-operations ^>= 1.1.1,-                      dataframe-parsing ^>= 1.0.2,-                      mmap >= 0.5.8 && < 0.6,-                      text >= 2.0 && < 3,+                      bytestring >= 0.11 && < 0.14,+                      containers >= 0.6.7 && < 0.10,+                      dataframe-core >= 2.1 && < 2.2,+                      dataframe-csv >= 2.1 && < 2.2,+                      dataframe-operations >= 2.1 && < 2.2,+                      dataframe-parsing >= 2.1 && < 2.2,+                      dataframe-parsing >= 2.1 && < 2.2,+                      mmap >= 0.5.8 && < 0.7,+                      text >= 2.1 && < 3,                       time >= 1.12 && < 2,-                      vector ^>= 0.13+                      vector >= 0.13 && < 0.15     hs-source-dirs:   src     c-sources:        cbits/process_csv.c     include-dirs:     cbits@@ -88,19 +89,18 @@                    Operations.TypedExtraction                    Properties.Csv     build-depends:  base >= 4 && < 5,-                    containers >= 0.6.7 && < 0.9,-                    dataframe >= 1 && < 3,-                    dataframe-core ^>= 1.1,-                    dataframe-csv ^>= 1.0.2,+                    containers >= 0.6.7 && < 0.10,+                    dataframe-core >= 2.1 && < 2.2,+                    dataframe-csv >= 2.1 && < 2.2,                     dataframe-fastcsv,-                    dataframe-operations ^>= 1.1.1,-                    dataframe-parsing ^>= 1.0.2,+                    dataframe-operations >= 2.1 && < 2.2,+                    dataframe-parsing >= 2.1 && < 2.2,                     directory >= 1.3.0.0 && < 2,-                    HUnit ^>= 1.6,+                    HUnit >= 1.6 && < 1.8,                     QuickCheck >= 2 && < 3,-                    text >= 2.0 && < 3,+                    text >= 2.1 && < 3,                     time >= 1.12 && < 2,-                    vector ^>= 0.13+                    vector >= 0.13 && < 0.15     hs-source-dirs: tests     default-language: Haskell2010     if flag(asan)
src/DataFrame/IO/CSV/Fast.hs view
@@ -1,13 +1,6 @@ {- | SIMD-accelerated CSV reader: a C scanner finds delimiter positions, then each column is parsed directly from the mmap'd byte slices into-unboxed column builders (Round-2 typed extraction). Reads are strict: the-returned 'DataFrame' is fully built, with no deferred parse work.--Extraction is chunk-parallel: inputs of 4 MB and up are split into one-row chunk per capability and parsed by 'Control.Concurrent.forkIO'-workers. The library never chooses RTS settings — to actually get-parallel reads, link the executable with @-threaded@ and run with-@+RTS -N@ (otherwise the reader falls back to the sequential path).+typed column builders. Chunk-parallel with @-threaded@ + @+RTS -N@. -} module DataFrame.IO.CSV.Fast (     fastReadCsv,@@ -50,7 +43,7 @@     tab,  ) import DataFrame.Internal.DataFrame (DataFrame (..))-import DataFrame.Internal.Schema (Schema (..))+import DataFrame.Schema (Schema (..))  readSeparatedDefault :: Word8 -> FilePath -> IO DataFrame readSeparatedDefault separator =@@ -80,10 +73,8 @@ fastReadTsvWithOpts = readSeparated tab  {- | Read a CSV and coerce each column to the type declared in the-supplied 'Schema'.  Schema columns bypass inference entirely: they are-parsed straight from the file bytes as the declared type, which is both-faster than inference and guards against the row-1 \"looks like Int\"-misclassification trap.+supplied 'Schema'. Schema columns bypass inference: parsed straight from+file bytes as the declared type, avoiding row-1 misclassification. -} fastReadCsvWithSchema :: Schema -> FilePath -> IO DataFrame fastReadCsvWithSchema schema =@@ -117,9 +108,8 @@     pure (projectColumns projection df)  {- | Filter a DataFrame's columns down to (and in the order of) the-supplied names.  Missing names are silently dropped rather than-raised; callers that want strict semantics can check the resulting-'columnIndices' map.+supplied names. Missing names are silently dropped; check the result's+'columnIndices' map for strict semantics. -} projectColumns :: [Text] -> DataFrame -> DataFrame projectColumns names df =@@ -129,14 +119,13 @@         (rows, _) = dataframeDimensions df      in DataFrame newCols newIndices (rows, length idxs) M.empty +-- | Reads via a private ('WriteCopy') mmap so the buffer is never copied. readSeparated ::     Word8 ->     ReadOptions ->     FilePath ->     IO DataFrame readSeparated separator opts filePath = do-    -- WriteCopy keeps the mapping private; the scanner never needs to-    -- grow or pad the buffer, so the file is no longer copied (audit F6).     (bufferPtr, offset, len) <-         mmapFileForeignPtr             filePath
src/DataFrame/IO/CSV/Fast/Columns.hs view
@@ -4,10 +4,8 @@ {-# LANGUAGE TypeApplications #-}  {- | Column-level dispatch for the fast CSV reader: resolve each column's-target type up front into a 'ColumnPlan' (schema hit -> typed pass directly;-miss -> sample classification + typed fallback chain; EitherRead \/ exotic-schema types -> the legacy Text pipeline). Plans are chunk-runnable so the-parallel reader (WS-E2) can execute them over row sub-ranges.+target type up front into a 'ColumnPlan' (schema hit, inference chain, or+legacy Text pipeline). Plans are chunk-runnable for the parallel reader. -} module DataFrame.IO.CSV.Fast.Columns (     ColumnEnv (..),@@ -46,7 +44,6 @@ import DataFrame.IO.CSV.Fast.Passes import DataFrame.IO.CSV.Fast.Slice (extractField) import DataFrame.Internal.Column (Column, ensureOptional, fromVector)-import DataFrame.Internal.Schema (SchemaType (..)) import DataFrame.Operations.Typing (     ParseOptions (..),     ParsingAssumption (..),@@ -56,6 +53,7 @@     makeParsingAssumption,     parseFromExamples,  )+import DataFrame.Schema (SchemaType (..))  -- | One step of a type-fallback chain (always ends in 'StepText'). data Step = StepBool | StepInt | StepDouble | StepDate | StepText@@ -212,8 +210,7 @@  {- | Build one column sequentially (single chunk over the full row range). Returns the column plus a flag telling the caller to re-apply the legacy-schema conversion ('parseWithTypes') afterwards — only set for schema'd-columns the typed passes cannot serve.+'parseWithTypes' conversion afterwards (schema types the passes can't serve). -} buildColumn :: ColumnEnv -> T.Text -> Int -> IO (Column, Bool) buildColumn env name col = case planColumn env name col of
tests/Operations/ChunkParallel.hs view
@@ -1,9 +1,9 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -{- | Round-2 (WS-E2) behavior pins for chunk-parallel extraction: a chunked-read must be indistinguishable from the sequential one — same column types-(promotion resolved across chunks), same values, same errors.+{- | Behavior pins for chunk-parallel extraction: a chunked read must be+indistinguishable from the sequential one — same column types (promotion+resolved across chunks), same values, and same errors. -} module Operations.ChunkParallel (tests) where @@ -23,8 +23,8 @@ import qualified DataFrame.IO.CSV.Fast as D import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (DataFrame, getColumn)-import DataFrame.Internal.Schema (Schema (..), SchemaType (..), elements) import DataFrame.Operations.Typing (SafeReadMode (..))+import DataFrame.Schema (Schema (..), SchemaType (..), elements) import Test.HUnit  comma :: Word8@@ -205,8 +205,6 @@ -- Date column with nulls: boxed chunk columns must splice correctly. testDateNullsAcrossChunks :: Test testDateNullsAcrossChunks = TestLabel "par_date_nulls" . TestCase $ do-    -- A second column keeps null rows from looking like blank lines-    -- (blank lines are skipped by documented behavior).     let cell k =             if k `mod` 17 == 9                 then ""@@ -240,9 +238,8 @@         Left D.CsvUnclosedQuote -> pure ()         other -> assertFailure ("expected CsvUnclosedQuote, got " <> show other) --- WS-E2 ingest pin: the parallel byte-level merge wraps the merged shared--- buffer as 'PackedText' (no Text spine), so the parallel path is also an--- RSS win. Values stay identical to the sequential read.+-- The parallel byte-level merge wraps the merged buffer as 'PackedText'+-- (no Text spine), an RSS win; values stay identical to the sequential read. testParallelTextFreezesPacked :: Test testParallelTextFreezesPacked = TestLabel "par_text_freezes_packed" . TestCase $ do     let body =
tests/Operations/ReadCsv.hs view
@@ -35,7 +35,7 @@     dataframeDimensions,     getColumn,  )-import DataFrame.Internal.Schema (Schema (..), SchemaType (..))+import DataFrame.Schema (Schema (..), SchemaType (..)) import System.Directory (removeFile) import System.IO (IOMode (..), withFile) import Test.HUnit@@ -64,7 +64,6 @@     TIO.hPutStrLn         handle         (T.intercalate (T.singleton sep) (map (escapeField sep) headers))-    -- Write data rows     mapM_         (TIO.hPutStrLn handle . T.intercalate (T.singleton sep) . getRowEscaped sep df)         [0 .. rows - 1]@@ -193,7 +192,6 @@ testTrailingBlankLine :: Test testTrailingBlankLine = TestLabel "malformed_trailing_blank_line" $ TestCase $ do     df <- D.readCsvFast (fixtureDir <> "trailing_blank_line.csv")-    -- blank line contributes 1 extra delimiter; (3+3+1) div 3 = 2, numRow=1     assertEqual         "trailing_blank_line.csv: 1 data row visible"         1@@ -252,11 +250,9 @@                 (DI.fromList @T.Text ["  New York", "  Los Angeles"])                 col --- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full).--- After the row-index refactor each line is classified individually:--- the short row survives with a null-padded column 'c'; the full row--- keeps its value.  This replaces the earlier "X bleeds from next row"--- behaviour, which was data corruption masquerading as a passing test.+-- File: a,b,c header; row "1,2" (short); row "X,Y,Z" (full). Each line is+-- classified individually: the short row survives with a null-padded 'c';+-- the full row keeps its value. testMissingFields :: Test testMissingFields = TestLabel "malformed_missing_fields" $ TestCase $ do     df <- D.readCsvFast (fixtureDir <> "missing_fields.csv")@@ -279,10 +275,9 @@                 (DI.fromList @(Maybe T.Text) [Nothing, Just "Z"])                 col --- File: a,b,c header; row "1,2,3,EXTRA" (over-long).--- Under the default ragged-row policy we read columns 0..numCol-1 and--- silently drop the EXTRA field.  A stricter `RaggedRowPolicy = Error`--- will come with Step 7's ReadOptions knob.+-- File: a,b,c header; row "1,2,3,EXTRA" (over-long). Under the default+-- ragged-row policy we read columns 0..numCol-1 and silently drop the+-- EXTRA field. testExtraFieldsTruncate :: Test testExtraFieldsTruncate =     TestLabel "malformed_extra_fields_truncate" $ TestCase $ do@@ -437,10 +432,9 @@             map fst . L.sortBy (compare `on` snd) . M.toList $ columnIndices df     assertEqual "projection preserves order" ["c", "a"] names --- An unmatched `"` used to silently swallow the rest of the file: the--- SIMD scanner's PCLMUL quote-parity chain never resets, so every byte--- after the stray quote becomes "inside quotes."  We now raise--- 'CsvUnclosedQuote' rather than returning a corrupted DataFrame.+-- An unmatched `"` used to silently swallow the rest of the file (the SIMD+-- quote-parity chain never resets). We now raise 'CsvUnclosedQuote' rather+-- than returning a corrupted DataFrame. testUnclosedQuote :: Test testUnclosedQuote = TestLabel "malformed_unclosed_quote" $ TestCase $ do     let path = fixtureDir <> "unclosed_quote.csv"
tests/Operations/TypedExtraction.hs view
@@ -2,9 +2,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Round-2 (WS-E1) behavior pins for the typed-extraction fastcsv path:-schema bypass, unified WS-B parser semantics, byte-level missing tests,-and the chunk-boundary scalar tail that replaced the full-file copy.+{- | Behavior pins for the typed-extraction fastcsv path: schema bypass,+parser semantics, byte-level missing tests, and the chunk-boundary scalar+tail that replaced the full-file copy. -} module Operations.TypedExtraction (tests) where @@ -25,8 +25,8 @@ import DataFrame.Internal.Column (Column) import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (DataFrame, dataframeDimensions, getColumn)-import DataFrame.Internal.Schema (Schema (..), SchemaType (..), elements) import DataFrame.Operations.Typing (SafeReadMode (..))+import DataFrame.Schema (Schema (..), SchemaType (..), elements) import System.Directory (removeFile) import Test.HUnit @@ -59,10 +59,9 @@                 df                 (DI.fromList @Double [1.0, 9.223372036854776e18]) --- Divergence #2 (pinned new behavior): doubles parse with strip-equivalent--- semantics, so a padded double past the 100-row inference sample no longer--- demotes the whole column to Text. (Inside the sample the Text-level--- classifier still rejects padding, by design.)+-- Divergence #2: doubles parse with strip-equivalent semantics, so a padded+-- double past the 100-row inference sample no longer demotes the column to+-- Text. (Inside the sample the classifier still rejects padding.) testPaddedDoubleStaysDouble :: Test testPaddedDoubleStaysDouble = TestLabel "typed_padded_double" $     TestCase $ do@@ -215,9 +214,8 @@                 )  -- The scalar tail (last <64 bytes after the SIMD chunks) must honour the--- quote state carried out of the SIMD region: a quoted field that opens--- before the 64-byte boundary and closes inside the tail may contain--- separators and newlines.+-- quote state from the SIMD region: a quoted field opening before the+-- boundary and closing in the tail may contain separators and newlines. testQuoteSpansChunkBoundary :: Test testQuoteSpansChunkBoundary = TestLabel "typed_quote_spans_boundary" $     TestCase $ do