dataframe-fastcsv 1.0.0.0 → 1.1.0.1

raw patch · 7 files changed

+145/−61 lines, 7 filesdep +dataframe-coredep +dataframe-csvdep +dataframe-operationsdep ~dataframePVP ok

version bump matches the API change (PVP)

Dependencies added: dataframe-core, dataframe-csv, dataframe-operations, dataframe-parsing

Dependency ranges changed: dataframe

API changes (from Hackage documentation)

Files

• cbits/process_csv.c +1/−1
• dataframe-fastcsv.cabal +11/−4
• src/DataFrame/IO/CSV/Fast.hs +61/−8
• tests/Main.hs +10/−8
• tests/Operations/ReadCsv.hs +3/−0
• tests/Properties/Csv.hs +50/−40
• tests/data/unstable_csv/either_read_mixed.csv +9/−0

@@ -200,7 +200,7 @@ #else   // SIMD not available or carryless multiplication not supported.   // Signal fallback to Haskell implementation.-  (void)buf; (void)len; (void)indices;+  (void)buf; (void)len; (void)separator; (void)indices;   return GDI_SIMD_UNAVAILABLE; #endif }

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-fastcsv-version:            1.0.0.0+version:            1.1.0.1  synopsis: SIMD-accelerated CSV reader for the dataframe library. @@ -14,8 +14,9 @@ author:             Michael Chavinda maintainer:         mschavinda@gmail.com -copyright: (c) 2024-2025 Michael Chavinda+copyright: (c) 2024-2026 Michael Chavinda category: Data+tested-with:        GHC ==9.4.8 || ==9.6.7 || ==9.8.4 || ==9.10.3 || ==9.12.2 extra-source-files: cbits/process_csv.h                     tests/data/unstable_csv/*.csv                     tests/data/unstable_csv/*.tsv@@ -49,7 +50,10 @@                       array >= 0.5.4.0 && < 0.6,                       bytestring >= 0.11 && < 0.13,                       containers >= 0.6.7 && < 0.9,-                      dataframe ^>= 1.1,+                      dataframe-core ^>= 1.0,+                      dataframe-csv ^>= 1.0,+                      dataframe-operations ^>= 1.1,+                      dataframe-parsing ^>= 1.0,                       mmap >= 0.5.8 && < 0.6,                       parallel >= 3.2.2.0 && < 5,                       text >= 2.0 && < 3,@@ -73,8 +77,11 @@                    Properties.Csv     build-depends:  base >= 4 && < 5,                     containers >= 0.6.7 && < 0.9,-                    dataframe ^>= 1.1,+                    dataframe >= 1 && < 3,+                    dataframe-core ^>= 1.0,+                    dataframe-csv ^>= 1.0,                     dataframe-fastcsv,+                    dataframe-parsing ^>= 1.0,                     directory >= 1.3.0.0 && < 2,                     HUnit ^>= 1.6,                     QuickCheck >= 2 && < 3,

@@ -11,7 +11,10 @@     fastReadTsvWithOpts,     fastReadCsvWithSchema,     fastReadCsvProj,+    fastReadCsvFromBytes,+    fastReadCsvFromBytesWithSchema,     readSeparated,+    readSeparatedFromBytes,     getDelimiterIndices,     CsvParseError (..), ) where@@ -35,12 +38,15 @@     castForeignPtr,     castPtr,  )+import qualified Foreign import Foreign.C.Types import Foreign.Marshal.Alloc (alloca)+import Foreign.Marshal.Utils (copyBytes) import Foreign.Storable (peek, poke)  import qualified Data.ByteString as BS import Data.ByteString.Internal (ByteString (PS))+import qualified Data.ByteString.Internal as BSI import qualified Data.Map as M import Data.Text (Text) import qualified Data.Text as Text@@ -121,6 +127,20 @@ unreferenced columns, so the end-to-end cost scales with the size of the projection rather than the row width. -}++-- | In-memory version of 'fastReadCsv'.+fastReadCsvFromBytes :: BS.ByteString -> IO DataFrame+fastReadCsvFromBytes = readSeparatedFromBytes comma defaultReadOptions++fastReadCsvFromBytesWithSchema :: Schema -> BS.ByteString -> IO DataFrame+fastReadCsvFromBytesWithSchema schema =+    readSeparatedFromBytes+        comma+        defaultReadOptions+            { typeSpec =+                SpecifyTypes (M.toList (elements schema)) (typeSpec defaultReadOptions)+            }+ fastReadCsvProj :: [Text] -> FilePath -> IO DataFrame fastReadCsvProj projection path = do     df <- fastReadCsv path@@ -153,6 +173,39 @@             WriteCopy             Nothing     let mutableFile = unsafeFromForeignPtr bufferPtr offset len+    readSeparatedFromMVec separator opts mutableFile len++{- | Identical to 'readSeparated' but takes an already-loaded byte buffer+(e.g. a slice of a memory-mapped file).  The bytes are copied once into+a fresh mutable storable vector so the SIMD code can grow it by 64+bytes of trailing padding without disturbing the caller's input.+-}+readSeparatedFromBytes ::+    Word8 ->+    ReadOptions ->+    BS.ByteString ->+    IO DataFrame+readSeparatedFromBytes separator opts bs = do+    let len = BS.length bs+    mvec <- VSM.unsafeNew len+    -- Copy ByteString → mutable vector (one pass, ~1 ns/byte).+    let (fp, off, _) = BSI.toForeignPtr bs+    Foreign.withForeignPtr fp $ \srcPtr ->+        VSM.unsafeWith mvec $ \dstPtr ->+            copyBytes (castPtr dstPtr) (srcPtr `Foreign.plusPtr` off) len+    readSeparatedFromMVec separator opts mvec len++{- | Shared core: takes a mutable storable vector containing exactly @len@+bytes of CSV (no trailing padding required by the caller — it is added+here) and produces a 'DataFrame'.+-}+readSeparatedFromMVec ::+    Word8 ->+    ReadOptions ->+    VSM.IOVector Word8 ->+    Int ->+    IO DataFrame+readSeparatedFromMVec separator opts mutableFile len = do     paddedMutableFile <- grow mutableFile 64     paddedCSVFileRaw <- VS.unsafeFreeze paddedMutableFile     let (paddedCSVFile, bomLen) = stripBom paddedCSVFileRaw@@ -224,21 +277,21 @@                             parseTypes (columnNames Vector.! col) $                                 Vector.generate numRow $ \i ->                                     extractAt (dataRows VS.! i) col-                        columns =+                        cols =                             Vector.fromListN                                 numCol                                 ( map generateColumn [0 .. numCol - 1]                                     `using` parList rpar                                 )-                        columnIndices =+                        colIndices =                             M.fromList $                                 zip (Vector.toList columnNames) [0 ..]-                        dataframeDimensions = (numRow, numCol)+                        dfDims = (numRow, numCol)                     let rawDf =                             DataFrame-                                columns-                                columnIndices-                                dataframeDimensions+                                cols+                                colIndices+                                dfDims                                 M.empty                         schemaMap = schemaTypeMap (typeSpec opts)                         resolveMode =@@ -284,8 +337,8 @@ -} {-# INLINE classifyRowEnds #-} classifyRowEnds :: VS.Vector Word8 -> Int -> VS.Vector CSize -> VS.Vector Int-classifyRowEnds file contentLen delimiters =-    VS.findIndices isRowBreak delimiters+classifyRowEnds file contentLen =+    VS.findIndices isRowBreak   where     isRowBreak pos =         let p = fromIntegral pos :: Int

@@ -1,9 +1,11 @@+{-# LANGUAGE ScopedTypeVariables #-}+ module Main where  import qualified System.Exit as Exit  import Test.HUnit-import Test.QuickCheck (Result, isSuccess, quickCheckWithResult, stdArgs)+import Test.QuickCheck  import qualified Operations.ReadCsv import qualified Properties.Csv@@ -11,8 +13,9 @@ tests :: Test tests = TestList Operations.ReadCsv.tests -allSuccessful :: [Result] -> Bool-allSuccessful = all isSuccess+isSuccessful :: Result -> Bool+isSuccessful (Success{}) = True+isSuccessful _ = False  main :: IO () main = do@@ -20,8 +23,7 @@     if failures result > 0 || errors result > 0         then Exit.exitFailure         else do-            propResults <--                mapM (quickCheckWithResult stdArgs) Properties.Csv.tests-            if allSuccessful propResults-                then Exit.exitSuccess-                else Exit.exitFailure+            propsRes <- mapM (quickCheckWithResult stdArgs) Properties.Csv.tests+            if not (all isSuccessful propsRes)+                then Exit.exitFailure+                else Exit.exitSuccess

@@ -448,6 +448,9 @@     removeFile path     case (result :: Either CsvParseError DataFrame) of         Left CsvUnclosedQuote -> return ()+        Left other ->+            assertFailure+                ("expected CsvUnclosedQuote, got " <> show other)         Right _ ->             assertFailure                 "readCsvFast should have thrown CsvUnclosedQuote on input with a stray quote"

@@ -1,5 +1,4 @@ {-# LANGUAGE GADTs #-}-{-# LANGUAGE MonoLocalBinds #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-}@@ -39,9 +38,10 @@ import Test.QuickCheck.Monadic (PropertyM, assert, monadicIO, run) import Type.Reflection (typeRep) --- | A generator for a single CSV cell.  Each cell starts with an ASCII--- letter (so type inference always picks 'Text') and may contain a small--- menagerie of special characters the parser must escape correctly.+{- | A generator for a single CSV cell.  Each cell starts with an ASCII+letter (so type inference always picks 'Text') and may contain a small+menagerie of special characters the parser must escape correctly.+-} newtype Cell = Cell {unCell :: T.Text}     deriving (Eq, Show) @@ -54,10 +54,11 @@         | T.null t || T.length t == 1 = []         | otherwise = [Cell (T.take (T.length t - 1) t)] --- | Like 'Cell' but guaranteed to contain neither @\\n@ nor @\\r@.  Used--- by properties that diff across line-ending dialects, where embedded--- line breaks become part of the data under CRLF transform and break--- the otherwise-clean invariant.+{- | Like 'Cell' but guaranteed to contain neither @\\n@ nor @\\r@.  Used+by properties that diff across line-ending dialects, where embedded+line breaks become part of the data under CRLF transform and break+the otherwise-clean invariant.+-} newtype SingleLineCell = SingleLineCell {unSingleLineCell :: T.Text}     deriving (Eq, Show) @@ -82,9 +83,10 @@         | length ns <= 1 = []         | otherwise = [ColumnNames (init ns)] --- | RFC 4180 cell encoding: wrap a field in @"@ and double embedded--- quotes when the cell contains the separator, a line break, or a--- quote.+{- | RFC 4180 cell encoding: wrap a field in @"@ and double embedded+quotes when the cell contains the separator, a line break, or a+quote.+-} encodeCell :: Char -> T.Text -> T.Text encodeCell sep t     | T.any needsQuote t = T.concat ["\"", T.replace "\"" "\"\"" t, "\""]@@ -92,20 +94,22 @@   where     needsQuote c = c == sep || c == '\n' || c == '\r' || c == '"' --- | Encode a grid @[[cell]]@ (rows of cells) as an RFC 4180 CSV.  The--- first row is written as the header, followed by one line per data--- row.  Always uses @\\n@ as the line terminator; callers that need--- CRLF can transform the output themselves.+{- | Encode a grid @[[cell]]@ (rows of cells) as an RFC 4180 CSV.  The+first row is written as the header, followed by one line per data+row.  Always uses @\\n@ as the line terminator; callers that need+CRLF can transform the output themselves.+-} encodeCsv :: Char -> [T.Text] -> [[T.Text]] -> T.Text encodeCsv sep header rows =     T.unlines (encodeRow header : map encodeRow rows)   where     encodeRow = T.intercalate (T.singleton sep) . map (encodeCell sep) --- | Pull the raw 'Text' values of the named column out of a DataFrame.--- Fails the property if the column is missing or is not a 'Text'--- column (which would signal a type-inference accident — the cell--- generator is designed to avoid that).+{- | Pull the raw 'Text' values of the named column out of a DataFrame.+Fails the property if the column is missing or is not a 'Text'+column (which would signal a type-inference accident — the cell+generator is designed to avoid that).+-} columnAsText :: T.Text -> DataFrame -> Maybe [T.Text] columnAsText name df = do     idx <- M.lookup name (columnIndices df)@@ -117,8 +121,9 @@                 Nothing -> Nothing         UnboxedColumn{} -> Nothing --- | Run a property in @IO@ against a generated CSV text, cleaning up--- the temp file afterwards no matter what.+{- | Run a property in @IO@ against a generated CSV text, cleaning up+the temp file afterwards no matter what.+-} withCsvFile :: String -> T.Text -> (FilePath -> IO a) -> IO a withCsvFile label body action = do     let path = "/tmp/fastcsv_prop_" <> label <> ".csv"@@ -130,8 +135,9 @@ -------------------------------------------------------------------------------- -- Properties --- | Any CSV parses the same whether or not a UTF-8 BOM (EF BB BF) is--- prepended.  Excel / PowerShell CSV exports all come with a BOM.+{- | Any CSV parses the same whether or not a UTF-8 BOM (EF BB BF) is+prepended.  Excel / PowerShell CSV exports all come with a BOM.+-} prop_bom_invariant :: ColumnNames -> [[Cell]] -> Property prop_bom_invariant (ColumnNames header) rows = monadicIO $ do     let cellRows = map (map unCell) rows@@ -142,11 +148,12 @@     assert (plain == withBom)     assert (dataframeDimensions plain == dataframeDimensions withBom) --- | A CSV with no embedded newlines inside quoted fields parses the--- same whether record separators are @\\n@ or @\\r\\n@.  When quoted--- cells themselves contain @\\n@, a CRLF-encoded file contains a literal--- @\\r\\n@ inside the quotes (that's data, not line ending), so the--- invariant correctly no longer holds; we filter such inputs out.+{- | A CSV with no embedded newlines inside quoted fields parses the+same whether record separators are @\\n@ or @\\r\\n@.  When quoted+cells themselves contain @\\n@, a CRLF-encoded file contains a literal+@\\r\\n@ inside the quotes (that's data, not line ending), so the+invariant correctly no longer holds; we filter such inputs out.+-} prop_crlf_invariant :: ColumnNames -> [[SingleLineCell]] -> Property prop_crlf_invariant (ColumnNames header) rows = monadicIO $ do     let cellRows = map (map unSingleLineCell) rows@@ -156,10 +163,11 @@     dfCrlf <- run $ withCsvFile "crlf_crlf" crlfCsv D.fastReadCsv     assert (dfLf == dfCrlf) --- | A DataFrame produced by the RFC 4180 encoder round-trips through--- the fast reader: column names survive, row count survives, and every--- cell comes back bit-for-bit.  Restricted to ASCII-letter-prefixed--- cells so that type inference always stays on 'Text'.+{- | A DataFrame produced by the RFC 4180 encoder round-trips through+the fast reader: column names survive, row count survives, and every+cell comes back bit-for-bit.  Restricted to ASCII-letter-prefixed+cells so that type inference always stays on 'Text'.+-} prop_roundtrip_ascii :: Property prop_roundtrip_ascii = forAll (chooseInt (0, 12)) $ \nRows ->     forAll (chooseInt (1, 4)) $ \nCols ->@@ -185,10 +193,11 @@         )         (zip header expected) --- | A file that ends with an unmatched @"@ must raise 'CsvUnclosedQuote'--- under the default policy.  The generator deliberately builds a valid--- prefix so the property is checking the error path, not a random--- parse failure.+{- | A file that ends with an unmatched @"@ must raise 'CsvUnclosedQuote'+under the default policy.  The generator deliberately builds a valid+prefix so the property is checking the error path, not a random+parse failure.+-} prop_unclosed_quote_throws :: Property prop_unclosed_quote_throws = forAll (listOf1 arbitrary) $ \(cells :: [Cell]) ->     monadicIO $ do@@ -205,10 +214,11 @@             Left CsvUnclosedQuote -> assert True             _ -> assert False --- | Files whose length sits exactly at a SIMD chunk boundary must parse--- without an off-by-one.  We generate inputs at the key critical sizes--- (64, 127, 128, 129, 192 bytes after the 'v\n' header) by padding a--- single column with ASCII letters.+{- | Files whose length sits exactly at a SIMD chunk boundary must parse+without an off-by-one.  We generate inputs at the key critical sizes+(64, 127, 128, 129, 192 bytes after the 'v\n' header) by padding a+single column with ASCII letters.+-} prop_boundary_sizes :: Property prop_boundary_sizes =     forAll (elements [64, 127, 128, 129, 192]) $ \targetBytes ->

@@ -0,0 +1,9 @@+id,score,name+1,10,alice+2,20,bob+3,30,carol+4,40,dave+5,50,eve+6,abc,frank+7,,grace+8,80,N/A