diff --git a/cbits/process_csv.c b/cbits/process_csv.c --- a/cbits/process_csv.c +++ b/cbits/process_csv.c @@ -200,7 +200,7 @@ #else // SIMD not available or carryless multiplication not supported. // Signal fallback to Haskell implementation. - (void)buf; (void)len; (void)indices; + (void)buf; (void)len; (void)separator; (void)indices; return GDI_SIMD_UNAVAILABLE; #endif } diff --git a/dataframe-fastcsv.cabal b/dataframe-fastcsv.cabal --- a/dataframe-fastcsv.cabal +++ b/dataframe-fastcsv.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe-fastcsv -version: 1.0.0.0 +version: 1.1.0.1 synopsis: SIMD-accelerated CSV reader for the dataframe library. @@ -14,8 +14,9 @@ author: Michael Chavinda maintainer: mschavinda@gmail.com -copyright: (c) 2024-2025 Michael Chavinda +copyright: (c) 2024-2026 Michael Chavinda category: Data +tested-with: GHC ==9.4.8 || ==9.6.7 || ==9.8.4 || ==9.10.3 || ==9.12.2 extra-source-files: cbits/process_csv.h tests/data/unstable_csv/*.csv tests/data/unstable_csv/*.tsv @@ -49,7 +50,10 @@ array >= 0.5.4.0 && < 0.6, bytestring >= 0.11 && < 0.13, containers >= 0.6.7 && < 0.9, - dataframe ^>= 1.1, + dataframe-core ^>= 1.0, + dataframe-csv ^>= 1.0, + dataframe-operations ^>= 1.1, + dataframe-parsing ^>= 1.0, mmap >= 0.5.8 && < 0.6, parallel >= 3.2.2.0 && < 5, text >= 2.0 && < 3, @@ -73,8 +77,11 @@ Properties.Csv build-depends: base >= 4 && < 5, containers >= 0.6.7 && < 0.9, - dataframe ^>= 1.1, + dataframe >= 1 && < 3, + dataframe-core ^>= 1.0, + dataframe-csv ^>= 1.0, dataframe-fastcsv, + dataframe-parsing ^>= 1.0, directory >= 1.3.0.0 && < 2, HUnit ^>= 1.6, QuickCheck >= 2 && < 3, diff --git a/src/DataFrame/IO/CSV/Fast.hs b/src/DataFrame/IO/CSV/Fast.hs --- a/src/DataFrame/IO/CSV/Fast.hs +++ b/src/DataFrame/IO/CSV/Fast.hs @@ -11,7 +11,10 @@ fastReadTsvWithOpts, fastReadCsvWithSchema, fastReadCsvProj, + fastReadCsvFromBytes, + fastReadCsvFromBytesWithSchema, readSeparated, + readSeparatedFromBytes, getDelimiterIndices, CsvParseError (..), ) where @@ -35,12 +38,15 @@ castForeignPtr, castPtr, ) +import qualified Foreign import Foreign.C.Types import Foreign.Marshal.Alloc (alloca) +import Foreign.Marshal.Utils (copyBytes) import Foreign.Storable (peek, poke) import qualified Data.ByteString as BS import Data.ByteString.Internal (ByteString (PS)) +import qualified Data.ByteString.Internal as BSI import qualified Data.Map as M import Data.Text (Text) import qualified Data.Text as Text @@ -121,6 +127,20 @@ unreferenced columns, so the end-to-end cost scales with the size of the projection rather than the row width. -} + +-- | In-memory version of 'fastReadCsv'. +fastReadCsvFromBytes :: BS.ByteString -> IO DataFrame +fastReadCsvFromBytes = readSeparatedFromBytes comma defaultReadOptions + +fastReadCsvFromBytesWithSchema :: Schema -> BS.ByteString -> IO DataFrame +fastReadCsvFromBytesWithSchema schema = + readSeparatedFromBytes + comma + defaultReadOptions + { typeSpec = + SpecifyTypes (M.toList (elements schema)) (typeSpec defaultReadOptions) + } + fastReadCsvProj :: [Text] -> FilePath -> IO DataFrame fastReadCsvProj projection path = do df <- fastReadCsv path @@ -153,6 +173,39 @@ WriteCopy Nothing let mutableFile = unsafeFromForeignPtr bufferPtr offset len + readSeparatedFromMVec separator opts mutableFile len + +{- | Identical to 'readSeparated' but takes an already-loaded byte buffer +(e.g. a slice of a memory-mapped file). The bytes are copied once into +a fresh mutable storable vector so the SIMD code can grow it by 64 +bytes of trailing padding without disturbing the caller's input. +-} +readSeparatedFromBytes :: + Word8 -> + ReadOptions -> + BS.ByteString -> + IO DataFrame +readSeparatedFromBytes separator opts bs = do + let len = BS.length bs + mvec <- VSM.unsafeNew len + -- Copy ByteString → mutable vector (one pass, ~1 ns/byte). + let (fp, off, _) = BSI.toForeignPtr bs + Foreign.withForeignPtr fp $ \srcPtr -> + VSM.unsafeWith mvec $ \dstPtr -> + copyBytes (castPtr dstPtr) (srcPtr `Foreign.plusPtr` off) len + readSeparatedFromMVec separator opts mvec len + +{- | Shared core: takes a mutable storable vector containing exactly @len@ +bytes of CSV (no trailing padding required by the caller — it is added +here) and produces a 'DataFrame'. +-} +readSeparatedFromMVec :: + Word8 -> + ReadOptions -> + VSM.IOVector Word8 -> + Int -> + IO DataFrame +readSeparatedFromMVec separator opts mutableFile len = do paddedMutableFile <- grow mutableFile 64 paddedCSVFileRaw <- VS.unsafeFreeze paddedMutableFile let (paddedCSVFile, bomLen) = stripBom paddedCSVFileRaw @@ -224,21 +277,21 @@ parseTypes (columnNames Vector.! col) $ Vector.generate numRow $ \i -> extractAt (dataRows VS.! i) col - columns = + cols = Vector.fromListN numCol ( map generateColumn [0 .. numCol - 1] `using` parList rpar ) - columnIndices = + colIndices = M.fromList $ zip (Vector.toList columnNames) [0 ..] - dataframeDimensions = (numRow, numCol) + dfDims = (numRow, numCol) let rawDf = DataFrame - columns - columnIndices - dataframeDimensions + cols + colIndices + dfDims M.empty schemaMap = schemaTypeMap (typeSpec opts) resolveMode = @@ -284,8 +337,8 @@ -} {-# INLINE classifyRowEnds #-} classifyRowEnds :: VS.Vector Word8 -> Int -> VS.Vector CSize -> VS.Vector Int -classifyRowEnds file contentLen delimiters = - VS.findIndices isRowBreak delimiters +classifyRowEnds file contentLen = + VS.findIndices isRowBreak where isRowBreak pos = let p = fromIntegral pos :: Int diff --git a/tests/Main.hs b/tests/Main.hs --- a/tests/Main.hs +++ b/tests/Main.hs @@ -1,9 +1,11 @@ +{-# LANGUAGE ScopedTypeVariables #-} + module Main where import qualified System.Exit as Exit import Test.HUnit -import Test.QuickCheck (Result, isSuccess, quickCheckWithResult, stdArgs) +import Test.QuickCheck import qualified Operations.ReadCsv import qualified Properties.Csv @@ -11,8 +13,9 @@ tests :: Test tests = TestList Operations.ReadCsv.tests -allSuccessful :: [Result] -> Bool -allSuccessful = all isSuccess +isSuccessful :: Result -> Bool +isSuccessful (Success{}) = True +isSuccessful _ = False main :: IO () main = do @@ -20,8 +23,7 @@ if failures result > 0 || errors result > 0 then Exit.exitFailure else do - propResults <- - mapM (quickCheckWithResult stdArgs) Properties.Csv.tests - if allSuccessful propResults - then Exit.exitSuccess - else Exit.exitFailure + propsRes <- mapM (quickCheckWithResult stdArgs) Properties.Csv.tests + if not (all isSuccessful propsRes) + then Exit.exitFailure + else Exit.exitSuccess diff --git a/tests/Operations/ReadCsv.hs b/tests/Operations/ReadCsv.hs --- a/tests/Operations/ReadCsv.hs +++ b/tests/Operations/ReadCsv.hs @@ -448,6 +448,9 @@ removeFile path case (result :: Either CsvParseError DataFrame) of Left CsvUnclosedQuote -> return () + Left other -> + assertFailure + ("expected CsvUnclosedQuote, got " <> show other) Right _ -> assertFailure "readCsvFast should have thrown CsvUnclosedQuote on input with a stray quote" diff --git a/tests/Properties/Csv.hs b/tests/Properties/Csv.hs --- a/tests/Properties/Csv.hs +++ b/tests/Properties/Csv.hs @@ -1,5 +1,4 @@ {-# LANGUAGE GADTs #-} -{-# LANGUAGE MonoLocalBinds #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} @@ -39,9 +38,10 @@ import Test.QuickCheck.Monadic (PropertyM, assert, monadicIO, run) import Type.Reflection (typeRep) --- | A generator for a single CSV cell. Each cell starts with an ASCII --- letter (so type inference always picks 'Text') and may contain a small --- menagerie of special characters the parser must escape correctly. +{- | A generator for a single CSV cell. Each cell starts with an ASCII +letter (so type inference always picks 'Text') and may contain a small +menagerie of special characters the parser must escape correctly. +-} newtype Cell = Cell {unCell :: T.Text} deriving (Eq, Show) @@ -54,10 +54,11 @@ | T.null t || T.length t == 1 = [] | otherwise = [Cell (T.take (T.length t - 1) t)] --- | Like 'Cell' but guaranteed to contain neither @\\n@ nor @\\r@. Used --- by properties that diff across line-ending dialects, where embedded --- line breaks become part of the data under CRLF transform and break --- the otherwise-clean invariant. +{- | Like 'Cell' but guaranteed to contain neither @\\n@ nor @\\r@. Used +by properties that diff across line-ending dialects, where embedded +line breaks become part of the data under CRLF transform and break +the otherwise-clean invariant. +-} newtype SingleLineCell = SingleLineCell {unSingleLineCell :: T.Text} deriving (Eq, Show) @@ -82,9 +83,10 @@ | length ns <= 1 = [] | otherwise = [ColumnNames (init ns)] --- | RFC 4180 cell encoding: wrap a field in @"@ and double embedded --- quotes when the cell contains the separator, a line break, or a --- quote. +{- | RFC 4180 cell encoding: wrap a field in @"@ and double embedded +quotes when the cell contains the separator, a line break, or a +quote. +-} encodeCell :: Char -> T.Text -> T.Text encodeCell sep t | T.any needsQuote t = T.concat ["\"", T.replace "\"" "\"\"" t, "\""] @@ -92,20 +94,22 @@ where needsQuote c = c == sep || c == '\n' || c == '\r' || c == '"' --- | Encode a grid @[[cell]]@ (rows of cells) as an RFC 4180 CSV. The --- first row is written as the header, followed by one line per data --- row. Always uses @\\n@ as the line terminator; callers that need --- CRLF can transform the output themselves. +{- | Encode a grid @[[cell]]@ (rows of cells) as an RFC 4180 CSV. The +first row is written as the header, followed by one line per data +row. Always uses @\\n@ as the line terminator; callers that need +CRLF can transform the output themselves. +-} encodeCsv :: Char -> [T.Text] -> [[T.Text]] -> T.Text encodeCsv sep header rows = T.unlines (encodeRow header : map encodeRow rows) where encodeRow = T.intercalate (T.singleton sep) . map (encodeCell sep) --- | Pull the raw 'Text' values of the named column out of a DataFrame. --- Fails the property if the column is missing or is not a 'Text' --- column (which would signal a type-inference accident — the cell --- generator is designed to avoid that). +{- | Pull the raw 'Text' values of the named column out of a DataFrame. +Fails the property if the column is missing or is not a 'Text' +column (which would signal a type-inference accident — the cell +generator is designed to avoid that). +-} columnAsText :: T.Text -> DataFrame -> Maybe [T.Text] columnAsText name df = do idx <- M.lookup name (columnIndices df) @@ -117,8 +121,9 @@ Nothing -> Nothing UnboxedColumn{} -> Nothing --- | Run a property in @IO@ against a generated CSV text, cleaning up --- the temp file afterwards no matter what. +{- | Run a property in @IO@ against a generated CSV text, cleaning up +the temp file afterwards no matter what. +-} withCsvFile :: String -> T.Text -> (FilePath -> IO a) -> IO a withCsvFile label body action = do let path = "/tmp/fastcsv_prop_" <> label <> ".csv" @@ -130,8 +135,9 @@ -------------------------------------------------------------------------------- -- Properties --- | Any CSV parses the same whether or not a UTF-8 BOM (EF BB BF) is --- prepended. Excel / PowerShell CSV exports all come with a BOM. +{- | Any CSV parses the same whether or not a UTF-8 BOM (EF BB BF) is +prepended. Excel / PowerShell CSV exports all come with a BOM. +-} prop_bom_invariant :: ColumnNames -> [[Cell]] -> Property prop_bom_invariant (ColumnNames header) rows = monadicIO $ do let cellRows = map (map unCell) rows @@ -142,11 +148,12 @@ assert (plain == withBom) assert (dataframeDimensions plain == dataframeDimensions withBom) --- | A CSV with no embedded newlines inside quoted fields parses the --- same whether record separators are @\\n@ or @\\r\\n@. When quoted --- cells themselves contain @\\n@, a CRLF-encoded file contains a literal --- @\\r\\n@ inside the quotes (that's data, not line ending), so the --- invariant correctly no longer holds; we filter such inputs out. +{- | A CSV with no embedded newlines inside quoted fields parses the +same whether record separators are @\\n@ or @\\r\\n@. When quoted +cells themselves contain @\\n@, a CRLF-encoded file contains a literal +@\\r\\n@ inside the quotes (that's data, not line ending), so the +invariant correctly no longer holds; we filter such inputs out. +-} prop_crlf_invariant :: ColumnNames -> [[SingleLineCell]] -> Property prop_crlf_invariant (ColumnNames header) rows = monadicIO $ do let cellRows = map (map unSingleLineCell) rows @@ -156,10 +163,11 @@ dfCrlf <- run $ withCsvFile "crlf_crlf" crlfCsv D.fastReadCsv assert (dfLf == dfCrlf) --- | A DataFrame produced by the RFC 4180 encoder round-trips through --- the fast reader: column names survive, row count survives, and every --- cell comes back bit-for-bit. Restricted to ASCII-letter-prefixed --- cells so that type inference always stays on 'Text'. +{- | A DataFrame produced by the RFC 4180 encoder round-trips through +the fast reader: column names survive, row count survives, and every +cell comes back bit-for-bit. Restricted to ASCII-letter-prefixed +cells so that type inference always stays on 'Text'. +-} prop_roundtrip_ascii :: Property prop_roundtrip_ascii = forAll (chooseInt (0, 12)) $ \nRows -> forAll (chooseInt (1, 4)) $ \nCols -> @@ -185,10 +193,11 @@ ) (zip header expected) --- | A file that ends with an unmatched @"@ must raise 'CsvUnclosedQuote' --- under the default policy. The generator deliberately builds a valid --- prefix so the property is checking the error path, not a random --- parse failure. +{- | A file that ends with an unmatched @"@ must raise 'CsvUnclosedQuote' +under the default policy. The generator deliberately builds a valid +prefix so the property is checking the error path, not a random +parse failure. +-} prop_unclosed_quote_throws :: Property prop_unclosed_quote_throws = forAll (listOf1 arbitrary) $ \(cells :: [Cell]) -> monadicIO $ do @@ -205,10 +214,11 @@ Left CsvUnclosedQuote -> assert True _ -> assert False --- | Files whose length sits exactly at a SIMD chunk boundary must parse --- without an off-by-one. We generate inputs at the key critical sizes --- (64, 127, 128, 129, 192 bytes after the 'v\n' header) by padding a --- single column with ASCII letters. +{- | Files whose length sits exactly at a SIMD chunk boundary must parse +without an off-by-one. We generate inputs at the key critical sizes +(64, 127, 128, 129, 192 bytes after the 'v\n' header) by padding a +single column with ASCII letters. +-} prop_boundary_sizes :: Property prop_boundary_sizes = forAll (elements [64, 127, 128, 129, 192]) $ \targetBytes -> diff --git a/tests/data/unstable_csv/either_read_mixed.csv b/tests/data/unstable_csv/either_read_mixed.csv new file mode 100644 --- /dev/null +++ b/tests/data/unstable_csv/either_read_mixed.csv @@ -0,0 +1,9 @@ +id,score,name +1,10,alice +2,20,bob +3,30,carol +4,40,dave +5,50,eve +6,abc,frank +7,,grace +8,80,N/A