packages feed

dataframe-csv 2.3.1.0 → 2.3.1.1

raw patch · 9 files changed

+78/−459 lines, 9 filesdep ~dataframe-coredep ~dataframe-operations

Dependency ranges changed: dataframe-core, dataframe-operations

Files

dataframe-csv.cabal view
@@ -1,6 +1,6 @@ cabal-version:      3.4 name:               dataframe-csv-version:            2.3.1.0+version:            2.3.1.1 synopsis:           CSV reader and writer for the dataframe ecosystem. description:     @DataFrame.IO.CSV@ — strict single-pass CSV read/write (pure@@ -39,8 +39,8 @@     build-depends:      base >= 4 && < 5,                         bytestring >= 0.11 && < 0.14,                         containers >= 0.6.7 && < 0.10,-                        dataframe-core >= 2.5 && < 2.6,-                        dataframe-operations >= 2.5 && < 2.6,+                        dataframe-core >= 2.6 && < 2.7,+                        dataframe-operations >= 2.6 && < 2.7,                         dataframe-parsing >= 2.2 && < 2.3,                         text >= 2.1 && < 3,                         time >= 1.12 && < 2,
src-internal/DataFrame/IO/Internal/MutableColumn.hs view
@@ -4,11 +4,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- |-Mutable-column ingest helpers used by CSV readers. Kept out of-@DataFrame.Internal.Column@ so that the core column type does not need to-depend on @DataFrame.Internal.Parsing@.--}+-- | Mutable-column ingest helpers used by the CSV readers. module DataFrame.IO.Internal.MutableColumn (     writeColumn,     freezeColumn',
src/DataFrame/IO/CSV.hs view
@@ -1,10 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | CSV reading and writing for dataframes. A strict, single-pass-RFC 4180 scanner (cassava-compatible) parses fields into typed column-builders; ragged rows are padded with nulls and extra fields dropped.--}+-- | Strict single-pass RFC 4180 CSV reading and writing for dataframes. module DataFrame.IO.CSV (     -- * Reading     readCsv,@@ -46,43 +43,16 @@ import DataFrame.Internal.DataFrame (DataFrame (..), toSeparated) import DataFrame.Schema (Schema, elements) -{- | Read CSV file from path and load it into a dataframe.--==== __Example__-@-ghci> D.readCsv ".\/data\/taxi.csv"--@--}+-- | Read a CSV file from a path into a dataframe. readCsv :: FilePath -> IO DataFrame readCsv = readSeparated defaultReadOptions -{- | A reader the lazy scan can drive: 'readSeparated' here, or-@fastReadCsvWithOpts@ from @dataframe-fastcsv@. The scan owns the-'ReadOptions', so every reader it accepts honours the projection it asks-for.--==== __Example__-@-myReader :: CsvReader-myReader opts path = D.readCsvWithOpts opts path-@--}+-- | A reader the lazy scan can drive, such as 'readSeparated'. type CsvReader = ReadOptions -> FilePath -> IO DataFrame  type CsvBytesReader = ReadOptions -> BS.ByteString -> IO DataFrame -{- | Read options a scan derives from its 'Schema': the schema assigns the-column types /and/ selects the columns, so a scan reads only what its-schema names — the same contract as @scanParquet@.--==== __Example__-@-ghci> schema = D.makeSchema [("id", D.schemaType \@Int), ("name", D.schemaType \@Text)]-ghci> D.readCsvWithOpts (schemaReadOptions schema) ".\/data\/customers.csv"--@--}+-- | Read options derived from a 'Schema', which sets types and columns. schemaReadOptions :: Schema -> ReadOptions schemaReadOptions schema =     defaultReadOptions@@ -91,16 +61,7 @@         , readColumns = Just (M.keys (elements schema))         } -{- | Schema-driven CSV reader. Coerces each column to the type declared-in 'Schema'; columns absent from the schema fall back to inference and are-still returned. To read /only/ the schema's columns, pass-'schemaReadOptions' to 'readCsvWithOpts'.--@-import qualified DataFrame as D-df <- D.readCsvWithSchema schema "input.csv"-@--}+-- | Read a CSV, coercing columns to the types declared in a 'Schema'. readCsvWithSchema :: Schema -> FilePath -> IO DataFrame readCsvWithSchema schema =     readSeparated@@ -111,36 +72,15 @@                     (typeSpec defaultReadOptions)             } -{- | Read CSV file from path and load it into a dataframe.--==== __Example__-@-ghci> D.readCsvWithOpts ".\/data\/taxi.csv" (D.defaultReadOptions { dateFormat = "%d/%-m/%-Y" })--@--}+-- | Read a CSV file from a path using the given 'ReadOptions'. readCsvWithOpts :: ReadOptions -> FilePath -> IO DataFrame readCsvWithOpts = readSeparated -{- | Read TSV (tab separated) file from path and load it into a dataframe.--==== __Example__-@-ghci> D.readTsv ".\/data\/taxi.tsv"--@--}+-- | Read a tab-separated file into a dataframe. readTsv :: FilePath -> IO DataFrame readTsv = readSeparated (defaultReadOptions{columnSeparator = '\t'}) -{- | Read text file with specified delimiter into a dataframe.--==== __Example__-@-ghci> D.readSeparated (D.defaultReadOptions { columnSeparator = ';' }) ".\/data\/taxi.txt"--@--}+-- | Read a text file with a chosen delimiter into a dataframe. readSeparated :: ReadOptions -> FilePath -> IO DataFrame readSeparated opts path = do     validateReadOptions opts@@ -148,52 +88,23 @@     csvData <- stripUtf8Bom <$> BS.readFile path     decodeSeparatedStrict opts csvData -{- | Decode in-memory CSV bytes into a dataframe. The result is fully-forced. (Note: unlike 'readSeparated', no UTF-8 BOM is stripped.)--==== __Example__-@-ghci> D.decodeSeparated D.defaultReadOptions "id,name\\n1,Ada\\n"--@--}+-- | Decode in-memory CSV bytes into a fully forced dataframe. decodeSeparated :: ReadOptions -> BL.ByteString -> IO DataFrame decodeSeparated opts csvData = decodeSeparatedStrict opts (BL.toStrict csvData) -{- | Decode a strict in-memory CSV buffer into a dataframe. The result is-fully forced. As with 'decodeSeparated', this function does not strip a UTF-8-BOM; callers reading the start of a file should strip it first.--}+-- | Decode a strict in-memory CSV buffer into a fully forced dataframe. decodeSeparatedStrict :: CsvBytesReader decodeSeparatedStrict = decodeCsvStrict -{- | Write a dataframe to a comma-separated file.--==== __Example__-@-ghci> D.writeCsv ".\/out.csv" df-@--}+-- | Write a dataframe to a comma-separated file. writeCsv :: FilePath -> DataFrame -> IO () writeCsv = writeSeparated ',' -{- | Write a dataframe to a tab-separated file.--==== __Example__-@-ghci> D.writeTsv ".\/out.tsv" df-@--}+-- | Write a dataframe to a tab-separated file. writeTsv :: FilePath -> DataFrame -> IO () writeTsv = writeSeparated '\t' -{- | Write a dataframe to a file using the given field separator.--==== __Example__-@-ghci> D.writeSeparated ';' ".\/out.txt" df-@--}+-- | Write a dataframe to a file using the given field separator. writeSeparated ::     -- | Separator     Char ->@@ -203,41 +114,18 @@     IO () writeSeparated c filepath df = TIO.writeFile filepath (toSeparated c df) -{- | Parse a CSV string into a DataFrame using default options.--==== __Example__-@-ghci> D.fromCsv "id,name\\n1,Ada\\n"-Right (DataFrame ...)--@--}+-- | Parse a CSV string into a DataFrame using default options. fromCsv :: String -> IO (Either String DataFrame) fromCsv s = do     let bs = BL.fromStrict (TE.encodeUtf8 (T.pack s))     (Right <$> decodeSeparated defaultReadOptions bs)         `catch` (\(e :: SomeException) -> pure (Left (show e))) -{- | Parse a lazy 'ByteString' containing CSV data into a DataFrame using-default options.--==== __Example__-@-ghci> D.fromCsvBytes "id,name\\n1,Ada\\n"--@--}+-- | Parse a lazy 'ByteString' of CSV data using default options. fromCsvBytes :: BL.ByteString -> IO DataFrame fromCsvBytes = decodeSeparated defaultReadOptions -{- | Strip one layer of double-quote delimiters from a field, if present.--==== __Example__->>> stripQuotes "\"hello\""-"hello"->>> stripQuotes "hello"-"hello"--}+-- | Strip one layer of double-quote delimiters from a field, if present. stripQuotes :: T.Text -> T.Text stripQuotes txt =     case T.uncons txt of
src/DataFrame/IO/CSV/Internal/Infer.hs view
@@ -3,13 +3,7 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | Column inference over raw field bytes for the default CSV reader:-classify a bounded sample with the shared lattice-("DataFrame.Operations.Inference"), then build the column in one fused-pass with in-place Int -> Double promotion (audit T2). Cells arrive as-'Cells' — unboxed offsets into the input buffer plus a builder validity-vector — so the passes parse slices without materializing cell values.--}+-- | Column inference over raw CSV field bytes for the default reader. module DataFrame.IO.CSV.Internal.Infer (     inferColumnFromBS, ) where@@ -65,8 +59,7 @@ inferColumnFromBS mode opts cells =     let sampleN = let n = typeInferenceSampleSize (typeSpec opts) in if n == 0 then 100 else n         dfmt = dateFormat opts-        -- The sample IS still Maybe-wrapped; it's bounded at 100 rows-        -- by default, so the allocation is ignorable.+        -- The sample is Maybe-wrapped but capped at 100 rows by default.         samples = V.generate (min sampleN (cLen cells)) $ \i ->             if validAt cells i then Just (cellAt cells i) else Nothing         assumption = makeParsingAssumptionBytes dfmt samples@@ -85,27 +78,20 @@ validAt cells i = VU.unsafeIndex (cValid cells) i /= 0 {-# INLINE validAt #-} -{- | @%Y-%m-%d@ takes the WS-B slice fast path; custom formats keep the-reference parser over a materialized cell.--}+-- | Fast slice path for @%Y-%m-%d@; other formats use the reference parser. dateSliceParser :: String -> Cells -> Int -> Maybe Day dateSliceParser "%Y-%m-%d" cells i = withCell cells i parseDateFieldSlice dateSliceParser fmt cells i = readByteStringDate fmt (cellAt cells i) {-# INLINE dateSliceParser #-} -{- | 'EitherRead' wrap for the ByteString inference path: produce an-@Either Text a@ column. Raw input bytes are preserved verbatim; rows that were-marked invalid by the builder (e.g. empty cells before EitherRead disabled-missing-indicator detection) become @Left \"\"@.--}+-- | Wrap a ByteString-inferred column as @Either Text a@, keeping raw bytes. handleBSEither :: String -> ParsingAssumption -> Cells -> Column handleBSEither dfmt assumption cells = case assumption of     BoolAssumption -> wrap readByteStringBool     IntAssumption -> wrap readByteStringInt     DoubleAssumption -> wrap readByteStringDouble     DateAssumption -> wrap (readByteStringDate dfmt)-    -- Text / No assumption: column is Either Text Text; empty cells become-    -- Left "" to keep the "Left means missing/failure" convention.+    -- Empty cells become Left "" ("Left means missing/failure").     TextAssumption -> fromVector (V.generate (cLen cells) textEither)     NoAssumption -> fromVector (V.generate (cLen cells) textEither)   where@@ -133,11 +119,7 @@     uncurry UnboxedColumn         <$> parseUnboxedCells False (\i -> withCell cells i parseBoolFieldSlice) cells -{- | Int columns: one fused pass with in-place Int -> Double promotion-('promoteIntColumnIndexed', audit T2) instead of the full-column Double-re-parse; both parsers are the WS-B slice parsers (bit-exact with the-old @readByteString*@ pair).--}+-- | Int columns: one fused pass with in-place Int to Double promotion. handleBSInt :: Cells -> Maybe Column handleBSInt cells =     promoteIntColumnIndexed@@ -151,9 +133,7 @@     uncurry UnboxedColumn         <$> parseUnboxedCells 0 (\i -> withCell cells i parseDoubleFieldSlice) cells --- Dates are boxed ('Day' isn't 'VU.Unbox'), so fuse into a V.Vector--- (Maybe Day) in one pass.  Bails on the first non-null cell that--- fails to parse.+-- Dates are boxed, so fuse into a V.Vector (Maybe Day); bails on a bad cell. handleBSDate :: String -> Cells -> Maybe Column handleBSDate dfmt cells =     case parseBoxedMaybeCells (dateSliceParser dfmt cells) cells of@@ -162,10 +142,7 @@             | otherwise -> Just (fromVector (V.mapMaybe id out))         Nothing -> Nothing -{- | Text columns build through the WS-C 'TextBuilder': one shared byte-array + offsets, 'Data.Text.Text' values sliced off it at freeze (UTF-8-validated once, lenient per-field decode only when invalid).--}+-- | Text columns build through a shared-array 'TextBuilder'. textColumn :: Cells -> IO Column textColumn cells = do     let n = cLen cells@@ -206,8 +183,7 @@         pure (UnboxedColumn mbm out)     | otherwise = maybe (textColumn cells) pure (handleBSDate dfmt cells) --- Boxed counterpart to 'parseUnboxedCells' for types that aren't--- 'VU.Unbox' (e.g. 'Day').  Same one-pass + early-bail shape.+-- Boxed counterpart to 'parseUnboxedCells' for non-Unbox types (e.g. 'Day'). parseBoxedMaybeCells ::     (Int -> Maybe a) -> Cells -> Maybe (Bool, V.Vector (Maybe a)) parseBoxedMaybeCells parser cells = runST $ do@@ -227,11 +203,7 @@                 Nothing -> return Nothing     loop 0 False -{- | One-pass fused parse into a typed unboxed column, reading-nullability from the builder's validity vector. Returns @Just@ only-when every non-null cell parses; the first unparseable cell-short-circuits so the caller can fall back to the next assumption.--}+-- | Fused parse into a typed unboxed column; Just only if every cell parses. parseUnboxedCells ::     forall a.     (VU.Unbox a) =>
src/DataFrame/IO/CSV/Internal/Options.hs view
@@ -1,9 +1,6 @@ {-# LANGUAGE OverloadedStrings #-} -{- | Configuration types for the CSV readers. Re-exported through-"DataFrame.IO.CSV"; shared by the default reader, @dataframe-fastcsv@-and @dataframe-lazy@.--}+-- | Configuration types for the CSV readers. module DataFrame.IO.CSV.Internal.Options (     HeaderSpec (..),     TypeSpec (..),@@ -30,14 +27,7 @@ import DataFrame.Operations.Typing (SafeReadMode (..)) import DataFrame.Schema (SchemaType) -{- | Where a reader's column names come from.--==== __Example__-@-ghci> D.readCsvWithOpts D.defaultReadOptions{D.headerSpec = D.ProvideNames ["a", "b"]} "no_header.csv"--@--}+-- | Where a reader's column names come from. data HeaderSpec     = NoHeader     | -- | Take names from the first row (the default).@@ -46,42 +36,22 @@       ProvideNames [T.Text]     deriving (Eq, Show) -{- | How a reader decides each column's type.--==== __Example__-@-ghci> D.readCsvWithOpts D.defaultReadOptions{D.typeSpec = D.InferFromSample 500} "wide.csv"--@--}+-- | How a reader decides each column's type. data TypeSpec     = -- | Infer each column's type from the first @n@ rows.       InferFromSample Int-    | {- | Pin the listed columns to their given types; fall back to the-      nested 'TypeSpec' for the rest.-      -}+    | -- | Pin the listed columns to their given types, with a fallback.       SpecifyTypes [(T.Text, SchemaType)] TypeSpec     | -- | Every column is read as 'Data.Text.Text', no inference at all.       NoInference -{- | How the fast reader should treat a row whose field count does not-match the header row.  Only consulted by @dataframe-fastcsv@; the pure-Haskell reader always pads short rows with nulls and drops extras.--}+-- | How the fast reader treats a row with the wrong field count. data RaggedRowPolicy-    = {- | Fill missing cells with nulls; silently drop extras.-      Matches pandas / polars lenient defaults.-      -}+    = -- | Fill missing cells with nulls and silently drop extras.       PadWithNull-    | {- | Fill missing cells with nulls; silently drop extras.  Alias-      kept for ergonomic naming when the caller only cares about the-      "don't raise, just forget the extras" half of 'PadWithNull'.-      -}+    | -- | Alias of 'PadWithNull' for callers who only care about extras.       Truncate-    | {- | Raise a 'DataFrame.IO.CSV.Fast.CsvParseError' on any row whose-      field count differs from the header.  Matches polars' strict-      schema-bound mode.-      -}+    | -- | Raise a 'CsvParseError' on any row whose field count differs.       RaiseOnRagged     deriving (Eq, Show) @@ -89,9 +59,7 @@ data UnclosedQuotePolicy     = -- | Raise a 'DataFrame.IO.CSV.Fast.CsvParseError'.  Default.       RaiseOnUnclosedQuote-    | {- | Return whatever rows were parsed before the stray quote; the-      remainder is silently dropped.-      -}+    | -- | Return the rows parsed before the stray quote; drop the rest.       BestEffort     deriving (Eq, Show) @@ -102,96 +70,39 @@     , typeSpec :: TypeSpec     -- ^ Whether/how to infer types. (default: InferFromSample 100)     , safeRead :: SafeReadMode-    {- ^ Default 'SafeReadMode' for columns without an entry in-    'safeReadOverrides'. (default: 'NoSafeRead')-    -}+    -- ^ Default 'SafeReadMode' for columns without an override.     , safeReadOverrides :: [(T.Text, SafeReadMode)]     -- ^ Per-column 'SafeReadMode' overrides; takes precedence over 'safeRead'.     , dateFormat :: String-    {- ^ Format of date fields as recognized by the Data.Time.Format module.--    __Examples:__--    @-    > parseTimeM True defaultTimeLocale "%Y/%-m/%-d" "2010/3/04" :: Maybe Day-    Just 2010-03-04-    > parseTimeM True defaultTimeLocale "%d/%-m/%-Y" "04/3/2010" :: Maybe Day-    Just 2010-03-04-    @-    -}+    -- ^ Format of date fields, as recognized by Data.Time.Format.     , columnSeparator :: Char     -- ^ Character that separates column values.     , numRowsToRead :: Maybe Int-    {- ^ Maximum number of data rows to read ('Nothing' = all).--    __Example:__--    @-    ghci> D.readCsvWithOpts D.defaultReadOptions{D.numRowsToRead = Just 5} "big.csv"-    @-    -}+    -- ^ Maximum number of data rows to read ('Nothing' = all).     , readColumns :: Maybe [T.Text]-    {- ^ Columns to read ('Nothing' = all). Unselected columns are never-    decoded, parsed or allocated; the returned frame carries the selected-    columns in the order given here. Naming a column the file does not have-    raises a 'ColumnsNotFoundException'.--    __Example:__--    @-    ghci> D.readCsvWithOpts D.defaultReadOptions{D.readColumns = Just ["id", "name"]} "customers.csv"-    @-    -}+    -- ^ Columns to read ('Nothing' = all); unselected ones are never decoded.     , missingIndicators :: [T.Text]     -- ^ Values that should be read as `Nothing`.     , fastCsvOnRaggedRow :: RaggedRowPolicy-    {- ^ @dataframe-fastcsv@: how to treat rows with a non-header field count.-    (default: 'PadWithNull')-    -}+    -- ^ @dataframe-fastcsv@: how to treat rows with a non-header field count.     , fastCsvOnUnclosedQuote :: UnclosedQuotePolicy-    {- ^ @dataframe-fastcsv@: how to treat an unclosed quoted field at EOF.-    (default: 'RaiseOnUnclosedQuote')-    -}+    -- ^ @dataframe-fastcsv@: how to treat an unclosed quoted field at EOF.     , fastCsvTrimUnquoted :: Bool-    {- ^ @dataframe-fastcsv@: if 'True', leading/trailing whitespace is-    stripped from unquoted fields after decoding.  RFC 4180 preserves-    this whitespace, and that is the default ('False').-    -}+    -- ^ @dataframe-fastcsv@: strip whitespace from unquoted fields if True.     } -{- | Whether a 'TypeSpec' still infers types from a sample, once any-'SpecifyTypes' fallback chain is followed to its end.--==== __Example__->>> shouldInferFromSample (InferFromSample 100)-True->>> shouldInferFromSample NoInference-False--}+-- | Whether a 'TypeSpec' still infers types from a sample. shouldInferFromSample :: TypeSpec -> Bool shouldInferFromSample (InferFromSample _) = True shouldInferFromSample (SpecifyTypes _ fallback) = shouldInferFromSample fallback shouldInferFromSample _ = False -{- | The explicit column\/type pairs a 'TypeSpec' declares, if any.--==== __Example__->>> :set -XTypeApplications->>> M.toList (schemaTypeMap (SpecifyTypes [("id", schemaType @Int)] (InferFromSample 100)))-[("id",Int)]--}+-- | The explicit column/type pairs a 'TypeSpec' declares, if any. schemaTypeMap :: TypeSpec -> M.Map T.Text SchemaType schemaTypeMap (SpecifyTypes xs _) = M.fromList xs schemaTypeMap _ = M.empty -{- | A 'ReadOptions' value that contradicts itself, with every problem listed.-Thrown by 'validateReadOptions'; see 'readOptionsErrors' for the check-without the exception.--==== __Example__->>> show (InvalidReadOptions ["readColumns is an empty selection; omit it to read every column"])-"Invalid ReadOptions:\n  - readColumns is an empty selection; omit it to read every column"--}+-- | A 'ReadOptions' value that contradicts itself, listing every problem. newtype InvalidReadOptions = InvalidReadOptions [T.Text]  instance Show InvalidReadOptions where@@ -201,16 +112,7 @@  instance Exception InvalidReadOptions -{- | Everything self-contradictory about @opts@, judged without looking at a-file. A type or safe-read override naming an unread column is redundant, not-contradictory, and is not reported.--==== __Example__->>> readOptionsErrors defaultReadOptions{readColumns = Just []}-["readColumns is an empty selection; omit it to read every column"]->>> readOptionsErrors defaultReadOptions-[]--}+-- | Everything self-contradictory about @opts@, judged without a file. readOptionsErrors :: ReadOptions -> [T.Text] readOptionsErrors opts =     concat@@ -237,32 +139,14 @@   where     dups = maybe [] (\cs -> L.nub (cs L.\\ L.nub cs)) (readColumns opts) -{- | Throw 'InvalidReadOptions' if @opts@ contradicts itself. Both readers-call this before opening the file.--==== __Example__-@-ghci> D.validateReadOptions D.defaultReadOptions{D.readColumns = Just []}-*** Exception: Invalid ReadOptions:-  - readColumns is an empty selection; omit it to read every column-@--}+-- | Throw 'InvalidReadOptions' if @opts@ contradicts itself. validateReadOptions :: ReadOptions -> IO () validateReadOptions opts =     unless (null problems) (throwIO (InvalidReadOptions problems))   where     problems = readOptionsErrors opts -{- | Resolve 'readColumns' against a file's header names: the columns to-build, each paired with its field index within a row. 'Nothing' keeps every-column in file order; a selection keeps the order it was written in.--==== __Example__->>> resolveSelection defaultReadOptions{readColumns = Just ["b", "a"]} ["a", "b", "c"]-[("b",1),("a",0)]->>> resolveSelection defaultReadOptions ["a", "b"]-[("a",0),("b",1)]--}+-- | Resolve 'readColumns' against header names to (name, field index) pairs. resolveSelection :: ReadOptions -> [T.Text] -> [(T.Text, Int)] resolveSelection opts names = case readColumns opts of     Nothing -> indexed@@ -273,29 +157,13 @@     indexed = zip names [0 ..]     fieldIndex = M.fromList indexed -{- | The sample size a 'TypeSpec' infers from, once any 'SpecifyTypes'-fallback chain is followed to its end. @0@ when the chain ends in-'NoInference'.--==== __Example__->>> typeInferenceSampleSize (InferFromSample 100)-100->>> typeInferenceSampleSize NoInference-0--}+-- | The sample size a 'TypeSpec' infers from; 0 if it never infers. typeInferenceSampleSize :: TypeSpec -> Int typeInferenceSampleSize (InferFromSample n) = n typeInferenceSampleSize (SpecifyTypes _ fallback) = typeInferenceSampleSize fallback typeInferenceSampleSize _ = 0 -{- | The default 'ReadOptions': infer types from a 100-row sample, treat the-first row as a header, read every row and every column.--==== __Example__-@-ghci> D.readCsvWithOpts D.defaultReadOptions{D.columnSeparator = ';'} "data.csv"-@--}+-- | The default 'ReadOptions': infer from 100 rows, header row, all data. defaultReadOptions :: ReadOptions defaultReadOptions =     ReadOptions
src/DataFrame/IO/CSV/Internal/Read.hs view
@@ -4,10 +4,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Driver for the default CSV reader: a single strict scan feeding-per-column sinks, with cassava-parity semantics. Ragged rows pad-trailing columns with null, drop extra fields, and force the result.--}+-- | Driver for the default CSV reader: one strict scan feeding column sinks. module DataFrame.IO.CSV.Internal.Read (decodeCsvStrict) where  import qualified Data.ByteString as BS@@ -142,10 +139,7 @@             forceDataFrame                 (parseWithTypes resolveMode (schemaTypeMap (typeSpec opts)) df) -{- | Resolve a column's sink: EitherRead and to-be-inferred columns-collect raw slices; schema'd Int/Double parse straight into builders;-every other schema type builds Text ('parseWithTypes' converts after).--}+-- | Resolve a column's sink from its schema type or inference. newSink ::     ReadOptions -> (T.Text -> SafeReadMode) -> Int -> T.Text -> IO Sink newSink opts resolveMode rowHint name
src/DataFrame/IO/CSV/Internal/Scanner.hs view
@@ -1,12 +1,6 @@ {-# LANGUAGE BangPatterns #-} -{- | RFC 4180 field scanner for the default CSV reader (Round-2 WS-D).-One pass over a strict 'BS.ByteString'; cassava-parity semantics pinned-by the @IO.CsvGolden@ suite: quotes open fields only at the first byte,-a double quote inside an unquoted field is an error, garbage after a-closing quote is an error, an unclosed quote consumes to EOF, and rows-end at @\\n@, @\\r\\n@ or a lone @\\r@.--}+-- | RFC 4180 field scanner for the default CSV reader. module DataFrame.IO.CSV.Internal.Scanner (     Term,     termSep,@@ -44,11 +38,7 @@ parseError off what =     error ("CSV Parse Error: " ++ what ++ " at byte " ++ show off) -{- | Scan one field starting at @pos@ and continue with-@k contentStart contentEnd needsUnescape term nextPos@. Quoted fields-yield the bytes between the outer quotes; @needsUnescape@ is set when-the content contains doubled quotes ('unescapeQuotes' collapses them).--}+-- | Scan one field at @pos@ and pass its content bounds to a continuation. {-# INLINE withField #-} withField ::     BS.ByteString ->@@ -88,25 +78,20 @@                 | b == quote -> parseError i "stray double quote in unquoted field"                 | otherwise -> unquoted (i + 1)     quoted !i !unesc-        | i >= len = k (pos + 1) len unesc termEof len -- unclosed: rest of input+        | i >= len = k (pos + 1) len unesc termEof len -- unclosed quote         | BSU.unsafeIndex bs i == quote =             if i + 1 < len && BSU.unsafeIndex bs (i + 1) == quote                 then quoted (i + 2) True                 else finish (pos + 1) i unesc (i + 1)         | otherwise = quoted (i + 1) unesc -{- | Whether the record starting at @pos@ is a blank record (a single-empty field, quoted or not, then end of row) — cassava drops these.-Assumes @pos < len@. Errors propagate from 'withField'.--}+-- | Whether the record at @pos@ is a blank record, which cassava drops. {-# INLINE isBlankRecordAt #-} isBlankRecordAt :: BS.ByteString -> Int -> Word8 -> Int -> Bool isBlankRecordAt bs !len !sep !pos =     withField bs len sep pos (\cs ce _ term _ -> cs == ce && term /= termSep) -{- | Collapse doubled quotes (@\"\"@ → @\"@) in @[s, e)@ into a fresh strict-'BS.ByteString'. Every quote in the range is the first of a pair.--}+-- | Collapse doubled quotes in a range into a fresh strict ByteString. unescapeQuotes :: BS.ByteString -> Int -> Int -> BS.ByteString unescapeQuotes bs !s !e = BSI.unsafeCreateUptoN (e - s) (go s 0)   where@@ -117,10 +102,7 @@             pokeByteOff (dst :: Ptr Word8) j w             if w == quote then go (i + 2) (j + 1) dst else go (i + 1) (j + 1) dst -{- | Strip with @Data.ByteString.Char8.strip@ semantics (Latin-1-whitespace: HT..CR, space and NBSP 0xA0) and continue with the-stripped range.--}+-- | Strip Latin-1 whitespace like ByteString strip and continue with the range. {-# INLINE withStripC8 #-} withStripC8 :: BS.ByteString -> Int -> Int -> (Int -> Int -> r) -> r withStripC8 bs !s0 !e0 k = k s e@@ -135,10 +117,7 @@         | i > s && isC8Space (BSU.unsafeIndex bs (i - 1)) = skipB (i - 1)         | otherwise = i -{- | Strip ASCII whitespace (the ASCII subset of @Data.Text.strip@) and-continue with the stripped range. Callers must fall back to a full-'Data.Text.strip' when the stripped range still has non-ASCII edges.--}+-- | Strip ASCII whitespace and continue with the stripped range. {-# INLINE withStripAscii #-} withStripAscii :: BS.ByteString -> Int -> Int -> (Int -> Int -> r) -> r withStripAscii bs !s0 !e0 k = k s e
src/DataFrame/IO/CSV/Internal/Sink.hs view
@@ -1,11 +1,6 @@ {-# LANGUAGE BangPatterns #-} -{- | Per-column sinks for the default CSV reader. Typed columns (explicit-schema) parse straight from the input slice into the WS-C builders;-inferred and EitherRead columns record unboxed @(start, end)@ offsets-into the input buffer (plus a cold side map for quote-unescaped cells)-for the inference pass ("DataFrame.IO.CSV.Internal.Infer").--}+-- | Per-column sinks for the default CSV reader. module DataFrame.IO.CSV.Internal.Sink (     Env (..),     MissingMode (..),@@ -41,9 +36,7 @@  ) import Foreign.Ptr (Ptr, castPtr, plusPtr) -{- | How missing tokens are detected (audit D2: canonical list goes-through the WS-B memcmp fast path; custom lists decode and compare).--}+-- | How missing tokens are detected. data MissingMode = MissNone | MissCanonical | MissCustom  -- | Read-time environment shared by every sink.@@ -63,11 +56,7 @@     | SinkText !(TextBuilder RealWorld)     | SinkBS !SliceCol -{- | Growable cell store for inferred columns: per cell a @(start, end)@-offset pair into the input (already Char8-stripped) and a 0\/1 validity-byte. Quote-unescaped cells (cold) live in an 'IM.IntMap' keyed by row;-their offset pair is @(-1, row)@.--}+-- | Growable cell store of offset pairs for inferred columns. data SliceCol = SliceCol     { scOffs :: !(IORef (VUM.IOVector Int))     , scValid :: !(IORef (VUM.IOVector Word8))@@ -83,9 +72,7 @@     , cLen :: !Int     } -{- | Resolve cell @i@ as a slice and continue (no 'BS.ByteString' header-is allocated for the common in-buffer case).--}+-- | Resolve cell @i@ as a slice and continue with it. withCell :: Cells -> Int -> (BS.ByteString -> Int -> Int -> r) -> r withCell c !i k =     let s = VU.unsafeIndex (cOffs c) (2 * i)@@ -147,10 +134,7 @@ sliceBS bs s e = BSU.unsafeTake (e - s) (BSU.unsafeDrop s bs) {-# INLINE sliceBS #-} -{- | Feed one field slice @[s, e)@ of the input into a sink. @unesc@-fields (quoted with embedded @\"\"@) are unescaped into a fresh strict-'BS.ByteString' first (cold path).--}+-- | Feed one field slice of the input into a sink. feedSink :: Env -> Sink -> Int -> Int -> Int -> Bool -> IO () feedSink env sink !row !s !e !unesc = case sink of     SinkBS sc@@ -194,11 +178,7 @@     SinkBS _ -> error "feedTyped: SinkBS handled by feedSink" {-# INLINE feedTyped #-} -{- | Text cells reproduce @T.strip . decodeUtf8Lenient@ plus the-missing-indicator test. Fast path: ASCII-edge slices append raw bytes-(the shared-buffer freeze decodes leniently per field); anything with-non-ASCII edges or a custom missing list goes through real 'T.strip'.--}+-- | Append a text cell, with a fast path for ASCII-edge slices. appendTextCell ::     Env ->     TextBuilder RealWorld ->
src/DataFrame/Typed/IO/CSV.hs view
@@ -2,19 +2,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Typed CSV reading.--The schema is the whole specification of the read: it names the columns to-fetch and gives their types, so these readers touch only the columns @cols@-declares and never infer a type the schema already knows.--@-type Customer = '[ '(\"customer_id\", Int), '(\"customer_name\", Text)]--customers <- readCsv \@Customer \"customers.csv\"  -- reads 2 columns, whatever-                                                 -- else the file holds-@--}+-- | Typed CSV reading driven entirely by the type-level schema. module DataFrame.Typed.IO.CSV (     readCsv,     readCsvWithError,@@ -37,10 +25,7 @@  import qualified Data.Map.Strict as M -{- | Fold a type-level schema into read options: fetch exactly its columns,-and parse them at its types. Anything else the caller asked for is kept,-including a wider 'typeSpec' for columns the schema does not name.--}+-- | Fold a type-level schema into read options: its columns and types. schemaOptions :: forall cols. (RuntimeSchema cols) => ReadOptions -> ReadOptions schemaOptions opts =     opts@@ -50,64 +35,33 @@   where     declared = elements (runtimeSchema @cols) -{- | Read a CSV file into a typed DataFrame, throwing on schema mismatch.-Reads only the columns @cols@ names, typed as @cols@ says.--==== __Example__-@-ghci> type Customer = '[ '(\"id\", Int), '(\"name\", Text)]-ghci> customers <- readCsv \@Customer \"customers.csv\"-@--}+-- | Read a CSV into a typed DataFrame, throwing on schema mismatch. readCsv ::     forall cols.     (KnownSchema cols, RuntimeSchema cols) =>     FilePath -> IO (TypedDataFrame cols) readCsv = readCsvWithOpts @cols defaultReadOptions -{- | Read a CSV file, returning a descriptive error on schema mismatch or a-missing column instead of throwing.--==== __Example__-@-ghci> readCsvWithError \@Customer \"customers.csv\"-Right (TDF ...)-@--}+-- | Read a CSV, returning an error on schema mismatch instead of throwing. readCsvWithError ::     forall cols.     (KnownSchema cols, RuntimeSchema cols) =>     FilePath -> IO (Either T.Text (TypedDataFrame cols)) readCsvWithError path = do-    -- The projection can fail before the freeze does (a column the file does-    -- not have); this reader reports rather than throws either way.+    -- A missing column can fail the projection before the freeze does.     r <- try (CSV.readCsvWithOpts (schemaOptions @cols defaultReadOptions) path)     pure $ case r of         Left (e :: SomeException) -> Left (T.pack (show e))         Right df -> freezeWithError df -{- | Read a tab-separated file into a typed DataFrame, throwing on mismatch.--==== __Example__-@-ghci> customers <- readTsv \@Customer \"customers.tsv\"-@--}+-- | Read a tab-separated file into a typed DataFrame, throwing on mismatch. readTsv ::     forall cols.     (KnownSchema cols, RuntimeSchema cols) =>     FilePath -> IO (TypedDataFrame cols) readTsv = readCsvWithOpts @cols defaultReadOptions{columnSeparator = '\t'} -{- | Read a CSV file with custom options, throwing on schema mismatch. The-schema still supplies the column selection and types; an explicit-'readColumns' takes precedence, and is then checked by the freeze.--==== __Example__-@-ghci> customers <- readCsvWithOpts \@Customer defaultReadOptions{safeRead = MaybeRead} \"customers.csv\"-@--}+-- | Read a CSV with custom options, throwing on schema mismatch. readCsvWithOpts ::     forall cols.     (KnownSchema cols, RuntimeSchema cols) =>@@ -115,22 +69,10 @@ readCsvWithOpts opts path =     CSV.readCsvWithOpts (schemaOptions @cols opts) path >>= freezeOrThrow @cols -{- | Write a typed DataFrame to a CSV file.--==== __Example__-@-ghci> writeCsv \"customers.csv\" customers-@--}+-- | Write a typed DataFrame to a CSV file. writeCsv :: FilePath -> TypedDataFrame cols -> IO () writeCsv path = CSV.writeCsv path . thaw -{- | Write a typed DataFrame to a tab-separated file.--==== __Example__-@-ghci> writeTsv \"customers.tsv\" customers-@--}+-- | Write a typed DataFrame to a tab-separated file. writeTsv :: FilePath -> TypedDataFrame cols -> IO () writeTsv path = CSV.writeTsv path . thaw