diff --git a/dataframe-csv.cabal b/dataframe-csv.cabal --- a/dataframe-csv.cabal +++ b/dataframe-csv.cabal @@ -1,6 +1,6 @@ -cabal-version: 2.4 +cabal-version: 3.4 name: dataframe-csv -version: 1.0.2.0 +version: 2.0.0.0 synopsis: CSV reader and writer for the dataframe ecosystem. description: @DataFrame.IO.CSV@ — strict single-pass CSV read/write (pure @@ -24,24 +24,39 @@ -Wunused-local-binds -Wunused-packages +library internal + import: warnings + visibility: public + exposed-modules: DataFrame.IO.Internal.MutableColumn + build-depends: base >= 4 && < 5, + dataframe-core:internal >= 2.0 && < 2.1, + dataframe-parsing:internal >= 2.0 && < 2.1, + text >= 2.1 && < 3, + vector >= 0.13 && < 0.15 + hs-source-dirs: src-internal + default-language: Haskell2010 + library import: warnings exposed-modules: DataFrame.IO.CSV + DataFrame.Typed.IO.CSV + other-modules: DataFrame.IO.CSV.Internal.Infer DataFrame.IO.CSV.Internal.Options DataFrame.IO.CSV.Internal.Read DataFrame.IO.CSV.Internal.Scanner DataFrame.IO.CSV.Internal.Sink - DataFrame.IO.Internal.MutableColumn build-depends: base >= 4 && < 5, - bytestring >= 0.11 && < 0.13, - containers >= 0.6.7 && < 0.9, - dataframe-core ^>= 1.1, - dataframe-operations ^>= 1.1.1, - dataframe-parsing ^>= 1.0.2, - text >= 2.0 && < 3, + bytestring >= 0.11 && < 0.14, + containers >= 0.6.7 && < 0.10, + dataframe-core >= 2.0 && < 2.1, + dataframe-core:internal >= 2.0 && < 2.1, + dataframe-operations >= 2.0 && < 2.1, + dataframe-parsing >= 2.0 && < 2.1, + dataframe-parsing:internal >= 2.0 && < 2.1, + text >= 2.1 && < 3, time >= 1.12 && < 2, - vector ^>= 0.13 + vector >= 0.13 && < 0.15 hs-source-dirs: src default-language: Haskell2010 diff --git a/src-internal/DataFrame/IO/Internal/MutableColumn.hs b/src-internal/DataFrame/IO/Internal/MutableColumn.hs new file mode 100644 --- /dev/null +++ b/src-internal/DataFrame/IO/Internal/MutableColumn.hs @@ -0,0 +1,89 @@ +{-# LANGUAGE ExplicitNamespaces #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +{- | +Mutable-column ingest helpers used by CSV readers. Kept out of +@DataFrame.Internal.Column@ so that the core column type does not need to +depend on @DataFrame.Internal.Parsing@. +-} +module DataFrame.IO.Internal.MutableColumn ( + writeColumn, + freezeColumn', +) where + +import qualified Data.Text as T +import qualified Data.Vector as VB +import qualified Data.Vector.Mutable as VBM +import qualified Data.Vector.Unboxed as VU +import qualified Data.Vector.Unboxed.Mutable as VUM + +import Data.Maybe (fromMaybe) +import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) +import Type.Reflection (typeRep) + +import DataFrame.Internal.Column ( + Column (..), + MutableColumn (..), + buildBitmapFromNulls, + ) +import DataFrame.Internal.Parsing (isNullish, readDouble, readInt) + +writeColumn :: Int -> T.Text -> MutableColumn -> IO (Either T.Text Bool) +writeColumn i value (MBoxedColumn (col :: VBM.IOVector a)) = + case testEquality (typeRep @a) (typeRep @T.Text) of + Just Refl -> + if isNullish value + then VBM.unsafeWrite col i "" >> return (Left $! value) + else VBM.unsafeWrite col i value >> return (Right True) + Nothing -> return (Left value) +writeColumn i value (MUnboxedColumn (col :: VUM.IOVector a)) = + case testEquality (typeRep @a) (typeRep @Int) of + Just Refl -> case readInt value of + Just v -> VUM.unsafeWrite col i v >> return (Right True) + Nothing -> VUM.unsafeWrite col i 0 >> return (Left value) + Nothing -> case testEquality (typeRep @a) (typeRep @Double) of + Nothing -> return (Left $! value) + Just Refl -> case readDouble value of + Just v -> VUM.unsafeWrite col i v >> return (Right True) + Nothing -> VUM.unsafeWrite col i 0 >> return (Left $! value) +{-# INLINE writeColumn #-} + +freezeColumn' :: [(Int, T.Text)] -> MutableColumn -> IO Column +freezeColumn' nulls (MBoxedColumn col) + | null nulls = BoxedColumn Nothing <$> VB.unsafeFreeze col + | all (isNullish . snd) nulls = do + frozen <- VB.unsafeFreeze col + let n = VB.length frozen + bm = buildBitmapFromNulls n (map fst nulls) + return $ BoxedColumn (Just bm) frozen + | otherwise = + BoxedColumn Nothing + . VB.imap + ( \i v -> + if i `elem` map fst nulls + then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) + else Right v + ) + <$> VB.unsafeFreeze col +freezeColumn' nulls (MUnboxedColumn col) + | null nulls = UnboxedColumn Nothing <$> VU.unsafeFreeze col + | all (isNullish . snd) nulls = do + c <- VU.unsafeFreeze col + let n = VU.length c + bm = buildBitmapFromNulls n (map fst nulls) + return $ UnboxedColumn (Just bm) c + | otherwise = do + c <- VU.unsafeFreeze col + return $ + BoxedColumn Nothing $ + VB.generate + (VU.length c) + ( \i -> + if i `elem` map fst nulls + then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) + else Right (c VU.! i) + ) +{-# INLINE freezeColumn' #-} diff --git a/src/DataFrame/IO/CSV.hs b/src/DataFrame/IO/CSV.hs --- a/src/DataFrame/IO/CSV.hs +++ b/src/DataFrame/IO/CSV.hs @@ -1,11 +1,9 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | CSV reading and writing for dataframes. The reader is a strict, -single-pass RFC 4180 scanner (cassava-compatible semantics, pinned by -the golden suite in @tests/IO/CsvGolden.hs@) that parses fields straight -into typed column builders. Ragged rows are padded with nulls (short -rows) and extra fields dropped. +{- | CSV reading and writing for dataframes. A strict, single-pass +RFC 4180 scanner (cassava-compatible) parses fields into typed column +builders; ragged rows are padded with nulls and extra fields dropped. -} module DataFrame.IO.CSV ( -- * Reading @@ -43,7 +41,7 @@ import DataFrame.IO.CSV.Internal.Options import DataFrame.IO.CSV.Internal.Read (decodeCsvStrict) import DataFrame.Internal.DataFrame (DataFrame (..), toSeparated) -import DataFrame.Internal.Schema (Schema, elements) +import DataFrame.Schema (Schema, elements) {- | Read CSV file from path and load it into a dataframe. @@ -58,10 +56,8 @@ type CsvReader = Schema -> FilePath -> IO DataFrame -{- | Schema-driven CSV reader. Coerces each column to the type declared -in 'Schema'; columns absent from the schema fall back to the default -inference path. Defined in terms of 'readSeparated' with the 'TypeSpec' -filled in. +{- | Schema-driven CSV reader. Coerces each column to the type declared +in 'Schema'; columns absent from the schema fall back to inference. @ import qualified DataFrame as D diff --git a/src/DataFrame/IO/CSV/Internal/Options.hs b/src/DataFrame/IO/CSV/Internal/Options.hs --- a/src/DataFrame/IO/CSV/Internal/Options.hs +++ b/src/DataFrame/IO/CSV/Internal/Options.hs @@ -19,8 +19,8 @@ import qualified Data.Map.Strict as M import qualified Data.Text as T -import DataFrame.Internal.Schema (SchemaType) import DataFrame.Operations.Typing (SafeReadMode (..)) +import DataFrame.Schema (SchemaType) data HeaderSpec = NoHeader | UseFirstRow | ProvideNames [T.Text] deriving (Eq, Show) diff --git a/src/DataFrame/IO/CSV/Internal/Read.hs b/src/DataFrame/IO/CSV/Internal/Read.hs --- a/src/DataFrame/IO/CSV/Internal/Read.hs +++ b/src/DataFrame/IO/CSV/Internal/Read.hs @@ -4,10 +4,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Driver for the default CSV reader (Round-2 WS-D): a single strict -scan feeding per-column sinks, with cassava-parity semantics pinned by -@IO.CsvGolden@. Ragged rows pad trailing columns with null and drop -extra fields (audit D6). The result is fully forced (audit D7). +{- | Driver for the default CSV reader: a single strict scan feeding +per-column sinks, with cassava-parity semantics. Ragged rows pad +trailing columns with null, drop extra fields, and force the result. -} module DataFrame.IO.CSV.Internal.Read (decodeCsvStrict) where @@ -32,12 +31,12 @@ import DataFrame.Internal.Column (Column, ensureOptional) import DataFrame.Internal.ColumnBuilder import DataFrame.Internal.DataFrame (DataFrame (..), forceDataFrame) -import DataFrame.Internal.Schema (SchemaType (..), schemaType) import DataFrame.Operations.Typing ( SafeReadMode (..), effectiveSafeRead, parseWithTypes, ) +import DataFrame.Schema (SchemaType (..), schemaType) import Foreign.Ptr (castPtr) import Type.Reflection (typeRep) @@ -47,8 +46,6 @@ let !len = BS.length bs !sep = fromIntegral (ord (columnSeparator opts)) :: Word8 - -- Position of the next non-blank record at/after @pos@ (cassava - -- drops single-empty-field records); @len@ when none remain. findRecord !pos | pos >= len = len | otherwise = withField bs len sep pos $ \cs ce _ term next -> @@ -56,8 +53,6 @@ then if term == termEof then len else findRecord next else pos - -- Decode the record at @pos@ as header fields; returns the - -- position after the record. headerFields !pos = go pos id where go !p acc = withField bs len sep p $ \cs ce unesc term next -> @@ -82,8 +77,6 @@ when (dataPos >= len) (error "Empty CSV file") let resolveMode = effectiveSafeRead (safeRead opts) (safeReadOverrides opts) - -- If ANY column is EitherRead we keep every raw cell (including - -- "N/A" etc.) verbatim; otherwise the missing-indicator list applies. anyEither = any (\n -> resolveMode n == EitherRead) names missing = if anyEither then [] else missingIndicators opts missMode diff --git a/src/DataFrame/IO/Internal/MutableColumn.hs b/src/DataFrame/IO/Internal/MutableColumn.hs deleted file mode 100644 --- a/src/DataFrame/IO/Internal/MutableColumn.hs +++ /dev/null @@ -1,89 +0,0 @@ -{-# LANGUAGE ExplicitNamespaces #-} -{-# LANGUAGE GADTs #-} -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE ScopedTypeVariables #-} -{-# LANGUAGE TypeApplications #-} - -{- | -Mutable-column ingest helpers used by CSV readers. Kept out of -@DataFrame.Internal.Column@ so that the core column type does not need to -depend on @DataFrame.Internal.Parsing@. --} -module DataFrame.IO.Internal.MutableColumn ( - writeColumn, - freezeColumn', -) where - -import qualified Data.Text as T -import qualified Data.Vector as VB -import qualified Data.Vector.Mutable as VBM -import qualified Data.Vector.Unboxed as VU -import qualified Data.Vector.Unboxed.Mutable as VUM - -import Data.Maybe (fromMaybe) -import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) -import Type.Reflection (typeRep) - -import DataFrame.Internal.Column ( - Column (..), - MutableColumn (..), - buildBitmapFromNulls, - ) -import DataFrame.Internal.Parsing (isNullish, readDouble, readInt) - -writeColumn :: Int -> T.Text -> MutableColumn -> IO (Either T.Text Bool) -writeColumn i value (MBoxedColumn (col :: VBM.IOVector a)) = - case testEquality (typeRep @a) (typeRep @T.Text) of - Just Refl -> - if isNullish value - then VBM.unsafeWrite col i "" >> return (Left $! value) - else VBM.unsafeWrite col i value >> return (Right True) - Nothing -> return (Left value) -writeColumn i value (MUnboxedColumn (col :: VUM.IOVector a)) = - case testEquality (typeRep @a) (typeRep @Int) of - Just Refl -> case readInt value of - Just v -> VUM.unsafeWrite col i v >> return (Right True) - Nothing -> VUM.unsafeWrite col i 0 >> return (Left value) - Nothing -> case testEquality (typeRep @a) (typeRep @Double) of - Nothing -> return (Left $! value) - Just Refl -> case readDouble value of - Just v -> VUM.unsafeWrite col i v >> return (Right True) - Nothing -> VUM.unsafeWrite col i 0 >> return (Left $! value) -{-# INLINE writeColumn #-} - -freezeColumn' :: [(Int, T.Text)] -> MutableColumn -> IO Column -freezeColumn' nulls (MBoxedColumn col) - | null nulls = BoxedColumn Nothing <$> VB.unsafeFreeze col - | all (isNullish . snd) nulls = do - frozen <- VB.unsafeFreeze col - let n = VB.length frozen - bm = buildBitmapFromNulls n (map fst nulls) - return $ BoxedColumn (Just bm) frozen - | otherwise = - BoxedColumn Nothing - . VB.imap - ( \i v -> - if i `elem` map fst nulls - then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) - else Right v - ) - <$> VB.unsafeFreeze col -freezeColumn' nulls (MUnboxedColumn col) - | null nulls = UnboxedColumn Nothing <$> VU.unsafeFreeze col - | all (isNullish . snd) nulls = do - c <- VU.unsafeFreeze col - let n = VU.length c - bm = buildBitmapFromNulls n (map fst nulls) - return $ UnboxedColumn (Just bm) c - | otherwise = do - c <- VU.unsafeFreeze col - return $ - BoxedColumn Nothing $ - VB.generate - (VU.length c) - ( \i -> - if i `elem` map fst nulls - then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls)) - else Right (c VU.! i) - ) -{-# INLINE freezeColumn' #-} diff --git a/src/DataFrame/Typed/IO/CSV.hs b/src/DataFrame/Typed/IO/CSV.hs new file mode 100644 --- /dev/null +++ b/src/DataFrame/Typed/IO/CSV.hs @@ -0,0 +1,51 @@ +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +module DataFrame.Typed.IO.CSV ( + readCsv, + readCsvWithError, + readTsv, + readCsvWithOpts, + writeCsv, + writeTsv, +) where + +import qualified Data.Text as T + +import DataFrame.IO.CSV (ReadOptions) +import qualified DataFrame.IO.CSV as CSV +import DataFrame.Typed.Freeze (freezeOrThrow, freezeWithError, thaw) +import DataFrame.Typed.Schema (KnownSchema) +import DataFrame.Typed.Types (TypedDataFrame) + +-- | Read a CSV file into a typed DataFrame, throwing on schema mismatch. +readCsv :: + forall cols. (KnownSchema cols) => FilePath -> IO (TypedDataFrame cols) +readCsv path = CSV.readCsv path >>= freezeOrThrow @cols + +-- | Read a CSV file, returning a descriptive error on schema mismatch. +readCsvWithError :: + forall cols. + (KnownSchema cols) => + FilePath -> IO (Either T.Text (TypedDataFrame cols)) +readCsvWithError path = freezeWithError <$> CSV.readCsv path + +-- | Read a tab-separated file into a typed DataFrame, throwing on mismatch. +readTsv :: + forall cols. (KnownSchema cols) => FilePath -> IO (TypedDataFrame cols) +readTsv path = CSV.readTsv path >>= freezeOrThrow @cols + +-- | Read a CSV file with custom options, throwing on schema mismatch. +readCsvWithOpts :: + forall cols. + (KnownSchema cols) => + ReadOptions -> FilePath -> IO (TypedDataFrame cols) +readCsvWithOpts opts path = CSV.readCsvWithOpts opts path >>= freezeOrThrow @cols + +-- | Write a typed DataFrame to a CSV file. +writeCsv :: FilePath -> TypedDataFrame cols -> IO () +writeCsv path = CSV.writeCsv path . thaw + +-- | Write a typed DataFrame to a tab-separated file. +writeTsv :: FilePath -> TypedDataFrame cols -> IO () +writeTsv path = CSV.writeTsv path . thaw