packages feed

dataframe-csv 1.0.2.0 → 2.0.0.0

raw patch · 7 files changed

+176/−121 lines, 7 filesdep ~bytestringdep ~containersdep ~dataframe-core

Dependency ranges changed: bytestring, containers, dataframe-core, dataframe-operations, dataframe-parsing, vector

Files

dataframe-csv.cabal view
@@ -1,6 +1,6 @@-cabal-version:      2.4+cabal-version:      3.4 name:               dataframe-csv-version:            1.0.2.0+version:            2.0.0.0 synopsis:           CSV reader and writer for the dataframe ecosystem. description:     @DataFrame.IO.CSV@ — strict single-pass CSV read/write (pure@@ -24,24 +24,39 @@         -Wunused-local-binds         -Wunused-packages +library internal+    import:             warnings+    visibility:         public+    exposed-modules:    DataFrame.IO.Internal.MutableColumn+    build-depends:      base >= 4 && < 5,+                        dataframe-core:internal >= 2.0 && < 2.1,+                        dataframe-parsing:internal >= 2.0 && < 2.1,+                        text >= 2.1 && < 3,+                        vector >= 0.13 && < 0.15+    hs-source-dirs:     src-internal+    default-language:   Haskell2010+ library     import:             warnings     exposed-modules:                         DataFrame.IO.CSV+                        DataFrame.Typed.IO.CSV+    other-modules:                         DataFrame.IO.CSV.Internal.Infer                         DataFrame.IO.CSV.Internal.Options                         DataFrame.IO.CSV.Internal.Read                         DataFrame.IO.CSV.Internal.Scanner                         DataFrame.IO.CSV.Internal.Sink-                        DataFrame.IO.Internal.MutableColumn     build-depends:      base >= 4 && < 5,-                        bytestring >= 0.11 && < 0.13,-                        containers >= 0.6.7 && < 0.9,-                        dataframe-core ^>= 1.1,-                        dataframe-operations ^>= 1.1.1,-                        dataframe-parsing ^>= 1.0.2,-                        text >= 2.0 && < 3,+                        bytestring >= 0.11 && < 0.14,+                        containers >= 0.6.7 && < 0.10,+                        dataframe-core >= 2.0 && < 2.1,+                        dataframe-core:internal >= 2.0 && < 2.1,+                        dataframe-operations >= 2.0 && < 2.1,+                        dataframe-parsing >= 2.0 && < 2.1,+                        dataframe-parsing:internal >= 2.0 && < 2.1,+                        text >= 2.1 && < 3,                         time >= 1.12 && < 2,-                        vector ^>= 0.13+                        vector >= 0.13 && < 0.15     hs-source-dirs:     src     default-language:   Haskell2010
+ src-internal/DataFrame/IO/Internal/MutableColumn.hs view
@@ -0,0 +1,89 @@+{-# LANGUAGE ExplicitNamespaces #-}+{-# LANGUAGE GADTs #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++{- |+Mutable-column ingest helpers used by CSV readers. Kept out of+@DataFrame.Internal.Column@ so that the core column type does not need to+depend on @DataFrame.Internal.Parsing@.+-}+module DataFrame.IO.Internal.MutableColumn (+    writeColumn,+    freezeColumn',+) where++import qualified Data.Text as T+import qualified Data.Vector as VB+import qualified Data.Vector.Mutable as VBM+import qualified Data.Vector.Unboxed as VU+import qualified Data.Vector.Unboxed.Mutable as VUM++import Data.Maybe (fromMaybe)+import Data.Type.Equality (TestEquality (..), type (:~:) (Refl))+import Type.Reflection (typeRep)++import DataFrame.Internal.Column (+    Column (..),+    MutableColumn (..),+    buildBitmapFromNulls,+ )+import DataFrame.Internal.Parsing (isNullish, readDouble, readInt)++writeColumn :: Int -> T.Text -> MutableColumn -> IO (Either T.Text Bool)+writeColumn i value (MBoxedColumn (col :: VBM.IOVector a)) =+    case testEquality (typeRep @a) (typeRep @T.Text) of+        Just Refl ->+            if isNullish value+                then VBM.unsafeWrite col i "" >> return (Left $! value)+                else VBM.unsafeWrite col i value >> return (Right True)+        Nothing -> return (Left value)+writeColumn i value (MUnboxedColumn (col :: VUM.IOVector a)) =+    case testEquality (typeRep @a) (typeRep @Int) of+        Just Refl -> case readInt value of+            Just v -> VUM.unsafeWrite col i v >> return (Right True)+            Nothing -> VUM.unsafeWrite col i 0 >> return (Left value)+        Nothing -> case testEquality (typeRep @a) (typeRep @Double) of+            Nothing -> return (Left $! value)+            Just Refl -> case readDouble value of+                Just v -> VUM.unsafeWrite col i v >> return (Right True)+                Nothing -> VUM.unsafeWrite col i 0 >> return (Left $! value)+{-# INLINE writeColumn #-}++freezeColumn' :: [(Int, T.Text)] -> MutableColumn -> IO Column+freezeColumn' nulls (MBoxedColumn col)+    | null nulls = BoxedColumn Nothing <$> VB.unsafeFreeze col+    | all (isNullish . snd) nulls = do+        frozen <- VB.unsafeFreeze col+        let n = VB.length frozen+            bm = buildBitmapFromNulls n (map fst nulls)+        return $ BoxedColumn (Just bm) frozen+    | otherwise =+        BoxedColumn Nothing+            . VB.imap+                ( \i v ->+                    if i `elem` map fst nulls+                        then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls))+                        else Right v+                )+            <$> VB.unsafeFreeze col+freezeColumn' nulls (MUnboxedColumn col)+    | null nulls = UnboxedColumn Nothing <$> VU.unsafeFreeze col+    | all (isNullish . snd) nulls = do+        c <- VU.unsafeFreeze col+        let n = VU.length c+            bm = buildBitmapFromNulls n (map fst nulls)+        return $ UnboxedColumn (Just bm) c+    | otherwise = do+        c <- VU.unsafeFreeze col+        return $+            BoxedColumn Nothing $+                VB.generate+                    (VU.length c)+                    ( \i ->+                        if i `elem` map fst nulls+                            then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls))+                            else Right (c VU.! i)+                    )+{-# INLINE freezeColumn' #-}
src/DataFrame/IO/CSV.hs view
@@ -1,11 +1,9 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | CSV reading and writing for dataframes. The reader is a strict,-single-pass RFC 4180 scanner (cassava-compatible semantics, pinned by-the golden suite in @tests/IO/CsvGolden.hs@) that parses fields straight-into typed column builders. Ragged rows are padded with nulls (short-rows) and extra fields dropped.+{- | CSV reading and writing for dataframes. A strict, single-pass+RFC 4180 scanner (cassava-compatible) parses fields into typed column+builders; ragged rows are padded with nulls and extra fields dropped. -} module DataFrame.IO.CSV (     -- * Reading@@ -43,7 +41,7 @@ import DataFrame.IO.CSV.Internal.Options import DataFrame.IO.CSV.Internal.Read (decodeCsvStrict) import DataFrame.Internal.DataFrame (DataFrame (..), toSeparated)-import DataFrame.Internal.Schema (Schema, elements)+import DataFrame.Schema (Schema, elements)  {- | Read CSV file from path and load it into a dataframe. @@ -58,10 +56,8 @@  type CsvReader = Schema -> FilePath -> IO DataFrame -{- | Schema-driven CSV reader.  Coerces each column to the type declared-in 'Schema'; columns absent from the schema fall back to the default-inference path.  Defined in terms of 'readSeparated' with the 'TypeSpec'-filled in.+{- | Schema-driven CSV reader. Coerces each column to the type declared+in 'Schema'; columns absent from the schema fall back to inference.  @ import qualified DataFrame as D
src/DataFrame/IO/CSV/Internal/Options.hs view
@@ -19,8 +19,8 @@ import qualified Data.Map.Strict as M import qualified Data.Text as T -import DataFrame.Internal.Schema (SchemaType) import DataFrame.Operations.Typing (SafeReadMode (..))+import DataFrame.Schema (SchemaType)  data HeaderSpec = NoHeader | UseFirstRow | ProvideNames [T.Text]     deriving (Eq, Show)
src/DataFrame/IO/CSV/Internal/Read.hs view
@@ -4,10 +4,9 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} -{- | Driver for the default CSV reader (Round-2 WS-D): a single strict-scan feeding per-column sinks, with cassava-parity semantics pinned by-@IO.CsvGolden@. Ragged rows pad trailing columns with null and drop-extra fields (audit D6). The result is fully forced (audit D7).+{- | Driver for the default CSV reader: a single strict scan feeding+per-column sinks, with cassava-parity semantics. Ragged rows pad+trailing columns with null, drop extra fields, and force the result. -} module DataFrame.IO.CSV.Internal.Read (decodeCsvStrict) where @@ -32,12 +31,12 @@ import DataFrame.Internal.Column (Column, ensureOptional) import DataFrame.Internal.ColumnBuilder import DataFrame.Internal.DataFrame (DataFrame (..), forceDataFrame)-import DataFrame.Internal.Schema (SchemaType (..), schemaType) import DataFrame.Operations.Typing (     SafeReadMode (..),     effectiveSafeRead,     parseWithTypes,  )+import DataFrame.Schema (SchemaType (..), schemaType) import Foreign.Ptr (castPtr) import Type.Reflection (typeRep) @@ -47,8 +46,6 @@     let !len = BS.length bs         !sep = fromIntegral (ord (columnSeparator opts)) :: Word8 -        -- Position of the next non-blank record at/after @pos@ (cassava-        -- drops single-empty-field records); @len@ when none remain.         findRecord !pos             | pos >= len = len             | otherwise = withField bs len sep pos $ \cs ce _ term next ->@@ -56,8 +53,6 @@                     then if term == termEof then len else findRecord next                     else pos -        -- Decode the record at @pos@ as header fields; returns the-        -- position after the record.         headerFields !pos = go pos id           where             go !p acc = withField bs len sep p $ \cs ce unesc term next ->@@ -82,8 +77,6 @@     when (dataPos >= len) (error "Empty CSV file")      let resolveMode = effectiveSafeRead (safeRead opts) (safeReadOverrides opts)-        -- If ANY column is EitherRead we keep every raw cell (including-        -- "N/A" etc.) verbatim; otherwise the missing-indicator list applies.         anyEither = any (\n -> resolveMode n == EitherRead) names         missing = if anyEither then [] else missingIndicators opts         missMode
− src/DataFrame/IO/Internal/MutableColumn.hs 
@@ -1,89 +0,0 @@-{-# LANGUAGE ExplicitNamespaces #-}-{-# LANGUAGE GADTs #-}-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE ScopedTypeVariables #-}-{-# LANGUAGE TypeApplications #-}--{- |-Mutable-column ingest helpers used by CSV readers. Kept out of-@DataFrame.Internal.Column@ so that the core column type does not need to-depend on @DataFrame.Internal.Parsing@.--}-module DataFrame.IO.Internal.MutableColumn (-    writeColumn,-    freezeColumn',-) where--import qualified Data.Text as T-import qualified Data.Vector as VB-import qualified Data.Vector.Mutable as VBM-import qualified Data.Vector.Unboxed as VU-import qualified Data.Vector.Unboxed.Mutable as VUM--import Data.Maybe (fromMaybe)-import Data.Type.Equality (TestEquality (..), type (:~:) (Refl))-import Type.Reflection (typeRep)--import DataFrame.Internal.Column (-    Column (..),-    MutableColumn (..),-    buildBitmapFromNulls,- )-import DataFrame.Internal.Parsing (isNullish, readDouble, readInt)--writeColumn :: Int -> T.Text -> MutableColumn -> IO (Either T.Text Bool)-writeColumn i value (MBoxedColumn (col :: VBM.IOVector a)) =-    case testEquality (typeRep @a) (typeRep @T.Text) of-        Just Refl ->-            if isNullish value-                then VBM.unsafeWrite col i "" >> return (Left $! value)-                else VBM.unsafeWrite col i value >> return (Right True)-        Nothing -> return (Left value)-writeColumn i value (MUnboxedColumn (col :: VUM.IOVector a)) =-    case testEquality (typeRep @a) (typeRep @Int) of-        Just Refl -> case readInt value of-            Just v -> VUM.unsafeWrite col i v >> return (Right True)-            Nothing -> VUM.unsafeWrite col i 0 >> return (Left value)-        Nothing -> case testEquality (typeRep @a) (typeRep @Double) of-            Nothing -> return (Left $! value)-            Just Refl -> case readDouble value of-                Just v -> VUM.unsafeWrite col i v >> return (Right True)-                Nothing -> VUM.unsafeWrite col i 0 >> return (Left $! value)-{-# INLINE writeColumn #-}--freezeColumn' :: [(Int, T.Text)] -> MutableColumn -> IO Column-freezeColumn' nulls (MBoxedColumn col)-    | null nulls = BoxedColumn Nothing <$> VB.unsafeFreeze col-    | all (isNullish . snd) nulls = do-        frozen <- VB.unsafeFreeze col-        let n = VB.length frozen-            bm = buildBitmapFromNulls n (map fst nulls)-        return $ BoxedColumn (Just bm) frozen-    | otherwise =-        BoxedColumn Nothing-            . VB.imap-                ( \i v ->-                    if i `elem` map fst nulls-                        then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls))-                        else Right v-                )-            <$> VB.unsafeFreeze col-freezeColumn' nulls (MUnboxedColumn col)-    | null nulls = UnboxedColumn Nothing <$> VU.unsafeFreeze col-    | all (isNullish . snd) nulls = do-        c <- VU.unsafeFreeze col-        let n = VU.length c-            bm = buildBitmapFromNulls n (map fst nulls)-        return $ UnboxedColumn (Just bm) c-    | otherwise = do-        c <- VU.unsafeFreeze col-        return $-            BoxedColumn Nothing $-                VB.generate-                    (VU.length c)-                    ( \i ->-                        if i `elem` map fst nulls-                            then Left (fromMaybe (error "UNEXPECTED ERROR DURING FREEZE") (lookup i nulls))-                            else Right (c VU.! i)-                    )-{-# INLINE freezeColumn' #-}
+ src/DataFrame/Typed/IO/CSV.hs view
@@ -0,0 +1,51 @@+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++module DataFrame.Typed.IO.CSV (+    readCsv,+    readCsvWithError,+    readTsv,+    readCsvWithOpts,+    writeCsv,+    writeTsv,+) where++import qualified Data.Text as T++import DataFrame.IO.CSV (ReadOptions)+import qualified DataFrame.IO.CSV as CSV+import DataFrame.Typed.Freeze (freezeOrThrow, freezeWithError, thaw)+import DataFrame.Typed.Schema (KnownSchema)+import DataFrame.Typed.Types (TypedDataFrame)++-- | Read a CSV file into a typed DataFrame, throwing on schema mismatch.+readCsv ::+    forall cols. (KnownSchema cols) => FilePath -> IO (TypedDataFrame cols)+readCsv path = CSV.readCsv path >>= freezeOrThrow @cols++-- | Read a CSV file, returning a descriptive error on schema mismatch.+readCsvWithError ::+    forall cols.+    (KnownSchema cols) =>+    FilePath -> IO (Either T.Text (TypedDataFrame cols))+readCsvWithError path = freezeWithError <$> CSV.readCsv path++-- | Read a tab-separated file into a typed DataFrame, throwing on mismatch.+readTsv ::+    forall cols. (KnownSchema cols) => FilePath -> IO (TypedDataFrame cols)+readTsv path = CSV.readTsv path >>= freezeOrThrow @cols++-- | Read a CSV file with custom options, throwing on schema mismatch.+readCsvWithOpts ::+    forall cols.+    (KnownSchema cols) =>+    ReadOptions -> FilePath -> IO (TypedDataFrame cols)+readCsvWithOpts opts path = CSV.readCsvWithOpts opts path >>= freezeOrThrow @cols++-- | Write a typed DataFrame to a CSV file.+writeCsv :: FilePath -> TypedDataFrame cols -> IO ()+writeCsv path = CSV.writeCsv path . thaw++-- | Write a typed DataFrame to a tab-separated file.+writeTsv :: FilePath -> TypedDataFrame cols -> IO ()+writeTsv path = CSV.writeTsv path . thaw