dataframe 3.1.0.0 → 3.2.0.0

raw patch · 9 files changed

+287/−21 lines, 9 filesdep +dataframe-fastcsvdep ~dataframedep ~dataframe-csvdep ~dataframe-lazyPVP ok

version bump matches the API change (PVP)

Dependencies added: dataframe-fastcsv

Dependency ranges changed: dataframe, dataframe-csv, dataframe-lazy, dataframe-parquet

API changes (from Hackage documentation)

Files

• CHANGELOG.md +4/−0
• dataframe.cabal +19/−17
• src/DataFrame.hs +2/−0
• src/DataFrame/Typed.hs +1/−0
• tests/IO/CsvGolden.hs +47/−2
• tests/LazyProjection.hs +169/−0
• tests/Main.hs +2/−0
• tests/Operations/WriteCsv.hs +4/−1
• tests/Typed/IOReaders.hs +39/−1

@@ -1,5 +1,9 @@ # Revision history for dataframe +## 3.2.0.0++* Restrict CSV reads to selected columns for untyped and typed readers.+ ## 3.1.0.0 * Fix toSeparated's handling of nulls * Fix build failing.

@@ -1,6 +1,6 @@ cabal-version:      3.4 name:               dataframe-version:            3.1.0.0+version:            3.2.0.0 synopsis: A fast, safe, and intuitive DataFrame library.  description: A fast, safe, and intuitive DataFrame library for exploratory data analysis.@@ -139,7 +139,7 @@     if !flag(no-csv)         reexported-modules: DataFrame.IO.CSV,                             DataFrame.Typed.IO.CSV-        build-depends:   dataframe-csv >= 2.1 && < 2.2+        build-depends:   dataframe-csv >= 2.2 && < 2.3         cpp-options:     -DWITH_CSV      if !flag(no-parquet)@@ -157,7 +157,7 @@                             DataFrame.IO.Parquet.Time,                             DataFrame.IO.Utils.RandomAccess,                             DataFrame.Typed.IO.Parquet-        build-depends:   dataframe-parquet >= 1.3 && < 1.4+        build-depends:   dataframe-parquet >= 1.4 && < 1.5         cpp-options:     -DWITH_PARQUET      -- The lazy executor calls both CSV and Parquet readers directly, so@@ -167,7 +167,7 @@                             DataFrame.Lazy.IO.Binary,                             DataFrame.Lazy.IO.CSV,                             DataFrame.Typed.Lazy-        build-depends:   dataframe-lazy >= 2.1 && < 2.2+        build-depends:   dataframe-lazy >= 2.2 && < 2.3         cpp-options:     -DWITH_LAZY      if !flag(no-th)@@ -205,11 +205,11 @@         base        >= 4   && < 5,         dataframe-core >= 2.1 && < 2.2,         dataframe-expr-serializer >= 1.2 && < 1.3,-        dataframe-csv >= 2.1 && < 2.2,+        dataframe-csv >= 2.2 && < 2.3,         dataframe-json >= 1.2 && < 1.3,-        dataframe-lazy >= 2.1 && < 2.2,+        dataframe-lazy >= 2.2 && < 2.3,         dataframe-operations >= 2.1 && < 2.2,-        dataframe-parquet >= 1.3 && < 1.4,+        dataframe-parquet >= 1.4 && < 1.5,         dataframe-parsing >= 2.1 && < 2.2,         text        >= 2.1 && < 3,         aeson       >= 0.11 && < 3,@@ -223,7 +223,7 @@     import: warnings     main-is: Benchmark.hs     build-depends:    base >= 4 && < 5,-                      dataframe >= 3.1 && < 3.2,+                      dataframe >= 3.2 && < 3.3,                       dataframe-operations >= 2.1 && < 2.2,                       random >= 1 && < 2,                       time >= 1.12 && < 2,@@ -236,7 +236,7 @@     import: warnings     main-is: Synthesis.hs     build-depends:    base >= 4 && < 5,-                      dataframe >= 3.1 && < 3.2,+                      dataframe >= 3.2 && < 3.3,                       dataframe-core >= 2.1 && < 2.2,                       dataframe-learn >= 2.1 && < 2.2,                       dataframe-operations >= 2.1 && < 2.2,@@ -270,9 +270,9 @@     build-depends:    base >= 4 && < 5,                       bytestring >= 0.11 && < 0.14,                       containers >= 0.6.7 && < 0.10,-                      dataframe >= 3.1 && < 3.2,+                      dataframe >= 3.2 && < 3.3,                       dataframe-core >= 2.1 && < 2.2,-                      dataframe-lazy >= 2.1 && < 2.2,+                      dataframe-lazy >= 2.2 && < 2.3,                       dataframe-parsing >= 2.1 && < 2.2,                       directory >= 1.3.0.0 && < 2,                       random >= 1 && < 2,@@ -291,7 +291,7 @@                    criterion >= 1 && < 2,                    deepseq >= 1.4 && < 2,                    process >= 1.6 && < 2,-                   dataframe >= 3.1 && < 3.2,+                   dataframe >= 3.2 && < 3.3,                    dataframe-core >= 2.1 && < 2.2,                    dataframe-operations >= 2.1 && < 2.2,                    random >= 1 && < 2,@@ -364,6 +364,7 @@                    Operations.Record,                    LazyParquet,                    LazyParity,+                   LazyProjection,                    Parquet,                    ParquetTestData,                    Plotting,@@ -376,16 +377,17 @@     build-depends:  base >= 4 && < 5,                     aeson >= 0.11.0.0 && < 3,                     bytestring >= 0.11 && < 0.14,-                    dataframe >= 3.1 && < 3.2,+                    dataframe >= 3.2 && < 3.3,                     dataframe-core >= 2.1 && < 2.2,                     dataframe-core >= 2.1 && < 2.2,-                    dataframe-csv >= 2.1 && < 2.2,+                    dataframe-csv >= 2.2 && < 2.3,                     dataframe-expr-serializer >= 1.2 && < 1.3,+                    dataframe-fastcsv >= 1.4 && < 1.5,                     dataframe-json >= 1.2 && < 1.3,-                    dataframe-lazy >= 2.1 && < 2.2,+                    dataframe-lazy >= 2.2 && < 2.3,                     dataframe-learn >= 2.1 && < 2.2,                     dataframe-operations >= 2.1 && < 2.2,-                    dataframe-parquet >= 1.3 && < 1.4,+                    dataframe-parquet >= 1.4 && < 1.5,                     dataframe-parsing >= 2.1 && < 2.2,                     HUnit >= 1.6 && < 1.8,                     QuickCheck >= 2 && < 3,@@ -411,7 +413,7 @@     other-modules: Internal.PackedText     build-depends:  base >= 4 && < 5,                     bytestring >= 0.11 && < 0.14,-                    dataframe >= 3.1 && < 3.2,+                    dataframe >= 3.2 && < 3.3,                     dataframe-core >= 2.1 && < 2.2,                     dataframe-operations >= 2.1 && < 2.2,                     HUnit >= 1.6 && < 1.8,

@@ -67,6 +67,7 @@      -- * Record bridge     module Record,+    module TypedSchema,      -- * Template Haskell column-binding splices #ifdef WITH_TH@@ -300,3 +301,4 @@     fromRecords,     toRecords,  )+import DataFrame.Typed.Schema as TypedSchema (schemaColumnNames)

@@ -318,6 +318,7 @@      -- * Constraints     KnownSchema (..),+    schemaColumnNames,     AllKnownSymbol (..), ) where 

@@ -310,15 +310,60 @@         )     ,         ( "row_cap"-        , defaultReadOptions{numColumns = Just 2}+        , defaultReadOptions{numRowsToRead = Just 2}         , "a\n1\n2\n3\n4\n"         , Cols (2, 1) [("a", ints [1, 2])]         )     ,         ( "row_cap_zero"-        , defaultReadOptions{numColumns = Just 0}+        , defaultReadOptions{numRowsToRead = Just 0}         , "a,b\n1,2\n"         , Cols (0, 2) [("a", mtexts []), ("b", mtexts [])]+        )+    ,+        ( "selected_subset"+        , defaultReadOptions{readColumns = Just ["c", "a"]}+        , "a,b,c\n1,x,2.5\n3,y,4.5\n"+        , Cols (2, 2) [("c", dbls [2.5, 4.5]), ("a", ints [1, 3])]+        )+    ,+        ( "selected_single_trailing"+        , defaultReadOptions{readColumns = Just ["c"]}+        , "a,b,c\n1,x,2\n3,y,4\n"+        , Cols (2, 1) [("c", ints [2, 4])]+        )+    ,+        ( "selected_all_reordered"+        , defaultReadOptions{readColumns = Just ["b", "a"]}+        , "a,b\n1,x\n2,y\n"+        , Cols (2, 2) [("b", texts ["x", "y"]), ("a", ints [1, 2])]+        )+    ,+        ( "selected_ignores_unselected_values"+        , defaultReadOptions{readColumns = Just ["a"]}+        , "a,b\n1,notanumber\n2,\n"+        , Cols (2, 1) [("a", ints [1, 2])]+        )+    ,+        ( "selected_noheader_positional"+        , defaultReadOptions{headerSpec = NoHeader, readColumns = Just ["2", "0"]}+        , "1,x,9\n2,y,8\n"+        , Cols (2, 2) [("2", ints [9, 8]), ("0", ints [1, 2])]+        )+    ,+        ( "selected_with_specified_type"+        , defaultReadOptions+            { readColumns = Just ["b"]+            , typeSpec = SpecifyTypes [("b", schemaTypeText)] (InferFromSample 100)+            }+        , "a,b\n1,2\n3,4\n"+        , Cols (2, 1) [("b", texts ["2", "4"])]+        )+    ,+        ( "selected_missing"+        , defaultReadOptions{readColumns = Just ["a", "nope"]}+        , "a,b\n1,2\n"+        , Err "Column not found"         )     ,         ( "trailing_sep"

@@ -0,0 +1,169 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE TypeApplications #-}++{- | A lazy scan reads only the columns its 'Schema' names, and rejects+self-contradictory 'ReadOptions' before opening the file.+-}+module LazyProjection (tests) where++import qualified Data.Map.Strict as M+import qualified Data.Text as T+import qualified Data.Text.IO as TIO++import Control.Exception (SomeException, evaluate, try)+import Data.List (isInfixOf)+import qualified DataFrame as D+import qualified DataFrame.IO.CSV as Csv+import qualified DataFrame.IO.CSV.Fast as Fast+import DataFrame.Internal.DataFrame (forceDataFrame)+import qualified DataFrame.Lazy as L+import DataFrame.Schema (Schema (..), schemaType)+import System.Directory (removeFile)+import System.IO.Temp (emptySystemTempFile)+import Test.HUnit++-- | Four columns; every schema below names at most two of them.+withCsv :: Char -> (FilePath -> IO a) -> IO a+withCsv sep k = do+    csvPath <- emptySystemTempFile "lazy_projection_.csv"+    TIO.writeFile csvPath (T.unlines (map (T.intercalate (T.singleton sep)) rows))+    r <- k csvPath+    removeFile csvPath+    pure r+  where+    rows =+        [ ["id", "name", "surname", "dob"]+        , ["1", "Ada", "Lovelace", "1815"]+        , ["2", "Alan", "Turing", "1912"]+        ]++twoCols :: Schema+twoCols =+    Schema $ M.fromList [("id", schemaType @Int), ("dob", schemaType @Int)]++{- | The scan keeps only the schema's columns — and 'dob' is not a prefix of+the header, so a positional scheme could not express this.+-}+scanProjects :: Test+scanProjects = TestLabel "scan_projects" $ TestCase $ withCsv ',' $ \csvPath -> do+    df <- L.runDataFrame (L.scanCsv twoCols (T.pack csvPath))+    assertEqual "dims" (2, 2) (D.dimensions df)+    assertEqual "columns" ["dob", "id"] (D.columnNames df)++-- | Same schema, same projection, through the SIMD reader.+scanProjectsWithFastReader :: Test+scanProjectsWithFastReader =+    TestLabel "scan_projects_fast_reader" $ TestCase $ withCsv ',' $ \csvPath -> do+        df <-+            L.runDataFrame+                (L.scanCsvWith Fast.fastReadCsvWithOpts twoCols (T.pack csvPath))+        assertEqual "dims" (2, 2) (D.dimensions df)+        assertEqual "columns" ["dob", "id"] (D.columnNames df)++-- | Both readers must agree on what a scan produces.+readersAgree :: Test+readersAgree = TestLabel "scan_readers_agree" $ TestCase $ withCsv ',' $ \csvPath -> do+    slow <- L.runDataFrame (L.scanCsv twoCols (T.pack csvPath))+    fast <-+        L.runDataFrame (L.scanCsvWith Fast.fastReadCsvWithOpts twoCols (T.pack csvPath))+    assertEqual "same frame" slow fast++{- | 'scanSeparated' used to drop its separator on the floor: the reader was+built from the schema alone and always split on commas.+-}+scanHonoursSeparator :: Test+scanHonoursSeparator =+    TestLabel "scan_honours_separator" $ TestCase $ withCsv ';' $ \csvPath -> do+        -- With the separator ignored, the whole line is one field and the+        -- schema's columns would not be found at all.+        df <- L.runDataFrame (L.scanSeparated ';' twoCols (T.pack csvPath))+        assertEqual "dims" (2, 2) (D.dimensions df)+        assertEqual "columns" ["dob", "id"] (D.columnNames df)++-- | A schema naming a column the file lacks is an error, not a short frame.+scanMissingColumn :: Test+scanMissingColumn = TestLabel "scan_missing_column" $ TestCase $ withCsv ',' $ \csvPath -> do+    let bogus = Schema $ M.fromList [("nope", schemaType @Int)]+    r <-+        try+            (L.runDataFrame (L.scanCsv bogus (T.pack csvPath)) >>= evaluate . forceDataFrame)+    case r of+        Left (e :: SomeException) ->+            assertBool+                ("expected Column not found, got " <> show e)+                ("Column not found" `isInfixOf` show e)+        Right _ -> assertFailure "expected a missing-column error"++-- | Contradictory options are rejected before the file is opened.+invalidOptionsCase :: String -> Csv.ReadOptions -> String -> Test+invalidOptionsCase label opts needle = TestLabel label $ TestCase $ do+    let expectFail which act = do+            r <- try (act >>= evaluate . forceDataFrame)+            case r of+                Left (e :: SomeException) ->+                    assertBool+                        ( label+                            <> ": "+                            <> which+                            <> " should mention "+                            <> show needle+                            <> ", got "+                            <> show e+                        )+                        (needle `isInfixOf` show e)+                Right _ -> assertFailure (label <> ": " <> which <> " should have failed")+    -- "no-such-file" proves the failure precedes any read.+    expectFail "pure reader" (Csv.readSeparated opts "no-such-file.csv")+    expectFail "fast reader" (Fast.fastReadCsvWithOpts opts "no-such-file.csv")++invalidOptions :: [Test]+invalidOptions =+    [ invalidOptionsCase+        "reject_empty_selection"+        Csv.defaultReadOptions{Csv.readColumns = Just []}+        "empty selection"+    , invalidOptionsCase+        "reject_duplicate_selection"+        Csv.defaultReadOptions{Csv.readColumns = Just ["a", "b", "a"]}+        "more than once"+    , invalidOptionsCase+        "reject_negative_row_cap"+        Csv.defaultReadOptions{Csv.numRowsToRead = Just (-1)}+        "numRowsToRead is negative"+    , invalidOptionsCase+        "reject_nameless_providenames"+        Csv.defaultReadOptions{Csv.headerSpec = Csv.ProvideNames []}+        "ProvideNames with no names"+    , invalidOptionsCase+        "reject_quote_separator"+        Csv.defaultReadOptions{Csv.columnSeparator = '"'}+        "cannot delimit fields"+    ]++{- | The SIMD reader's low-level entry takes its separator as a scanner byte,+so a 'columnSeparator' saying something else would be silently ignored.+Reject it instead.+-}+rejectSeparatorMismatch :: Test+rejectSeparatorMismatch = TestLabel "reject_separator_mismatch" $ TestCase $ do+    let opts = Csv.defaultReadOptions{Csv.columnSeparator = ';'}+        htab = 0x09+    r <- try (Fast.readSeparated htab opts "no-such-file.csv")+    case r of+        Left (e :: SomeException) ->+            assertBool+                ("expected a separator complaint, got " <> show e)+                ("columnSeparator" `isInfixOf` show e)+        Right (_ :: D.DataFrame) -> assertFailure "expected a separator mismatch error"++tests :: [Test]+tests =+    [ scanProjects+    , scanProjectsWithFastReader+    , readersAgree+    , scanHonoursSeparator+    , scanMissingColumn+    , rejectSeparatorMismatch+    ]+        <> invalidOptions

@@ -20,6 +20,7 @@ import qualified Internal.Parsing import qualified LazyParity import qualified LazyParquet+import qualified LazyProjection import qualified Learn.Denotation import qualified Learn.Ensembles import qualified Learn.Metamorphic@@ -119,6 +120,7 @@             ++ Parquet.tests             ++ LazyParquet.tests             ++ LazyParity.tests+            ++ LazyProjection.tests             ++ Plotting.tests             ++ Simplify.tests             ++ PackedTextMigration.tests

@@ -25,7 +25,10 @@ toCsvWithNulls = TestLabel "toCsv_withNulls" $ TestCase $ do     let df =             D.fromNamedColumns-                [ ("name", DI.fromList @(Maybe T.Text) [Nothing, Just "Alice", Just "Bob", Just "Charlie"])+                [+                    ( "name"+                    , DI.fromList @(Maybe T.Text) [Nothing, Just "Alice", Just "Bob", Just "Charlie"]+                    )                 , ("age", DI.fromList @(Maybe Int) [Just 30, Nothing, Just 25, Just 35])                 ]         expected = "name,age\nnull,30\nAlice,null\nBob,25\nCharlie,35\n"

@@ -18,6 +18,7 @@ import qualified DataFrame as D import DataFrame.Errors (DataFrameException) import qualified DataFrame.Internal.Column as DI+import DataFrame.Internal.DataFrame (getColumn) import qualified DataFrame.Typed as DT import qualified DataFrame.Typed.IO.CSV as TCSV @@ -61,5 +62,42 @@             IO (Either DataFrameException Int)     assertBool "wrong schema => throws DataFrameException" (isLeft r) +{- | The schema is the read specification: only its columns are fetched, out+of a file that holds more.+-}+typedReadProjects :: Test+typedReadProjects = TestCase $ withSystemTempFile "typed_proj.csv" $ \fp h -> do+    hClose h+    D.writeCsv fp sampleDF+    narrow <- DT.thaw <$> TCSV.readCsv @'[DT.Column "g" T.Text] fp+    assertEqual "only the schema's column is read" ["g"] (D.columnNames narrow)+    assertEqual "rows intact" (3, 1) (D.dimensions narrow)++{- | The schema's types win over inference: a column of digits declared 'Text'+comes back as 'Text', which a plain read would have called 'Int'.+-}+typedReadSharesTypes :: Test+typedReadSharesTypes = TestCase $ withSystemTempFile "typed_types.csv" $ \fp h -> do+    hClose h+    D.writeCsv fp digits+    inferred <- D.readCsv fp+    assertEqual+        "untyped read infers Int"+        (Just (DI.fromList [1, 2, 3 :: Int]))+        (getColumn "n" inferred)+    typed <- DT.thaw <$> TCSV.readCsv @'[DT.Column "n" T.Text] fp+    assertEqual+        "schema types the column as Text"+        (Just (DI.fromList ["1", "2", "3" :: T.Text]))+        (getColumn "n" typed)+  where+    digits = D.fromNamedColumns [("n", DI.fromList [1, 2, 3 :: Int])]+ tests :: [Test]-tests = [roundTrip, wrongSchemaEither, wrongSchemaThrows]+tests =+    [ roundTrip+    , wrongSchemaEither+    , wrongSchemaThrows+    , typedReadProjects+    , typedReadSharesTypes+    ]