diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Revision history for dataframe +## 3.2.0.0 + +* Restrict CSV reads to selected columns for untyped and typed readers. + ## 3.1.0.0 * Fix toSeparated's handling of nulls * Fix build failing. diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: dataframe -version: 3.1.0.0 +version: 3.2.0.0 synopsis: A fast, safe, and intuitive DataFrame library. description: A fast, safe, and intuitive DataFrame library for exploratory data analysis. @@ -139,7 +139,7 @@ if !flag(no-csv) reexported-modules: DataFrame.IO.CSV, DataFrame.Typed.IO.CSV - build-depends: dataframe-csv >= 2.1 && < 2.2 + build-depends: dataframe-csv >= 2.2 && < 2.3 cpp-options: -DWITH_CSV if !flag(no-parquet) @@ -157,7 +157,7 @@ DataFrame.IO.Parquet.Time, DataFrame.IO.Utils.RandomAccess, DataFrame.Typed.IO.Parquet - build-depends: dataframe-parquet >= 1.3 && < 1.4 + build-depends: dataframe-parquet >= 1.4 && < 1.5 cpp-options: -DWITH_PARQUET -- The lazy executor calls both CSV and Parquet readers directly, so @@ -167,7 +167,7 @@ DataFrame.Lazy.IO.Binary, DataFrame.Lazy.IO.CSV, DataFrame.Typed.Lazy - build-depends: dataframe-lazy >= 2.1 && < 2.2 + build-depends: dataframe-lazy >= 2.2 && < 2.3 cpp-options: -DWITH_LAZY if !flag(no-th) @@ -205,11 +205,11 @@ base >= 4 && < 5, dataframe-core >= 2.1 && < 2.2, dataframe-expr-serializer >= 1.2 && < 1.3, - dataframe-csv >= 2.1 && < 2.2, + dataframe-csv >= 2.2 && < 2.3, dataframe-json >= 1.2 && < 1.3, - dataframe-lazy >= 2.1 && < 2.2, + dataframe-lazy >= 2.2 && < 2.3, dataframe-operations >= 2.1 && < 2.2, - dataframe-parquet >= 1.3 && < 1.4, + dataframe-parquet >= 1.4 && < 1.5, dataframe-parsing >= 2.1 && < 2.2, text >= 2.1 && < 3, aeson >= 0.11 && < 3, @@ -223,7 +223,7 @@ import: warnings main-is: Benchmark.hs build-depends: base >= 4 && < 5, - dataframe >= 3.1 && < 3.2, + dataframe >= 3.2 && < 3.3, dataframe-operations >= 2.1 && < 2.2, random >= 1 && < 2, time >= 1.12 && < 2, @@ -236,7 +236,7 @@ import: warnings main-is: Synthesis.hs build-depends: base >= 4 && < 5, - dataframe >= 3.1 && < 3.2, + dataframe >= 3.2 && < 3.3, dataframe-core >= 2.1 && < 2.2, dataframe-learn >= 2.1 && < 2.2, dataframe-operations >= 2.1 && < 2.2, @@ -270,9 +270,9 @@ build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.14, containers >= 0.6.7 && < 0.10, - dataframe >= 3.1 && < 3.2, + dataframe >= 3.2 && < 3.3, dataframe-core >= 2.1 && < 2.2, - dataframe-lazy >= 2.1 && < 2.2, + dataframe-lazy >= 2.2 && < 2.3, dataframe-parsing >= 2.1 && < 2.2, directory >= 1.3.0.0 && < 2, random >= 1 && < 2, @@ -291,7 +291,7 @@ criterion >= 1 && < 2, deepseq >= 1.4 && < 2, process >= 1.6 && < 2, - dataframe >= 3.1 && < 3.2, + dataframe >= 3.2 && < 3.3, dataframe-core >= 2.1 && < 2.2, dataframe-operations >= 2.1 && < 2.2, random >= 1 && < 2, @@ -364,6 +364,7 @@ Operations.Record, LazyParquet, LazyParity, + LazyProjection, Parquet, ParquetTestData, Plotting, @@ -376,16 +377,17 @@ build-depends: base >= 4 && < 5, aeson >= 0.11.0.0 && < 3, bytestring >= 0.11 && < 0.14, - dataframe >= 3.1 && < 3.2, + dataframe >= 3.2 && < 3.3, dataframe-core >= 2.1 && < 2.2, dataframe-core >= 2.1 && < 2.2, - dataframe-csv >= 2.1 && < 2.2, + dataframe-csv >= 2.2 && < 2.3, dataframe-expr-serializer >= 1.2 && < 1.3, + dataframe-fastcsv >= 1.4 && < 1.5, dataframe-json >= 1.2 && < 1.3, - dataframe-lazy >= 2.1 && < 2.2, + dataframe-lazy >= 2.2 && < 2.3, dataframe-learn >= 2.1 && < 2.2, dataframe-operations >= 2.1 && < 2.2, - dataframe-parquet >= 1.3 && < 1.4, + dataframe-parquet >= 1.4 && < 1.5, dataframe-parsing >= 2.1 && < 2.2, HUnit >= 1.6 && < 1.8, QuickCheck >= 2 && < 3, @@ -411,7 +413,7 @@ other-modules: Internal.PackedText build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.14, - dataframe >= 3.1 && < 3.2, + dataframe >= 3.2 && < 3.3, dataframe-core >= 2.1 && < 2.2, dataframe-operations >= 2.1 && < 2.2, HUnit >= 1.6 && < 1.8, diff --git a/src/DataFrame.hs b/src/DataFrame.hs --- a/src/DataFrame.hs +++ b/src/DataFrame.hs @@ -67,6 +67,7 @@ -- * Record bridge module Record, + module TypedSchema, -- * Template Haskell column-binding splices #ifdef WITH_TH @@ -300,3 +301,4 @@ fromRecords, toRecords, ) +import DataFrame.Typed.Schema as TypedSchema (schemaColumnNames) diff --git a/src/DataFrame/Typed.hs b/src/DataFrame/Typed.hs --- a/src/DataFrame/Typed.hs +++ b/src/DataFrame/Typed.hs @@ -318,6 +318,7 @@ -- * Constraints KnownSchema (..), + schemaColumnNames, AllKnownSymbol (..), ) where diff --git a/tests/IO/CsvGolden.hs b/tests/IO/CsvGolden.hs --- a/tests/IO/CsvGolden.hs +++ b/tests/IO/CsvGolden.hs @@ -310,15 +310,60 @@ ) , ( "row_cap" - , defaultReadOptions{numColumns = Just 2} + , defaultReadOptions{numRowsToRead = Just 2} , "a\n1\n2\n3\n4\n" , Cols (2, 1) [("a", ints [1, 2])] ) , ( "row_cap_zero" - , defaultReadOptions{numColumns = Just 0} + , defaultReadOptions{numRowsToRead = Just 0} , "a,b\n1,2\n" , Cols (0, 2) [("a", mtexts []), ("b", mtexts [])] + ) + , + ( "selected_subset" + , defaultReadOptions{readColumns = Just ["c", "a"]} + , "a,b,c\n1,x,2.5\n3,y,4.5\n" + , Cols (2, 2) [("c", dbls [2.5, 4.5]), ("a", ints [1, 3])] + ) + , + ( "selected_single_trailing" + , defaultReadOptions{readColumns = Just ["c"]} + , "a,b,c\n1,x,2\n3,y,4\n" + , Cols (2, 1) [("c", ints [2, 4])] + ) + , + ( "selected_all_reordered" + , defaultReadOptions{readColumns = Just ["b", "a"]} + , "a,b\n1,x\n2,y\n" + , Cols (2, 2) [("b", texts ["x", "y"]), ("a", ints [1, 2])] + ) + , + ( "selected_ignores_unselected_values" + , defaultReadOptions{readColumns = Just ["a"]} + , "a,b\n1,notanumber\n2,\n" + , Cols (2, 1) [("a", ints [1, 2])] + ) + , + ( "selected_noheader_positional" + , defaultReadOptions{headerSpec = NoHeader, readColumns = Just ["2", "0"]} + , "1,x,9\n2,y,8\n" + , Cols (2, 2) [("2", ints [9, 8]), ("0", ints [1, 2])] + ) + , + ( "selected_with_specified_type" + , defaultReadOptions + { readColumns = Just ["b"] + , typeSpec = SpecifyTypes [("b", schemaTypeText)] (InferFromSample 100) + } + , "a,b\n1,2\n3,4\n" + , Cols (2, 1) [("b", texts ["2", "4"])] + ) + , + ( "selected_missing" + , defaultReadOptions{readColumns = Just ["a", "nope"]} + , "a,b\n1,2\n" + , Err "Column not found" ) , ( "trailing_sep" diff --git a/tests/LazyProjection.hs b/tests/LazyProjection.hs new file mode 100644 --- /dev/null +++ b/tests/LazyProjection.hs @@ -0,0 +1,169 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +{- | A lazy scan reads only the columns its 'Schema' names, and rejects +self-contradictory 'ReadOptions' before opening the file. +-} +module LazyProjection (tests) where + +import qualified Data.Map.Strict as M +import qualified Data.Text as T +import qualified Data.Text.IO as TIO + +import Control.Exception (SomeException, evaluate, try) +import Data.List (isInfixOf) +import qualified DataFrame as D +import qualified DataFrame.IO.CSV as Csv +import qualified DataFrame.IO.CSV.Fast as Fast +import DataFrame.Internal.DataFrame (forceDataFrame) +import qualified DataFrame.Lazy as L +import DataFrame.Schema (Schema (..), schemaType) +import System.Directory (removeFile) +import System.IO.Temp (emptySystemTempFile) +import Test.HUnit + +-- | Four columns; every schema below names at most two of them. +withCsv :: Char -> (FilePath -> IO a) -> IO a +withCsv sep k = do + csvPath <- emptySystemTempFile "lazy_projection_.csv" + TIO.writeFile csvPath (T.unlines (map (T.intercalate (T.singleton sep)) rows)) + r <- k csvPath + removeFile csvPath + pure r + where + rows = + [ ["id", "name", "surname", "dob"] + , ["1", "Ada", "Lovelace", "1815"] + , ["2", "Alan", "Turing", "1912"] + ] + +twoCols :: Schema +twoCols = + Schema $ M.fromList [("id", schemaType @Int), ("dob", schemaType @Int)] + +{- | The scan keeps only the schema's columns — and 'dob' is not a prefix of +the header, so a positional scheme could not express this. +-} +scanProjects :: Test +scanProjects = TestLabel "scan_projects" $ TestCase $ withCsv ',' $ \csvPath -> do + df <- L.runDataFrame (L.scanCsv twoCols (T.pack csvPath)) + assertEqual "dims" (2, 2) (D.dimensions df) + assertEqual "columns" ["dob", "id"] (D.columnNames df) + +-- | Same schema, same projection, through the SIMD reader. +scanProjectsWithFastReader :: Test +scanProjectsWithFastReader = + TestLabel "scan_projects_fast_reader" $ TestCase $ withCsv ',' $ \csvPath -> do + df <- + L.runDataFrame + (L.scanCsvWith Fast.fastReadCsvWithOpts twoCols (T.pack csvPath)) + assertEqual "dims" (2, 2) (D.dimensions df) + assertEqual "columns" ["dob", "id"] (D.columnNames df) + +-- | Both readers must agree on what a scan produces. +readersAgree :: Test +readersAgree = TestLabel "scan_readers_agree" $ TestCase $ withCsv ',' $ \csvPath -> do + slow <- L.runDataFrame (L.scanCsv twoCols (T.pack csvPath)) + fast <- + L.runDataFrame (L.scanCsvWith Fast.fastReadCsvWithOpts twoCols (T.pack csvPath)) + assertEqual "same frame" slow fast + +{- | 'scanSeparated' used to drop its separator on the floor: the reader was +built from the schema alone and always split on commas. +-} +scanHonoursSeparator :: Test +scanHonoursSeparator = + TestLabel "scan_honours_separator" $ TestCase $ withCsv ';' $ \csvPath -> do + -- With the separator ignored, the whole line is one field and the + -- schema's columns would not be found at all. + df <- L.runDataFrame (L.scanSeparated ';' twoCols (T.pack csvPath)) + assertEqual "dims" (2, 2) (D.dimensions df) + assertEqual "columns" ["dob", "id"] (D.columnNames df) + +-- | A schema naming a column the file lacks is an error, not a short frame. +scanMissingColumn :: Test +scanMissingColumn = TestLabel "scan_missing_column" $ TestCase $ withCsv ',' $ \csvPath -> do + let bogus = Schema $ M.fromList [("nope", schemaType @Int)] + r <- + try + (L.runDataFrame (L.scanCsv bogus (T.pack csvPath)) >>= evaluate . forceDataFrame) + case r of + Left (e :: SomeException) -> + assertBool + ("expected Column not found, got " <> show e) + ("Column not found" `isInfixOf` show e) + Right _ -> assertFailure "expected a missing-column error" + +-- | Contradictory options are rejected before the file is opened. +invalidOptionsCase :: String -> Csv.ReadOptions -> String -> Test +invalidOptionsCase label opts needle = TestLabel label $ TestCase $ do + let expectFail which act = do + r <- try (act >>= evaluate . forceDataFrame) + case r of + Left (e :: SomeException) -> + assertBool + ( label + <> ": " + <> which + <> " should mention " + <> show needle + <> ", got " + <> show e + ) + (needle `isInfixOf` show e) + Right _ -> assertFailure (label <> ": " <> which <> " should have failed") + -- "no-such-file" proves the failure precedes any read. + expectFail "pure reader" (Csv.readSeparated opts "no-such-file.csv") + expectFail "fast reader" (Fast.fastReadCsvWithOpts opts "no-such-file.csv") + +invalidOptions :: [Test] +invalidOptions = + [ invalidOptionsCase + "reject_empty_selection" + Csv.defaultReadOptions{Csv.readColumns = Just []} + "empty selection" + , invalidOptionsCase + "reject_duplicate_selection" + Csv.defaultReadOptions{Csv.readColumns = Just ["a", "b", "a"]} + "more than once" + , invalidOptionsCase + "reject_negative_row_cap" + Csv.defaultReadOptions{Csv.numRowsToRead = Just (-1)} + "numRowsToRead is negative" + , invalidOptionsCase + "reject_nameless_providenames" + Csv.defaultReadOptions{Csv.headerSpec = Csv.ProvideNames []} + "ProvideNames with no names" + , invalidOptionsCase + "reject_quote_separator" + Csv.defaultReadOptions{Csv.columnSeparator = '"'} + "cannot delimit fields" + ] + +{- | The SIMD reader's low-level entry takes its separator as a scanner byte, +so a 'columnSeparator' saying something else would be silently ignored. +Reject it instead. +-} +rejectSeparatorMismatch :: Test +rejectSeparatorMismatch = TestLabel "reject_separator_mismatch" $ TestCase $ do + let opts = Csv.defaultReadOptions{Csv.columnSeparator = ';'} + htab = 0x09 + r <- try (Fast.readSeparated htab opts "no-such-file.csv") + case r of + Left (e :: SomeException) -> + assertBool + ("expected a separator complaint, got " <> show e) + ("columnSeparator" `isInfixOf` show e) + Right (_ :: D.DataFrame) -> assertFailure "expected a separator mismatch error" + +tests :: [Test] +tests = + [ scanProjects + , scanProjectsWithFastReader + , readersAgree + , scanHonoursSeparator + , scanMissingColumn + , rejectSeparatorMismatch + ] + <> invalidOptions diff --git a/tests/Main.hs b/tests/Main.hs --- a/tests/Main.hs +++ b/tests/Main.hs @@ -20,6 +20,7 @@ import qualified Internal.Parsing import qualified LazyParity import qualified LazyParquet +import qualified LazyProjection import qualified Learn.Denotation import qualified Learn.Ensembles import qualified Learn.Metamorphic @@ -119,6 +120,7 @@ ++ Parquet.tests ++ LazyParquet.tests ++ LazyParity.tests + ++ LazyProjection.tests ++ Plotting.tests ++ Simplify.tests ++ PackedTextMigration.tests diff --git a/tests/Operations/WriteCsv.hs b/tests/Operations/WriteCsv.hs --- a/tests/Operations/WriteCsv.hs +++ b/tests/Operations/WriteCsv.hs @@ -25,7 +25,10 @@ toCsvWithNulls = TestLabel "toCsv_withNulls" $ TestCase $ do let df = D.fromNamedColumns - [ ("name", DI.fromList @(Maybe T.Text) [Nothing, Just "Alice", Just "Bob", Just "Charlie"]) + [ + ( "name" + , DI.fromList @(Maybe T.Text) [Nothing, Just "Alice", Just "Bob", Just "Charlie"] + ) , ("age", DI.fromList @(Maybe Int) [Just 30, Nothing, Just 25, Just 35]) ] expected = "name,age\nnull,30\nAlice,null\nBob,25\nCharlie,35\n" diff --git a/tests/Typed/IOReaders.hs b/tests/Typed/IOReaders.hs --- a/tests/Typed/IOReaders.hs +++ b/tests/Typed/IOReaders.hs @@ -18,6 +18,7 @@ import qualified DataFrame as D import DataFrame.Errors (DataFrameException) import qualified DataFrame.Internal.Column as DI +import DataFrame.Internal.DataFrame (getColumn) import qualified DataFrame.Typed as DT import qualified DataFrame.Typed.IO.CSV as TCSV @@ -61,5 +62,42 @@ IO (Either DataFrameException Int) assertBool "wrong schema => throws DataFrameException" (isLeft r) +{- | The schema is the read specification: only its columns are fetched, out +of a file that holds more. +-} +typedReadProjects :: Test +typedReadProjects = TestCase $ withSystemTempFile "typed_proj.csv" $ \fp h -> do + hClose h + D.writeCsv fp sampleDF + narrow <- DT.thaw <$> TCSV.readCsv @'[DT.Column "g" T.Text] fp + assertEqual "only the schema's column is read" ["g"] (D.columnNames narrow) + assertEqual "rows intact" (3, 1) (D.dimensions narrow) + +{- | The schema's types win over inference: a column of digits declared 'Text' +comes back as 'Text', which a plain read would have called 'Int'. +-} +typedReadSharesTypes :: Test +typedReadSharesTypes = TestCase $ withSystemTempFile "typed_types.csv" $ \fp h -> do + hClose h + D.writeCsv fp digits + inferred <- D.readCsv fp + assertEqual + "untyped read infers Int" + (Just (DI.fromList [1, 2, 3 :: Int])) + (getColumn "n" inferred) + typed <- DT.thaw <$> TCSV.readCsv @'[DT.Column "n" T.Text] fp + assertEqual + "schema types the column as Text" + (Just (DI.fromList ["1", "2", "3" :: T.Text])) + (getColumn "n" typed) + where + digits = D.fromNamedColumns [("n", DI.fromList [1, 2, 3 :: Int])] + tests :: [Test] -tests = [roundTrip, wrongSchemaEither, wrongSchemaThrows] +tests = + [ roundTrip + , wrongSchemaEither + , wrongSchemaThrows + , typedReadProjects + , typedReadSharesTypes + ]