diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Revision history for dataframe +## 3.3.0.0 + +### Breaking changes +* Typed schemas are now type-level lists of promoted pairs instead of the + `Column` phantom: `'[ '("name", Text), '("age", Int)]` replaces + `'[Column "name" Text, Column "age" Int]`. `Column` is no longer exported + from `DataFrame.Typed`; the schema kind is `[(Symbol, Type)]`. +* Coordinated major bumps: `dataframe-core` → `2.2.0.0`, + `dataframe-operations`/`dataframe-parsing`/`dataframe-learn`/`dataframe-th` + → `2.2.0.0`, `dataframe-csv`/`dataframe-lazy` → `2.3.0.0`, + `dataframe-parquet` → `1.5.0.0`, `dataframe-csv-th`/`dataframe-parquet-th` + → `1.3.0.0`, `dataframe-viz` → `1.3.0.0`, `dataframe-fusion` → `0.2.0.0`, + `dataframe-persistent` → `0.5.0.0`, with inter-package bounds raised to + match. + ## 3.2.0.0 * Restrict CSV reads to selected columns for untyped and typed readers. diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -289,9 +289,9 @@ ```haskell -- Generates: --- type HousingSchema = '[ DT.Column "longitude" Double --- , DT.Column "latitude" Double --- , DT.Column "total_rooms" Double +-- type HousingSchema = '[ '("longitude", Double) +-- , '("latitude", Double) +-- , '("total_rooms", Double) -- , ... -- ] $(DT.deriveSchemaFromCsvFile "HousingSchema" "./data/housing.csv") @@ -318,7 +318,7 @@ $(DT.deriveSchemaFromType ''Order) -- expands to: -- type OrderSchema = --- '[DT.Column "order_id" Int64, DT.Column "region" Text, DT.Column "amount" Double] +-- '[ '("order_id", Int64), '("region", Text), '("amount", Double)] -- instance DT.HasSchema Order where -- type Schema Order = OrderSchema -- toColumns = ... @@ -433,9 +433,9 @@ ```haskell type EmployeeSchema = - '[ DT.Column "name" Text - , DT.Column "department" Text - , DT.Column "salary" Double + '[ '("name", Text) + , '("department", Text) + , '("salary", Double) ] employees <- D.readCsv "./data/employees.csv" @@ -464,7 +464,7 @@ ```text -- Typo in column name -> compile error tdf |> DT.filterWhere (DT.col @"slary" DT..>. DT.lit 50000) --- error: Column "slary" not found in schema +-- error: Column 'slary' not found in schema -- Wrong type -> compile error tdf |> DT.filterWhere (DT.col @"name" DT..>. DT.lit 50000) @@ -476,7 +476,7 @@ ```haskell -type ScoreSchema = '[ DT.Column "name" Text, DT.Column "score" (Maybe Double) ] +type ScoreSchema = '[ '("name", Text), '("score", Maybe Double)] scoresDf = D.fromNamedColumns [ ("name", D.fromList ["a", "b", "c" :: Text]) diff --git a/app/Synthesis.hs b/app/Synthesis.hs --- a/app/Synthesis.hs +++ b/app/Synthesis.hs @@ -28,7 +28,7 @@ -- Survived is Maybe Int (safeRead = MaybeRead); prediction is Int (model output). type RawPredSchema = - '[DT.Column "Survived" (Maybe Int), DT.Column "prediction" Int] + '[ '("Survived", Maybe Int), '("prediction", Int)] prediction :: D.Expr Int prediction = F.col @Int "prediction" diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: dataframe -version: 3.2.0.1 +version: 3.3.0.0 synopsis: A fast, safe, and intuitive DataFrame library. description: A fast, safe, and intuitive DataFrame library for exploratory data analysis. @@ -128,18 +128,18 @@ DataFrame.Typed.Record, DataFrame.Typed.Generic build-depends: base >= 4 && <5, - dataframe-core >= 2.1 && < 2.2, - dataframe-json >= 1.2 && < 1.3, - dataframe-expr-serializer >= 1.2 && < 1.3, - dataframe-operations >= 2.1 && < 2.2, - dataframe-parsing >= 2.1 && < 2.2, - dataframe-viz >= 1.2 && < 1.3, - dataframe-learn >= 2.1 && < 2.2 + dataframe-core >= 2.2 && < 2.3, + dataframe-json >= 1.2.0.1 && < 1.3, + dataframe-expr-serializer >= 1.2.0.1 && < 1.3, + dataframe-operations >= 2.2 && < 2.3, + dataframe-parsing >= 2.2 && < 2.3, + dataframe-viz >= 1.3 && < 1.4, + dataframe-learn >= 2.2 && < 2.3 if !flag(no-csv) reexported-modules: DataFrame.IO.CSV, DataFrame.Typed.IO.CSV - build-depends: dataframe-csv >= 2.2 && < 2.3 + build-depends: dataframe-csv >= 2.3 && < 2.4 cpp-options: -DWITH_CSV if !flag(no-parquet) @@ -157,7 +157,7 @@ DataFrame.IO.Parquet.Time, DataFrame.IO.Utils.RandomAccess, DataFrame.Typed.IO.Parquet - build-depends: dataframe-parquet >= 1.4 && < 1.5 + build-depends: dataframe-parquet >= 1.5 && < 1.6 cpp-options: -DWITH_PARQUET -- The lazy executor calls both CSV and Parquet readers directly, so @@ -167,21 +167,21 @@ DataFrame.Lazy.IO.Binary, DataFrame.Lazy.IO.CSV, DataFrame.Typed.Lazy - build-depends: dataframe-lazy >= 2.2 && < 2.3 + build-depends: dataframe-lazy >= 2.3 && < 2.4 cpp-options: -DWITH_LAZY if !flag(no-th) - build-depends: dataframe-th >= 2.1 && < 2.2 + build-depends: dataframe-th >= 2.2 && < 2.3 cpp-options: -DWITH_TH exposed-modules: DataFrame.TH, DataFrame.Typed.TH if !flag(no-th) && !flag(no-csv) - build-depends: dataframe-csv-th >= 1.2 && < 1.3 + build-depends: dataframe-csv-th >= 1.3 && < 1.4 cpp-options: -DWITH_CSV_TH if !flag(no-th) && !flag(no-parquet) - build-depends: dataframe-parquet-th >= 1.2 && < 1.3 + build-depends: dataframe-parquet-th >= 1.3 && < 1.4 cpp-options: -DWITH_PARQUET_TH hs-source-dirs: src @@ -203,14 +203,14 @@ buildable: False build-depends: base >= 4 && < 5, - dataframe-core >= 2.1 && < 2.2, - dataframe-expr-serializer >= 1.2 && < 1.3, - dataframe-csv >= 2.2 && < 2.3, - dataframe-json >= 1.2 && < 1.3, - dataframe-lazy >= 2.2 && < 2.3, - dataframe-operations >= 2.1 && < 2.2, - dataframe-parquet >= 1.4 && < 1.5, - dataframe-parsing >= 2.1 && < 2.2, + dataframe-core >= 2.2 && < 2.3, + dataframe-expr-serializer >= 1.2.0.1 && < 1.3, + dataframe-csv >= 2.3 && < 2.4, + dataframe-json >= 1.2.0.1 && < 1.3, + dataframe-lazy >= 2.3 && < 2.4, + dataframe-operations >= 2.2 && < 2.3, + dataframe-parquet >= 1.5 && < 1.6, + dataframe-parsing >= 2.2 && < 2.3, text >= 2.1 && < 3, aeson >= 0.11 && < 3, bytestring >= 0.11 && < 0.14, @@ -223,8 +223,8 @@ import: warnings main-is: Benchmark.hs build-depends: base >= 4 && < 5, - dataframe >= 3.2 && < 3.3, - dataframe-operations >= 2.1 && < 2.2, + dataframe >= 3.3 && < 3.4, + dataframe-operations >= 2.2 && < 2.3, random >= 1 && < 2, time >= 1.12 && < 2, vector >= 0.13 && < 0.15, @@ -236,10 +236,10 @@ import: warnings main-is: Synthesis.hs build-depends: base >= 4 && < 5, - dataframe >= 3.2 && < 3.3, - dataframe-core >= 2.1 && < 2.2, - dataframe-learn >= 2.1 && < 2.2, - dataframe-operations >= 2.1 && < 2.2, + dataframe >= 3.3 && < 3.4, + dataframe-core >= 2.2 && < 2.3, + dataframe-learn >= 2.2 && < 2.3, + dataframe-operations >= 2.2 && < 2.3, random >= 1 && < 2, text >= 2.1 && < 3 hs-source-dirs: app @@ -270,10 +270,10 @@ build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.14, containers >= 0.6.7 && < 0.10, - dataframe >= 3.2 && < 3.3, - dataframe-core >= 2.1 && < 2.2, - dataframe-lazy >= 2.2 && < 2.3, - dataframe-parsing >= 2.1 && < 2.2, + dataframe >= 3.3 && < 3.4, + dataframe-core >= 2.2 && < 2.3, + dataframe-lazy >= 2.3 && < 2.4, + dataframe-parsing >= 2.2 && < 2.3, directory >= 1.3.0.0 && < 2, random >= 1 && < 2, text >= 2.1 && < 3, @@ -291,9 +291,9 @@ criterion >= 1 && < 2, deepseq >= 1.4 && < 2, process >= 1.6 && < 2, - dataframe >= 3.2 && < 3.3, - dataframe-core >= 2.1 && < 2.2, - dataframe-operations >= 2.1 && < 2.2, + dataframe >= 3.3 && < 3.4, + dataframe-core >= 2.2 && < 2.3, + dataframe-operations >= 2.2 && < 2.3, random >= 1 && < 2, default-language: Haskell2010 ghc-options: @@ -377,18 +377,18 @@ build-depends: base >= 4 && < 5, aeson >= 0.11.0.0 && < 3, bytestring >= 0.11 && < 0.14, - dataframe >= 3.2 && < 3.3, - dataframe-core >= 2.1 && < 2.2, - dataframe-core >= 2.1 && < 2.2, - dataframe-csv >= 2.2 && < 2.3, - dataframe-expr-serializer >= 1.2 && < 1.3, - dataframe-fastcsv >= 1.4 && < 1.5, - dataframe-json >= 1.2 && < 1.3, - dataframe-lazy >= 2.2 && < 2.3, - dataframe-learn >= 2.1 && < 2.2, - dataframe-operations >= 2.1 && < 2.2, - dataframe-parquet >= 1.4 && < 1.5, - dataframe-parsing >= 2.1 && < 2.2, + dataframe >= 3.3 && < 3.4, + dataframe-core >= 2.2 && < 2.3, + dataframe-core >= 2.2 && < 2.3, + dataframe-csv >= 2.3 && < 2.4, + dataframe-expr-serializer >= 1.2.0.1 && < 1.3, + dataframe-fastcsv >= 1.4.0.1 && < 1.5, + dataframe-json >= 1.2.0.1 && < 1.3, + dataframe-lazy >= 2.3 && < 2.4, + dataframe-learn >= 2.2 && < 2.3, + dataframe-operations >= 2.2 && < 2.3, + dataframe-parquet >= 1.5 && < 1.6, + dataframe-parsing >= 2.2 && < 2.3, HUnit >= 1.6 && < 1.8, QuickCheck >= 2 && < 3, random-shuffle >= 0.0.4 && < 1, @@ -413,9 +413,9 @@ other-modules: Internal.PackedText build-depends: base >= 4 && < 5, bytestring >= 0.11 && < 0.14, - dataframe >= 3.2 && < 3.3, - dataframe-core >= 2.1 && < 2.2, - dataframe-operations >= 2.1 && < 2.2, + dataframe >= 3.3 && < 3.4, + dataframe-core >= 2.2 && < 2.3, + dataframe-operations >= 2.2 && < 2.3, HUnit >= 1.6 && < 1.8, text >= 2.1 && < 3, vector >= 0.13 && < 0.15 diff --git a/src/DataFrame/Typed.hs b/src/DataFrame/Typed.hs --- a/src/DataFrame/Typed.hs +++ b/src/DataFrame/Typed.hs @@ -24,7 +24,7 @@ {\-\# LANGUAGE DataKinds, TypeApplications, TypeOperators \#-\} import qualified DataFrame.Typed as T -type People = '[T.Column \"name\" Text, T.Column \"age\" Int] +type People = '[ '(\"name\", Text), '(\"age\", Int)] main = do raw <- D.readCsv \"people.csv\" @@ -43,8 +43,8 @@ == filterAllJust tracks Maybe-stripping @ -df :: TypedDataFrame '[Column \"x\" (Maybe Double), Column \"y\" Int] -T.filterAllJust df :: TypedDataFrame '[Column \"x\" Double, Column \"y\" Int] +df :: TypedDataFrame '[ '(\"x\", Maybe Double), '(\"y\", Int)] +T.filterAllJust df :: TypedDataFrame '[ '(\"x\", Double), '(\"y\", Int)] @ == Typed aggregation @@ -60,7 +60,6 @@ module DataFrame.Typed ( -- * Core types TypedDataFrame, - Column, TypedGrouped, These (..), @@ -394,7 +393,6 @@ ) #endif import DataFrame.Typed.Types ( - Column, TSortOrder (..), These (..), TypedDataFrame, diff --git a/tests/Learn/TypedModel.hs b/tests/Learn/TypedModel.hs --- a/tests/Learn/TypedModel.hs +++ b/tests/Learn/TypedModel.hs @@ -33,9 +33,9 @@ -- | All-Double schema: two features and a Double target. type Houses = - '[ DT.Column "x1" Double - , DT.Column "x2" Double - , DT.Column "y" Double + '[ '("x1", Double) + , '("x2", Double) + , '("y", Double) ] regDF :: D.DataFrame diff --git a/tests/Operations/Aggregations.hs b/tests/Operations/Aggregations.hs --- a/tests/Operations/Aggregations.hs +++ b/tests/Operations/Aggregations.hs @@ -74,12 +74,12 @@ ( testData & either (error . show) id . DT.freezeWithError - @[ DT.Column "test1" Int - , DT.Column "test2" Int - , DT.Column "test3" Int - , DT.Column "test4" Char - , DT.Column "test5" String - , DT.Column "test6" Integer + @[ '("test1", Int) + , '("test2", Int) + , '("test3", Int) + , '("test4", Char) + , '("test5", String) + , '("test6", Integer) ] & DT.groupBy @'["test1"] & DT.aggregate (DT.as @"n" DT.countAll) @@ -101,12 +101,12 @@ ( testData & either (error . show) id . DT.freezeWithError - @[ DT.Column "test1" Int - , DT.Column "test2" Int - , DT.Column "test3" Int - , DT.Column "test4" Char - , DT.Column "test5" String - , DT.Column "test6" Integer + @[ '("test1", Int) + , '("test2", Int) + , '("test3", Int) + , '("test4", Char) + , '("test5", String) + , '("test6", Integer) ] & DT.groupBy @'["test1"] & DT.aggregate (DT.as @"test2_count" (DT.count (DT.col @"test2"))) @@ -145,12 +145,12 @@ ( testData & either (error . show) id . DT.freezeWithError - @[ DT.Column "test1" Int - , DT.Column "test2" Int - , DT.Column "test3" Int - , DT.Column "test4" Char - , DT.Column "test5" String - , DT.Column "test6" Integer + @[ '("test1", Int) + , '("test2", Int) + , '("test3", Int) + , '("test4", Char) + , '("test5", String) + , '("test6", Integer) ] & DT.groupBy @'["test1"] & DT.aggregate (DT.as @"test2_mean" (DT.mean (DT.col @"test2"))) diff --git a/tests/Operations/Derive.hs b/tests/Operations/Derive.hs --- a/tests/Operations/Derive.hs +++ b/tests/Operations/Derive.hs @@ -65,7 +65,7 @@ (error . show) id ( DT.freezeWithError - @[DT.Column "test1" Int, DT.Column "test2" String, DT.Column "test3" Char] + @['("test1", Int), '("test2", String), '("test3", Char)] testData ) ) diff --git a/tests/Operations/Join.hs b/tests/Operations/Join.hs --- a/tests/Operations/Join.hs +++ b/tests/Operations/Join.hs @@ -85,10 +85,10 @@ (D.sortBy [D.Asc (F.col @Text "key")] (rightJoin ["key"] df1 df2)) ) -tdf1 :: DT.TypedDataFrame [DT.Column "key" Text, DT.Column "A" Text] +tdf1 :: DT.TypedDataFrame ['("key", Text), '("A", Text)] tdf1 = either (error . show) id (DT.freezeWithError df1) -tdf2 :: DT.TypedDataFrame [DT.Column "key" Text, DT.Column "B" Text] +tdf2 :: DT.TypedDataFrame ['("key", Text), '("B", Text)] tdf2 = either (error . show) id (DT.freezeWithError df2) testInnerJoinTyped :: Test @@ -128,7 +128,7 @@ ] tdfOptional :: - DT.TypedDataFrame [DT.Column "key" Text, DT.Column "C" (Maybe Int)] + DT.TypedDataFrame ['("key", Text), '("C", Maybe Int)] tdfOptional = either (error . show) id (DT.freezeWithError dfOptional) {- | A left join over an already-optional column must not nest the Maybe: the @@ -155,7 +155,7 @@ where joined :: DT.TypedDataFrame - [DT.Column "key" Text, DT.Column "A" Text, DT.Column "C" (Maybe Int)] + ['("key", Text), '("A", Text), '("C", Maybe Int)] joined = DT.leftJoin @'["key"] tdf1 tdfOptional testRightJoinTyped :: Test diff --git a/tests/Operations/Nullable.hs b/tests/Operations/Nullable.hs --- a/tests/Operations/Nullable.hs +++ b/tests/Operations/Nullable.hs @@ -530,10 +530,10 @@ -- --------------------------------------------------------------------------- type CrossSchema = - '[ DT.Column "x" Int - , DT.Column "y" (Maybe Int) - , DT.Column "d" Double - , DT.Column "md" (Maybe Double) + '[ '("x", Int) + , '("y", Maybe Int) + , '("d", Double) + , '("md", Maybe Double) ] typedCrossData :: DT.TypedDataFrame CrossSchema @@ -617,7 +617,7 @@ -- Typed TExpr layer tests -- --------------------------------------------------------------------------- -type TestSchema = '[DT.Column "x" Int, DT.Column "y" (Maybe Int)] +type TestSchema = '[ '("x", Int), '("y", Maybe Int)] typedTestData :: DT.TypedDataFrame TestSchema typedTestData = diff --git a/tests/Plotting.hs b/tests/Plotting.hs --- a/tests/Plotting.hs +++ b/tests/Plotting.hs @@ -200,7 +200,7 @@ typedParity = TestCase $ do let tdf = DT.unsafeFreeze numFrame :: - DT.TypedDataFrame '[DT.Column "a" Double, DT.Column "b" Double] + DT.TypedDataFrame '[ '("a", Double), '("b", Double)] specU = C.toVegaSpec ( C.chart numFrame diff --git a/tests/Typed/IOReaders.hs b/tests/Typed/IOReaders.hs --- a/tests/Typed/IOReaders.hs +++ b/tests/Typed/IOReaders.hs @@ -25,9 +25,9 @@ import Test.HUnit type S = - '[ DT.Column "x" Int - , DT.Column "y" Double - , DT.Column "g" T.Text + '[ '("x", Int) + , '("y", Double) + , '("g", T.Text) ] sampleDF :: D.DataFrame @@ -50,7 +50,7 @@ wrongSchemaEither = TestCase $ withSystemTempFile "typed_err.csv" $ \fp h -> do hClose h D.writeCsv fp sampleDF - res <- TCSV.readCsvWithError @'[DT.Column "nope" Int] fp + res <- TCSV.readCsvWithError @'[ '("nope", Int)] fp assertBool "wrong schema => Left" (isLeft res) wrongSchemaThrows :: Test @@ -58,7 +58,7 @@ hClose h D.writeCsv fp sampleDF r <- - try (TCSV.readCsv @'[DT.Column "nope" Int] fp >>= evaluate . DT.nRows) :: + try (TCSV.readCsv @'[ '("nope", Int)] fp >>= evaluate . DT.nRows) :: IO (Either DataFrameException Int) assertBool "wrong schema => throws DataFrameException" (isLeft r) @@ -69,7 +69,7 @@ typedReadProjects = TestCase $ withSystemTempFile "typed_proj.csv" $ \fp h -> do hClose h D.writeCsv fp sampleDF - narrow <- DT.thaw <$> TCSV.readCsv @'[DT.Column "g" T.Text] fp + narrow <- DT.thaw <$> TCSV.readCsv @'[ '("g", T.Text)] fp assertEqual "only the schema's column is read" ["g"] (D.columnNames narrow) assertEqual "rows intact" (3, 1) (D.dimensions narrow) @@ -85,7 +85,7 @@ "untyped read infers Int" (Just (DI.fromList [1, 2, 3 :: Int])) (getColumn "n" inferred) - typed <- DT.thaw <$> TCSV.readCsv @'[DT.Column "n" T.Text] fp + typed <- DT.thaw <$> TCSV.readCsv @'[ '("n", T.Text)] fp assertEqual "schema types the column as Text" (Just (DI.fromList ["1", "2", "3" :: T.Text])) diff --git a/tests/Typed/Parity.hs b/tests/Typed/Parity.hs --- a/tests/Typed/Parity.hs +++ b/tests/Typed/Parity.hs @@ -18,9 +18,9 @@ import Test.HUnit type S = - '[ DT.Column "x" Int - , DT.Column "y" Double - , DT.Column "g" T.Text + '[ '("x", Int) + , '("y", Double) + , '("g", T.Text) ] baseDF :: D.DataFrame @@ -178,10 +178,10 @@ rightDF :: D.DataFrame rightDF = D.fromNamedColumns [("y", DI.fromList [1.0, 2.0 :: Double])] -leftT :: DT.TypedDataFrame '[DT.Column "x" Int] +leftT :: DT.TypedDataFrame '[ '("x", Int)] leftT = either (error . show) id (DT.freezeWithError leftDF) -rightT :: DT.TypedDataFrame '[DT.Column "y" Double] +rightT :: DT.TypedDataFrame '[ '("y", Double)] rightT = either (error . show) id (DT.freezeWithError rightDF) -- Numeric-only frame for the matrix test. @@ -192,7 +192,7 @@ , ("y", DI.fromList [1.5, 2.5, 3.5 :: Double]) ] -numericT :: DT.TypedDataFrame '[DT.Column "x" Int, DT.Column "y" Double] +numericT :: DT.TypedDataFrame '[ '("x", Int), '("y", Double)] numericT = either (error . show) id (DT.freezeWithError numericDF) tests :: [Test]