diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Revision history for dataframe +## 3.6.1.0 + +* splices for deriving schema +* writeParquet and friends added. +* skewness computed as g1 +* throw on empty dataset for statistical functions +* streaming reads for lazy CSV +* shuffling now uses fischer yates + ## 3.6.0.0 ### Breaking changes diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + +> > | product
Int | total
Int | orders
Int | > | ---------------|--------------|-------------- | > | 1 | 220 | 2 | -> | 3 | 70 | 2 | > | 2 | 70 | 2 | - +> | 3 | 70 | 2 | Reading from files works the same way: - ```haskell fileDf <- D.readCsv "./data/housing.csv" fileDf <- D.readParquet "./data/mtcars.parquet" @@ -129,30 +122,26 @@ D.dimensions fileDf ``` -> +> > (32,12) - ## Interactive REPL The `dataframe` REPL comes with all imports pre-loaded. Here's a typical exploration session (each block runs as a cell): - ```haskell df <- D.readCsv "./data/housing.csv" D.dimensions df ``` -> +> > (20640,10) - - ```haskell D.describeColumns df |> D.toMarkdown' ``` -> +> > | Column Name
Text | # Non-null Values
Int | # Null Values
Int | Type
Text | > | --------------------|--------------------------|----------------------|------------- | > | total_bedrooms | 20433 | 207 | Maybe Double | @@ -166,17 +155,13 @@ > | latitude | 20640 | 0 | Double | > | longitude | 20640 | 0 | Double | - The `:declareColumns` macro (`$(D.declareColumns df)` outside the REPL) generates typed column references from a dataframe, so you can use column names directly in expressions instead of writing `F.col @Double "median_income"` every time: - ```haskell $(D.declareColumns df) ``` -> - - +> ```haskell df |> D.groupBy ["ocean_proximity"] @@ -184,34 +169,30 @@ |> D.toMarkdown' ``` -> +> > | ocean_proximity
Text | avg_value
Double | > | ------------------------|-------------------- | +> | ISLAND | 380440.0 | > | NEAR BAY | 259212.31179039303 | -> | NEAR OCEAN | 249433.97742663656 | -> | INLAND | 124805.39200122119 | > | <1H OCEAN | 240084.28546409807 | -> | ISLAND | 380440.0 | - +> | INLAND | 124805.39200122119 | +> | NEAR OCEAN | 249433.97742663656 | Create new columns from existing ones: - ```haskell df |> D.derive "rooms_per_household" (total_rooms / households) |> D.take 3 |> D.toMarkdown' ``` -> +> > | longitude
Double | latitude
Double | housing_median_age
Double | total_rooms
Double | total_bedrooms
Maybe Double | population
Double | households
Double | median_income
Double | median_house_value
Double | ocean_proximity
Text | rooms_per_household
Double | > | --------------------|--------------------|------------------------------|-----------------------|--------------------------------|----------------------|----------------------|-------------------------|------------------------------|-------------------------|------------------------------ | -> | -122.23 | 37.88 | 41.0 | 880.0 | Just 129.0 | 322.0 | 126.0 | 8.3252 | 452600.0 | NEAR BAY | 6.984126984126984 | -> | -122.22 | 37.86 | 21.0 | 7099.0 | Just 1106.0 | 2401.0 | 1138.0 | 8.3014 | 358500.0 | NEAR BAY | 6.238137082601054 | -> | -122.24 | 37.85 | 52.0 | 1467.0 | Just 190.0 | 496.0 | 177.0 | 7.2574 | 352100.0 | NEAR BAY | 8.288135593220339 | - +> | -122.23 | 37.88 | 41.0 | 880.0 | 129.0 | 322.0 | 126.0 | 8.3252 | 452600.0 | NEAR BAY | 6.984126984126984 | +> | -122.22 | 37.86 | 21.0 | 7099.0 | 1106.0 | 2401.0 | 1138.0 | 8.3014 | 358500.0 | NEAR BAY | 6.238137082601054 | +> | -122.24 | 37.85 | 52.0 | 1467.0 | 190.0 | 496.0 | 177.0 | 7.2574 | 352100.0 | NEAR BAY | 8.288135593220339 | Type mismatches are caught as compile errors — adding a `Double` column to a `Text` column won't silently produce garbage: - ```text dataframe> df |> D.derive "nonsense" (latitude + ocean_proximity) @@ -224,7 +205,6 @@ '(latitude + ocean_proximity)' ``` - ## Template Haskell For scripts and projects, Template Haskell can generate column bindings at compile time. @@ -234,7 +214,6 @@ `declareColumnsFromCsvFile` (in `DataFrame.TH`, also re-exported from `DataFrame`) reads your CSV at compile time and generates typed `Expr` bindings for every column: - ```haskell -- Reads housing.csv at compile time and generates: -- latitude :: Expr Double @@ -251,18 +230,16 @@ |> D.toMarkdown' ``` -> +> > | ocean_proximity
Text | avg_value
Double | > | ------------------------|-------------------- | > | NEAR BAY | 361441.9354304636 | -> | NEAR OCEAN | 380041.63071895426 | -> | INLAND | 234817.86695906433 | > | <1H OCEAN | 333411.75125531096 | - +> | INLAND | 234817.86695906433 | +> | NEAR OCEAN | 380041.63071895426 | Compare this to the manual version which requires spelling out every column name and type: - ```haskell -- Without TH — every column needs its name and type spelled out df |> D.derive "rooms_per_household" @@ -272,22 +249,22 @@ |> D.toMarkdown' ``` -> +> > | longitude
Double | latitude
Double | housing_median_age
Double | total_rooms
Double | total_bedrooms
Maybe Double | population
Double | households
Double | median_income
Double | median_house_value
Double | ocean_proximity
Text | rooms_per_household
Double | > | --------------------|--------------------|------------------------------|-----------------------|--------------------------------|----------------------|----------------------|-------------------------|------------------------------|-------------------------|------------------------------ | -> | -122.23 | 37.88 | 41.0 | 880.0 | Just 129.0 | 322.0 | 126.0 | 8.3252 | 452600.0 | NEAR BAY | 6.984126984126984 | -> | -122.22 | 37.86 | 21.0 | 7099.0 | Just 1106.0 | 2401.0 | 1138.0 | 8.3014 | 358500.0 | NEAR BAY | 6.238137082601054 | -> | -122.24 | 37.85 | 52.0 | 1467.0 | Just 190.0 | 496.0 | 177.0 | 7.2574 | 352100.0 | NEAR BAY | 8.288135593220339 | -> | -122.25 | 37.85 | 52.0 | 1274.0 | Just 235.0 | 558.0 | 219.0 | 5.6431000000000004 | 341300.0 | NEAR BAY | 5.8173515981735155 | -> | -122.29 | 37.82 | 49.0 | 135.0 | Just 29.0 | 86.0 | 23.0 | 6.1183 | 75000.0 | NEAR BAY | 5.869565217391305 | - +> | -122.23 | 37.88 | 41.0 | 880.0 | 129.0 | 322.0 | 126.0 | 8.3252 | 452600.0 | NEAR BAY | 6.984126984126984 | +> | -122.22 | 37.86 | 21.0 | 7099.0 | 1106.0 | 2401.0 | 1138.0 | 8.3014 | 358500.0 | NEAR BAY | 6.238137082601054 | +> | -122.24 | 37.85 | 52.0 | 1467.0 | 190.0 | 496.0 | 177.0 | 7.2574 | 352100.0 | NEAR BAY | 8.288135593220339 | +> | -122.25 | 37.85 | 52.0 | 1274.0 | 235.0 | 558.0 | 219.0 | 5.6431000000000004 | 341300.0 | NEAR BAY | 5.8173515981735155 | +> | -122.29 | 37.82 | 49.0 | 135.0 | 29.0 | 86.0 | 23.0 | 6.1183 | 75000.0 | NEAR BAY | 5.869565217391305 | ### Generate a schema type from a CSV `deriveSchemaFromCsvFile` generates a type synonym for use with the typed API — instead of manually writing out every column name and type: - ```haskell +import qualified DataFrame.Typed as DT + -- Generates: -- type HousingSchema = '[ '("longitude", Double) -- , '("latitude", Double) @@ -297,8 +274,7 @@ $(DT.deriveSchemaFromCsvFile "HousingSchema" "./data/housing.csv") ``` -> - +> ### Generate a schema (and a row bridge) from a record ADT @@ -307,8 +283,10 @@ instance that converts between `[Order]` and a `DataFrame` (or `TypedDataFrame OrderSchema`) at runtime: - ```haskell +import Data.Text (Text) +import Data.Int (Int64) + data Order = Order { orderId :: Int64 , region :: Text @@ -334,49 +312,43 @@ ordersDf |> D.toMarkdown' ``` -> +> > | order_id
Int64 | region
Text | amount
Double | > | ------------------|----------------|----------------- | > | 1 | us | 10.0 | > | 2 | eu | 20.5 | - The runtime-checked round-trip back to records: - ```haskell D.toRecords ordersDf :: Either Text [Order] ``` -> +> > Right [Order {orderId = 1, region = "us", amount = 10.0},Order {orderId = 2, region = "eu", amount = 20.5}] - And the typed bridge — `[Order]` to `TypedDataFrame OrderSchema` and back: - ```haskell DT.thaw (DT.fromRecordsTyped xs :: DT.TypedDataFrame OrderSchema) |> D.toMarkdown' ``` -> +> > | order_id
Int64 | region
Text | amount
Double | > | ------------------|----------------|----------------- | > | 1 | us | 10.0 | > | 2 | eu | 20.5 | - Field names are translated `camelCase → snake_case` by default; override the translation with `deriveSchemaFromTypeWith defaultSchemaOptions{nameTransform = id}` (or any `String -> String`). If all you need is a runtime `Schema` to drive `readCsvWithSchema` (no -typed-dataframe machinery), there's a companion splice in -`DataFrame.Internal.Schema` (re-exported from `DataFrame`): - +typed-dataframe machinery), `deriveSchemaValues` (in +`DataFrame.Typed.TH.Records`, re-exported from `DataFrame`) is the companion splice: ```haskell -$(D.deriveSchema ''Order) +$(D.deriveSchemaValues ''Order) -- emits: -- orderSchema :: Schema -- orderSchema = makeSchema [("order_id", schemaType @Int64), ...] @@ -393,8 +365,7 @@ pure (D.filter orderAmount (> 100) raw) ``` -> - +> Each record field gets a typed accessor named ``, so `data Order { customerId :: Int }` yields `orderCustomerId :: Expr Int = col "customer_id"`. @@ -404,7 +375,6 @@ If you'd rather not depend on Template Haskell, the same schema is available via `GHC.Generics` (shown here on an equivalent record): - ```haskell import GHC.Generics (Generic) import DataFrame.Typed (Schema) @@ -423,14 +393,12 @@ fromColumns = DT.genericFromColumns ``` -> - +> ## Typed API When you want compile-time guarantees that column names exist and types match, wrap your `DataFrame` in a `TypedDataFrame`: - ```haskell type EmployeeSchema = '[ '("name", Text) @@ -449,7 +417,7 @@ |> D.toMarkdown' ``` -> +> > | name
Text | bonus
Double | > | -------------|---------------- | > | Alice | 8500.0 | @@ -457,10 +425,8 @@ > | Dave | 5200.0 | > | Frank | 6700.0 | - `DT.freeze` validates the runtime `DataFrame` against your schema once at the boundary. After that, every column access is checked at compile time: - ```text -- Typo in column name -> compile error tdf |> DT.filterWhere (DT.col @"slary" DT..>. DT.lit 50000) @@ -471,10 +437,8 @@ -- error: Couldn't match type 'Text' with 'Double' ``` - `filterAllJust` goes further — it strips `Maybe` from every column in the schema type, so downstream code can't accidentally treat cleaned columns as nullable: - ```haskell type ScoreSchema = '[ '("name", Text), '("score", Maybe Double)] @@ -490,13 +454,12 @@ DT.thaw (DT.filterAllJust stdf |> DT.derive @"scaled" (DT.col @"score" * DT.lit 100)) |> D.toMarkdown' ``` -> +> > | name
Text | score
Double | scaled
Double | > | -------------|-----------------|----------------- | > | a | 1.0 | 100.0 | > | c | 3.0 | 300.0 | - ## Features **I/O**: CSV, TSV, Parquet (Snappy, ZSTD, Gzip), JSON. Read Parquet from Hugging Face datasets (`hf://` URIs) via the `dataframe-huggingface` package. Column projection and predicate pushdown for Parquet reads. @@ -521,10 +484,9 @@ For files too large to fit in memory, `DataFrame.Lazy` provides a streaming query engine. Declare a schema, build a query plan with the same familiar operations, and `runDataFrame` runs it through an optimizer before streaming results batch-by-batch: - ```haskell import qualified DataFrame.Lazy as L -import DataFrame.Internal.Schema (schemaType, makeSchema) +import DataFrame.Schema (schemaType, makeSchema) housingSchema = makeSchema [ ("longitude", schemaType @Double) @@ -550,7 +512,7 @@ D.take 10 lazyResult |> D.toMarkdown' ``` -> +> > | ocean_proximity
Text | median_house_value
Double | value_per_income
Double | > | ------------------------|------------------------------|--------------------------- | > | NEAR BAY | 452600.0 | 54365.06029885168 | @@ -563,7 +525,6 @@ > | NEAR BAY | 347600.0 | 65748.65703260951 | > | NEAR BAY | 366100.0 | 61467.42780389524 | > | NEAR BAY | 373600.0 | 58895.860264211624 | - The optimizer pushes the filter into the scan, drops unreferenced columns before reading, and stops pulling batches once 1000 rows have been collected. diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: dataframe -version: 3.6.0.0 +version: 3.6.1.0 synopsis: A fast, safe, and intuitive DataFrame library. description: A fast, safe, and intuitive DataFrame library for exploratory data analysis. @@ -152,11 +152,12 @@ DataFrame.IO.Parquet.Page, DataFrame.IO.Parquet.Schema, DataFrame.IO.Parquet.Utils, + DataFrame.IO.Parquet.Writer, DataFrame.IO.Parquet.Seeking, DataFrame.IO.Parquet.Time, DataFrame.IO.Utils.RandomAccess, DataFrame.Typed.IO.Parquet - build-depends: dataframe-parquet >= 1.5 && < 1.6 + build-depends: dataframe-parquet >= 1.5.0.1 && < 1.6 cpp-options: -DWITH_PARQUET -- The lazy executor calls both CSV and Parquet readers directly, so @@ -170,17 +171,17 @@ cpp-options: -DWITH_LAZY if !flag(no-th) - build-depends: dataframe-th >= 2.2.1 && < 2.3 + build-depends: dataframe-th >= 2.3 && < 2.4 cpp-options: -DWITH_TH exposed-modules: DataFrame.TH, DataFrame.Typed.TH if !flag(no-th) && !flag(no-csv) - build-depends: dataframe-csv-th >= 1.3 && < 1.4 + build-depends: dataframe-csv-th >= 1.4 && < 1.5 cpp-options: -DWITH_CSV_TH if !flag(no-th) && !flag(no-parquet) - build-depends: dataframe-parquet-th >= 1.3 && < 1.4 + build-depends: dataframe-parquet-th >= 1.4 && < 1.5 cpp-options: -DWITH_PARQUET_TH hs-source-dirs: src @@ -354,7 +355,7 @@ dataframe-lazy >= 2.4.1 && < 2.5, dataframe-learn >= 2.4.2 && < 2.5, dataframe-operations >= 2.5 && < 2.6, - dataframe-parquet >= 1.5 && < 1.6, + dataframe-parquet >= 1.5.0.1 && < 1.6, dataframe-parsing >= 2.2 && < 2.3, HUnit >= 1.6 && < 1.8, QuickCheck >= 2 && < 3, diff --git a/src/DataFrame.hs b/src/DataFrame.hs --- a/src/DataFrame.hs +++ b/src/DataFrame.hs @@ -117,12 +117,18 @@ #ifdef WITH_PARQUET import DataFrame.IO.Parquet as Parquet ( ParquetReadOptions (..), + ParquetWriteOptions (..), + WriterStrategy (..), defaultParquetReadOptions, + defaultParquetWriteOptions, readParquet, readParquetFiles, readParquetFilesWithOpts, readParquetWithOpts, + writeParquet, + writeParquetWithOptions, ) + #endif import DataFrame.Core as CoreTypes ( Any, @@ -291,12 +297,16 @@ #ifdef WITH_CSV_TH declareColumnsFromCsvFile, declareColumnsFromCsvWithOpts, + deriveSchemaValuesFromCsvFile, + deriveSchemaValuesFromCsvWithOpts, #endif #ifdef WITH_PARQUET_TH declareColumnsFromParquetFile, + deriveSchemaValuesFromParquetFile, #endif declareColumnsWithPrefix, declareColumnsWithPrefix', + declareSchemaValues, ) #endif import DataFrame.Typed.Record as Record ( diff --git a/tests/Internal/DictEncode.hs b/tests/Internal/DictEncode.hs --- a/tests/Internal/DictEncode.hs +++ b/tests/Internal/DictEncode.hs @@ -115,6 +115,10 @@ "cap 3 bails" Nothing (dictEncodeColumnUpTo 3 (packedFromTexts sampleRows)) + assertEqual + "cap bails when the final row crosses it" + Nothing + (dictEncodeColumnUpTo 2 (packedFromTexts ["a", "b", "c"])) -- a generous cap still encodes. assertBool "cap 100 encodes" diff --git a/tests/Internal/Markdown.hs b/tests/Internal/Markdown.hs --- a/tests/Internal/Markdown.hs +++ b/tests/Internal/Markdown.hs @@ -8,7 +8,7 @@ import qualified Data.Text as T import DataFrame.Display.Terminal.PrettyPrint (escapeMarkdownCell) -import DataFrame.Internal.Column (fromList) +import DataFrame.Internal.Column (ensureOptional, fromList) import qualified DataFrame.Internal.DataFrame as D import Test.HUnit @@ -42,4 +42,14 @@ assertBool ("rows misaligned, delimiter counts: " ++ show delims) (case delims of [] -> False; (d : ds) -> all (== d) ds) + , TestLabel "nullable boxed Text renders without quotes" $ + TestCase $ do + let df = + D.fromNamedColumns + [("name", ensureOptional (fromList ["Ada" :: T.Text]))] + md = D.toMarkdown df + assertBool + "nullable Text value is missing" + ("Ada" `T.isInfixOf` md) + assertBool "nullable Text contains quotes" (not ("\"Ada\"" `T.isInfixOf` md)) ] diff --git a/tests/LazyParity.hs b/tests/LazyParity.hs --- a/tests/LazyParity.hs +++ b/tests/LazyParity.hs @@ -142,5 +142,46 @@ (show eager) (show lazy) +streamingMeanParity :: Test +streamingMeanParity = + TestCase $ + withCsv input $ \csvPath -> do + case DI.fromList [(0, 0) :: (Double, Int)] of + DI.UnboxedColumn{} -> pure () + _ -> assertFailure "streaming mean tuple must stay unboxed" + let valueD = F.col @Double "amount" + valueI = F.col @Int "order_id" + aggs = + [ F.mean valueD `as` "mean_double" + , F.mean valueI `as` "mean_int" + ] + query = + (L.scanCsvStreamingWith Csv.readSeparated ordersSchema (T.pack csvPath)) + { L.batchSize = 2 + } + |> L.groupBy ["customer_id"] aggs + |> L.sortBy [("customer_id", Descending)] + actual <- L.runDataFrame query + let expected = + Perm.sortBy [Perm.Desc (E.Col @Int "customer_id")] $ + Agg.aggregate aggs $ + Agg.groupBy ["customer_id"] input + assertEqual + "streaming partial mean == eager mean" + (show expected) + (show actual) + where + input = + D.fromNamedColumns + [ ("order_id", DI.fromList [1 .. 12 :: Int]) + , ("customer_id", DI.fromList [0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2 :: Int]) + , + ( "amount" + , DI.fromList + [0.5, 10.25, -3.0, 1.5, 11.25, -1.0, 2.5, 12.25, 1.0, 3.5, 13.25, 3.0 :: Double] + ) + , ("discount", DI.fromList (replicate 12 (0 :: Double))) + ] + tests :: [Test] -tests = [joinPipelineParity, groupByPipelineParity] +tests = [joinPipelineParity, groupByPipelineParity, streamingMeanParity] diff --git a/tests/Learn/MetricsTests.hs b/tests/Learn/MetricsTests.hs --- a/tests/Learn/MetricsTests.hs +++ b/tests/Learn/MetricsTests.hs @@ -1,8 +1,11 @@ {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-} module Learn.MetricsTests (tests) where +import qualified Control.Exception as E + import qualified DataFrame as D import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI @@ -43,6 +46,16 @@ assertBool "rmse" (close 1e-9 (rmse p t) 0.5) assertBool "mae" (close 1e-9 (mae p t) 0.25) assertBool "r2 in range" (r2 p t <= 1) + assertBool + "mse averages over compared pairs" + (close 1e-9 (mse (VU.fromList [2, 2]) (VU.fromList [0, 0, 0, 0])) 4) + assertBool + "mae averages over compared pairs" + (close 1e-9 (mae (VU.fromList [2, 2]) (VU.fromList [0, 0, 0, 0])) 2) + r <- E.try (E.evaluate (mse VU.empty (VU.fromList [5, 5, 5]))) + case r of + Left (_ :: E.SomeException) -> pure () + Right v -> assertFailure ("mse with no pairs returned " ++ show v) testMulticlassMetrics :: Test testMulticlassMetrics = TestCase $ do diff --git a/tests/Operations/Record.hs b/tests/Operations/Record.hs --- a/tests/Operations/Record.hs +++ b/tests/Operations/Record.hs @@ -42,6 +42,13 @@ $(DT.deriveSchemaFromType ''Order) $(D.deriveSchemaValues ''Order) +$(D.deriveSchemaValuesFromCsvFile "housing" "./tests/data/housing.csv") + +$( D.deriveSchemaValuesFromParquetFile + "alltypes" + "./tests/data/alltypes_plain.parquet" + ) + -- Nullable fields (Maybe Text -> RNullableBoxed; Maybe Int -> RNullableUnboxed). data User = User { userId :: Int64 @@ -353,6 +360,77 @@ [Order 1 "us" 10.0] xs +deriveSchemaValuesFromCsv :: Test +deriveSchemaValuesFromCsv = TestCase $ do + assertEqual + "csv-derived schema column names" + [ "households" + , "housing_median_age" + , "latitude" + , "longitude" + , "median_house_value" + , "median_income" + , "ocean_proximity" + , "population" + , "total_bedrooms" + , "total_rooms" + ] + (M.keys (IS.elements housingSchema)) + assertEqual + "csv-derived schema infers Double" + (Just (IS.schemaType @Double)) + (M.lookup "median_income" (IS.elements housingSchema)) + assertEqual + "csv-derived schema infers Text" + (Just (IS.schemaType @T.Text)) + (M.lookup "ocean_proximity" (IS.elements housingSchema)) + +deriveSchemaValuesFromCsvAccessors :: Test +deriveSchemaValuesFromCsvAccessors = TestCase $ do + df <- D.readCsvWithSchema housingSchema "./tests/data/housing.csv" + assertEqual + "prefixed accessor reads its column" + [8.3252, 8.3014] + (take 2 (D.columnAsList housingMedianIncome df)) + assertEqual + "snake_case column becomes a camelCased accessor" + ["NEAR BAY", "NEAR BAY"] + (take 2 (D.columnAsList housingOceanProximity df)) + +deriveSchemaValuesFromParquet :: Test +deriveSchemaValuesFromParquet = TestCase $ do + assertEqual + "parquet-derived schema column names" + [ "bigint_col" + , "bool_col" + , "date_string_col" + , "double_col" + , "float_col" + , "id" + , "int_col" + , "smallint_col" + , "string_col" + , "timestamp_col" + , "tinyint_col" + ] + (M.keys (IS.elements alltypesSchema)) + assertEqual + "parquet-derived schema infers Double" + (Just (IS.schemaType @Double)) + (M.lookup "double_col" (IS.elements alltypesSchema)) + +deriveSchemaValuesFromParquetAccessors :: Test +deriveSchemaValuesFromParquetAccessors = TestCase $ do + df <- D.readParquet "./tests/data/alltypes_plain.parquet" + assertEqual + "prefixed accessor reads its parquet column" + (D.columnAsList (D.col @Double "double_col") df) + (D.columnAsList alltypesDoubleCol df) + assertEqual + "snake_case parquet column becomes a camelCased accessor" + (D.columnAsList (D.col @T.Text "date_string_col") df) + (D.columnAsList alltypesDateStringCol df) + tests :: [Test] tests = [ TestLabel "basicTypedRoundTrip" basicTypedRoundTrip @@ -373,4 +451,12 @@ , TestLabel "deriveSchemaReadsCsv" deriveSchemaReadsCsv , TestLabel "deriveSchemaAccessorFilter" deriveSchemaAccessorFilter , TestLabel "deriveSchemaAccessorDerive" deriveSchemaAccessorDerive + , TestLabel "deriveSchemaValuesFromCsv" deriveSchemaValuesFromCsv + , TestLabel + "deriveSchemaValuesFromCsvAccessors" + deriveSchemaValuesFromCsvAccessors + , TestLabel "deriveSchemaValuesFromParquet" deriveSchemaValuesFromParquet + , TestLabel + "deriveSchemaValuesFromParquetAccessors" + deriveSchemaValuesFromParquetAccessors ] diff --git a/tests/Operations/Statistics.hs b/tests/Operations/Statistics.hs --- a/tests/Operations/Statistics.hs +++ b/tests/Operations/Statistics.hs @@ -57,6 +57,7 @@ 0 ) +-- g1, matching scipy.stats.skew skewnessOfSimpleDataSet :: Test skewnessOfSimpleDataSet = TestCase @@ -64,7 +65,7 @@ "Skewness of a simple data set" ( abs ( D.skewness' (VU.fromList [25 :: Int, 28, 26, 30, 40, 50, 40]) - - 0.566_731_633_676 + - 0.612_140_127_240_396_6 ) < 1e-12 ) diff --git a/tests/PrettyPrint.hs b/tests/PrettyPrint.hs --- a/tests/PrettyPrint.hs +++ b/tests/PrettyPrint.hs @@ -23,17 +23,18 @@ tests :: [Test] tests = - [ -- A short conditional stays compact; then/else still break onto own lines. + [ -- Conditionals lay out like Python statements: branches indented 4 under + -- their `if cond` / `else` header even when the whole thing is short. golden "fits on one line" - "if x .>=. 0.0\nthen \"pos\"\nelse \"neg\"" + "if x .>=. 0.0\n \"pos\"\nelse\n \"neg\"" ( prettyPrint (ifThenElse (col @Double "x" .>=. lit 0.0) (lit @T.Text "pos") (lit "neg")) ) , -- Nested else-if forms a flat ladder (no staircase indentation). golden "flat else-if ladder" - "if a .>. 1.0\nthen \"x\"\nelse if b .>. 2.0\nthen \"y\"\nelse \"z\"" + "if a .>. 1.0\n \"x\"\nelse if b .>. 2.0\n \"y\"\nelse\n \"z\"" ( prettyPrint ( ifThenElse (col @Double "a" .>. lit 1.0)