dataframe 2.2.0.0 → 2.3.0.0
raw patch · 6 files changed
+87/−19 lines, 6 filesdep ~dataframe-lazydep ~dataframe-parquetdep ~textPVP ok
version bump matches the API change (PVP)
Dependency ranges changed: dataframe-lazy, dataframe-parquet, text
API changes (from Hackage documentation)
Files
- CHANGELOG.md +14/−0
- README.md +4/−3
- dataframe.cabal +18/−15
- tests/Internal/Markdown.hs +44/−0
- tests/Main.hs +2/−0
- tests/Plotting.hs +5/−1
CHANGELOG.md view
@@ -1,5 +1,19 @@ # Revision history for dataframe +## 2.3.0.0++### Breaking changes+* HuggingFace (`hf://`) Parquet reading moved out of `dataframe-parquet` and+ `dataframe-lazy` into a new dedicated `dataframe-huggingface` package, so the+ heavy `aeson` and `http-conduit` dependencies are no longer pulled in by the+ Parquet/lazy stack (or by this meta-package). `D.readParquet "hf://..."` no+ longer fetches; depend on `dataframe-huggingface` and use+ `DataFrame.IO.HuggingFace.readParquet` (eager) or `scanParquet` (lazy,+ download-then-scan) instead.+* Coordinated major bumps: `dataframe-parquet` and `dataframe-lazy` →+ `1.1.0.0`, with inter-package lower bounds raised to `^>= 1.1`; umbrella+ `dataframe` → `2.3.0.0`.+ ## 2.2.0.0 A large performance and ML release.
README.md view
@@ -122,8 +122,9 @@ fileDf <- D.readCsv "./data/housing.csv" fileDf <- D.readParquet "./data/mtcars.parquet" --- Hugging Face datasets (needs network access):--- fileDf <- D.readParquet "hf://datasets/scikit-learn/iris/default/train/0000.parquet"+-- Hugging Face datasets (needs network access, via the dataframe-huggingface package):+-- import qualified DataFrame.IO.HuggingFace as HF+-- fileDf <- HF.readParquet "hf://datasets/scikit-learn/iris/default/train/0000.parquet" D.dimensions fileDf ```@@ -498,7 +499,7 @@ ## Features -**I/O**: CSV, TSV, Parquet (Snappy, ZSTD, Gzip), JSON. Read Parquet from HTTP URLs and Hugging Face datasets (`hf://` URIs). Column projection and predicate pushdown for Parquet reads.+**I/O**: CSV, TSV, Parquet (Snappy, ZSTD, Gzip), JSON. Read Parquet from Hugging Face datasets (`hf://` URIs) via the `dataframe-huggingface` package. Column projection and predicate pushdown for Parquet reads. **Operations**: filter, select, derive, groupBy, aggregate, joins (inner, left, right, full outer), sort, sample, stratified sample, distinct, k-fold splits.
dataframe.cabal view
@@ -1,6 +1,6 @@ cabal-version: 3.0 name: dataframe-version: 2.2.0.0+version: 2.3.0.0 synopsis: A fast, safe, and intuitive DataFrame library. description: A fast, safe, and intuitive DataFrame library for exploratory data analysis.@@ -54,8 +54,10 @@ default: False manual: True description: Exclude the Parquet reader/writer (@dataframe-parquet@- plus pinch, zstd, snappy, streamly, http-conduit).- Enable with @-f +no-parquet@.+ plus pinch, zstd, snappy, streamly). Enable with+ @-f +no-parquet@. (Reading Parquet from HuggingFace lives+ in the separate @dataframe-huggingface@ package, which is+ not a dependency of this meta-package.) flag no-th default: False@@ -147,7 +149,7 @@ DataFrame.IO.Parquet.Seeking, DataFrame.IO.Parquet.Time, DataFrame.IO.Utils.RandomAccess- build-depends: dataframe-parquet ^>= 1.0.1.1+ build-depends: dataframe-parquet ^>= 1.1 cpp-options: -DWITH_PARQUET -- The lazy executor calls both CSV and Parquet readers directly, so@@ -162,7 +164,7 @@ DataFrame.Lazy.Internal.Optimizer, DataFrame.Lazy.Internal.Executor, DataFrame.Typed.Lazy- build-depends: dataframe-lazy ^>= 1.0.2+ build-depends: dataframe-lazy ^>= 1.1 cpp-options: -DWITH_LAZY if !flag(no-th)@@ -198,11 +200,11 @@ dataframe-core ^>= 1.1, dataframe-csv ^>= 1.0.2, dataframe-json ^>= 1.0,- dataframe-lazy ^>= 1.0.2,+ dataframe-lazy ^>= 1.1, dataframe-operations >= 1.1.1 && < 1.2,- dataframe-parquet ^>= 1.0.1.1,+ dataframe-parquet ^>= 1.1, dataframe-parsing ^>= 1.0.2,- text >= 2.0 && < 3,+ text >= 2.1 && < 3, aeson >= 0.11 && < 3, bytestring >= 0.11 && < 0.13, containers >= 0.6.7 && < 0.9,@@ -232,7 +234,7 @@ dataframe-learn ^>= 1.1, dataframe-operations >= 1.1.1 && < 1.2, random >= 1 && < 2,- text >= 2.0 && < 3+ text >= 2.1 && < 3 hs-source-dirs: app default-language: Haskell2010 ghc-options: -rtsopts -threaded -with-rtsopts=-N@@ -263,11 +265,11 @@ containers >= 0.6.7 && < 0.9, dataframe >= 1 && < 3, dataframe-core ^>= 1.1,- dataframe-lazy ^>= 1.0.2,+ dataframe-lazy ^>= 1.1, dataframe-parsing ^>= 1.0.2, directory >= 1.3.0.0 && < 2, random >= 1 && < 2,- text >= 2.0 && < 3,+ text >= 2.1 && < 3, time >= 1.12 && < 2, hs-source-dirs: app default-language: Haskell2010@@ -308,6 +310,7 @@ GenDataFrame, Internal.ColumnBuilder, Internal.DictEncode,+ Internal.Markdown, Internal.PackedText, Internal.Parsing, PackedTextMigration,@@ -372,16 +375,16 @@ dataframe-core ^>= 1.1, dataframe-csv ^>= 1.0.2, dataframe-json ^>= 1.0,- dataframe-lazy ^>= 1.0.2,+ dataframe-lazy ^>= 1.1, dataframe-learn ^>= 1.1, dataframe-operations >= 1.1.1 && < 1.2,- dataframe-parquet ^>= 1.0,+ dataframe-parquet ^>= 1.1, dataframe-parsing ^>= 1.0.2, HUnit ^>= 1.6, QuickCheck >= 2 && < 3, random-shuffle >= 0.0.4 && < 1, random >= 1 && < 2,- text >= 2.0 && < 3,+ text >= 2.1 && < 3, time >= 1.12 && < 2, vector ^>= 0.13, temporary >= 1.3 && < 1.4,@@ -405,7 +408,7 @@ dataframe-core ^>= 1.1, dataframe-operations >= 1.1.1 && < 1.2, HUnit ^>= 1.6,- text >= 2.0 && < 3,+ text >= 2.1 && < 3, vector ^>= 0.13 hs-source-dirs: tests default-language: Haskell2010
+ tests/Internal/Markdown.hs view
@@ -0,0 +1,44 @@+{-# LANGUAGE OverloadedStrings #-}++-- | Markdown rendering must escape pipe-table metacharacters in cell values,+-- or an operator name like @<|>@ splits the row into the wrong columns.+module Internal.Markdown (tests) where++import qualified Data.Text as T++import DataFrame.Internal.Column (fromList)+import qualified DataFrame.Internal.DataFrame as D+import DataFrame.Display.Terminal.PrettyPrint (escapeMarkdownCell)+import Test.HUnit++-- Count the column-delimiter pipes in a rendered row: a bare '|' delimits a+-- cell; an escaped "\|" does not.+unescapedPipes :: T.Text -> Int+unescapedPipes = go . T.unpack+ where+ go ('\\' : '|' : rest) = go rest+ go ('|' : rest) = 1 + go rest+ go (_ : rest) = go rest+ go [] = 0++tests :: [Test]+tests =+ [ TestLabel "escapeMarkdownCell escapes a bare pipe" $+ TestCase (escapeMarkdownCell "f<|>g" @?= "f<\\|>g")+ , TestLabel "escapeMarkdownCell turns a newline into <br>" $+ TestCase (escapeMarkdownCell "a\nb" @?= "a<br>b")+ , TestLabel "toMarkdown keeps an operator name in one cell" $+ TestCase $ do+ let df =+ D.fromNamedColumns+ [ ("name", fromList ["plain" :: T.Text, "f<|>g"])+ , ("mb", fromList [1.0 :: Double, 2.0])+ ]+ md = D.toMarkdown df+ tableRows = filter (T.isPrefixOf "|") (T.lines md)+ delims = map unescapedPipes tableRows+ assertBool "pipe in a value left unescaped" ("f<\\|>g" `T.isInfixOf` md)+ assertBool+ ("rows misaligned, delimiter counts: " ++ show delims)+ (case delims of [] -> False; (d : ds) -> all (== d) ds)+ ]
tests/Main.hs view
@@ -16,6 +16,7 @@ import qualified IO.JSON import qualified Internal.ColumnBuilder import qualified Internal.DictEncode+import qualified Internal.Markdown import qualified Internal.PackedText import qualified Internal.Parsing import qualified LazyParity@@ -76,6 +77,7 @@ DecisionTree.tests ++ Internal.ColumnBuilder.tests ++ Internal.DictEncode.tests+ ++ Internal.Markdown.tests ++ Internal.PackedText.tests ++ Internal.Parsing.tests ++ Learn.Numerics.tests
tests/Plotting.hs view
@@ -30,6 +30,10 @@ import qualified DataFrame.Display.Web.Chart.Typed as CT import qualified DataFrame.Display.Web.Plot as P +-- Unqualified so the record update below has a single 'includeZero' field+-- label in scope; GHC < 9.8 can't disambiguate the qualified duplicate field.+import DataFrame.Display.Web.Plot (Scatter (includeZero))+ -- --------------------------------------------------------------------------- -- Fixtures + JSON helpers -- ---------------------------------------------------------------------------@@ -180,7 +184,7 @@ assertBool "scatter axes fit the data by default" ("\"zero\":false" `L.isInfixOf` html)- anchored <- P.scatter ((P.mkScatter "a" "b"){P.includeZero = True}) numFrame+ anchored <- P.scatter ((P.mkScatter "a" "b"){includeZero = True}) numFrame assertBool "includeZero = True anchors the axes at zero explicitly" ("\"zero\":true" `L.isInfixOf` anchored)