diff --git a/CHANGELOG.md b/CHANGELOG.md
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,19 @@
# Revision history for dataframe
+## 2.3.0.0
+
+### Breaking changes
+* HuggingFace (`hf://`) Parquet reading moved out of `dataframe-parquet` and
+ `dataframe-lazy` into a new dedicated `dataframe-huggingface` package, so the
+ heavy `aeson` and `http-conduit` dependencies are no longer pulled in by the
+ Parquet/lazy stack (or by this meta-package). `D.readParquet "hf://..."` no
+ longer fetches; depend on `dataframe-huggingface` and use
+ `DataFrame.IO.HuggingFace.readParquet` (eager) or `scanParquet` (lazy,
+ download-then-scan) instead.
+* Coordinated major bumps: `dataframe-parquet` and `dataframe-lazy` →
+ `1.1.0.0`, with inter-package lower bounds raised to `^>= 1.1`; umbrella
+ `dataframe` → `2.3.0.0`.
+
## 2.2.0.0
A large performance and ML release.
diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -122,8 +122,9 @@
fileDf <- D.readCsv "./data/housing.csv"
fileDf <- D.readParquet "./data/mtcars.parquet"
--- Hugging Face datasets (needs network access):
--- fileDf <- D.readParquet "hf://datasets/scikit-learn/iris/default/train/0000.parquet"
+-- Hugging Face datasets (needs network access, via the dataframe-huggingface package):
+-- import qualified DataFrame.IO.HuggingFace as HF
+-- fileDf <- HF.readParquet "hf://datasets/scikit-learn/iris/default/train/0000.parquet"
D.dimensions fileDf
```
@@ -498,7 +499,7 @@
## Features
-**I/O**: CSV, TSV, Parquet (Snappy, ZSTD, Gzip), JSON. Read Parquet from HTTP URLs and Hugging Face datasets (`hf://` URIs). Column projection and predicate pushdown for Parquet reads.
+**I/O**: CSV, TSV, Parquet (Snappy, ZSTD, Gzip), JSON. Read Parquet from Hugging Face datasets (`hf://` URIs) via the `dataframe-huggingface` package. Column projection and predicate pushdown for Parquet reads.
**Operations**: filter, select, derive, groupBy, aggregate, joins (inner, left, right, full outer), sort, sample, stratified sample, distinct, k-fold splits.
diff --git a/dataframe.cabal b/dataframe.cabal
--- a/dataframe.cabal
+++ b/dataframe.cabal
@@ -1,6 +1,6 @@
cabal-version: 3.0
name: dataframe
-version: 2.2.0.0
+version: 2.3.0.0
synopsis: A fast, safe, and intuitive DataFrame library.
description: A fast, safe, and intuitive DataFrame library for exploratory data analysis.
@@ -54,8 +54,10 @@
default: False
manual: True
description: Exclude the Parquet reader/writer (@dataframe-parquet@
- plus pinch, zstd, snappy, streamly, http-conduit).
- Enable with @-f +no-parquet@.
+ plus pinch, zstd, snappy, streamly). Enable with
+ @-f +no-parquet@. (Reading Parquet from HuggingFace lives
+ in the separate @dataframe-huggingface@ package, which is
+ not a dependency of this meta-package.)
flag no-th
default: False
@@ -147,7 +149,7 @@
DataFrame.IO.Parquet.Seeking,
DataFrame.IO.Parquet.Time,
DataFrame.IO.Utils.RandomAccess
- build-depends: dataframe-parquet ^>= 1.0.1.1
+ build-depends: dataframe-parquet ^>= 1.1
cpp-options: -DWITH_PARQUET
-- The lazy executor calls both CSV and Parquet readers directly, so
@@ -162,7 +164,7 @@
DataFrame.Lazy.Internal.Optimizer,
DataFrame.Lazy.Internal.Executor,
DataFrame.Typed.Lazy
- build-depends: dataframe-lazy ^>= 1.0.2
+ build-depends: dataframe-lazy ^>= 1.1
cpp-options: -DWITH_LAZY
if !flag(no-th)
@@ -198,11 +200,11 @@
dataframe-core ^>= 1.1,
dataframe-csv ^>= 1.0.2,
dataframe-json ^>= 1.0,
- dataframe-lazy ^>= 1.0.2,
+ dataframe-lazy ^>= 1.1,
dataframe-operations >= 1.1.1 && < 1.2,
- dataframe-parquet ^>= 1.0.1.1,
+ dataframe-parquet ^>= 1.1,
dataframe-parsing ^>= 1.0.2,
- text >= 2.0 && < 3,
+ text >= 2.1 && < 3,
aeson >= 0.11 && < 3,
bytestring >= 0.11 && < 0.13,
containers >= 0.6.7 && < 0.9,
@@ -232,7 +234,7 @@
dataframe-learn ^>= 1.1,
dataframe-operations >= 1.1.1 && < 1.2,
random >= 1 && < 2,
- text >= 2.0 && < 3
+ text >= 2.1 && < 3
hs-source-dirs: app
default-language: Haskell2010
ghc-options: -rtsopts -threaded -with-rtsopts=-N
@@ -263,11 +265,11 @@
containers >= 0.6.7 && < 0.9,
dataframe >= 1 && < 3,
dataframe-core ^>= 1.1,
- dataframe-lazy ^>= 1.0.2,
+ dataframe-lazy ^>= 1.1,
dataframe-parsing ^>= 1.0.2,
directory >= 1.3.0.0 && < 2,
random >= 1 && < 2,
- text >= 2.0 && < 3,
+ text >= 2.1 && < 3,
time >= 1.12 && < 2,
hs-source-dirs: app
default-language: Haskell2010
@@ -308,6 +310,7 @@
GenDataFrame,
Internal.ColumnBuilder,
Internal.DictEncode,
+ Internal.Markdown,
Internal.PackedText,
Internal.Parsing,
PackedTextMigration,
@@ -372,16 +375,16 @@
dataframe-core ^>= 1.1,
dataframe-csv ^>= 1.0.2,
dataframe-json ^>= 1.0,
- dataframe-lazy ^>= 1.0.2,
+ dataframe-lazy ^>= 1.1,
dataframe-learn ^>= 1.1,
dataframe-operations >= 1.1.1 && < 1.2,
- dataframe-parquet ^>= 1.0,
+ dataframe-parquet ^>= 1.1,
dataframe-parsing ^>= 1.0.2,
HUnit ^>= 1.6,
QuickCheck >= 2 && < 3,
random-shuffle >= 0.0.4 && < 1,
random >= 1 && < 2,
- text >= 2.0 && < 3,
+ text >= 2.1 && < 3,
time >= 1.12 && < 2,
vector ^>= 0.13,
temporary >= 1.3 && < 1.4,
@@ -405,7 +408,7 @@
dataframe-core ^>= 1.1,
dataframe-operations >= 1.1.1 && < 1.2,
HUnit ^>= 1.6,
- text >= 2.0 && < 3,
+ text >= 2.1 && < 3,
vector ^>= 0.13
hs-source-dirs: tests
default-language: Haskell2010
diff --git a/tests/Internal/Markdown.hs b/tests/Internal/Markdown.hs
new file mode 100644
--- /dev/null
+++ b/tests/Internal/Markdown.hs
@@ -0,0 +1,44 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+-- | Markdown rendering must escape pipe-table metacharacters in cell values,
+-- or an operator name like @<|>@ splits the row into the wrong columns.
+module Internal.Markdown (tests) where
+
+import qualified Data.Text as T
+
+import DataFrame.Internal.Column (fromList)
+import qualified DataFrame.Internal.DataFrame as D
+import DataFrame.Display.Terminal.PrettyPrint (escapeMarkdownCell)
+import Test.HUnit
+
+-- Count the column-delimiter pipes in a rendered row: a bare '|' delimits a
+-- cell; an escaped "\|" does not.
+unescapedPipes :: T.Text -> Int
+unescapedPipes = go . T.unpack
+ where
+ go ('\\' : '|' : rest) = go rest
+ go ('|' : rest) = 1 + go rest
+ go (_ : rest) = go rest
+ go [] = 0
+
+tests :: [Test]
+tests =
+ [ TestLabel "escapeMarkdownCell escapes a bare pipe" $
+ TestCase (escapeMarkdownCell "f<|>g" @?= "f<\\|>g")
+ , TestLabel "escapeMarkdownCell turns a newline into
" $
+ TestCase (escapeMarkdownCell "a\nb" @?= "a
b")
+ , TestLabel "toMarkdown keeps an operator name in one cell" $
+ TestCase $ do
+ let df =
+ D.fromNamedColumns
+ [ ("name", fromList ["plain" :: T.Text, "f<|>g"])
+ , ("mb", fromList [1.0 :: Double, 2.0])
+ ]
+ md = D.toMarkdown df
+ tableRows = filter (T.isPrefixOf "|") (T.lines md)
+ delims = map unescapedPipes tableRows
+ assertBool "pipe in a value left unescaped" ("f<\\|>g" `T.isInfixOf` md)
+ assertBool
+ ("rows misaligned, delimiter counts: " ++ show delims)
+ (case delims of [] -> False; (d : ds) -> all (== d) ds)
+ ]
diff --git a/tests/Main.hs b/tests/Main.hs
--- a/tests/Main.hs
+++ b/tests/Main.hs
@@ -16,6 +16,7 @@
import qualified IO.JSON
import qualified Internal.ColumnBuilder
import qualified Internal.DictEncode
+import qualified Internal.Markdown
import qualified Internal.PackedText
import qualified Internal.Parsing
import qualified LazyParity
@@ -76,6 +77,7 @@
DecisionTree.tests
++ Internal.ColumnBuilder.tests
++ Internal.DictEncode.tests
+ ++ Internal.Markdown.tests
++ Internal.PackedText.tests
++ Internal.Parsing.tests
++ Learn.Numerics.tests
diff --git a/tests/Plotting.hs b/tests/Plotting.hs
--- a/tests/Plotting.hs
+++ b/tests/Plotting.hs
@@ -30,6 +30,10 @@
import qualified DataFrame.Display.Web.Chart.Typed as CT
import qualified DataFrame.Display.Web.Plot as P
+-- Unqualified so the record update below has a single 'includeZero' field
+-- label in scope; GHC < 9.8 can't disambiguate the qualified duplicate field.
+import DataFrame.Display.Web.Plot (Scatter (includeZero))
+
-- ---------------------------------------------------------------------------
-- Fixtures + JSON helpers
-- ---------------------------------------------------------------------------
@@ -180,7 +184,7 @@
assertBool
"scatter axes fit the data by default"
("\"zero\":false" `L.isInfixOf` html)
- anchored <- P.scatter ((P.mkScatter "a" "b"){P.includeZero = True}) numFrame
+ anchored <- P.scatter ((P.mkScatter "a" "b"){includeZero = True}) numFrame
assertBool
"includeZero = True anchors the axes at zero explicitly"
("\"zero\":true" `L.isInfixOf` anchored)