dataframe-huggingface 1.0.0.0 → 1.2.0.2
raw patch · 2 files changed
+24/−30 lines, 2 filesdep ~bytestringdep ~dataframe-coredep ~dataframe-lazyPVP ok
version bump matches the API change (PVP)
Dependency ranges changed: bytestring, dataframe-core, dataframe-lazy, dataframe-operations, dataframe-parquet, dataframe-parsing
API changes (from Hackage documentation)
Files
dataframe-huggingface.cabal view
@@ -1,7 +1,6 @@ cabal-version: 2.4 name: dataframe-huggingface-version: 1.0.0.0-+version: 1.2.0.2 synopsis: Read Parquet datasets from HuggingFace into dataframes. description: @DataFrame.IO.HuggingFace@ — resolve and download Parquet files from@@ -32,12 +31,12 @@ exposed-modules: DataFrame.IO.HuggingFace build-depends: base >= 4 && < 5, aeson >= 0.11.0.0 && < 3,- bytestring >= 0.11 && < 0.13,- dataframe-core ^>= 1.1,- dataframe-lazy ^>= 1.1,- dataframe-operations ^>= 1.1.1,- dataframe-parquet ^>= 1.1,- dataframe-parsing ^>= 1.0.2,+ bytestring >= 0.11 && < 0.14,+ dataframe-core ^>= 2.3,+ dataframe-lazy ^>= 2.3,+ dataframe-operations ^>= 2.3,+ dataframe-parquet ^>= 1.5,+ dataframe-parsing ^>= 2.2, directory >= 1.3.0.0 && < 2, filepath >= 1.4 && < 2, Glob >= 0.10 && < 1,
src/DataFrame/IO/HuggingFace.hs view
@@ -1,16 +1,10 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | Read Parquet datasets from HuggingFace (@hf://@) into 'DataFrame's.--A @hf://@ URI has the form @hf:\/\/datasets\/{owner}\/{dataset}\/{glob}@.-When the glob contains wildcards the dataset's file list is resolved via the-HuggingFace datasets-server API; otherwise the file is fetched directly from-the repo. Resolved files are downloaded to a temporary location and then read-with the local "DataFrame.IO.Parquet" reader.--This module is the home for the heavier @aeson@ and @http-conduit@-dependencies, keeping @dataframe-parquet@ and @dataframe-lazy@ free of them.+{- | Read Parquet datasets from HuggingFace URIs+(@hf:\/\/datasets\/{owner}\/{dataset}\/{glob}@) into 'DataFrame's. Globs are+resolved via the datasets-server API, files downloaded to a temp dir, then read+with the local "DataFrame.IO.Parquet". -} module DataFrame.IO.HuggingFace ( -- * Eager readers@@ -49,16 +43,16 @@ import qualified Data.List as L import qualified Data.Text as T import Data.Text.Encoding (encodeUtf8)+import DataFrame.Core (DataFrame) import DataFrame.IO.Parquet ( ParquetReadOptions (..), defaultParquetReadOptions, ) import qualified DataFrame.IO.Parquet as Parquet-import DataFrame.Internal.DataFrame (DataFrame)-import DataFrame.Internal.Schema (Schema) import qualified DataFrame.Lazy as Lazy import DataFrame.Operations.Merge () import qualified DataFrame.Operations.Subset as DS+import DataFrame.Schema (Schema) import Network.HTTP.Simple ( getResponseBody, getResponseStatusCode,@@ -106,7 +100,8 @@ | otherwise = Parquet.readParquetFilesWithOpts opts path {- | Read a list of downloaded local files, concatenate them, and apply the-global row range once over the result. -}+global row range once over the result.+-} readDownloaded :: ParquetReadOptions -> [FilePath] -> IO DataFrame readDownloaded opts files = do let optsNoRange = opts{rowRange = Nothing}@@ -120,10 +115,8 @@ -- Lazy / streaming reader ------------------------------------------------- {- | Build a lazy 'Lazy.LazyDataFrame' over a @hf://@ dataset (or local path).--The HuggingFace files are resolved and downloaded to a temporary directory up-front, then scanned lazily from local disk — so the query runs in constant-memory but the whole dataset is fetched before scanning begins.+HuggingFace files are downloaded to a temp dir up front, then scanned lazily+from disk — constant-memory query, but the whole dataset is fetched first. -} scanParquet :: Schema -> T.Text -> IO Lazy.LazyDataFrame scanParquet schema uri@@ -206,7 +199,6 @@ case T.breakOn "/resolve/" (hfpUrl f) of (_, rest) | not (T.null rest) ->- -- Drop "/resolve/", then drop the ref component (up to and including "/") T.unpack $ T.drop 1 $ T.dropWhile (/= '/') $ T.drop (T.length "/resolve/") rest _ -> T.unpack (hfpConfig f) </> T.unpack (hfpSplit f) </> T.unpack (hfpFilename f)@@ -242,10 +234,10 @@ downloadHFFilesTo tmpDir mToken files -- | Download files into @destDir@, returning the local paths.-downloadHFFilesTo :: FilePath -> Maybe BS.ByteString -> [HFParquetFile] -> IO [FilePath]+downloadHFFilesTo ::+ FilePath -> Maybe BS.ByteString -> [HFParquetFile] -> IO [FilePath] downloadHFFilesTo destDir mToken files = forM files $ \f -> do- -- Derive a collision-resistant name from the URL path components let fname = case (hfpConfig f, hfpSplit f) of (c, s) | T.null c && T.null s -> T.unpack (hfpFilename f) (c, s) -> T.unpack c <> "_" <> T.unpack s <> "_" <> T.unpack (hfpFilename f)@@ -280,7 +272,8 @@ <> hfGlob ref {- | Resolve a @hf://@ URI to the token and the list of files to fetch, without-downloading. -}+downloading.+-} resolveFiles :: FilePath -> IO (Maybe BS.ByteString, [HFParquetFile]) resolveFiles uri = do ref <- case parseHFUri uri of@@ -292,7 +285,9 @@ then do hfFiles <- resolveHFUrls mToken ref when (null hfFiles) $- ioError $ userError $ "No parquet files found for " ++ uri+ ioError $+ userError $+ "No parquet files found for " ++ uri pure hfFiles else do let url = directHFUrl ref