packages feed

dataframe-parsing 2.2.0.0 → 2.2.0.1

raw patch · 12 files changed

+44/−157 lines, 12 filesdep ~dataframe-corePVP ok

version bump matches the API change (PVP)

Dependency ranges changed: dataframe-core

API changes (from Hackage documentation)

Files

bench/FieldParsers.hs view
@@ -2,13 +2,7 @@ {-# LANGUAGE MagicHash #-} {-# LANGUAGE UnboxedTuples #-} -{- | Per-field microbenchmark for "DataFrame.Internal.Parsing.Fast"-against the reference parsers (WS-B exit criterion: parseDoubleField->= 5x Data.Text.Read.double on doubles_100mb-shaped fields).--Style follows benchmarks/csv: monotonic-clock wall over a strict fold,-best of 5 runs, ns/field plus speedup printed per pairing.--}+-- | Per-field microbenchmark of the fast parsers against the reference ones. module Main (main) where  import qualified Data.ByteString.Char8 as C@@ -37,9 +31,7 @@         z2 = (z1 `xor` (z1 `shiftR` 33)) * 0xC4CEB9FE1A85EC53      in z2 `xor` (z2 `shiftR` 33) -{- | Round-trip-precision doubles, the doubles_100mb shape-(full 17-significant-digit mantissas from @show@).--}+-- | Round-trip-precision doubles in the doubles_100mb shape. doubleFields :: V.Vector C.ByteString doubleFields = V.generate fieldCount gen   where@@ -93,8 +85,7 @@ bench :: V.Vector a -> (a -> Double) -> IO Double bench fields step = minimum <$> mapM run [1 .. 5 :: Int]   where-    -- Seed the accumulator with the run index so the fold cannot be-    -- floated out and shared between the five runs.+    -- Seed the fold with the run index so it is not shared between runs.     run k = do         t0 <- getMonotonicTimeNSec         let !s = V.foldl' (\ !acc f -> acc + step f) (fromIntegral k) fields
dataframe-parsing.cabal view
@@ -1,6 +1,6 @@ cabal-version:      3.4 name:               dataframe-parsing-version:            2.2.0.0+version:            2.2.0.1 synopsis:           Shared text/binary parsing helpers for the dataframe ecosystem. description:     Parsing primitives used by the @dataframe@ family: CSV-friendly text@@ -45,7 +45,7 @@                         bytestring >= 0.11 && < 0.14,                         bytestring-lexing >= 0.5 && < 0.7,                         containers >= 0.6.7 && < 0.10,-                        dataframe-core >= 2.2 && < 2.3,+                        dataframe-core >= 2.6 && < 2.7,                         text >= 2.1 && < 3,                         time >= 1.12 && < 2,                         vector >= 0.12 && < 0.15
src-internal/DataFrame/Internal/Parsing.hs view
@@ -17,7 +17,15 @@ import Data.Text.Read (decimal, double, signed) import Data.Time (Day, defaultTimeLocale, parseTimeM) import GHC.Stack (HasCallStack)-import System.IO (Handle, IOMode (..), hIsEOF, hTell, withFile)+import System.IO (+    Handle,+    IOMode (..),+    hIsEOF,+    hSetEncoding,+    hTell,+    utf8,+    withFile,+ ) import Prelude hiding (takeWhile)  isNullish :: T.Text -> Bool@@ -75,9 +83,7 @@     Just (value, "") -> Just value     _ -> Nothing #else--- bytestring < 0.12: 'C.readInt' silently wraps on overflow. Fields of--- <= 18 characters fit in an 'Int' and keep the fast path; longer ones fall--- back to arbitrary-precision 'C.readInteger' with a range check.+-- bytestring < 0.12 'C.readInt' wraps on overflow; long fields use Integer. readByteStringInt s     | C.length t <= 18 = case C.readInt t of         Just (value, "") -> Just value@@ -134,10 +140,6 @@         Right (_value, _) -> Left s {-# INLINE readDoubleEither #-} --- ------------------------------------------------------------------------------ Attoparsec CSV parser combinators (shared between Lazy.IO.CSV and others)--- ---------------------------------------------------------------------------- parseSep :: Char -> T.Text -> [T.Text] parseSep c s = either error id (parseOnly (record c) s) {-# INLINE parseSep #-}@@ -187,7 +189,8 @@  -- | First pass to count rows for exact allocation. countRows :: Char -> FilePath -> IO Int-countRows c path = withFile path ReadMode $! go 0 ""+countRows c path = withFile path ReadMode $ \h ->+    hSetEncoding h utf8 >> go 0 "" h   where     go n input h = do         isEOF <- hIsEOF h
src-internal/DataFrame/Internal/Parsing/Fast.hs view
@@ -1,10 +1,7 @@ {-# LANGUAGE MagicHash #-} {-# LANGUAGE UnboxedTuples #-} -{- | Fast, non-backtracking field parsers for CSV ingest. Each is a pure-function over a @(buf, start, end)@ byte slice, bit-exact with the reference-parsers in "DataFrame.Internal.Parsing". Unboxed @#@ variants avoid boxing.--}+-- | Fast, non-backtracking field parsers for CSV ingest over byte slices. module DataFrame.Internal.Parsing.Fast (     -- * Int fields     parseIntField,@@ -45,9 +42,7 @@     parseDateFieldSlice,  ) -{- | Strip-tolerant @Int@ parse of a whole field; rejects overflow,-matching @readByteStringInt@ exactly.--}+-- | Strip-tolerant @Int@ parse of a whole field; rejects overflow. parseIntField :: BS.ByteString -> Maybe Int parseIntField bs = parseIntFieldSlice bs 0 (BS.length bs) {-# INLINE parseIntField #-}@@ -59,9 +54,7 @@         (# _, n #) -> Just (I# n) {-# INLINE parseIntFieldSlice #-} -{- | Strip-tolerant @Double@ parse of a whole field; bit-exact with-@readByteStringDouble@ (falls back to it outside the fast window).--}+-- | Strip-tolerant @Double@ parse of a whole field, bit-exact with reference. parseDoubleField :: BS.ByteString -> Maybe Double parseDoubleField bs = parseDoubleFieldSlice bs 0 (BS.length bs) {-# INLINE parseDoubleField #-}@@ -85,16 +78,12 @@         (# _, b #) -> Just (I# b /= 0) {-# INLINE parseBoolFieldSlice #-} -{- | @%Y-%m-%d@ date parse: byte-level fast path for the padded-10-byte shape, 'Data.Time.parseTimeM' fallback for everything else.--}+-- | @%Y-%m-%d@ date parse with a fast path for the padded 10-byte shape. parseDateField :: BS.ByteString -> Maybe Day parseDateField bs = parseDateFieldSlice bs 0 (BS.length bs) {-# INLINE parseDateField #-} -{- | Byte-level test against the canonical missing-token list-(@Nothing NULL \"\" \" \" nan null N\/A NaN NAN NA@), no Text decode.--}+-- | Byte-level test against the canonical missing-token list. isMissingField :: BS.ByteString -> Bool isMissingField bs = isMissingFieldSlice bs 0 (BS.length bs) {-# INLINE isMissingField #-}
src-internal/DataFrame/Internal/Parsing/Fast/Common.hs view
@@ -4,10 +4,7 @@ {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} -{- | Shared byte-level helpers for the fast slice parsers. All operate on-@(buf, start, end)@ slices; the caller guarantees-@0 <= start <= end <= length buf@ so everything below uses 'unsafeIndex'.--}+-- | Shared byte-level helpers for the fast slice parsers. module DataFrame.Internal.Parsing.Fast.Common (     isStripByte,     isDigitByte,@@ -36,10 +33,7 @@  ) import GHC.Word (Word64 (..)) -{- | Bytes removed by 'Data.ByteString.Char8.strip' (Latin-1 'isSpace'):-HT LF VT FF CR SP and NBSP 0xA0. Probed over all 256 bytes; parity-with the strip-based reference parsers depends on this exact set.--}+-- | Bytes removed by 'Data.ByteString.Char8.strip' (Latin-1 whitespace). isStripByte :: Word8 -> Bool isStripByte w = w == 0x20 || (w - 0x09) <= 4 || w == 0xA0 {-# INLINE isStripByte #-}@@ -75,10 +69,7 @@         | otherwise = i {-# INLINE skipZeroes #-} -{- | Consume ASCII digits from @i@, passing the stop index and the wrapping-'Word64' accumulation to the continuation. Callers must bound the-significant digit count before trusting the value.--}+-- | Consume ASCII digits from @i@, passing the stop index and Word64 sum on. takeDigits64 ::     forall (rep :: RuntimeRep) (r :: TYPE rep).     BS.ByteString ->
src-internal/DataFrame/Internal/Parsing/Fast/Double.hs view
@@ -2,10 +2,7 @@ {-# LANGUAGE MagicHash #-} {-# LANGUAGE UnboxedTuples #-} -{- | Fast @Double@ slice parser, bit-exact with @readByteStringDouble@.-Replays the reference parser's exact floating-point operations via 'Word64'-digit accumulation and 10^k tables, falling back when exactness is in doubt.--}+-- | Fast @Double@ slice parser, bit-exact with @readByteStringDouble@. module DataFrame.Internal.Parsing.Fast.Double (parseDoubleField#) where  import qualified Data.ByteString as BS@@ -18,10 +15,6 @@ import DataFrame.Internal.Parsing (readByteStringDouble) import DataFrame.Internal.Parsing.Fast.Common -{- | @10 ^ k@ for @k <= tableMax@, computed with the same @(^)@ the reference-parser uses, so every entry is bit-identical. Entries from @10^309@ up are-@Infinity@, so clamping larger exponents to 'tableMax' is exact.--} pow10Table :: VU.Vector Double pow10Table = VU.generate (tableMax + 1) (10 ^) {-# NOINLINE pow10Table #-}@@ -34,10 +27,7 @@ tableMax :: Int tableMax = 1024 -{- | 'Word64' to 'Double' exactly as the reference parser's-'fromInteger' rounds it; values up to @2^53@ take the exact-'Int' conversion (int2Double#), larger ones the 'Integer' route.--}+-- | 'Word64' to 'Double' rounded exactly as the reference parser does. w2d :: Word64 -> Double w2d w     | w <= 9007199254740991 = fromIntegral (fromIntegral w :: Int)@@ -52,9 +42,7 @@         Nothing -> (# 0#, 0.0## #) {-# NOINLINE referenceSlice #-} -{- | Result is @(# ok, value #)@ with @ok@ 0 or 1. Caller guarantees-@0 <= start <= end <= length buf@.--}+-- | Result is @(# ok, value #)@ with @ok@ 0 or 1. parseDoubleField# :: BS.ByteString -> Int -> Int -> (# Int#, Double# #) parseDoubleField# bs start end0     | i0 >= end = none
src-internal/DataFrame/Internal/Parsing/Fast/Int.hs view
@@ -2,10 +2,7 @@ {-# LANGUAGE MagicHash #-} {-# LANGUAGE UnboxedTuples #-} -{- | Non-backtracking @Int@ slice parser, bit-compatible with-@readByteStringInt@: grammar @WS* sign? digit+ WS*@, whole slice consumed,-'Nothing' on 64-bit overflow.--}+-- | Non-backtracking @Int@ slice parser matching @readByteStringInt@. module DataFrame.Internal.Parsing.Fast.Int (parseIntField#) where  import qualified Data.ByteString as BS@@ -15,9 +12,7 @@  import DataFrame.Internal.Parsing.Fast.Common -{- | Result is @(# ok, value #)@ with @ok@ 0 or 1. Caller guarantees-@0 <= start <= end <= length buf@.--}+-- | Result is @(# ok, value #)@ with @ok@ 0 or 1. parseIntField# :: BS.ByteString -> Int -> Int -> (# Int#, Int# #) parseIntField# bs start end0     | i0 >= end = none
src-internal/DataFrame/Internal/Parsing/Fast/Token.hs view
@@ -1,10 +1,7 @@ {-# LANGUAGE MagicHash #-} {-# LANGUAGE UnboxedTuples #-} -{- | Byte-level token tests: Bool fields, the canonical missing-token-set, and the default @%Y-%m-%d@ date shape. All length-bucketed with-first-byte dispatch; no Text decode, no list walk.--}+-- | Byte-level token tests: Bool fields, missing tokens and the date shape. module DataFrame.Internal.Parsing.Fast.Token (     parseBoolField#,     isMissingFieldSlice,@@ -22,9 +19,7 @@ import DataFrame.Internal.Parsing (readByteStringDate) import DataFrame.Internal.Parsing.Fast.Common (isDigitByte) -{- | Exactly @True|true|TRUE|False|false|FALSE@, no strip (the-'readByteStringBool' grammar). Result is @(# ok, bool #)@.--}+-- | Exactly @True|true|TRUE|False|false|FALSE@, with no strip. parseBoolField# :: BS.ByteString -> Int -> Int -> (# Int#, Int# #) parseBoolField# bs s e = case e - s of     4@@ -42,10 +37,7 @@     alse a b c d = ix 1 == a && ix 2 == b && ix 3 == c && ix 4 == d {-# INLINE parseBoolField# #-} -{- | Membership in the canonical missing list-@[\"Nothing\",\"NULL\",\"\",\" \",\"nan\",\"null\",\"N\/A\",\"NaN\",\"NAN\",\"NA\"]@-(case-sensitive, exact), dispatched on length then first byte.--}+-- | Membership in the canonical missing-token list, case-sensitive and exact. isMissingFieldSlice :: BS.ByteString -> Int -> Int -> Bool isMissingFieldSlice bs s e = case e - s of     0 -> True@@ -80,10 +72,7 @@ isMissingFieldIn toks f = f `elem` toks {-# INLINE isMissingFieldIn #-} -{- | @%Y-%m-%d@: byte-level fast path for the padded 10-byte shape-(@dddd-dd-dd@); anything else (unpadded, whitespace-tolerant, long-years, invalid) falls back to 'readByteStringDate'.--}+-- | @%Y-%m-%d@ with a fast path for the padded 10-byte shape. parseDateFieldSlice :: BS.ByteString -> Int -> Int -> Maybe Day parseDateFieldSlice bs s e     | e - s == 10
src/DataFrame/Internal/Schema.hs view
@@ -10,11 +10,7 @@ {-# LANGUAGE TypeOperators #-} {-# LANGUAGE UndecidableInstances #-} -{- |-Runtime schema representation. The Template-Haskell @deriveSchema@ splice-lives in "DataFrame.Internal.Schema.TH" so this module can be used from-packages that do not depend on @template-haskell@.--}+-- | Runtime schema representation, free of any @template-haskell@ dependency. module DataFrame.Internal.Schema (     SchemaType (..),     schemaType,@@ -38,55 +34,25 @@     -- | Constructor carrying a 'Proxy' of the element type.     SType :: (Columnable a, Read a) => P.Proxy a -> SchemaType -{- | Show the underlying element type using 'typeRep'.--==== __Examples__->>> :set -XTypeApplications->>> show (schemaType @Bool)-"Bool"--}+-- | Show the underlying element type using 'typeRep'. instance Show SchemaType where     show :: SchemaType -> String     show (SType (_ :: P.Proxy a)) = show (typeRep @a) -{- | Two 'SchemaType's are equal iff their element types are the same.--==== __Examples__->>> :set -XTypeApplications->>> schemaType @Int == schemaType @Int-True-->>> schemaType @Int == schemaType @Integer-False--}+-- | Two 'SchemaType's are equal iff their element types are the same. instance Eq SchemaType where     (==) :: SchemaType -> SchemaType -> Bool     (==) (SType (_ :: P.Proxy a)) (SType (_ :: P.Proxy b)) =         isJust (testEquality (typeRep @a) (typeRep @b)) -{- | Construct a 'SchemaType' for the given @a@.--==== __Examples__->>> :set -XTypeApplications->>> schemaType @T.Text == schemaType @T.Text-True-->>> show (schemaType @Double)-"Double"--}+-- | Construct a 'SchemaType' for the given @a@. schemaType :: forall a. (Columnable a, Read a) => SchemaType schemaType = SType (P.Proxy @a) -{- | Logical schema of a 'DataFrame': a mapping from column names to their-element types ('SchemaType').--}+-- | Logical schema of a 'DataFrame': column names mapped to element types. newtype Schema = Schema     { elements :: M.Map T.Text SchemaType-    {- ^ Mapping from /column name/ to its 'SchemaType'.--    Invariant: keys are unique column names. A missing key means the column-    is not present in the schema.-    -}+    -- ^ Mapping from column name to its 'SchemaType'; keys are unique.     }     deriving (Show, Eq) @@ -94,18 +60,7 @@ makeSchema :: [(T.Text, SchemaType)] -> Schema makeSchema = Schema . M.fromList -{- | The runtime 'Schema' behind a type-level schema — names /and/ element-types — so a reader can project to a schema's columns and skip inference for-them in one step.--Every column type must have a 'Read' instance, which 'Columnable' does not-imply; that is what lets the names carry their types across to a reader.--==== __Examples__->>> :set -XTypeApplications -XDataKinds->>> elements (runtimeSchema @'[ '("n", Int)])-fromList [("n",Int)]--}+-- | The runtime 'Schema' behind a type-level schema, names and element types. class RuntimeSchema (cols :: [(Symbol, Type)]) where     runtimeSchema :: Schema
src/DataFrame/Schema.hs view
@@ -1,7 +1,4 @@-{- | The runtime schema surface for the @dataframe@ ecosystem: the 'Schema'-tag, its element types, and builders to describe a frame's columns by name.-Re-exported so callers never reach into @DataFrame.Internal.Schema@.--}+-- | The runtime schema surface for the @dataframe@ ecosystem. module DataFrame.Schema (     module DataFrame.Internal.Schema, ) where
tests/Properties/FastParsing.hs view
@@ -1,13 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -{- | QuickCheck parity properties for "DataFrame.Internal.Parsing.Fast".--Every fast slice parser must agree with the existing reference parser-(`readByteStringInt` / `readByteStringDouble` / `readByteStringBool` /-`readByteStringDate` / `isNullishBS`) on arbitrary byte fields, with-Doubles compared bit-for-bit ('castDoubleToWord64').--}+-- | QuickCheck parity properties for "DataFrame.Internal.Parsing.Fast". module Properties.FastParsing (tests) where  import qualified Data.ByteString as BS
tests/Unit/FastParsing.hs view
@@ -1,11 +1,6 @@ {-# LANGUAGE OverloadedStrings #-} -{- | Pinned adversarial cases for "DataFrame.Internal.Parsing.Fast".--Each case asserts parity with the reference parsers plus, where the-parser-semantics contract (benchmarks/csv/results/code-audit.md section 2)-pins a concrete value, the value itself.--}+-- | Pinned adversarial parity cases for "DataFrame.Internal.Parsing.Fast". module Unit.FastParsing (tests) where  import qualified Data.ByteString as BS