dataframe-core 2.3.0.0 → 2.4.0.0

raw patch · 10 files changed

+398/−85 lines, 10 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• dataframe-core.cabal +1/−1
• src-internal/DataFrame/Internal/Column.hs +137/−46
• src-internal/DataFrame/Internal/ColumnMerge.hs +9/−0
• src-internal/DataFrame/Internal/DataFrame.hs +6/−0
• src-internal/DataFrame/Internal/DictEncode.hs +43/−1
• src-internal/DataFrame/Internal/Grouping.hs +57/−3
• src-internal/DataFrame/Internal/Interpreter.hs +5/−0
• src-internal/DataFrame/Internal/PackedText.hs +120/−31
• src-internal/DataFrame/Internal/Row.hs +10/−0
• src-internal/DataFrame/Internal/RowHash.hs +10/−3

@@ -1,6 +1,6 @@ cabal-version:      3.4 name:               dataframe-core-version:            2.3.0.0+version:            2.4.0.0 synopsis:           Core data structures for the dataframe library. description:     Minimal interchange-format types for the @dataframe@ ecosystem:

@@ -47,13 +47,11 @@     packedGather,     packedIndexText,     packedLength,-    packedRowOffsetVec,     packedSlice,     packedTake,     sliceEqBytes,  ) import DataFrame.Internal.Types-import DataFrame.Internal.Utf8 (sliceTextVector) import System.IO.Unsafe (unsafePerformIO) import System.Random import Type.Reflection@@ -69,10 +67,15 @@     BoxedColumn :: (Columnable a) => Maybe Bitmap -> VB.Vector a -> Column     UnboxedColumn ::         (Columnable a, VU.Unbox a) => Maybe Bitmap -> VU.Vector a -> Column+    -- Efficient intermediate formats.     -- Bit-packed Text: shared UTF-8 byte buffer + row offsets + optional bitmap;     -- Text is materialized on demand. Only CSV ingest emits this; user-built     -- Text columns stay 'BoxedColumn'.     PackedText :: Maybe Bitmap -> {-# UNPACK #-} !PackedTextData -> Column+    -- A join's same-named non-key column pair ('mergeColumns'): both sides+    -- keep their native (packed/dict/unboxed) representation; per-row 'These'+    -- values only materialize on element access ('materializeMerged').+    MergedColumn :: !Column -> !Column -> Column  {- | A mutable companion struct to dataframe columns. @@ -204,14 +207,14 @@ columnBitmap (BoxedColumn bm _) = bm columnBitmap (UnboxedColumn bm _) = bm columnBitmap (PackedText bm _) = bm+columnBitmap (MergedColumn _ _) = Nothing  {- | Decode a 'PackedText' into a @BoxedColumn Text@ (bit-identical to materializing at freeze). Identity on every other column. -} materializePacked :: Column -> Column-materializePacked (PackedText bm p) = case packedRowOffsetVec p of-    Just (arr, offs) -> BoxedColumn bm (sliceTextVector arr offs)-    Nothing -> BoxedColumn bm (VB.generate (packedLength p) (packedIndexText p))+materializePacked (PackedText bm p) =+    BoxedColumn bm (VB.generate (packedLength p) (packedIndexText p)) materializePacked c = c {-# INLINE materializePacked #-} @@ -221,6 +224,28 @@ isPackedText _ = False {-# INLINE isPackedText #-} +-- | Whether a column is a 'MergedColumn'.+isMergedColumn :: Column -> Bool+isMergedColumn (MergedColumn _ _) = True+isMergedColumn _ = False+{-# INLINE isMergedColumn #-}++{- | 'MergedColumn' defers element construction, so forcing must still surface+the one deferred error — a row null on both sides — inside strict IO/executor+boundaries. O(rows) bitmap walk, no allocation; both-null needs a bitmap on+each side, so anything else passes immediately.+-}+checkMergedNoBothNull :: Column -> Column -> ()+checkMergedNoBothNull a b = case (columnBitmap a, columnBitmap b) of+    (Just ba, Just bb) ->+        let !n = min (columnLength a) (columnLength b)+            go !i+                | i >= n = ()+                | bitmapTestBit ba i || bitmapTestBit bb i = go (i + 1)+                | otherwise = error "mergeColumns: both null"+         in go 0+    _ -> ()+ -- --------------------------------------------------------------------------- -- End bitmap helpers -- ---------------------------------------------------------------------------@@ -260,6 +285,7 @@  -- | Checks if a column contains numeric values. isNumeric :: Column -> Bool+isNumeric c@(MergedColumn _ _) = isNumeric (mergedHead c) isNumeric (UnboxedColumn _ (_vec :: VU.Vector a)) = case sNumeric @a of     STrue -> True     _ -> False@@ -276,6 +302,7 @@     BoxedColumn bm (_column :: VB.Vector b) -> checkBoxed bm (typeRep @b)     UnboxedColumn bm (_column :: VU.Vector b) -> checkUnboxed bm (typeRep @b)     PackedText bm _ -> checkBoxed bm (typeRep @T.Text)+    c@(MergedColumn _ _) -> hasElemType @a (mergedHead c)   where     directMatch :: forall (b :: Type). TypeRep b -> Bool     directMatch = isJust . testEquality (typeRep @a)@@ -299,6 +326,7 @@     UnboxedColumn (Just _) _ -> "NullableUnboxed"     PackedText Nothing _ -> "Boxed"     PackedText (Just _) _ -> "NullableBoxed"+    MergedColumn _ _ -> columnVersionString (mergedHead column)  {- | An internal/debugging function to get the type stored in the outermost vector of a column.@@ -311,6 +339,7 @@     UnboxedColumn (Just _) (_ :: VU.Vector a) -> showMaybeType @a     PackedText Nothing _ -> show (typeRep @T.Text)     PackedText (Just _) _ -> showMaybeType @T.Text+    MergedColumn _ _ -> columnTypeString (mergedHead column)   where     showMaybeType :: forall a. (Typeable a) => String     showMaybeType =@@ -334,10 +363,13 @@             | otherwise = go (i + 1)      in go 0 forceColumn (UnboxedColumn _ v) = v `seq` ()-forceColumn (PackedText _ (PackedTextData arr offs sel)) = arr `seq` offs `seq` sel `seq` ()+forceColumn (PackedText _ (PackedTextData arr offs sel _)) = arr `seq` offs `seq` sel `seq` ()+forceColumn (MergedColumn a b) =+    forceColumn a `seq` forceColumn b `seq` checkMergedNoBothNull a b  instance Show Column where     show :: Column -> String+    show c@(MergedColumn _ _) = show (materializeMerged c)     show (BoxedColumn Nothing column) = show column     show (BoxedColumn (Just bm) column) =         let n = VB.length column@@ -396,6 +428,8 @@                             )                             a                         )+    (==) lhs@(MergedColumn _ _) rhs = materializeMerged lhs == rhs+    (==) lhs rhs@(MergedColumn _ _) = lhs == materializeMerged rhs     (==) (PackedText bm1 p1) (PackedText bm2 p2) = eqPackedCols bm1 p1 bm2 p2     (==) lhs@(PackedText _ _) rhs = materializePacked lhs == rhs     (==) lhs rhs@(PackedText _ _) = lhs == materializePacked rhs@@ -555,6 +589,7 @@     BoxedColumn bm (col :: VB.Vector a) -> runBoxed bm col     UnboxedColumn bm (col :: VU.Vector a) -> runUnboxed bm col     c@(PackedText _ _) -> mapColumn f (materializePacked c)+    c@(MergedColumn _ _) -> mapColumn f (materializeMerged c)   where     runBoxed ::         forall a.@@ -627,6 +662,7 @@     BoxedColumn bm (col :: VB.Vector a) -> runBoxed bm col     UnboxedColumn bm (col :: VU.Vector a) -> runUnboxed bm col     c@(PackedText _ _) -> imapColumn f (materializePacked c)+    c@(MergedColumn _ _) -> imapColumn f (materializeMerged c)   where     runBoxed ::         forall a.@@ -653,6 +689,7 @@  -- | O(1) Gets the number of elements in the column. columnLength :: Column -> Int+columnLength (MergedColumn a b) = min (columnLength a) (columnLength b) columnLength (BoxedColumn _ xs) = VB.length xs columnLength (UnboxedColumn _ xs) = VU.length xs columnLength (PackedText _ p) = packedLength p@@ -660,6 +697,7 @@  -- | O(n) Gets the number of non-null elements in the column. numElements :: Column -> Int+numElements (MergedColumn a b) = min (columnLength a) (columnLength b) numElements (BoxedColumn Nothing xs) = VB.length xs numElements (BoxedColumn (Just bm) _xs) = VU.foldl' (\acc b -> acc + popCount b) 0 bm numElements (UnboxedColumn Nothing xs) = VU.length xs@@ -670,6 +708,7 @@  -- | O(n) Takes the first n values of a column. takeColumn :: Int -> Column -> Column+takeColumn n (MergedColumn a b) = MergedColumn (takeColumn n a) (takeColumn n b) takeColumn n (BoxedColumn bm xs) =     BoxedColumn (fmap (bitmapSlice 0 n) bm) (VG.take n xs) takeColumn n (UnboxedColumn bm xs) =@@ -685,6 +724,8 @@  -- | O(n) Takes n values after a given column index. sliceColumn :: Int -> Int -> Column -> Column+sliceColumn start n (MergedColumn a b) =+    MergedColumn (sliceColumn start n a) (sliceColumn start n b) sliceColumn start n (BoxedColumn bm xs) =     BoxedColumn (fmap (bitmapSlice start n) bm) (VG.slice start n xs) sliceColumn start n (UnboxedColumn bm xs) =@@ -717,6 +758,8 @@             bm         )         (VU.unsafeBackpermute column indexes)+atIndicesStable indexes (MergedColumn a b) =+    MergedColumn (atIndicesStable indexes a) (atIndicesStable indexes b) atIndicesStable indexes (PackedText bm p) =     PackedText         ( fmap@@ -733,6 +776,8 @@ Keeps the index vector fully unboxed (no @VB.Vector (Maybe Int)@). -} gatherWithSentinel :: VU.Vector Int -> Column -> Column+gatherWithSentinel indices c@(MergedColumn _ _) =+    gatherWithSentinel indices (materializeMerged c) gatherWithSentinel indices col =     let !n = VU.length indices         newBm = buildBitmapFromValid $ VU.generate n $ \i ->@@ -807,6 +852,7 @@     BoxedColumn _ (v :: VB.Vector b) -> run v VG.convert     UnboxedColumn _ (v :: VU.Vector b) -> run v id     c@(PackedText _ _) -> findIndices predicate (materializePacked c)+    c@(MergedColumn _ _) -> findIndices predicate (materializeMerged c)   where     run ::         forall b v.@@ -835,6 +881,7 @@     BoxedColumn _ column -> foldrWorker column     UnboxedColumn _ column -> foldrWorker column     c@(PackedText _ _) -> ifoldrColumn f acc (materializePacked c)+    c@(MergedColumn _ _) -> ifoldrColumn f acc (materializeMerged c)   where     foldrWorker ::         forall c v.@@ -862,6 +909,7 @@     BoxedColumn _ column -> foldlWorker column     UnboxedColumn _ column -> foldlWorker column     c@(PackedText _ _) -> foldlColumn f acc (materializePacked c)+    c@(MergedColumn _ _) -> foldlColumn f acc (materializeMerged c)   where     foldlWorker ::         forall c v.@@ -889,6 +937,7 @@     BoxedColumn _ column -> foldl1Worker column     UnboxedColumn _ column -> foldl1Worker column     c@(PackedText _ _) -> foldl1Column f (materializePacked c)+    c@(MergedColumn _ _) -> foldl1Column f (materializeMerged c)   where     foldl1Worker ::         forall c v.@@ -925,6 +974,7 @@         UnboxedColumn _ (vec :: VU.Vector d) -> UnboxedColumn Nothing <$> foldl1Worker vec         BoxedColumn _ (vec :: VB.Vector d) -> BoxedColumn Nothing <$> foldl1Worker vec         PackedText _ _ -> foldl1DirectGroups f (materializePacked col) valueIndices offsets+        MergedColumn _ _ -> foldl1DirectGroups f (materializeMerged col) valueIndices offsets   where     foldl1Worker ::         forall c v.@@ -977,6 +1027,8 @@         BoxedColumn _ (vec :: VB.Vector d) -> foldLinearWorker vec         PackedText _ _ ->             foldLinearGroups f seed (materializePacked col) rowToGroup nGroups+        MergedColumn _ _ ->+            foldLinearGroups f seed (materializeMerged col) rowToGroup nGroups   where     foldLinearWorker ::         forall c v.@@ -1022,6 +1074,7 @@     BoxedColumn _ col -> headWorker col     UnboxedColumn _ col -> headWorker col     c@(PackedText _ _) -> headColumn (materializePacked c)+    c@(MergedColumn _ _) -> headColumn (mergedHead c)   where     headWorker ::         forall c v.@@ -1046,6 +1099,8 @@  -- | An internal, column version of zip. zipColumns :: Column -> Column -> Column+zipColumns l@(MergedColumn _ _) r = zipColumns (materializeMerged l) r+zipColumns l r@(MergedColumn _ _) = zipColumns l (materializeMerged r) zipColumns l@(PackedText _ _) r = zipColumns (materializePacked l) r zipColumns l r@(PackedText _ _) = zipColumns l (materializePacked r) zipColumns (BoxedColumn _ column) (BoxedColumn _ other) = BoxedColumn Nothing (VG.zip column other)@@ -1066,49 +1121,62 @@ zipColumns (UnboxedColumn _ column) (UnboxedColumn _ other) = UnboxedColumn Nothing (VG.zip column other) {-# INLINE zipColumns #-} --- | Merge two columns using `These`.+{- | Merge two columns using `These`. O(1): the sides are kept in their+native representation and 'These' values materialize on element access.+-} mergeColumns :: Column -> Column -> Column-mergeColumns colA colB = case (colA, colB) of-    (PackedText _ _, _) -> mergeColumns (materializePacked colA) colB-    (_, PackedText _ _) -> mergeColumns colA (materializePacked colB)-    (BoxedColumn bmA c1, BoxedColumn bmB c2) -> case (bmA, bmB) of-        (Just ba, Just bb) ->-            BoxedColumn Nothing $ mkVec c1 c2 $ \i v1 v2 ->-                let nullA = not (bitmapTestBit ba i)-                    nullB = not (bitmapTestBit bb i)-                 in case (nullA, nullB) of-                        (True, True) -> error "mergeColumns: both null"-                        (False, True) -> This v1-                        (True, False) -> That v2-                        (False, False) -> These v1 v2-        (Just ba, Nothing) ->-            BoxedColumn Nothing $ mkVec c1 c2 $ \i v1 v2 ->-                if not (bitmapTestBit ba i) then That v2 else These v1 v2-        (Nothing, Just bb) ->-            BoxedColumn Nothing $ mkVec c1 c2 $ \i v1 v2 ->-                if not (bitmapTestBit bb i) then This v1 else These v1 v2-        (Nothing, Nothing) ->-            BoxedColumn Nothing $ mkVecSimple c1 c2 These-    (BoxedColumn _ c1, UnboxedColumn _ c2) ->-        BoxedColumn Nothing $ mkVecSimple c1 c2 These-    (UnboxedColumn _ c1, BoxedColumn _ c2) ->-        BoxedColumn Nothing $ mkVecSimple c1 c2 These-    (UnboxedColumn _ c1, UnboxedColumn _ c2) ->-        BoxedColumn Nothing $ mkVecSimple c1 c2 These-  where-    mkVec c1 c2 combineElements =-        VB.generate-            (min (VG.length c1) (VG.length c2))-            (\i -> combineElements i (c1 VG.! i) (c2 VG.! i))-    {-# INLINE mkVec #-}--    mkVecSimple c1 c2 f =-        VB.generate-            (min (VG.length c1) (VG.length c2))-            (\i -> f (c1 VG.! i) (c2 VG.! i))-    {-# INLINE mkVecSimple #-}+mergeColumns = MergedColumn {-# INLINE mergeColumns #-} +-- | Decode a 'MergedColumn' into the eager @BoxedColumn (These a b)@ form.+materializeMerged :: Column -> Column+materializeMerged (MergedColumn colA colB) =+    mergeEager (materializeMerged colA) (materializeMerged colB)+materializeMerged c = c++mergedHead :: Column -> Column+mergedHead (MergedColumn a b) =+    materializeMerged (MergedColumn (takeColumn 1 a) (takeColumn 1 b))+mergedHead c = c++{- | The eager element-wise merge ('These' per row, boxed). Bitmaps are+honored for every representation pair: a null side yields 'This'/'That',+both-null is an error (the join kernels never produce such a row).+-}+mergeEager :: Column -> Column -> Column+mergeEager colA colB = case (colA, colB) of+    (MergedColumn a b, _) -> mergeEager (mergeEager a b) colB+    (_, MergedColumn a b) -> mergeEager colA (mergeEager a b)+    (PackedText _ _, _) -> mergeEager (materializePacked colA) colB+    (_, PackedText _ _) -> mergeEager colA (materializePacked colB)+    (BoxedColumn bmA c1, BoxedColumn bmB c2) ->+        merged bmA bmB (VG.length c1) (VG.length c2) (c1 VG.!) (c2 VG.!)+    (BoxedColumn bmA c1, UnboxedColumn bmB c2) ->+        merged bmA bmB (VG.length c1) (VG.length c2) (c1 VG.!) (c2 VG.!)+    (UnboxedColumn bmA c1, BoxedColumn bmB c2) ->+        merged bmA bmB (VG.length c1) (VG.length c2) (c1 VG.!) (c2 VG.!)+    (UnboxedColumn bmA c1, UnboxedColumn bmB c2) ->+        merged bmA bmB (VG.length c1) (VG.length c2) (c1 VG.!) (c2 VG.!)+  where+    merged ::+        (Columnable a, Columnable b) =>+        Maybe Bitmap ->+        Maybe Bitmap ->+        Int ->+        Int ->+        (Int -> a) ->+        (Int -> b) ->+        Column+    merged bmA bmB lenA lenB atA atB =+        BoxedColumn Nothing $ VB.generate (min lenA lenB) $ \i ->+            case (validAt bmA i, validAt bmB i) of+                (True, True) -> These (atA i) (atB i)+                (True, False) -> This (atA i)+                (False, True) -> That (atB i)+                (False, False) -> error "mergeColumns: both null"+    validAt mbm i = maybe True (`bitmapTestBit` i) mbm+    {-# INLINE validAt #-}+ -- | An internal, column version of zipWith. zipWithColumns ::     forall a b c.@@ -1181,6 +1249,7 @@ No-op when already nullable. -} ensureOptional :: Column -> Column+ensureOptional c@(MergedColumn _ _) = ensureOptional (materializeMerged c) ensureOptional c@(BoxedColumn (Just _) _) = c ensureOptional (BoxedColumn Nothing col) =     BoxedColumn (Just (allValidBitmap (VB.length col))) col@@ -1193,6 +1262,9 @@  -- | Fills the end of a column, up to n, with null rows. Does nothing if column has length >= n. expandColumn :: Int -> Column -> Column+expandColumn n c@(MergedColumn a b)+    | n <= min (columnLength a) (columnLength b) = c+    | otherwise = expandColumn n (materializeMerged c) expandColumn n c@(PackedText _ p)     | n <= packedLength p = c     | otherwise = expandColumn n (materializePacked c)@@ -1224,6 +1296,9 @@  -- | Fills the beginning of a column, up to n, with null rows. Does nothing if column has length >= n. leftExpandColumn :: Int -> Column -> Column+leftExpandColumn n c@(MergedColumn a b)+    | n <= min (columnLength a) (columnLength b) = c+    | otherwise = leftExpandColumn n (materializeMerged c) leftExpandColumn n c@(PackedText _ p)     | n <= packedLength p = c     | otherwise = leftExpandColumn n (materializePacked c)@@ -1261,6 +1336,8 @@ -} concatColumns :: Column -> Column -> Either DataFrameException Column concatColumns left right = case (left, right) of+    (MergedColumn _ _, _) -> concatColumns (materializeMerged left) right+    (_, MergedColumn _ _) -> concatColumns left (materializeMerged right)     (PackedText _ _, _) -> concatColumns (materializePacked left) right     (_, PackedText _ _) -> concatColumns left (materializePacked right)     (BoxedColumn bmL l, BoxedColumn bmR r) -> case testEquality (typeOf l) (typeOf r) of@@ -1318,6 +1395,8 @@ concatManyColumns [] = fromList ([] :: [Maybe Int]) concatManyColumns [c] = c concatManyColumns all'+    | any isMergedColumn all' =+        concatManyColumns (map materializeMerged all')     | any isPackedText all' =         concatManyColumns (map materializePacked all') concatManyColumns (c0 : cs) = case c0 of@@ -1364,8 +1443,13 @@                      in Just $ concatBms (zip expandedBms allVecs)          in UnboxedColumn newBm (VU.concat allVecs)     PackedText _ _ -> concatManyColumns (map materializePacked (c0 : cs))+    MergedColumn _ _ -> concatManyColumns (map materializeMerged (c0 : cs))  concatColumnsEither :: Column -> Column -> Column+concatColumnsEither l@(MergedColumn _ _) r =+    concatColumnsEither (materializeMerged l) r+concatColumnsEither l r@(MergedColumn _ _) =+    concatColumnsEither l (materializeMerged r) concatColumnsEither l@(PackedText _ _) r = concatColumnsEither (materializePacked l) r concatColumnsEither l r@(PackedText _ _) = concatColumnsEither l (materializePacked r) concatColumnsEither (BoxedColumn bmL left) (BoxedColumn bmR right) = case testEquality (typeOf left) (typeOf right) of@@ -1429,9 +1513,12 @@ newMutableColumn n (UnboxedColumn _ (_ :: VU.Vector a)) =     MUnboxedColumn <$> (VUM.new n :: IO (VUM.IOVector a)) newMutableColumn n c@(PackedText _ _) = newMutableColumn n (materializePacked c)+newMutableColumn n c@(MergedColumn _ _) = newMutableColumn n (materializeMerged c)  -- | Copy a column chunk into a mutable column starting at offset @off@. copyIntoMutableColumn :: MutableColumn -> Int -> Column -> IO ()+copyIntoMutableColumn mv off c@(MergedColumn _ _) =+    copyIntoMutableColumn mv off (materializeMerged c) copyIntoMutableColumn (MBoxedColumn (mv :: VBM.IOVector b)) off (BoxedColumn _ (v :: VB.Vector a)) =     case testEquality (typeRep @a) (typeRep @b) of         Just Refl -> VG.imapM_ (\i x -> VBM.unsafeWrite mv (off + i) x) v@@ -1481,6 +1568,7 @@     (VG.Vector v a, Columnable a) => Column -> Either DataFrameException (v a) toVector col = case col of     PackedText _ _ -> toVector (materializePacked col)+    MergedColumn _ _ -> toVector (materializeMerged col)     BoxedColumn bm (inner :: VB.Vector c) ->         -- Check if user wants Maybe c (nullable) or c directly         case testEquality (typeRep @a) (typeRep @c) of@@ -1538,6 +1626,7 @@ toDoubleVector column =     case column of         PackedText _ _ -> toDoubleVector (materializePacked column)+        MergedColumn _ _ -> toDoubleVector (materializeMerged column)         UnboxedColumn bm (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Double) of             Just Refl -> case bm of                 Nothing -> Right f@@ -1602,6 +1691,7 @@ toFloatVector column =     case column of         PackedText _ _ -> toFloatVector (materializePacked column)+        MergedColumn _ _ -> toFloatVector (materializeMerged column)         UnboxedColumn bm (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Float) of             Just Refl -> case bm of                 Nothing -> Right f@@ -1666,6 +1756,7 @@ toIntVector column =     case column of         PackedText _ _ -> toIntVector (materializePacked column)+        MergedColumn _ _ -> toIntVector (materializeMerged column)         UnboxedColumn _ (f :: VU.Vector a) -> case testEquality (typeRep @a) (typeRep @Int) of             Just Refl -> Right f             Nothing -> case sFloating @a of

@@ -34,6 +34,9 @@     Column (..),     Columnable,     allValidBitmap,+    isMergedColumn,+    isPackedText,+    materializeMerged,     materializePacked,  ) import DataFrame.Internal.PackedText (mkPackedContiguous)@@ -94,8 +97,14 @@ mergeColumns :: [Column] -> Column mergeColumns [] = error "DataFrame.Internal.ColumnBuilder.mergeColumns: empty list" mergeColumns [c] = c+-- Normalize on the whole list, not the head: a packed or merged chunk in any+-- position must demote every chunk to the common boxed form.+mergeColumns cols@(c0 : _)+    | any isMergedColumn cols = mergeColumns (map materializeMerged cols)+    | any isPackedText cols = mergeColumns (map materializePacked cols) mergeColumns cols@(c0 : _) = case c0 of     PackedText _ _ -> mergeColumns (map materializePacked cols)+    MergedColumn _ _ -> mergeColumns (map materializeMerged cols)     UnboxedColumn _ (_ :: VU.Vector a) ->         let parts = map (unboxedPart @a) cols             !merged = VU.concat (map snd parts)

@@ -177,8 +177,10 @@         getType (UnboxedColumn (Just _) (_ :: VU.Vector a)) = T.pack $ showMaybeType @a         getType (PackedText Nothing _) = T.pack $ show (typeRep @T.Text)         getType (PackedText (Just _) _) = T.pack $ showMaybeType @T.Text+        getType c@(MergedColumn _ _) = getType (mergedHead c)          get :: Maybe Column -> V.Vector T.Text+        get (Just c@(MergedColumn _ _)) = get (Just (materializeMerged c))         get (Just (BoxedColumn (Just bm) (column :: V.Vector a))) =             V.generate (V.length column) $ \i ->                 if bitmapTestBit bm i@@ -341,6 +343,10 @@ getRowAsText df i = map (`showElement` i) (V.toList (columns df))  showElement :: Column -> Int -> T.Text+showElement (MergedColumn a b) i =+    showElement+        (materializeMerged (MergedColumn (sliceColumn i 1 a) (sliceColumn i 1 b)))+        0 showElement (BoxedColumn bm (c :: V.Vector a)) i = case bm of     Just b | not (bitmapTestBit b i) -> "null"     _ -> case c V.!? i of

@@ -11,11 +11,14 @@ module DataFrame.Internal.DictEncode (     dictEncodeColumn,     dictEncodeColumnUpTo,+    dictCompactColumn,     dictMaxCardinality, ) where +import Control.Monad (when) import Control.Monad.ST (runST) import qualified Data.Text as T+import qualified Data.Text.Array as A import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU@@ -26,7 +29,9 @@ import DataFrame.Internal.Hash (fnvOffset, mixBytes, mixText, nullSalt) import DataFrame.Internal.HashTable (htInsert, newHashTable) import DataFrame.Internal.PackedText (-    PackedTextData,+    PackedTextData (..),+    mkOffsets,+    mkSel,     packedLength,     packedSlice,     sliceEqBytes,@@ -127,3 +132,40 @@             Just card -> do                 frozen <- VU.unsafeFreeze codes                 pure (Just (frozen, card))++dictCompactColumn :: Column -> Column+dictCompactColumn col@(PackedText bm p) =+    case encodePacked dictMaxCardinality bm p of+        Just (codes, card)+            | 2 * card <= packedLength p ->+                PackedText bm (dictPacked p codes card)+        _ -> col+dictCompactColumn col = col++dictPacked :: PackedTextData -> VU.Vector Int -> Int -> PackedTextData+dictPacked p codes card = runST $ do+    let n = VU.length codes+    reps <- VUM.replicate card (-1)+    let findReps !i !remaining+            | remaining <= 0 || i >= n = pure ()+            | otherwise = do+                let c = VU.unsafeIndex codes i+                cur <- VUM.unsafeRead reps c+                if cur < 0+                    then VUM.unsafeWrite reps c i >> findReps (i + 1) (remaining - 1)+                    else findReps (i + 1) remaining+    findReps 0 card+    repsV <- VU.unsafeFreeze reps+    let lens = VU.map (\r -> let (_, _, l) = packedSlice p r in l) repsV+        offs = VU.scanl' (+) 0 lens+        total = VU.last offs+    marr <- A.new (max 1 total)+    let copyRep !c =+            when (c < card) $ do+                let r = VU.unsafeIndex repsV c+                    (arr, o, l) = packedSlice p r+                A.copyI l marr (VU.unsafeIndex offs c) arr o+                copyRep (c + 1)+    copyRep 0+    arr <- A.unsafeFreeze marr+    pure (PackedTextData arr (mkOffsets offs) (Just (mkSel card codes)) True)

@@ -31,20 +31,27 @@     Bitmap,     Column (..),     bitmapTestBit,+    materializeMerged,  ) import DataFrame.Internal.DataFrame (DataFrame (..), GroupedDataFrame (..)) import DataFrame.Internal.DictEncode (dictEncodeColumnUpTo) import DataFrame.Internal.GroupingDirect (     DirectGrouping (..),+    directGroupThreshold,     tryDirectGroupColumn,  ) import DataFrame.Internal.GroupingPar (parallelAssignGroups, shouldParallelize) import DataFrame.Internal.Hash import DataFrame.Internal.HashTable (htInsert, newHashTable) import DataFrame.Internal.PackedText (-    PackedTextData,+    PackedSel,+    PackedTextData (..),+    offAt,+    offCount,     packedLength,     packedSlice,+    selAt,+    selLength,     sliceEqBytes,  ) import DataFrame.Internal.RadixRank (rankByHash)@@ -90,9 +97,52 @@     case tryDirectGroupColumn col of         Just dg ->             Just (Grouped df [name] (dgValueIndices dg) (dgOffsets dg) (dgRowToGroup dg))-        Nothing -> tryDictGroup (nRows df) df [name] col+        Nothing -> case col of+            PackedText Nothing p -> dictCodesGroup df [name] p+            _ -> tryDictGroup (nRows df) df [name] col tryDirectGroup _ _ = Nothing +dictCodesGroup ::+    DataFrame -> [T.Text] -> PackedTextData -> Maybe GroupedDataFrame+dictCodesGroup df names p = do+    sel <- ptSel p+    guard (ptCanonicalSel p)+    let offs = ptOffsets p+        card = offCount offs - 1+        n = selLength sel+    guard (card > 0 && card <= directGroupThreshold && n > 0)+    counts <- codeHistogram card sel+    let occupied = VU.filter (\g -> VU.unsafeIndex counts g > 0) (VU.enumFromN 0 card)+        nGroups = VU.length occupied+        entryHash g =+            let o = offAt offs g+             in mixBytes fnvOffset (ptBytes p) o (offAt offs (g + 1) - o)+        rank =+            runST (rankByHash (pure . entryHash . VU.unsafeIndex occupied) nGroups)+        remap = runST $ do+            m <- VUM.new card+            VU.imapM_ (\j g -> VUM.unsafeWrite m g (VU.unsafeIndex rank j)) occupied+            VU.unsafeFreeze m+        rtg = VU.generate n (VU.unsafeIndex remap . selAt sel)+        (vis, os) = indicesFromGroups rtg nGroups+    pure (Grouped df names vis os rtg)++-- | Per-code occupancy counts; 'Nothing' as soon as any code is negative.+codeHistogram :: Int -> PackedSel -> Maybe (VU.Vector Int)+codeHistogram card sel = runST $ do+    counts <- VUM.replicate card 0+    let n = selLength sel+        go i+            | i >= n = Just <$> VU.unsafeFreeze counts+            | otherwise = do+                let c = selAt sel i+                if c < 0 || c >= card+                    then pure Nothing+                    else do+                        VUM.unsafeModify counts (+ 1) c+                        go (i + 1)+    go 0+ {- | Dictionary-encode a single text key to dense int codes, then derive @valueIndices@/@offsets@ by counting sort. Profiled slower than the fused hash group-by on every db-benchmark question, so it always falls back ('dictGroupEnabled').@@ -197,7 +247,8 @@ computeHashes :: DataFrame -> [Int] -> Int -> ST s (VU.Vector Int) computeHashes df indicesToGroup n = do     mh <- VUM.replicate n fnvOffset-    let selectedCols = map (columns df V.!) indicesToGroup+    -- Merged key columns are exotic; hash their eager form.+    let selectedCols = map (materializeMerged . (columns df V.!)) indicesToGroup     forM_ selectedCols $ \case         UnboxedColumn ubm (v :: VU.Vector a) ->             case testEquality (typeRep @a) (typeRep @Int) of@@ -238,6 +289,8 @@                         )                         v         PackedText bm p -> hashPacked mh bm p+        MergedColumn _ _ ->+            error "computeHashes: MergedColumn is normalized before hashing"     VU.unsafeFreeze mh  {- | Build the row-key equality predicate over the selected key columns.@@ -255,6 +308,7 @@ when both are null, or both are valid and their values compare equal. -} colEqRow :: Column -> (Int -> Int -> Bool)+colEqRow c@(MergedColumn _ _) = colEqRow (materializeMerged c) colEqRow (UnboxedColumn bm v) =     let eqV a b = VU.unsafeIndex v a == VU.unsafeIndex v b      in withNulls bm eqV

@@ -471,6 +471,7 @@ sliceGroups :: Column -> VU.Vector Int -> VU.Vector Int -> V.Vector Column sliceGroups col os indices = case col of     PackedText _ _ -> sliceGroups (materializePacked col) os indices+    MergedColumn _ _ -> sliceGroups (materializeMerged col) os indices     BoxedColumn bm vec ->         let !sorted =                 V.generate@@ -570,6 +571,7 @@                 SFalse -> castMismatch @c @b     BoxedColumn _ _ -> tryParseWith @Double onResult col     PackedText _ _ -> promoteToDoubleWith onResult (materializePacked col)+    MergedColumn _ _ -> promoteToDoubleWith onResult (materializeMerged col)  promoteToFloatWith ::     forall b.@@ -610,6 +612,7 @@                 SFalse -> castMismatch @c @b     BoxedColumn _ _ -> tryParseWith @Float onResult col     PackedText _ _ -> promoteToFloatWith onResult (materializePacked col)+    MergedColumn _ _ -> promoteToFloatWith onResult (materializeMerged col)  promoteToIntWith ::     forall b.@@ -650,6 +653,7 @@                 SFalse -> castMismatch @c @b     BoxedColumn _ _ -> tryParseWith @Int onResult col     PackedText _ _ -> promoteToIntWith onResult (materializePacked col)+    MergedColumn _ _ -> promoteToIntWith onResult (materializeMerged col)  -- | Single parse primitive: apply @onResult@ to the result of 'reads'. parseWith :: (Read a) => (Either String a -> b) -> String -> b@@ -665,6 +669,7 @@     (Either String a -> b) -> Column -> Either DataFrameException Column tryParseWith onResult col = case col of     PackedText _ _ -> tryParseWith onResult (materializePacked col)+    MergedColumn _ _ -> tryParseWith onResult (materializeMerged col)     BoxedColumn bm (v :: V.Vector c) ->         case testEquality (typeRep @c) (typeRep @String) of             Just Refl -> case bm of

@@ -3,13 +3,24 @@ {- | Packed-text payload + byte-slice primitives. A 'PackedTextData' shares one UTF-8 byte buffer across all rows of a string column, with @n+1@ row offsets, so no per-row 'Data.Text.Text' header is materialized until decode is demanded.+Offsets and selection vectors are stored 'Int32' whenever their values fit+(Arrow-style), halving the per-row footprint of large string columns. -} module DataFrame.Internal.PackedText (     PackedTextData (..),+    PackedOffsets (..),+    PackedSel (..),+    offAt,+    offCount,+    selAt,+    selLength,     mkPackedContiguous,+    mkPackedContiguous32,+    mkOffsets,+    mkSel,     packedGather,     packedTake,-    packedRowOffsetVec,+    packedRowOffsets,     packedLength,     packedSlice,     packedIndexText,@@ -21,40 +32,116 @@ import qualified Data.Text.Array as A import qualified Data.Vector.Unboxed as VU +import Data.Int (Int32) import Data.Ord (comparing) import Data.Text.Internal (Text (Text)) import DataFrame.Internal.Utf8 (isValidUtf8Slice, lenientDecodeSlice) +{- | Row byte-offsets, physically 'Int32' when every value fits (total buffer+bytes < 2^31) and 'Int' otherwise. Values are non-negative byte positions.+-}+data PackedOffsets+    = Offs32 {-# UNPACK #-} !(VU.Vector Int32)+    | Offs64 {-# UNPACK #-} !(VU.Vector Int)++-- | Offset at index @i@, widened to 'Int'.+offAt :: PackedOffsets -> Int -> Int+offAt (Offs32 v) i = fromIntegral (VU.unsafeIndex v i)+offAt (Offs64 v) i = VU.unsafeIndex v i+{-# INLINE offAt #-}++-- | Number of offset entries (row count + 1).+offCount :: PackedOffsets -> Int+offCount (Offs32 v) = VU.length v+offCount (Offs64 v) = VU.length v+{-# INLINE offCount #-}++{- | A selection layer mapping logical rows to base rows; @-1@ marks an+invalid/null row. 'Int32' when the base row count fits.+-}+data PackedSel+    = Sel32 {-# UNPACK #-} !(VU.Vector Int32)+    | Sel64 {-# UNPACK #-} !(VU.Vector Int)++-- | Base row for logical row @i@ (may be @-1@).+selAt :: PackedSel -> Int -> Int+selAt (Sel32 v) i = fromIntegral (VU.unsafeIndex v i)+selAt (Sel64 v) i = VU.unsafeIndex v i+{-# INLINE selAt #-}++selLength :: PackedSel -> Int+selLength (Sel32 v) = VU.length v+selLength (Sel64 v) = VU.length v+{-# INLINE selLength #-}+ {- | A shared UTF-8 byte buffer plus @n+1@ row offsets (base row @r@ spans bytes @[offsets!r, offsets!(r+1))@); validity lives in the column's bitmap. @ptSel@ is an optional selection layer letting a gather/join/sort result share the buffer.++@ptCanonicalSel@ marks a selection that is a canonical dictionary encoding:+equal byte slices always map to the same base row (codes). Set by dictionary+compaction; preserved by gather/take over an already-canonical selection (a+row keeps its code); 'False' for a gather over an unselected base, where two+logical rows can select different but equal-byted base rows. Grouping keys on+codes directly when it holds. -} data PackedTextData = PackedTextData     { ptBytes :: {-# UNPACK #-} !A.Array-    , ptOffsets :: {-# UNPACK #-} !(VU.Vector Int)-    , ptSel :: !(Maybe (VU.Vector Int))+    , ptOffsets :: !PackedOffsets+    , ptSel :: !(Maybe PackedSel)+    , ptCanonicalSel :: !Bool     } +int32Max :: Int+int32Max = fromIntegral (maxBound :: Int32)++-- | Narrow an 'Int' offset vector when the final offset (total bytes) fits.+mkOffsets :: VU.Vector Int -> PackedOffsets+mkOffsets offs+    | not (VU.null offs) && VU.last offs <= int32Max =+        Offs32 (VU.map fromIntegral offs)+    | otherwise = Offs64 offs+{-# INLINE mkOffsets #-}++{- | Narrow an 'Int' base-row vector (@-1@ sentinels allowed) when the base+row count fits in 'Int32'.+-}+mkSel :: Int -> VU.Vector Int -> PackedSel+mkSel base rows+    | base <= int32Max = Sel32 (VU.map fromIntegral rows)+    | otherwise = Sel64 rows+{-# INLINE mkSel #-}+ -- | Build a contiguous packed payload (no selection): the freeze-path shape. mkPackedContiguous :: A.Array -> VU.Vector Int -> PackedTextData-mkPackedContiguous arr offs = PackedTextData arr offs Nothing+mkPackedContiguous arr offs = PackedTextData arr (mkOffsets offs) Nothing False {-# INLINE mkPackedContiguous #-} +-- | 'mkPackedContiguous' from offsets already produced at 'Int32' width.+mkPackedContiguous32 :: A.Array -> VU.Vector Int32 -> PackedTextData+mkPackedContiguous32 arr offs = PackedTextData arr (Offs32 offs) Nothing False+{-# INLINE mkPackedContiguous32 #-}+ {- | Reindex a packed payload by a selection vector, sharing the byte buffer; logical row @i@ becomes base row @indices!i@. A negative or out-of-range index-decodes to the empty slice. Composes with an existing selection.+decodes to the empty slice. Composes with an existing selection; canonicality+survives composition (a kept row keeps its code) but not a first selection+over the unselected base. -} packedGather :: VU.Vector Int -> PackedTextData -> PackedTextData-packedGather indices (PackedTextData arr offs msel) =-    let !base = VU.length offs - 1+packedGather indices (PackedTextData arr offs msel canon) =+    let !base = offCount offs - 1         clamp r = if r >= 0 && r < base then r else -1-        sel' = case msel of-            Nothing -> VU.map clamp indices+        (sel', canon') = case msel of+            Nothing -> (VU.map clamp indices, False)             Just s ->-                VU.map-                    (\i -> if i >= 0 && i < VU.length s then clamp (VU.unsafeIndex s i) else -1)-                    indices-     in PackedTextData arr offs (Just sel')+                let !sn = selLength s+                 in ( VU.map+                        (\i -> if i >= 0 && i < sn then clamp (selAt s i) else -1)+                        indices+                    , canon+                    )+     in PackedTextData arr offs (Just (mkSel base sel')) canon' {-# INLINE packedGather #-}  {- | Take the first @k@ logical rows, sharing the byte buffer via a capped@@ -62,47 +149,49 @@ large packed column. -} packedTake :: Int -> PackedTextData -> PackedTextData-packedTake k (PackedTextData arr offs msel) =-    let !base = VU.length offs - 1+packedTake k (PackedTextData arr offs msel canon) =+    let !base = offCount offs - 1         !k' = max 0 k-     in case msel of-            Just s -> PackedTextData arr offs (Just (VU.take k' s))-            Nothing -> PackedTextData arr offs (Just (VU.enumFromN 0 (min k' base)))+        (sel', canon') = case msel of+            Just (Sel32 s) -> (Sel32 (VU.take k' s), canon)+            Just (Sel64 s) -> (Sel64 (VU.take k' s), canon)+            Nothing -> (mkSel base (VU.enumFromN 0 (min k' base)), False)+     in PackedTextData arr offs (Just sel') canon' {-# INLINE packedTake #-}  -- | Map a logical row index to its base row, honoring any selection layer. baseRow :: PackedTextData -> Int -> Int-baseRow (PackedTextData _ _ Nothing) i = i-baseRow (PackedTextData _ _ (Just sel)) i = VU.unsafeIndex sel i+baseRow (PackedTextData _ _ Nothing _) i = i+baseRow (PackedTextData _ _ (Just sel) _) i = selAt sel i {-# INLINE baseRow #-}  -- | Row count: @length sel@ when selected, else @length offsets - 1@. packedLength :: PackedTextData -> Int-packedLength (PackedTextData _ offs Nothing) = VU.length offs - 1-packedLength (PackedTextData _ _ (Just sel)) = VU.length sel+packedLength (PackedTextData _ offs Nothing _) = offCount offs - 1+packedLength (PackedTextData _ _ (Just sel) _) = selLength sel {-# INLINE packedLength #-}  -- | Raw byte slice for logical row @i@: @(buffer, offset, length)@. The hot accessor. packedSlice :: PackedTextData -> Int -> (A.Array, Int, Int)-packedSlice p@(PackedTextData arr offs _) i =+packedSlice p@(PackedTextData arr offs _ _) i =     let !r = baseRow p i      in if r < 0             then (arr, 0, 0)             else-                let o = VU.unsafeIndex offs r in (arr, o, VU.unsafeIndex offs (r + 1) - o)+                let o = offAt offs r in (arr, o, offAt offs (r + 1) - o) {-# INLINE packedSlice #-}  {- | The shared buffer + contiguous @n+1@ offsets when the payload is the unselected base; a selected (gathered) payload returns 'Nothing' (its rows are-non-contiguous). Lets the boxed-Text fallback take the fast contiguous path.+non-contiguous). Lets contiguous consumers skip the selection indirection. -}-packedRowOffsetVec :: PackedTextData -> Maybe (A.Array, VU.Vector Int)-packedRowOffsetVec (PackedTextData arr offs Nothing) = Just (arr, offs)-packedRowOffsetVec _ = Nothing-{-# INLINE packedRowOffsetVec #-}+packedRowOffsets :: PackedTextData -> Maybe (A.Array, PackedOffsets)+packedRowOffsets (PackedTextData arr offs Nothing _) = Just (arr, offs)+packedRowOffsets _ = Nothing+{-# INLINE packedRowOffsets #-}  {- | On-demand single 'Data.Text.Text' for row @i@, using the same-validate-or-lenient decode as 'sliceTextVector' so output is bit-identical.+validate-or-lenient decode as the freeze path so output is bit-identical. -} packedIndexText :: PackedTextData -> Int -> T.Text packedIndexText p i =@@ -110,7 +199,7 @@      in decodeField arr o l {-# INLINE packedIndexText #-} --- Decode one field exactly as 'sliceTextVector' does per row.+-- Decode one field exactly as the boxed freeze path does per row. decodeField :: A.Array -> Int -> Int -> T.Text decodeField arr o l     | l == 0 = T.empty

@@ -167,6 +167,10 @@         Just (BoxedColumn bm column) -> cellAny bm i (column V.! i)         Just (UnboxedColumn bm column) -> cellAny bm i (column VU.! i)         Just (PackedText bm p) -> cellAny bm i (packedIndexText p i)+        Just c@(MergedColumn _ _) ->+            case materializeMerged (sliceColumn i 1 c) of+                BoxedColumn bm column -> cellAny bm 0 (column V.! 0)+                _ -> error "mkRowFromArgs: materializeMerged is boxed"  -- Returns row values in the caller's requested column order, not the -- dataframe's storage order.@@ -190,6 +194,12 @@             Nothing -> throwError name         Just (PackedText bm p)             | i < packedLength p -> cellAny bm i (packedIndexText p i)+            | otherwise -> throwError name+        Just c@(MergedColumn _ _)+            | i < columnLength c ->+                case materializeMerged (sliceColumn i 1 c) of+                    BoxedColumn bm column -> cellAny bm 0 (column V.! 0)+                    _ -> error "mkRowRep: materializeMerged is boxed"             | otherwise -> throwError name         Nothing ->             throw $ ColumnsNotFoundException [name] "mkRowRep" (M.keys $ columnIndices df)

@@ -26,7 +26,12 @@ import System.IO.Unsafe (unsafePerformIO) import Type.Reflection (typeRep) -import DataFrame.Internal.Column (Bitmap, Column (..), bitmapTestBit)+import DataFrame.Internal.Column (+    Bitmap,+    Column (..),+    bitmapTestBit,+    materializeMerged,+ ) import DataFrame.Internal.Hash (     fnvOffset,     mixBytes,@@ -38,6 +43,7 @@  ) import DataFrame.Internal.PackedText (     PackedTextData (..),+    offAt,     packedSlice,  ) import DataFrame.Internal.Types (@@ -103,6 +109,7 @@ -} mixColumnRange :: VUM.IOVector Int -> Int -> Int -> Column -> IO () mixColumnRange mv lo hi = \case+    c@(MergedColumn _ _) -> mixColumnRange mv lo hi (materializeMerged c)     UnboxedColumn ubm (v :: VU.Vector a) ->         case testEquality (typeRep @a) (typeRep @Int) of             Just Refl -> unboxedRange mv lo hi ubm mixInt v@@ -196,8 +203,8 @@             | i >= hi = pure ()             | otherwise = do                 h <- VUM.unsafeRead mv i-                let !o = VU.unsafeIndex offs i-                    !l = VU.unsafeIndex offs (i + 1) - o+                let !o = offAt offs i+                    !l = offAt offs (i + 1) - o                     !h' = if valid i then mixBytes h arr o l else mixInt h nullSalt                 VUM.unsafeWrite mv i h'                 go (i + 1)