dataframe 0.3.3.3 → 0.3.3.4

raw patch · 21 files changed

+437/−205 lines, 21 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• CHANGELOG.md +9/−0
• app/Main.hs +3/−2
• dataframe.cabal +2/−1
• src/DataFrame.hs +16/−0
• src/DataFrame/Display.hs +26/−0
• src/DataFrame/Functions.hs +97/−16
• src/DataFrame/IO/CSV.hs +57/−25
• src/DataFrame/IO/Parquet/Levels.hs +2/−2
• src/DataFrame/IO/Parquet/Thrift.hs +1/−3
• src/DataFrame/Internal/Column.hs +4/−4
• src/DataFrame/Internal/DataFrame.hs +18/−10
• src/DataFrame/Internal/Expression.hs +11/−8
• src/DataFrame/Internal/Schema.hs +66/−3
• src/DataFrame/Internal/Statistics.hs +9/−4
• src/DataFrame/Operations/Aggregation.hs +3/−3
• src/DataFrame/Operations/Subset.hs +15/−22
• src/DataFrame/Operations/Typing.hs +74/−80
• tests/Functions.hs +7/−6
• tests/Main.hs +3/−0
• tests/Operations/Aggregations.hs +14/−15
• tests/Operations/Core.hs +0/−1

@@ -1,5 +1,14 @@ # Revision history for dataframe +## 0.3.3.4+* Add linting CI step + fix existing lint errors.+* Show now only prints 10 row. To print more you should use the new `display` function that takes the number of rows as a parameter in its configuration.+* Add `toDouble`, `div`, and, `mod` functions.+* Define an `IsString` instance for columns so you can use string literals without `F.lit`.+* Include variance expression.+* Improved filter performance.+* Make beam search loss function configurable for synthesizing features.+ ## 0.3.3.3 * Split `toMatrix` into more specific `to<Type>Matrix` functions. 

@@ -12,12 +12,13 @@ main = do     let n = 100_000_000     g <- newIOGenM =<< newStdGen-    let range = (-20.0 :: Double, 20.0 :: Double)+    let range = (0 :: Double, 1 :: Double)     startGeneration <- getCurrentTime     ns <- VU.replicateM n (uniformRM range g)     xs <- VU.replicateM n (uniformRM range g)     ys <- VU.replicateM n (uniformRM range g)     let df = D.fromUnnamedColumns (map D.fromUnboxedVector [ns, xs, ys])+    print df     endGeneration <- getCurrentTime     let generationTime = diffUTCTime endGeneration startGeneration     putStrLn $ "Data generation Time: " ++ show generationTime@@ -29,7 +30,7 @@     let calculationTime = diffUTCTime endCalculation startCalculation     putStrLn $ "Calculation Time: " ++ show calculationTime     startFilter <- getCurrentTime-    print $ D.filter "0" (>= (19.9 :: Double)) df D.|> D.take 10+    print $ D.filter "0" (> (0.971 :: Double)) df D.|> D.take 10     endFilter <- getCurrentTime     let filterTime = diffUTCTime endFilter startFilter     putStrLn $ "Filter Time: " ++ show filterTime

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            0.3.3.3+version:            0.3.3.4  synopsis: A fast, safe, and intuitive DataFrame library. @@ -57,6 +57,7 @@                     DataFrame.Operations.Transformations,                     DataFrame.Operations.Typing,                     DataFrame.Operations.Aggregation,+                    DataFrame.Display,                     DataFrame.Display.Terminal.Plot,                     DataFrame.IO.CSV,                     DataFrame.IO.Parquet,

@@ -197,9 +197,15 @@     module Row,     module Expression, +    -- * Display operations+    module Display,+     -- * Core dataframe operations     module Core, +    -- * Types+    module Schema,+     -- * I/O     module CSV,     module Parquet,@@ -224,10 +230,17 @@ ) where +import DataFrame.Display as Display (+    DisplayOptions (..),+    defaultDisplayOptions,+    display,+ ) import DataFrame.Display.Terminal.Plot as Plot import DataFrame.Errors as Errors import DataFrame.IO.CSV as CSV (+    HeaderSpec (..),     ReadOptions (..),+    TypeSpec (..),     defaultReadOptions,     readCsv,     readCsvWithOpts,@@ -269,6 +282,9 @@     toAny,     toRowList,     toRowVector,+ )+import DataFrame.Internal.Schema as Schema (+    schemaType,  ) import DataFrame.Operations.Aggregation as Aggregation (     aggregate,

@@ -0,0 +1,26 @@+{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}++{-# HLINT ignore "Use newtype instead of data" #-}+module DataFrame.Display where++import qualified Data.Text.IO as T+import qualified DataFrame.Internal.DataFrame as D+import qualified DataFrame.Operations.Subset as D++import Data.Function++data DisplayOptions = DisplayOptions+    { {-- | Maximum number of rows to render.+      --+      --   * If this value is less than or equal to 0, no rows are printed.+      --   * If it is greater than the number of rows in the frame, all rows are printed.+      --}+      displayRows :: Int+    }++defaultDisplayOptions :: DisplayOptions+defaultDisplayOptions = DisplayOptions 10++-- | Render a 'DataFrame' to stdout according to 'DisplayOptions'.+display :: DisplayOptions -> D.DataFrame -> IO ()+display opts df = df & D.take (displayRows opts) & (`D.asText` False) & T.putStrLn

@@ -72,6 +72,15 @@     (c -> b -> a) -> Expr c -> Expr b -> Expr a lift2 = BinaryOp "udf" +toDouble :: (Columnable a, Real a) => Expr a -> Expr Double+toDouble = UnaryOp "toDouble" realToFrac++div :: (Integral a, Columnable a) => Expr a -> Expr a -> Expr a+div = BinaryOp "div" Prelude.div++mod :: (Integral a, Columnable a) => Expr a -> Expr a -> Expr a+mod = BinaryOp "mod" Prelude.mod+ (==) :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool (==) = BinaryOp "eq" (Prelude.==) @@ -126,9 +135,12 @@ mean :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double mean expr = AggNumericVector expr "mean" mean' -median :: Expr Double -> Expr Double-median expr = AggNumericVector expr "mean" median'+variance :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double+variance expr = AggNumericVector expr "variance" variance' +median :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double+median expr = AggNumericVector expr "median" median'+ percentile :: Int -> Expr Double -> Expr Double percentile n expr =     AggNumericVector@@ -312,13 +324,59 @@ synthesizeFeatureExpr ::     -- | Target expression     T.Text ->+    BeamConfig ->+    DataFrame ->+    Either String (Expr Double)+synthesizeFeatureExpr target cfg df =+    let+        df' = exclude [target] df+        t = case interpret df (Col target) of+            Left e -> throw e+            Right v -> v+     in+        case beamSearch+            df'+            cfg+            t+            [] of+            Nothing -> Left "No programs found"+            Just p -> Right p++f1FromBinary :: VU.Vector Double -> VU.Vector Double -> Maybe Double+f1FromBinary trues preds =+    let (!tp, !fp, !fn) =+            VU.foldl' step (0 :: Int, 0 :: Int, 0 :: Int) $+                VU.zip (VU.map (Prelude.> 0) preds) (VU.map (Prelude.> 0) trues)+     in f1FromCounts tp fp fn+  where+    step (!tp, !fp, !fn) (!p, !t) =+        case (p, t) of+            (True, True) -> (tp + 1, fp, fn)+            (True, False) -> (tp, fp + 1, fn)+            (False, True) -> (tp, fp, fn + 1)+            (False, False) -> (tp, fp, fn)++f1FromCounts :: Int -> Int -> Int -> Maybe Double+f1FromCounts tp fp fn =+    let tp' = fromIntegral tp+        fp' = fromIntegral fp+        fn' = fromIntegral fn+        precision = if tp' + fp' Prelude.== 0 then 0 else tp' / (tp' + fp')+        recall = if tp' + fn' Prelude.== 0 then 0 else tp' / (tp' + fn')+     in if precision + recall Prelude.== 0+            then Nothing+            else Just (2 * precision * recall / (precision + recall))++fitClassifier ::+    -- | Target expression+    T.Text ->     -- | Depth of search (Roughly, how many terms in the final expression)     Int ->     -- | Beam size - the number of candidate expressions to consider at a time.     Int ->     DataFrame ->-    Either String (Expr Double)-synthesizeFeatureExpr target d b df =+    Either String (Expr Int)+fitClassifier target d b df =     let         df' = exclude [target] df         t = case interpret df (Col target) of@@ -327,11 +385,11 @@      in         case beamSearch             df'-            (BeamConfig d b (\l r -> (^ 2) <$> correlation' l r))+            (BeamConfig d b F1)             t             [] of             Nothing -> Left "No programs found"-            Just p -> Right p+            Just p -> Right (ifThenElse (p DataFrame.Functions.> 0) 1 0)  fitRegression ::     -- | Target expression@@ -355,12 +413,7 @@             ( BeamConfig                 d                 b-                ( \l r ->-                    mutualInformationBinned-                        (Prelude.max 10 (ceiling (sqrt (fromIntegral (VU.length l)))))-                        l-                        r-                )+                MutualInformation             )             t             [] of@@ -372,18 +425,41 @@                             ( D.derive "_generated_regression_feature_" p df                                 & select ["_generated_regression_feature_"]                             )-                            (BeamConfig d b (\l r -> fmap negate (meanSquaredError l r)))+                            (BeamConfig d b MeanSquaredError)                             t                             [Col "_generated_regression_feature_", lit targetMean, lit 10] of                             Nothing -> Left "Could not find coefficients"                             Just p' -> Right (replaceExpr p (Col @Double "_generated_regression_feature_") p') +data LossFunction+    = PearsonCorrelation+    | MutualInformation+    | MeanSquaredError+    | F1++getLossFunction ::+    LossFunction -> (VU.Vector Double -> VU.Vector Double -> Maybe Double)+getLossFunction f = case f of+    MutualInformation ->+        ( \l r ->+            mutualInformationBinned+                (Prelude.max 10 (ceiling (sqrt (fromIntegral (VU.length l)))))+                l+                r+        )+    PearsonCorrelation -> (\l r -> (^ 2) <$> correlation' l r)+    MeanSquaredError -> (\l r -> fmap negate (meanSquaredError l r))+    F1 -> f1FromBinary+ data BeamConfig = BeamConfig     { searchDepth :: Int     , beamLength :: Int-    , rankingFunction :: VU.Vector Double -> VU.Vector Double -> Maybe Double+    , lossFunction :: LossFunction     } +defaultBeamConfig :: BeamConfig+defaultBeamConfig = BeamConfig 2 100 PearsonCorrelation+ beamSearch ::     DataFrame ->     -- | Parameters of the beam search.@@ -432,7 +508,10 @@                                 then LT                                 else compare c1 c2                         )-                        (map (\(e, res) -> (e, rankingFunction cfg l (asDoubleVector res))) ps)+                        ( map+                            (\(e, res) -> (e, getLossFunction (lossFunction cfg) l (asDoubleVector res)))+                            ps+                        )                 )         asDoubleVector c =             let@@ -454,7 +533,9 @@         trace             ( "Best loss: "                 ++ show-                    (rankingFunction cfg l <$> (interpretDoubleVector <$> (listToMaybe ordered)))+                    ( getLossFunction (lossFunction cfg) l . interpretDoubleVector+                        <$> listToMaybe ordered+                    )                 ++ " "                 ++ (if null ordered then "empty" else show (listToMaybe ordered))             )

@@ -15,6 +15,7 @@ import qualified Data.ByteString.Char8 as C8 import qualified Data.List as L import qualified Data.Map.Strict as M+import qualified Data.Proxy as P import qualified Data.Text as T import qualified Data.Text.Encoding as TE import qualified Data.Text.IO as TIO@@ -24,7 +25,7 @@ import qualified Data.Vector.Unboxed.Mutable as VUM  import Control.Applicative (many, (<|>))-import Control.Monad (forM_, unless, zipWithM_)+import Control.Monad (forM_, unless, zipWithM, zipWithM_) import Data.Attoparsec.ByteString.Char8 hiding (endOfLine) import Data.Bits (shiftL) import Data.Char@@ -37,6 +38,7 @@ import DataFrame.Internal.Column (Column (..), columnLength) import DataFrame.Internal.DataFrame (DataFrame (..)) import DataFrame.Internal.Parsing+import DataFrame.Internal.Schema import DataFrame.Operations.Typing import System.IO import Type.Reflection@@ -59,12 +61,34 @@     | GrowingDouble !(GrowingUnboxedVector Double) !(IORef [Int])     | GrowingText !(GrowingVector T.Text) !(IORef [Int]) +data HeaderSpec+    = -- | File has no header row+      NoHeader+    | -- | Use first row as column names+      UseFirstRow+    | -- | Supply names for a no-header file+      ProvideNames [T.Text]+    deriving (Eq, Show)++data TypeSpec+    = InferFromSample Int+    | SpecifyTypes [SchemaType]+    | NoInference++shouldInferFromSample :: TypeSpec -> Bool+shouldInferFromSample (InferFromSample _) = True+shouldInferFromSample _ = False++typeInferenceSampleSize :: TypeSpec -> Int+typeInferenceSampleSize (InferFromSample n) = n+typeInferenceSampleSize _ = 0+ -- | CSV read parameters. data ReadOptions = ReadOptions-    { hasHeader :: Bool-    -- ^ Whether or not the CSV file has a header. (default: True)-    , inferTypes :: Bool-    -- ^ Whether to try and infer types. (default: True)+    { headerSpec :: HeaderSpec+    -- ^ Where to get the headers from. (default: UseFirstRow)+    , typeSpec :: TypeSpec+    -- ^ Whether/how to infer types. (default: InferFromSample 100)     , safeRead :: Bool     -- ^ Whether to partially parse values into `Maybe`/`Either`. (default: True)     , chunkSize :: Int@@ -86,8 +110,8 @@ defaultReadOptions :: ReadOptions defaultReadOptions =     ReadOptions-        { hasHeader = True-        , inferTypes = True+        { headerSpec = UseFirstRow+        , typeSpec = InferFromSample 100         , safeRead = True         , chunkSize = 512_000         , dateFormat = "%Y-%m-%d"@@ -175,7 +199,7 @@ @ -} readCsvWithOpts :: ReadOptions -> FilePath -> IO DataFrame-readCsvWithOpts opts = readSeparated ',' opts+readCsvWithOpts = readSeparated ','  {- | Read TSV (tab separated) file from path and load it into a dataframe. @@ -202,12 +226,12 @@      firstLine <- C8.hGetLine handle     let firstRow = parseLine sep firstLine-        columnNames =-            if hasHeader opts-                then map (T.filter (/= '\"') . TE.decodeUtf8Lenient) firstRow-                else map (T.singleton . intToDigit) [0 .. length firstRow - 1]+        columnNames = case headerSpec opts of+            NoHeader -> map (T.pack . show) [0 .. length firstRow - 1]+            UseFirstRow -> map (T.filter (/= '\"') . TE.decodeUtf8Lenient) firstRow+            ProvideNames ns -> ns -    unless (hasHeader opts) $ hSeek handle AbsoluteSeek 0+    unless (headerSpec opts == UseFirstRow) $ hSeek handle AbsoluteSeek 0      dataLine <- C8.hGetLine handle     let dataRow = parseLine sep dataLine@@ -227,23 +251,31 @@                 , dataframeDimensions = (numRows, V.length frozenCols)                 }     return $-        if inferTypes opts-            then parseDefaults (safeRead opts) (dateFormat opts) df+        if shouldInferFromSample (typeSpec opts)+            then+                parseDefaults+                    (typeInferenceSampleSize (typeSpec opts))+                    (safeRead opts)+                    (dateFormat opts)+                    df             else df  initializeColumns :: [BS.ByteString] -> ReadOptions -> IO [GrowingColumn]-initializeColumns row opts = mapM initColumn row+initializeColumns row opts = case typeSpec opts of+    NoInference -> zipWithM initColumn row (expandTypes [])+    InferFromSample _ -> zipWithM initColumn row (expandTypes [])+    SpecifyTypes ts -> zipWithM initColumn row (expandTypes ts)   where-    initColumn :: BS.ByteString -> IO GrowingColumn-    initColumn bs = do+    expandTypes xs = xs ++ replicate (length row - length xs) (schemaType @T.Text)+    initColumn :: BS.ByteString -> SchemaType -> IO GrowingColumn+    initColumn bs t = do         nullsRef <- newIORef []-        let val = TE.decodeUtf8Lenient bs-        if inferTypes opts-            then case inferType val of-                IntType -> GrowingInt <$> newGrowingUnboxedVector 1_024 <*> pure nullsRef-                DoubleType -> GrowingDouble <$> newGrowingUnboxedVector 1_024 <*> pure nullsRef-                TextType -> GrowingText <$> newGrowingVector 1_024 <*> pure nullsRef-            else GrowingText <$> newGrowingVector 1_024 <*> pure nullsRef+        case t of+            SType (_ :: P.Proxy a) -> case testEquality (typeRep @a) (typeRep @Int) of+                Just Refl -> GrowingInt <$> newGrowingUnboxedVector 1_024 <*> pure nullsRef+                Nothing -> case testEquality (typeRep @a) (typeRep @Double) of+                    Just Refl -> GrowingDouble <$> newGrowingUnboxedVector 1_024 <*> pure nullsRef+                    Nothing -> GrowingText <$> newGrowingVector 1_024 <*> pure nullsRef  data InferredType = IntType | DoubleType | TextType 

@@ -54,7 +54,7 @@      in (map fromIntegral defLvlsU32, map fromIntegral repLvlsU32, afterDefBytes)  stitchNullable :: Int -> [Int] -> [a] -> [Maybe a]-stitchNullable maxDef defLvls vals = go defLvls vals+stitchNullable maxDef = go   where     go [] _ = []     go (d : ds) vs@@ -95,7 +95,7 @@ parseAll xs = let (n, xs') = parseOne xs in n : parseAll xs'  levelsForPath :: [SchemaElement] -> [String] -> (Int, Int)-levelsForPath schemaTail path = go 0 0 (parseAll schemaTail) path+levelsForPath schemaTail = go 0 0 (parseAll schemaTail)   where     go defC repC _ [] = (defC, repC)     go defC repC nodes (p : ps) =

@@ -256,9 +256,7 @@     let lastFieldId = 0     let fieldStack = []     bufferPos <- newIORef (0 :: Int)-    metadata <--        readFileMetaData defaultMetadata metadataBytes bufferPos lastFieldId fieldStack-    return metadata+    readFileMetaData defaultMetadata metadataBytes bufferPos lastFieldId fieldStack  readFileMetaData ::     FileMetadata ->

@@ -1064,11 +1064,11 @@             Nothing -> case sFloating @a of                 STrue ->                     Right-                        (VB.convert $ VB.map (fromMaybe (read @Double "NaN") . (fmap realToFrac)) f)+                        (VB.convert $ VB.map (maybe (read @Double "NaN") realToFrac) f)                 SFalse -> case sIntegral @a of                     STrue ->                         Right-                            (VB.convert $ VB.map (fromMaybe (read @Double "NaN") . (fmap fromIntegral)) f)+                            (VB.convert $ VB.map (maybe (read @Double "NaN") fromIntegral) f)                     SFalse ->                         Left $                             TypeMismatchException@@ -1139,11 +1139,11 @@             Nothing -> case sFloating @a of                 STrue ->                     Right-                        (VB.convert $ VB.map (fromMaybe (read @Float "NaN") . (fmap realToFrac)) f)+                        (VB.convert $ VB.map (maybe (read @Float "NaN") realToFrac) f)                 SFalse -> case sIntegral @a of                     STrue ->                         Right-                            (VB.convert $ VB.map (fromMaybe (read @Float "NaN") . (fmap fromIntegral)) f)+                            (VB.convert $ VB.map (maybe (read @Float "NaN") fromIntegral) f)                     SFalse ->                         Left $                             TypeMismatchException

@@ -68,7 +68,18 @@  instance Show DataFrame where     show :: DataFrame -> String-    show d = T.unpack (asText d False)+    show d =+        let+            d' =+                d+                    { columns = V.map (takeColumn 10) (columns d)+                    , dataframeDimensions = (10, snd (dataframeDimensions d))+                    }+         in+            T.unpack (asText d' False)+                ++ "\n"+                ++ "Showing 10 rows out of "+                ++ show (fst (dataframeDimensions d))  -- | For showing the dataframe as markdown in notebooks. toMarkdownTable :: DataFrame -> T.Text@@ -76,7 +87,7 @@  asText :: DataFrame -> Bool -> T.Text asText d properMarkdown =-    let header = "index" : map fst (sortBy (compare `on` snd) $ M.toList (columnIndices d))+    let header = map fst (sortBy (compare `on` snd) $ M.toList (columnIndices d))         types = V.toList $ V.filter (/= "") $ V.map getType (columns d)         getType :: Column -> T.Text         getType (BoxedColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a)@@ -93,14 +104,11 @@         get (Just (UnboxedColumn column)) = V.map (T.pack . show) (V.convert column)         get (Just (OptionalColumn column)) = V.map (T.pack . show) column         get Nothing = V.empty-        getTextColumnFromFrame df (i, name) =-            if i == 0-                then V.fromList (map (T.pack . show) [0 .. (fst (dataframeDimensions df) - 1)])-                else get $ (V.!?) (columns d) ((M.!) (columnIndices d) name)+        getTextColumnFromFrame df (i, name) = get $ (V.!?) (columns d) ((M.!) (columnIndices d) name)         rows =             transpose $                 zipWith (curry (V.toList . getTextColumnFromFrame d)) [0 ..] header-     in showTable properMarkdown header ("Int" : types) rows+     in showTable properMarkdown header types rows  -- | O(1) Creates an empty dataframe empty :: DataFrame@@ -162,7 +170,7 @@ toFloatMatrix ::     DataFrame -> Either DataFrameException (V.Vector (VU.Vector Float)) toFloatMatrix df = case V.foldl'-    (\acc c -> V.snoc <$> acc <*> (toFloatVector c))+    (\acc c -> V.snoc <$> acc <*> toFloatVector c)     (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Float)))     (columns df) of     Left e -> Left e@@ -189,7 +197,7 @@ toDoubleMatrix ::     DataFrame -> Either DataFrameException (V.Vector (VU.Vector Double)) toDoubleMatrix df = case V.foldl'-    (\acc c -> V.snoc <$> acc <*> (toDoubleVector c))+    (\acc c -> V.snoc <$> acc <*> toDoubleVector c)     (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Double)))     (columns df) of     Left e -> Left e@@ -215,7 +223,7 @@ -} toIntMatrix :: DataFrame -> Either DataFrameException (V.Vector (VU.Vector Int)) toIntMatrix df = case V.foldl'-    (\acc c -> V.snoc <$> acc <*> (toIntVector c))+    (\acc c -> V.snoc <$> acc <*> toIntVector c)     (Right V.empty :: Either DataFrameException (V.Vector (VU.Vector Int)))     (columns df) of     Left e -> Left e

@@ -1,5 +1,4 @@ {-# LANGUAGE AllowAmbiguousTypes #-}-{-# LANGUAGE BangPatterns #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-}@@ -17,6 +16,7 @@  import qualified Data.Map as M import Data.Maybe (fromMaybe, isJust)+import Data.String import qualified Data.Text as T import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) import qualified Data.Vector as V@@ -425,12 +425,11 @@     V.generate         (VU.length os - 1)         ( \i ->-            ( VG.generate+            VG.generate                 (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i))                 ( \j ->                     col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))                 )-            )         )  nestedTypeException ::@@ -504,7 +503,7 @@                 Just Refl -> case testEquality (typeRep @n) (typeRep @(VU.Vector d)) of                     Just Refl -> case (sUnbox @c, sUnbox @d, sUnbox @e) of                         (STrue, STrue, STrue) ->-                            Right $ UnAggregated $ fromVector $ V.zipWith (\l' r' -> VU.zipWith f l' r') l r+                            Right $ UnAggregated $ fromVector $ V.zipWith (VU.zipWith f) l r                         (STrue, STrue, SFalse) ->                             Right $                                 UnAggregated $@@ -517,7 +516,7 @@                                 Right $                                     UnAggregated $                                         fromVector $-                                            V.zipWith (\l' r' -> V.zipWith f (V.convert l') r') l r+                                            V.zipWith (V.zipWith f . V.convert) l r                             SFalse -> Left $ InternalException "Unboxed vectors contain boxed types"                         Nothing -> Left $ nestedTypeException @n @d (show right)                 Nothing -> case testEquality (typeRep @m) (typeRep @(V.Vector c)) of@@ -545,7 +544,7 @@                                         Right $                                             UnAggregated $                                                 fromVector $-                                                    V.zipWith (\l' r' -> V.zipWith f (V.convert l') r') l r+                                                    V.zipWith (V.zipWith f . V.convert) l r                                     STrue ->                                         Right $                                             UnAggregated $@@ -849,11 +848,11 @@                     )         (Left e) -> Left e         Right (UnAggregated (BoxedColumn (col :: V.Vector d))) -> case testEquality (typeRep @(V.Vector b)) (typeRep @d) of-            Just Refl -> Right $ Aggregated $ TColumn $ fromVector $ V.map (\v -> V.foldl' f s v) col+            Just Refl -> Right $ Aggregated $ TColumn $ fromVector $ V.map (V.foldl' f s) col             Nothing -> case testEquality (typeRep @(VU.Vector b)) (typeRep @d) of                 Just Refl -> case sUnbox @b of                     STrue ->-                        Right $ Aggregated $ TColumn $ fromVector $ V.map (\v -> VU.foldl' f s v) col+                        Right $ Aggregated $ TColumn $ fromVector $ V.map (VU.foldl' f s) col                     SFalse -> Left $ InternalException "Boxed type inside an unboxed column"                 Nothing -> Left $ nestedTypeException @d @b (show expr)         Right (UnAggregated _) -> Left $ InternalException "Aggregated into non-boxed column"@@ -898,6 +897,10 @@  divide :: (Fractional a, Columnable a) => Expr a -> Expr a -> Expr a divide = (/)++instance (IsString a, Columnable a) => IsString (Expr a) where+    fromString :: String -> Expr a+    fromString s = Lit (fromString s)  instance (Floating a, Columnable a) => Floating (Expr a) where     pi :: (Floating a, Columnable a) => Expr a

@@ -1,6 +1,7 @@ {-# LANGUAGE AllowAmbiguousTypes #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-}+{-# LANGUAGE InstanceSigs #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeApplications #-}@@ -12,24 +13,86 @@ import qualified Data.Proxy as P import qualified Data.Text as T -import Data.Maybe+import Data.Maybe (isJust) import Data.Type.Equality (TestEquality (..))-import DataFrame.Internal.Column+import DataFrame.Internal.Column (Columnable) import Type.Reflection (typeRep) +-- | A runtime tag for a column’s element type. data SchemaType where+    -- | Constructor carrying a 'Proxy' of the element type.     SType :: (Columnable a) => P.Proxy a -> SchemaType +{- | Show the underlying element type using 'typeRep'.++==== __Examples__+>>> :set -XTypeApplications+>>> show (schemaType @Bool)+"Bool"+-} instance Show SchemaType where+    show :: SchemaType -> String     show (SType (_ :: P.Proxy a)) = show (typeRep @a) +{- | Two 'SchemaType's are equal iff their element types are the same.++==== __Examples__+>>> :set -XTypeApplications+>>> schemaType @Int == schemaType @Int+True++>>> schemaType @Int == schemaType @Integer+False+-} instance Eq SchemaType where-    (==) (SType (_ :: P.Proxy a)) (SType (_ :: P.Proxy b)) = isJust (testEquality (typeRep @a) (typeRep @b))+    (==) :: SchemaType -> SchemaType -> Bool+    (==) (SType (_ :: P.Proxy a)) (SType (_ :: P.Proxy b)) =+        isJust (testEquality (typeRep @a) (typeRep @b)) +{- | Construct a 'SchemaType' for the given @a@.++==== __Examples__+>>> :set -XTypeApplications+>>> schemaType @T.Text == schemaType @T.Text+True++>>> show (schemaType @Double)+"Double"+-} schemaType :: forall a. (Columnable a) => SchemaType schemaType = SType (P.Proxy @a) +{- | Logical schema of a 'DataFrame': a mapping from column names to their+element types ('SchemaType').++==== __Examples__+Constructing and querying a schema:++>>> import qualified Data.Map as M+>>> import qualified Data.Text as T+>>> let s = Schema (M.fromList [("country", schemaType @T.Text), ("amount", schemaType @Double)])+>>> M.lookup "amount" (elements s) == Just (schemaType @Double)+True++Extending a schema:++>>> let s' = Schema (M.insert "discount" (schemaType @Double) (elements s))+>>> M.member "discount" (elements s')+True++Equality is structural over the map contents:++>>> let a = Schema (M.fromList [("x", schemaType @Int), ("y", schemaType @Double)])+>>> let b = Schema (M.fromList [("y", schemaType @Double), ("x", schemaType @Int)])+>>> a == b+True+-} newtype Schema = Schema     { elements :: M.Map T.Text SchemaType+    {- ^ Mapping from /column name/ to its 'SchemaType'.++    Invariant: keys are unique column names. A missing key means the column+    is not present in the schema.+    -}     }     deriving (Show, Eq)

@@ -79,7 +79,9 @@ skewness' = computeSkewness . VU.foldl' skewnessStep (SkewAcc 0 0 0 0) {-# INLINE skewness' #-} -correlation' :: VU.Vector Double -> VU.Vector Double -> Maybe Double+correlation' ::+    (Real a, VU.Unbox a, Real b, VU.Unbox b) =>+    VU.Vector a -> VU.Vector b -> Maybe Double correlation' xs ys     | VU.length xs /= VU.length ys = Nothing     | nI < 2 = Nothing@@ -93,8 +95,8 @@     !nI = VU.length xs     go !i !sumX !sumY !sumSquaredX !sumSquaredY !sumXY         | i < nI =-            let !x = VU.unsafeIndex xs i-                !y = VU.unsafeIndex ys i+            let !x = realToFrac (VU.unsafeIndex xs i)+                !y = realToFrac (VU.unsafeIndex ys i)                 !sumX' = sumX + x                 !sumY' = sumY + y                 !sumSquaredX' = sumSquaredX + x * x@@ -187,7 +189,10 @@ bincount k bs = VU.create $ do     mv <- VU.thaw (VU.replicate k 0)     VU.forM_ bs $ \b -> do-        let i = if b < 0 then 0 else if b >= k then k - 1 else b+        let i+                | b < 0 = 0+                | b >= k = k - 1+                | otherwise = b         x <- VUM.read mv i         VUM.write mv i (x + 1)     pure mv

@@ -65,7 +65,7 @@         VG.unsafeFreeze withIndexes  changingPoints :: (Eq a, VU.Unbox a) => VU.Vector (Int, a) -> [Int]-changingPoints vs = VG.length vs : (fst (VU.ifoldl findChangePoints initialState vs))+changingPoints vs = VG.length vs : fst (VU.ifoldl findChangePoints initialState vs)   where     initialState = ([0], snd (VG.head vs))     findChangePoints (offsets, currentVal) index (_, newVal)@@ -74,10 +74,10 @@  mkRowRep :: [Int] -> DataFrame -> Int -> Int mkRowRep groupColumnIndices df i = case h of-    (x : []) -> x+    [x] -> x     xs -> hash h   where-    h = (map mkHash groupColumnIndices)+    h = map mkHash groupColumnIndices     getHashedElem :: Column -> Int -> Int     getHashedElem (BoxedColumn (c :: V.Vector a)) j = hash' @a (c V.! j)     getHashedElem (UnboxedColumn (c :: VU.Vector a)) j = hash' @a (c VU.! j)

@@ -1,4 +1,3 @@-{-# LANGUAGE BangPatterns #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-}@@ -15,11 +14,9 @@ import qualified Data.Vector as V import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU-import qualified Data.Vector.Unboxed.Mutable as VUM import qualified Prelude  import Control.Exception (throw)-import Control.Monad.ST import Data.Function ((&)) import Data.Maybe (fromJust, fromMaybe, isJust, isNothing) import Data.Type.Equality (TestEquality (..))@@ -131,23 +128,19 @@                 }  indexes :: (VG.Vector v a) => (a -> Bool) -> v a -> VU.Vector Int-indexes condition cols = runST $ do-    ixs <- VUM.new 8192-    (!icount, _, _, !ixs') <--        VG.foldM-            ( \(!icount, !vcount, !cap, mv) v -> do-                if not (condition v)-                    then-                        pure (icount, vcount + 1, cap, mv)-                    else do-                        let shouldGrow = icount == cap-                        mv' <- if shouldGrow then VUM.grow mv cap else pure mv-                        VUM.write mv' icount vcount-                        pure (icount + 1, vcount + 1, cap + (cap * fromEnum shouldGrow), mv')-            )-            (0, 0, 8192, ixs)-            cols-    VU.freeze (VUM.slice 0 icount ixs')+indexes condition cols =+    let+        (ixs, n) =+            VG.ifoldl'+                ( \(acc, sz) i v -> do+                    if not (condition v)+                        then (acc, sz)+                        else (i : acc, sz + 1)+                )+                ([], 0)+                cols+     in+        VU.fromListN n ixs  {- | O(k) a version of filter where the predicate comes first. @@ -166,7 +159,7 @@         (TColumn col) = case interpret @Bool df expr of             Left e -> throw e             Right c -> c-        indexes = case findIndices (== True) col of+        indexes = case findIndices id col of             Right ixs -> ixs             Left e -> throw e         c' = snd $ dataframeDimensions df@@ -368,7 +361,7 @@     let         rand = generateRandomVector pureGen (fst (dataframeDimensions df))         withRand = df & insertUnboxedVector "__rand__" rand-        partitionSize = 1 / (fromIntegral folds)+        partitionSize = 1 / fromIntegral folds         singleFold n d =             d                 & filterWhere

@@ -19,84 +19,78 @@ import DataFrame.Internal.Parsing import Type.Reflection (typeRep) -parseDefaults :: Bool -> String -> DataFrame -> DataFrame-parseDefaults safeRead dateFormat df = df{columns = V.map (parseDefault safeRead dateFormat) (columns df)}+parseDefaults :: Int -> Bool -> String -> DataFrame -> DataFrame+parseDefaults n safeRead dateFormat df = df{columns = V.map (parseDefault n safeRead dateFormat) (columns df)} -parseDefault :: Bool -> String -> Column -> Column-parseDefault safeRead dateFormat (BoxedColumn (c :: V.Vector a)) =-    let-        parseTimeOpt s =-            parseTimeM {- Accept leading/trailing whitespace -}-                True-                defaultTimeLocale-                dateFormat-                (T.unpack s) ::-                Maybe Day-        unsafeParseTime s =-            parseTimeOrError {- Accept leading/trailing whitespace -}-                True-                defaultTimeLocale-                dateFormat-                (T.unpack s) ::-                Day-     in-        case (typeRep @a) `testEquality` (typeRep @T.Text) of-            Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of-                Just Refl ->-                    let-                        emptyToNothing v = if isNullish (T.pack v) then Nothing else Just v-                        safeVector = V.map emptyToNothing c-                        hasNulls =-                            V.foldl' (\acc v -> if isNothing v then acc || True else acc) False safeVector-                     in-                        if safeRead && hasNulls-                            then BoxedColumn safeVector-                            else BoxedColumn (V.map T.pack c)-                Nothing -> BoxedColumn c-            Just Refl ->-                let example = T.strip (V.head c)-                    emptyToNothing v = if isNullish v then Nothing else Just v-                 in case readInt example of-                        Just _ ->-                            let safeVector = V.map ((=<<) readInt . emptyToNothing) c-                                hasNulls = V.elem Nothing safeVector-                             in if safeRead && hasNulls-                                    then BoxedColumn safeVector-                                    else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!)))-                        Nothing -> case readDouble example of-                            Just _ ->-                                let safeVector = V.map ((=<<) readDouble . emptyToNothing) c-                                    hasNulls = V.elem Nothing safeVector-                                 in if safeRead && hasNulls-                                        then BoxedColumn safeVector-                                        else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!)))-                            Nothing -> case parseTimeOpt example of-                                Just d ->-                                    let-                                        -- failed parse should be Either, nullish should be Maybe-                                        emptyToNothing' v = if isNullish v then Left v else Right v-                                        parseTimeEither v = case parseTimeOpt v of-                                            Just v' -> Right v'-                                            Nothing -> Left v-                                        safeVector = V.map ((=<<) parseTimeEither . emptyToNothing') c-                                        toMaybe (Left _) = Nothing-                                        toMaybe (Right value) = Just value-                                        lefts = V.filter isLeft safeVector-                                        onlyNulls = (not (V.null lefts) && V.all (isNullish . fromLeft "non-null") lefts)-                                     in-                                        if safeRead-                                            then-                                                if onlyNulls-                                                    then BoxedColumn (V.map toMaybe safeVector)-                                                    else-                                                        if V.any isLeft safeVector-                                                            then BoxedColumn safeVector-                                                            else BoxedColumn (V.map unsafeParseTime c)-                                            else BoxedColumn (V.map unsafeParseTime c)-                                Nothing ->-                                    let-                                        safeVector = V.map emptyToNothing c-                                        hasNulls = V.any isNullish c-                                     in-                                        if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c-parseDefault _ _ column = column+parseDefault :: Int -> Bool -> String -> Column -> Column+parseDefault n safeRead dateFormat (BoxedColumn (c :: V.Vector a)) =+    case (typeRep @a) `testEquality` (typeRep @T.Text) of+        Nothing -> case (typeRep @a) `testEquality` (typeRep @String) of+            Just Refl -> parseFromExamples n dateFormat safeRead (V.map T.pack c)+            Nothing -> BoxedColumn c+        Just Refl -> parseFromExamples n dateFormat safeRead c+parseDefault _ _ _ column = column++emptyToNothing :: T.Text -> Maybe T.Text+emptyToNothing v = if isNullish v then Nothing else Just v++parseTimeOpt :: String -> T.Text -> Maybe Day+parseTimeOpt dateFormat s =+    parseTimeM {- Accept leading/trailing whitespace -}+        True+        defaultTimeLocale+        dateFormat+        (T.unpack s)++unsafeParseTime :: String -> T.Text -> Day+unsafeParseTime dateFormat s =+    parseTimeOrError {- Accept leading/trailing whitespace -}+        True+        defaultTimeLocale+        dateFormat+        (T.unpack s)++parseFromExamples :: Int -> String -> Bool -> V.Vector T.Text -> Column+parseFromExamples n dateFormat safeRead c+    | V.all isJust (V.map readInt examples) =+        let safeVector = V.map ((=<<) readInt . emptyToNothing) c+            hasNulls = V.elem Nothing safeVector+         in if safeRead && hasNulls+                then BoxedColumn safeVector+                else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!)))+    | V.all isJust (V.map readDouble examples) =+        let safeVector = V.map ((=<<) readDouble . emptyToNothing) c+            hasNulls = V.elem Nothing safeVector+         in if safeRead && hasNulls+                then BoxedColumn safeVector+                else UnboxedColumn (VU.generate (V.length c) (fromMaybe 0 . (safeVector V.!)))+    | V.any isJust (V.map (parseTimeOpt dateFormat) examples) =+        let+            -- failed parse should be Either, nullish should be Maybe+            emptyToNothing' v = if isNullish v then Left v else Right v+            parseTimeEither v = case parseTimeOpt dateFormat v of+                Just v' -> Right v'+                Nothing -> Left v+            safeVector = V.map ((=<<) parseTimeEither . emptyToNothing') c+            toMaybe (Left _) = Nothing+            toMaybe (Right value) = Just value+            lefts = V.filter isLeft safeVector+            onlyNulls = (not (V.null lefts) && V.all (isNullish . fromLeft "non-null") lefts)+         in+            if safeRead+                then+                    if onlyNulls+                        then BoxedColumn (V.map toMaybe safeVector)+                        else+                            if V.any isLeft safeVector+                                then BoxedColumn safeVector+                                else BoxedColumn (V.map (unsafeParseTime dateFormat) c)+                else BoxedColumn (V.map (unsafeParseTime dateFormat) c)+    | otherwise =+        let+            safeVector = V.map emptyToNothing c+            hasNulls = V.any isNullish c+         in+            if safeRead && hasNulls then BoxedColumn safeVector else BoxedColumn c+  where+    examples = V.take n c

@@ -11,6 +11,7 @@     lit,     max,     mean,+    median,     min,     percentile,     pow,@@ -87,7 +88,7 @@             , log (add (col @Double "x") (lit 1.0))             , exp (col @Double "x")             , mean (col @Double "x")-            , mean (col @Double "x")+            , median (col @Double "x")             , stddev (col @Double "x")             , sin (col @Double "x")             , cos (col @Double "x")@@ -119,7 +120,7 @@             , log (add (col @Double "x") (lit 1.0))             , exp (col @Double "x")             , mean (col @Double "x")-            , mean (col @Double "x")+            , median (col @Double "x")             , stddev (col @Double "x")             , sin (col @Double "x")             , cos (col @Double "x")@@ -133,7 +134,7 @@             , log (add (col @Double "y") (lit 1.0))             , exp (col @Double "y")             , mean (col @Double "y")-            , mean (col @Double "y")+            , median (col @Double "y")             , stddev (col @Double "y")             , sin (col @Double "y")             , cos (col @Double "y")@@ -341,7 +342,7 @@             , log (add (col @Double "x") (lit 1.0))             , exp (col @Double "x")             , mean (col @Double "x")-            , mean (col @Double "x")+            , median (col @Double "x")             , stddev (col @Double "x")             , sin (col @Double "x")             , cos (col @Double "x")@@ -355,7 +356,7 @@             , log (add (col @Double "y") (lit 1.0))             , exp (col @Double "y")             , mean (col @Double "y")-            , mean (col @Double "y")+            , median (col @Double "y")             , stddev (col @Double "y")             , sin (col @Double "y")             , cos (col @Double "y")@@ -372,7 +373,7 @@             , log (add (add (lit 1.0) (col @Double "z")) (lit 1.0))             , exp (add (lit 1.0) (col @Double "z"))             , mean (add (lit 1.0) (col @Double "z"))-            , mean (add (lit 1.0) (col @Double "z"))+            , median (add (lit 1.0) (col @Double "z"))             , stddev (add (lit 1.0) (col @Double "z"))             , sin (add (lit 1.0) (col @Double "z"))             , cos (add (lit 1.0) (col @Double "z"))

@@ -58,6 +58,7 @@                 )         actual =             D.parseDefault+                10                 True                 "%Y-%m-%d"                 (DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "2021-02-14", "2022-02-14"]))@@ -77,6 +78,7 @@                 )         actual =             D.parseDefault+                10                 True                 "%Y-%m-%d"                 (DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "2021-02-", "2022-02-14"]))@@ -93,6 +95,7 @@                 )         actual =             D.parseDefault+                10                 True                 "%Y-%m-%d"                 (DI.fromVector (V.fromList ["2020-02-14" :: T.Text, "", "2022-02-14"]))

@@ -1,5 +1,4 @@ {-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeApplications #-}  module Operations.Aggregations where@@ -31,8 +30,8 @@         ( assertEqual             "Counting elements after grouping gives correct numbers"             ( D.fromNamedColumns-                [ ("test1", DI.fromList [(1 :: Int), 2, 3])-                , ("test2", DI.fromList [(6 :: Int), 3, 3])+                [ ("test1", DI.fromList [1 :: Int, 2, 3])+                , ("test2", DI.fromList [6 :: Int, 3, 3])                 ]             )             ( testData@@ -47,8 +46,8 @@         ( assertEqual             "Mean works for ints"             ( D.fromNamedColumns-                [ ("test1", DI.fromList [(1 :: Int), 2, 3])-                , ("test2", DI.fromList [(6.5 :: Double), 8.0, 5.0])+                [ ("test1", DI.fromList [1 :: Int, 2, 3])+                , ("test2", DI.fromList [6.5 :: Double, 8.0, 5.0])                 ]             )             ( testData@@ -63,8 +62,8 @@         ( assertEqual             "Mean works for ints"             ( D.fromNamedColumns-                [ ("test1", DI.fromList [(1 :: Int), 2, 3])-                , ("test2", DI.fromList [(6.5 :: Double), 8.0, 5.0])+                [ ("test1", DI.fromList [1 :: Int, 2, 3])+                , ("test2", DI.fromList [6.5 :: Double, 8.0, 5.0])                 ]             )             ( testData@@ -81,13 +80,13 @@         ( assertEqual             "Mean works for ints"             ( D.fromNamedColumns-                [ ("test1", DI.fromList [(1 :: Int), 2, 3])-                , ("test2", DI.fromList [(13 :: Double), 16, 10])+                [ ("test1", DI.fromList [1 :: Int, 2, 3])+                , ("test2", DI.fromList [13 :: Double, 16, 10])                 ]             )             ( testData                 & D.groupBy ["test1"]-                & D.aggregate [F.mean (F.col @Int "test2" + (F.col @Int "test2")) `F.as` "test2"]+                & D.aggregate [F.mean (F.col @Int "test2" + F.col @Int "test2") `F.as` "test2"]             )         ) @@ -97,8 +96,8 @@         ( assertEqual             "Mean works for ints"             ( D.fromNamedColumns-                [ ("test1", DI.fromList [(1 :: Int), 2, 3])-                , ("test2", DI.fromList [(12 :: Double), 9, 6])+                [ ("test1", DI.fromList [1 :: Int, 2, 3])+                , ("test2", DI.fromList [12 :: Double, 9, 6])                 ]             )             ( testData@@ -116,14 +115,14 @@         ( assertEqual             "Mean works for ints"             ( D.fromNamedColumns-                [ ("test1", DI.fromList [(1 :: Int), 2, 3])-                , ("test2", DI.fromList [(24 :: Int), 18, 12])+                [ ("test1", DI.fromList [1 :: Int, 2, 3])+                , ("test2", DI.fromList [24 :: Int, 18, 12])                 ]             )             ( testData                 & D.groupBy ["test1"]                 & D.aggregate-                    [F.maximum (F.col @Int "test2" + (F.col @Int "test2")) `F.as` "test2"]+                    [F.maximum (F.col @Int "test2" + F.col @Int "test2") `F.as` "test2"]             )         ) 

@@ -1,5 +1,4 @@ {-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE TypeApplications #-}  module Operations.Core where