dataframe 1.0.0.0 → 1.0.0.1

raw patch · 14 files changed

+115/−83 lines, 14 filesdep ~streamly-bytestringdep ~streamly-corePVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: streamly-bytestring, streamly-core

API changes (from Hackage documentation)

Files

• CHANGELOG.md +8/−0
• dataframe.cabal +4/−5
• src/DataFrame.hs +28/−5
• src/DataFrame/Functions.hs +2/−2
• src/DataFrame/IO/CSV.hs +2/−1
• src/DataFrame/Internal/Column.hs +2/−2
• src/DataFrame/Internal/DataFrame.hs +12/−5
• src/DataFrame/Internal/Expression.hs +0/−4
• src/DataFrame/Internal/Row.hs +0/−4
• src/DataFrame/Internal/Types.hs +1/−2
• src/DataFrame/Operations/Core.hs +52/−47
• src/DataFrame/Operators.hs +3/−3
• src/DataFrame/Typed/Expr.hs +1/−1
• tests/Operations/Provenance.hs +0/−2

@@ -1,5 +1,13 @@ # Revision history for dataframe +## 1.0.0.1+* toMarkdownTable is now toMarkdown (mostly used internally)+* Provide toMarkdown' that outputs string+* Add associativity to nullable operators+* Better null dataframe handling/error messages for core operations.+* Fix some function display names.+* Examples now build with CI+ ## 1.0.0.0 * Fix mappend to respect schema of empty columns. * Add cast operators that force column schema

@@ -1,6 +1,6 @@ cabal-version:      2.4 name:               dataframe-version:            1.0.0.0+version:            1.0.0.1  synopsis: A fast, safe, and intuitive DataFrame library. @@ -145,10 +145,9 @@                       stm >= 2.5 && < 3,                       filepath >= 1.4 && < 2,                       Glob >= 0.10 && < 1,-                      http-conduit    >= 2.3 && < 3,-                      streamly-core,-                      streamly-bytestring,-+                      http-conduit >= 2.3 && < 3,+                      streamly-core >= 0.2.3 && < 0.4,+                      streamly-bytestring >= 0.2.0 && < 0.4     hs-source-dirs:   src     c-sources:        cbits/process_csv.c     include-dirs:     cbits

@@ -288,7 +288,8 @@     GroupedDataFrame,     empty,     null,-    toMarkdownTable,+    toMarkdown,+    toMarkdown',  ) import DataFrame.Internal.Expression as Expression (Expr, prettyPrint) import DataFrame.Internal.Row as Row (@@ -312,10 +313,16 @@ import DataFrame.Operations.Core as Core hiding (     ColumnInfo (..),     nulls,-    partiallyParsed,     renameSafe,  )-import DataFrame.Operations.Join as Join+import DataFrame.Operations.Join as Join (+    JoinType (..),+    fullOuterJoin,+    innerJoin,+    join,+    leftJoin,+    rightJoin,+ ) import DataFrame.Operations.Merge as Merge import DataFrame.Operations.Permutation as Permutation (     SortOrder (..),@@ -368,6 +375,22 @@     take,     takeLast,  )-import DataFrame.Operations.Transformations as Transformations-import DataFrame.Operations.Typing as Typing+import DataFrame.Operations.Transformations as Transformations (+    apply,+    applyAtIndex,+    applyDouble,+    applyInt,+    applyMany,+    applyWhere,+    derive,+    deriveMany,+    deriveWithExpr,+    impute,+    safeApply,+ )+import DataFrame.Operations.Typing as Typing (+    ParseOptions (..),+    defaultParseOptions,+    parseDefaults,+ ) import DataFrame.Operators as Operators

@@ -309,13 +309,13 @@ zScore c = (c - mean c) / stddev c  pow :: (Columnable a, Num a) => Expr a -> Int -> Expr a-pow expr i = lift2Decorated (^) "max" (Just "^") True 8 expr (Lit i)+pow expr i = lift2Decorated (^) "pow" (Just "^") True 8 expr (Lit i)  relu :: (Columnable a, Num a, Ord a) => Expr a -> Expr a relu = liftDecorated (Prelude.max 0) "relu" Nothing  min :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr a-min = lift2Decorated Prelude.min "max" Nothing True 1+min = lift2Decorated Prelude.min "min" Nothing True 1  max :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr a max = lift2Decorated Prelude.max "max" Nothing True 1

@@ -27,6 +27,7 @@ import Data.Csv.Streaming (Records (..)) import qualified Data.Csv.Streaming as CsvStream +import Control.DeepSeq import Control.Monad import Data.Char import qualified Data.Csv as Csv@@ -272,7 +273,7 @@ readSeparated opts !path = do     let stripUtf8Bom bs = fromMaybe bs (BL.stripPrefix "\xEF\xBB\xBF" bs)     csvData <- stripUtf8Bom <$> BL.readFile path-    decodeSeparated opts csvData+    fmap force (decodeSeparated opts csvData)  decodeSeparated :: ReadOptions -> BL.ByteString -> IO DataFrame decodeSeparated !opts csvData = do

@@ -134,9 +134,9 @@     show (TColumn col) = show col  instance NFData Column where-    rnf (BoxedColumn (v :: VB.Vector a)) = rnf v+    rnf (BoxedColumn (v :: VB.Vector a)) = VB.foldl' (const (`seq` ())) () v     rnf (UnboxedColumn v) = v `seq` ()-    rnf (OptionalColumn (v :: VB.Vector (Maybe a))) = rnf v+    rnf (OptionalColumn (v :: VB.Vector (Maybe a))) = VB.foldl' (const (`seq` ())) () v  instance Show Column where     show :: Column -> String

@@ -24,6 +24,7 @@ import DataFrame.Internal.Expression import Text.Printf import Type.Reflection (typeRep)+import Prelude hiding (null)  data DataFrame = DataFrame     { columns :: V.Vector Column@@ -97,9 +98,13 @@             T.unpack (asText d' False) ++ (if r > rows then truncationInfo else "")  -- | For showing the dataframe as markdown in notebooks.-toMarkdownTable :: DataFrame -> T.Text-toMarkdownTable df = asText df True+toMarkdown :: DataFrame -> T.Text+toMarkdown df = asText df True +-- | For showing the dataframe as a string markdown in notebooks.+toMarkdown' :: DataFrame -> String+toMarkdown' = T.unpack . toMarkdown+ asText :: DataFrame -> Bool -> T.Text asText d properMarkdown =     let header = map fst (sortBy (compare `on` snd) $ M.toList (columnIndices d))@@ -148,9 +153,11 @@ Nothing -} getColumn :: T.Text -> DataFrame -> Maybe Column-getColumn name df = do-    i <- columnIndices df M.!? name-    columns df V.!? i+getColumn name df+    | null df = Nothing+    | otherwise = do+        i <- columnIndices df M.!? name+        columns df V.!? i  {- | Retrieves a column by name from the dataframe, throwing an exception if not found. 

@@ -13,7 +13,6 @@  module DataFrame.Internal.Expression where -import Control.DeepSeq (NFData (..)) import Data.String import qualified Data.Text as T import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl))@@ -37,9 +36,6 @@  data MeanAcc = MeanAcc {-# UNPACK #-} !Double {-# UNPACK #-} !Int     deriving (Show, Eq, Ord, Read)--instance NFData MeanAcc where-    rnf (MeanAcc _ _) = ()  data AggStrategy a b where     CollectAgg ::

@@ -17,7 +17,6 @@ import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU -import Control.DeepSeq (NFData (..)) import Control.Exception (throw) import Control.Monad.ST (runST) import Data.Function (on)@@ -57,9 +56,6 @@ instance Show Any where     show :: Any -> String     show (Value a) = T.unpack (showValue a)--instance NFData Any where-    rnf (Value a) = rnf a  showValue :: forall a. (Columnable a) => a -> T.Text showValue v = case testEquality (typeRep @a) (typeRep @T.Text) of

@@ -13,14 +13,13 @@  module DataFrame.Internal.Types where -import Control.DeepSeq (NFData) import Data.Int (Int16, Int32, Int64, Int8) import Data.Kind (Constraint, Type) import Data.Typeable (Typeable) import qualified Data.Vector.Unboxed as VU import Data.Word (Word16, Word32, Word64, Word8) -type Columnable' a = (Typeable a, Show a, Ord a, Eq a, Read a, NFData a)+type Columnable' a = (Typeable a, Show a, Ord a, Eq a, Read a)  {- | A type with column representations used to select the "right" representation when specializing the `toColumn` function.

@@ -44,6 +44,7 @@     derivingExpressions,     empty,     getColumn,+    null,  ) import DataFrame.Internal.Expression import DataFrame.Internal.Interpreter@@ -389,13 +390,15 @@ @ -} cloneColumn :: T.Text -> T.Text -> DataFrame -> DataFrame-cloneColumn original new df = fromMaybe-    ( throw $-        ColumnNotFoundException original "cloneColumn" (M.keys $ columnIndices df)-    )-    $ do-        column <- getColumn original df-        return $ insertColumn new column df+cloneColumn original new df+    | null df = throw (EmptyDataSetException "cloneColumn")+    | otherwise = fromMaybe+        ( throw $+            ColumnNotFoundException original "cloneColumn" (M.keys $ columnIndices df)+        )+        $ do+            column <- getColumn original df+            return $ insertColumn new column df  {- | /O(n)/ Renames a single column. @@ -479,13 +482,15 @@  renameSafe ::     T.Text -> T.Text -> DataFrame -> Either DataFrameException DataFrame-renameSafe orig new df = fromMaybe-    (Left $ ColumnNotFoundException orig "rename" (M.keys $ columnIndices df))-    $ do-        columnIndex <- M.lookup orig (columnIndices df)-        let origRemoved = M.delete orig (columnIndices df)-        let newAdded = M.insert new columnIndex origRemoved-        return (Right df{columnIndices = newAdded})+renameSafe orig new df+    | null df = throw (EmptyDataSetException "rename")+    | otherwise = fromMaybe+        (Left $ ColumnNotFoundException orig "rename" (M.keys $ columnIndices df))+        $ do+            columnIndex <- M.lookup orig (columnIndices df)+            let origRemoved = M.delete orig (columnIndices df)+            let newAdded = M.insert new columnIndex origRemoved+            return (Right df{columnIndices = newAdded})  data ColumnInfo = ColumnInfo     { nameOfColumn :: !T.Text@@ -579,15 +584,6 @@             _ -> 0 nulls _ = 0 -partiallyParsed :: Column -> Int-partiallyParsed (BoxedColumn (xs :: V.Vector a)) =-    case typeRep @a of-        App (App tycon t1) t2 -> case eqTypeRep tycon (typeRep @Either) of-            Just HRefl -> VG.length $ VG.filter isLeft xs-            Nothing -> 0-        _ -> 0-partiallyParsed _ = 0- {- | Creates a dataframe from a list of tuples with name and column.  ==== __Example__@@ -684,13 +680,15 @@ -} valueCounts ::     forall a. (Ord a, Columnable a) => Expr a -> DataFrame -> [(a, Int)]-valueCounts expr df = case columnAsVector expr df of-    Left e -> throw e-    Right column' ->-        let-            column = V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column'-         in-            M.toAscList column+valueCounts expr df+    | null df = throw (EmptyDataSetException "valueCounts")+    | otherwise = case columnAsVector expr df of+        Left e -> throw e+        Right column' ->+            let+                column = V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column'+             in+                M.toAscList column  {- | O (k * n) Shows the proportions of each value in a given column. @@ -706,16 +704,18 @@ -} valueProportions ::     forall a. (Ord a, Columnable a) => Expr a -> DataFrame -> [(a, Double)]-valueProportions expr df = case columnAsVector expr df of-    Left e -> throw e-    Right column' ->-        let-            counts =-                M.toAscList-                    (V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column')-            total = fromIntegral (sum (map snd counts))-         in-            map (fmap ((/ total) . fromIntegral)) counts+valueProportions expr df+    | null df = throw (EmptyDataSetException "valueCounts")+    | otherwise = case columnAsVector expr df of+        Left e -> throw e+        Right column' ->+            let+                counts =+                    M.toAscList+                        (V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column')+                total = fromIntegral (sum (map snd counts))+             in+                map (fmap ((/ total) . fromIntegral)) counts  {- | A left fold for dataframes that takes the dataframe as the last object. This makes it easier to chain operations.@@ -853,13 +853,16 @@ columnAsVector ::     forall a.     (Columnable a) => Expr a -> DataFrame -> Either DataFrameException (V.Vector a)-columnAsVector (Col name) df = case getColumn name df of-    Just col -> toVector col-    Nothing ->-        Left $ ColumnNotFoundException name "columnAsVector" (M.keys $ columnIndices df)-columnAsVector expr df = case interpret df expr of-    Left e -> throw e-    Right (TColumn col) -> toVector col+columnAsVector expr df+    | null df = throw (EmptyDataSetException "columnAsVector")+    | otherwise = case expr of+        (Col name) -> case getColumn name df of+            Just col -> toVector col+            Nothing ->+                Left $ ColumnNotFoundException name "columnAsVector" (M.keys $ columnIndices df)+        _ -> case interpret df expr of+            Left e -> throw e+            Right (TColumn col) -> toVector col  {- | Retrieves a column as an unboxed vector of 'Int' values. @@ -952,6 +955,8 @@ @(name, expression)@ pairs. Derived columns show their expression; raw columns show an identity @col \@type name@ expression. -}++-- TODO: mchavinda - Expand out these expressions if possible. showDerivedExpressions :: DataFrame -> [NamedExpr] showDerivedExpressions df =     let exprs = derivingExpressions df

@@ -36,9 +36,9 @@  ) import DataFrame.Internal.Types (Promote, PromoteDiv) -infix 8 .^^, .^^., .^, .^.-infix 6 .+, .--infix 7 .*, ./+infixr 8 .^^, .^^., .^, .^.+infixl 7 .*, ./, .*., ./.+infixl 6 .+, .-, .+., .-. infix 4 .==, .==., .<, .<., .<=, .<=., .>=, .>=., .>, .>., ./=, ./=. infixr 3 .&&, .&&. infixr 2 .||, .||.

@@ -274,7 +274,7 @@ infixr 2 .||., .|| infixl 6 .+., .-. infixl 7 .*., ./.-infix 8 .^^., .^^, .^., .^+infixr 8 .^^., .^^, .^., .^  (.==.) ::     (Columnable a, Eq a) => TExpr cols a -> TExpr cols a -> TExpr cols Bool

@@ -3,9 +3,7 @@  module Operations.Provenance where -import qualified Data.List as L import qualified Data.Map as M-import qualified Data.Text as T import qualified DataFrame as D import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI