diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Revision history for dataframe +## 1.0.0.1 +* toMarkdownTable is now toMarkdown (mostly used internally) +* Provide toMarkdown' that outputs string +* Add associativity to nullable operators +* Better null dataframe handling/error messages for core operations. +* Fix some function display names. +* Examples now build with CI + ## 1.0.0.0 * Fix mappend to respect schema of empty columns. * Add cast operators that force column schema diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe -version: 1.0.0.0 +version: 1.0.0.1 synopsis: A fast, safe, and intuitive DataFrame library. @@ -145,10 +145,9 @@ stm >= 2.5 && < 3, filepath >= 1.4 && < 2, Glob >= 0.10 && < 1, - http-conduit >= 2.3 && < 3, - streamly-core, - streamly-bytestring, - + http-conduit >= 2.3 && < 3, + streamly-core >= 0.2.3 && < 0.4, + streamly-bytestring >= 0.2.0 && < 0.4 hs-source-dirs: src c-sources: cbits/process_csv.c include-dirs: cbits diff --git a/src/DataFrame.hs b/src/DataFrame.hs --- a/src/DataFrame.hs +++ b/src/DataFrame.hs @@ -288,7 +288,8 @@ GroupedDataFrame, empty, null, - toMarkdownTable, + toMarkdown, + toMarkdown', ) import DataFrame.Internal.Expression as Expression (Expr, prettyPrint) import DataFrame.Internal.Row as Row ( @@ -312,10 +313,16 @@ import DataFrame.Operations.Core as Core hiding ( ColumnInfo (..), nulls, - partiallyParsed, renameSafe, ) -import DataFrame.Operations.Join as Join +import DataFrame.Operations.Join as Join ( + JoinType (..), + fullOuterJoin, + innerJoin, + join, + leftJoin, + rightJoin, + ) import DataFrame.Operations.Merge as Merge import DataFrame.Operations.Permutation as Permutation ( SortOrder (..), @@ -368,6 +375,22 @@ take, takeLast, ) -import DataFrame.Operations.Transformations as Transformations -import DataFrame.Operations.Typing as Typing +import DataFrame.Operations.Transformations as Transformations ( + apply, + applyAtIndex, + applyDouble, + applyInt, + applyMany, + applyWhere, + derive, + deriveMany, + deriveWithExpr, + impute, + safeApply, + ) +import DataFrame.Operations.Typing as Typing ( + ParseOptions (..), + defaultParseOptions, + parseDefaults, + ) import DataFrame.Operators as Operators diff --git a/src/DataFrame/Functions.hs b/src/DataFrame/Functions.hs --- a/src/DataFrame/Functions.hs +++ b/src/DataFrame/Functions.hs @@ -309,13 +309,13 @@ zScore c = (c - mean c) / stddev c pow :: (Columnable a, Num a) => Expr a -> Int -> Expr a -pow expr i = lift2Decorated (^) "max" (Just "^") True 8 expr (Lit i) +pow expr i = lift2Decorated (^) "pow" (Just "^") True 8 expr (Lit i) relu :: (Columnable a, Num a, Ord a) => Expr a -> Expr a relu = liftDecorated (Prelude.max 0) "relu" Nothing min :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr a -min = lift2Decorated Prelude.min "max" Nothing True 1 +min = lift2Decorated Prelude.min "min" Nothing True 1 max :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr a max = lift2Decorated Prelude.max "max" Nothing True 1 diff --git a/src/DataFrame/IO/CSV.hs b/src/DataFrame/IO/CSV.hs --- a/src/DataFrame/IO/CSV.hs +++ b/src/DataFrame/IO/CSV.hs @@ -27,6 +27,7 @@ import Data.Csv.Streaming (Records (..)) import qualified Data.Csv.Streaming as CsvStream +import Control.DeepSeq import Control.Monad import Data.Char import qualified Data.Csv as Csv @@ -272,7 +273,7 @@ readSeparated opts !path = do let stripUtf8Bom bs = fromMaybe bs (BL.stripPrefix "\xEF\xBB\xBF" bs) csvData <- stripUtf8Bom <$> BL.readFile path - decodeSeparated opts csvData + fmap force (decodeSeparated opts csvData) decodeSeparated :: ReadOptions -> BL.ByteString -> IO DataFrame decodeSeparated !opts csvData = do diff --git a/src/DataFrame/Internal/Column.hs b/src/DataFrame/Internal/Column.hs --- a/src/DataFrame/Internal/Column.hs +++ b/src/DataFrame/Internal/Column.hs @@ -134,9 +134,9 @@ show (TColumn col) = show col instance NFData Column where - rnf (BoxedColumn (v :: VB.Vector a)) = rnf v + rnf (BoxedColumn (v :: VB.Vector a)) = VB.foldl' (const (`seq` ())) () v rnf (UnboxedColumn v) = v `seq` () - rnf (OptionalColumn (v :: VB.Vector (Maybe a))) = rnf v + rnf (OptionalColumn (v :: VB.Vector (Maybe a))) = VB.foldl' (const (`seq` ())) () v instance Show Column where show :: Column -> String diff --git a/src/DataFrame/Internal/DataFrame.hs b/src/DataFrame/Internal/DataFrame.hs --- a/src/DataFrame/Internal/DataFrame.hs +++ b/src/DataFrame/Internal/DataFrame.hs @@ -24,6 +24,7 @@ import DataFrame.Internal.Expression import Text.Printf import Type.Reflection (typeRep) +import Prelude hiding (null) data DataFrame = DataFrame { columns :: V.Vector Column @@ -97,9 +98,13 @@ T.unpack (asText d' False) ++ (if r > rows then truncationInfo else "") -- | For showing the dataframe as markdown in notebooks. -toMarkdownTable :: DataFrame -> T.Text -toMarkdownTable df = asText df True +toMarkdown :: DataFrame -> T.Text +toMarkdown df = asText df True +-- | For showing the dataframe as a string markdown in notebooks. +toMarkdown' :: DataFrame -> String +toMarkdown' = T.unpack . toMarkdown + asText :: DataFrame -> Bool -> T.Text asText d properMarkdown = let header = map fst (sortBy (compare `on` snd) $ M.toList (columnIndices d)) @@ -148,9 +153,11 @@ Nothing -} getColumn :: T.Text -> DataFrame -> Maybe Column -getColumn name df = do - i <- columnIndices df M.!? name - columns df V.!? i +getColumn name df + | null df = Nothing + | otherwise = do + i <- columnIndices df M.!? name + columns df V.!? i {- | Retrieves a column by name from the dataframe, throwing an exception if not found. diff --git a/src/DataFrame/Internal/Expression.hs b/src/DataFrame/Internal/Expression.hs --- a/src/DataFrame/Internal/Expression.hs +++ b/src/DataFrame/Internal/Expression.hs @@ -13,7 +13,6 @@ module DataFrame.Internal.Expression where -import Control.DeepSeq (NFData (..)) import Data.String import qualified Data.Text as T import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) @@ -37,9 +36,6 @@ data MeanAcc = MeanAcc {-# UNPACK #-} !Double {-# UNPACK #-} !Int deriving (Show, Eq, Ord, Read) - -instance NFData MeanAcc where - rnf (MeanAcc _ _) = () data AggStrategy a b where CollectAgg :: diff --git a/src/DataFrame/Internal/Row.hs b/src/DataFrame/Internal/Row.hs --- a/src/DataFrame/Internal/Row.hs +++ b/src/DataFrame/Internal/Row.hs @@ -17,7 +17,6 @@ import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU -import Control.DeepSeq (NFData (..)) import Control.Exception (throw) import Control.Monad.ST (runST) import Data.Function (on) @@ -57,9 +56,6 @@ instance Show Any where show :: Any -> String show (Value a) = T.unpack (showValue a) - -instance NFData Any where - rnf (Value a) = rnf a showValue :: forall a. (Columnable a) => a -> T.Text showValue v = case testEquality (typeRep @a) (typeRep @T.Text) of diff --git a/src/DataFrame/Internal/Types.hs b/src/DataFrame/Internal/Types.hs --- a/src/DataFrame/Internal/Types.hs +++ b/src/DataFrame/Internal/Types.hs @@ -13,14 +13,13 @@ module DataFrame.Internal.Types where -import Control.DeepSeq (NFData) import Data.Int (Int16, Int32, Int64, Int8) import Data.Kind (Constraint, Type) import Data.Typeable (Typeable) import qualified Data.Vector.Unboxed as VU import Data.Word (Word16, Word32, Word64, Word8) -type Columnable' a = (Typeable a, Show a, Ord a, Eq a, Read a, NFData a) +type Columnable' a = (Typeable a, Show a, Ord a, Eq a, Read a) {- | A type with column representations used to select the "right" representation when specializing the `toColumn` function. diff --git a/src/DataFrame/Operations/Core.hs b/src/DataFrame/Operations/Core.hs --- a/src/DataFrame/Operations/Core.hs +++ b/src/DataFrame/Operations/Core.hs @@ -44,6 +44,7 @@ derivingExpressions, empty, getColumn, + null, ) import DataFrame.Internal.Expression import DataFrame.Internal.Interpreter @@ -389,13 +390,15 @@ @ -} cloneColumn :: T.Text -> T.Text -> DataFrame -> DataFrame -cloneColumn original new df = fromMaybe - ( throw $ - ColumnNotFoundException original "cloneColumn" (M.keys $ columnIndices df) - ) - $ do - column <- getColumn original df - return $ insertColumn new column df +cloneColumn original new df + | null df = throw (EmptyDataSetException "cloneColumn") + | otherwise = fromMaybe + ( throw $ + ColumnNotFoundException original "cloneColumn" (M.keys $ columnIndices df) + ) + $ do + column <- getColumn original df + return $ insertColumn new column df {- | /O(n)/ Renames a single column. @@ -479,13 +482,15 @@ renameSafe :: T.Text -> T.Text -> DataFrame -> Either DataFrameException DataFrame -renameSafe orig new df = fromMaybe - (Left $ ColumnNotFoundException orig "rename" (M.keys $ columnIndices df)) - $ do - columnIndex <- M.lookup orig (columnIndices df) - let origRemoved = M.delete orig (columnIndices df) - let newAdded = M.insert new columnIndex origRemoved - return (Right df{columnIndices = newAdded}) +renameSafe orig new df + | null df = throw (EmptyDataSetException "rename") + | otherwise = fromMaybe + (Left $ ColumnNotFoundException orig "rename" (M.keys $ columnIndices df)) + $ do + columnIndex <- M.lookup orig (columnIndices df) + let origRemoved = M.delete orig (columnIndices df) + let newAdded = M.insert new columnIndex origRemoved + return (Right df{columnIndices = newAdded}) data ColumnInfo = ColumnInfo { nameOfColumn :: !T.Text @@ -579,15 +584,6 @@ _ -> 0 nulls _ = 0 -partiallyParsed :: Column -> Int -partiallyParsed (BoxedColumn (xs :: V.Vector a)) = - case typeRep @a of - App (App tycon t1) t2 -> case eqTypeRep tycon (typeRep @Either) of - Just HRefl -> VG.length $ VG.filter isLeft xs - Nothing -> 0 - _ -> 0 -partiallyParsed _ = 0 - {- | Creates a dataframe from a list of tuples with name and column. ==== __Example__ @@ -684,13 +680,15 @@ -} valueCounts :: forall a. (Ord a, Columnable a) => Expr a -> DataFrame -> [(a, Int)] -valueCounts expr df = case columnAsVector expr df of - Left e -> throw e - Right column' -> - let - column = V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column' - in - M.toAscList column +valueCounts expr df + | null df = throw (EmptyDataSetException "valueCounts") + | otherwise = case columnAsVector expr df of + Left e -> throw e + Right column' -> + let + column = V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column' + in + M.toAscList column {- | O (k * n) Shows the proportions of each value in a given column. @@ -706,16 +704,18 @@ -} valueProportions :: forall a. (Ord a, Columnable a) => Expr a -> DataFrame -> [(a, Double)] -valueProportions expr df = case columnAsVector expr df of - Left e -> throw e - Right column' -> - let - counts = - M.toAscList - (V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column') - total = fromIntegral (sum (map snd counts)) - in - map (fmap ((/ total) . fromIntegral)) counts +valueProportions expr df + | null df = throw (EmptyDataSetException "valueCounts") + | otherwise = case columnAsVector expr df of + Left e -> throw e + Right column' -> + let + counts = + M.toAscList + (V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column') + total = fromIntegral (sum (map snd counts)) + in + map (fmap ((/ total) . fromIntegral)) counts {- | A left fold for dataframes that takes the dataframe as the last object. This makes it easier to chain operations. @@ -853,13 +853,16 @@ columnAsVector :: forall a. (Columnable a) => Expr a -> DataFrame -> Either DataFrameException (V.Vector a) -columnAsVector (Col name) df = case getColumn name df of - Just col -> toVector col - Nothing -> - Left $ ColumnNotFoundException name "columnAsVector" (M.keys $ columnIndices df) -columnAsVector expr df = case interpret df expr of - Left e -> throw e - Right (TColumn col) -> toVector col +columnAsVector expr df + | null df = throw (EmptyDataSetException "columnAsVector") + | otherwise = case expr of + (Col name) -> case getColumn name df of + Just col -> toVector col + Nothing -> + Left $ ColumnNotFoundException name "columnAsVector" (M.keys $ columnIndices df) + _ -> case interpret df expr of + Left e -> throw e + Right (TColumn col) -> toVector col {- | Retrieves a column as an unboxed vector of 'Int' values. @@ -952,6 +955,8 @@ @(name, expression)@ pairs. Derived columns show their expression; raw columns show an identity @col \@type name@ expression. -} + +-- TODO: mchavinda - Expand out these expressions if possible. showDerivedExpressions :: DataFrame -> [NamedExpr] showDerivedExpressions df = let exprs = derivingExpressions df diff --git a/src/DataFrame/Operators.hs b/src/DataFrame/Operators.hs --- a/src/DataFrame/Operators.hs +++ b/src/DataFrame/Operators.hs @@ -36,9 +36,9 @@ ) import DataFrame.Internal.Types (Promote, PromoteDiv) -infix 8 .^^, .^^., .^, .^. -infix 6 .+, .- -infix 7 .*, ./ +infixr 8 .^^, .^^., .^, .^. +infixl 7 .*, ./, .*., ./. +infixl 6 .+, .-, .+., .-. infix 4 .==, .==., .<, .<., .<=, .<=., .>=, .>=., .>, .>., ./=, ./=. infixr 3 .&&, .&&. infixr 2 .||, .||. diff --git a/src/DataFrame/Typed/Expr.hs b/src/DataFrame/Typed/Expr.hs --- a/src/DataFrame/Typed/Expr.hs +++ b/src/DataFrame/Typed/Expr.hs @@ -274,7 +274,7 @@ infixr 2 .||., .|| infixl 6 .+., .-. infixl 7 .*., ./. -infix 8 .^^., .^^, .^., .^ +infixr 8 .^^., .^^, .^., .^ (.==.) :: (Columnable a, Eq a) => TExpr cols a -> TExpr cols a -> TExpr cols Bool diff --git a/tests/Operations/Provenance.hs b/tests/Operations/Provenance.hs --- a/tests/Operations/Provenance.hs +++ b/tests/Operations/Provenance.hs @@ -3,9 +3,7 @@ module Operations.Provenance where -import qualified Data.List as L import qualified Data.Map as M -import qualified Data.Text as T import qualified DataFrame as D import qualified DataFrame.Functions as F import qualified DataFrame.Internal.Column as DI