diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Revision history for dataframe +## 0.3.1.1 +* Aggregation now works on expressions rather than just column references. +* Export writeCsv +* Loosen bounds for dependencies to keep library on stackage. +* Add `filterNothing` function that returns all empty rows of a column. +* Add `IfThenElse` function for conditional expressions. +* Add `synthesizeFeatureExpr` function that does a search for a predictive variable in a `Double` dataframe. + +## 0.3.1.0 +* Add new `selectBy` function which subsumes all the other select functions. Specifically we can: + * `selectBy [byName "x"] df`: normal select. + * `selectBy [byProperty isNumeric] df`: all columns with a given property. + * `selectBy [byNameProperty (T.isPrefixOf "weight")] df`: select by column name predicate. + * `selectBy [byIndexRange (0, 5)] df`: picks the first size columns. + * `selectBy [byNameRange ("a", "c")] df`: select names within a range. +* Cut down dependencies to reduce binary/installation size. +* Add module for web plots that uses chartjs. +* Web plots can open in the browser. + ## 0.3.0.4 * Fix bug with parquet reader. @@ -87,7 +106,7 @@ We don't have good test coverage right now. This will help us determine where to invest. @oforero provided a script to make an HPC HTML report for coverage. -### Convenience functions for comparisons +### Convenience functions for comparisons Instead of lifting all bool operations we provide `eq`, `leq` etc. ## 0.1.0.3 diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -2,15 +2,8 @@ -- Useful Haskell extensions. -- Allow string literal to be interpreted as any other string type. {-# LANGUAGE OverloadedStrings #-} --- Convenience syntax for specifiying the type `sum a b :: Int` vs `sum @Int a b'. -{-# LANGUAGE TypeApplications #-} import qualified DataFrame as D -- import for general functionality. -import qualified DataFrame.Functions as F -- import for column expressions. - -import DataFrame ((|>)) -- import chaining operator with unqualified. - -import Control.Monad (replicateM) import Data.Time import qualified Data.Vector.Unboxed as VU import System.Random.Stateful @@ -27,18 +20,18 @@ let df = D.fromUnnamedColumns (map D.fromUnboxedVector [ns, xs, ys]) endGeneration <- getCurrentTime let generationTime = diffUTCTime endGeneration startGeneration - putStrLn $ "Data generation Time: " ++ (show generationTime) + putStrLn $ "Data generation Time: " ++ show generationTime startCalculation <- getCurrentTime print $ D.mean "0" df print $ D.variance "1" df print $ D.correlation "1" "2" df endCalculation <- getCurrentTime let calculationTime = diffUTCTime endCalculation startCalculation - putStrLn $ "Calculation Time: " ++ (show calculationTime) + putStrLn $ "Calculation Time: " ++ show calculationTime startFilter <- getCurrentTime print $ D.filter "0" (>= (19.9 :: Double)) df D.|> D.take 10 endFilter <- getCurrentTime let filterTime = diffUTCTime endFilter startFilter - putStrLn $ "Filter Time: " ++ (show filterTime) + putStrLn $ "Filter Time: " ++ show filterTime let totalTime = diffUTCTime endFilter startGeneration - putStrLn $ "Total Time: " ++ (show totalTime) + putStrLn $ "Total Time: " ++ show totalTime diff --git a/benchmark/Main.hs b/benchmark/Main.hs --- a/benchmark/Main.hs +++ b/benchmark/Main.hs @@ -2,17 +2,13 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -import qualified Data.Vector.Unboxed as VU -import qualified Data.Vector.Unboxed.Mutable as VUM import qualified DataFrame as D import qualified DataFrame.Functions as F -import Control.Monad (replicateM) +import Control.Monad (void) import Criterion.Main -import Data.Time import DataFrame ((|>)) import System.Process -import System.Random.Stateful haskell :: IO () haskell = do @@ -29,6 +25,12 @@ output <- readProcess "./benchmark/dataframe_benchmark/bin/python3" ["./benchmark/pandas/pandas_benchmark.py"] "" putStrLn output +explorer :: IO () +explorer = do + _ <- readProcess "./benchmark/dataframe_benchmark/bin/mix" ["deps.get"] "" + output <- readProcess "./benchmark/dataframe_benchmark/bin/mix" ["run", "./benchmark/explorer/explorer_benchmark.exs"] "" + putStrLn output + groupByHaskell :: IO () groupByHaskell = do df <- D.readCsv "./data/housing.csv" @@ -36,8 +38,8 @@ df |> D.groupBy ["ocean_proximity"] |> D.aggregate - [ (F.minimum (F.col @Double "median_house_value")) `F.as` "minimum_median_house_value" - , (F.maximum (F.col @Double "median_house_value")) `F.as` "maximum_median_house_value" + [ F.minimum (F.col @Double "median_house_value") `F.as` "minimum_median_house_value" + , F.maximum (F.col @Double "median_house_value") `F.as` "maximum_median_house_value" ] groupByPolars :: IO () @@ -50,17 +52,50 @@ output <- readProcess "./benchmark/dataframe_benchmark/bin/python3" ["./benchmark/pandas/group_by.py"] "" putStrLn output -main = do - output <- readProcess "cabal" ["build", "-O2"] "" +groupByExplorer :: IO () +groupByExplorer = do + output <- readProcess "./benchmark/dataframe_benchmark/bin/mix" ["run", "./benchmark/explorer/group_by.exs"] "" putStrLn output - defaultMain - [ bgroup - "stats" - [ bench "simpleStatsHaskell" $ nfIO haskell - , bench "simpleStatsPandas" $ nfIO pandas - , bench "simpleStatsPolars" $ nfIO polars - , bench "groupByHaskell" $ nfIO groupByHaskell - , bench "groupByPolars" $ nfIO groupByPolars - , bench "groupByPandas" $ nfIO groupByPandas - ] + +parseFile :: String -> IO () +parseFile path = void $ D.readCsv path + +parseCovidEffectsCSV :: IO () +parseCovidEffectsCSV = parseFile "./data/effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv" + +parseHousingCSV :: IO () +parseHousingCSV = parseFile "./data/housing.csv" + +parseStarWarsCSV :: IO () +parseStarWarsCSV = parseFile "./data/starwars.csv" + +parseChipotleTSV :: IO () +parseChipotleTSV = void $ D.readTsv "./data/chipotle.tsv" + +parseMeasurementsTXT :: IO () +parseMeasurementsTXT = parseFile "./data/measurements.txt" + +main :: IO () +main = do + output <- readProcess "cabal" ["build", "-O2"] "" + putStrLn output + defaultMain + [ bgroup + "stats" + [ bench "simpleStatsHaskell" $ nfIO haskell + , bench "simpleStatsPandas" $ nfIO pandas + , bench "simpleStatsPolars" $ nfIO polars + , bench "groupByHaskell" $ nfIO groupByHaskell + , bench "groupByPolars" $ nfIO groupByPolars + , bench "groupByPandas" $ nfIO groupByPandas + , bench "groupByExplorer" $ nfIO groupByExplorer + ], + bgroup + "Parsing" + [ bench "effects-of-covid-19-on-trade-at-15-december-2021-provisional.csv (9.0 MB)" $ nfIO parseCovidEffectsCSV + , bench "housing.csv (1.4 MB)" $ nfIO parseHousingCSV + , bench "starwars.csv (10 KB)" $ nfIO parseStarWarsCSV + , bench "chipotle.tsv (356 KB)" $ nfIO parseChipotleTSV + , bench "measurements.txt (135 KB)" $ nfIO parseMeasurementsTXT ] + ] diff --git a/dataframe.cabal b/dataframe.cabal --- a/dataframe.cabal +++ b/dataframe.cabal @@ -1,6 +1,6 @@ cabal-version: 2.4 name: dataframe -version: 0.3.0.4 +version: 0.3.1.1 synopsis: A fast, safe, and intuitive DataFrame library. @@ -21,140 +21,106 @@ type: git location: https://github.com/mchav/dataframe +common warnings + -- TODO add more warnings, eventually -Wall + ghc-options: -Wunused-packages -Wunused-imports + library + import: warnings default-extensions: Strict exposed-modules: DataFrame, - DataFrame.Lazy, - DataFrame.Functions - other-modules: DataFrame.Internal.Types, - DataFrame.Internal.Expression, - DataFrame.Internal.Parsing, - DataFrame.Internal.Column, - DataFrame.Display.Terminal.PrettyPrint, - DataFrame.Display.Terminal.Colours, - DataFrame.Internal.DataFrame, - DataFrame.Internal.Row, - DataFrame.Internal.Schema, - DataFrame.Errors, - DataFrame.Operations.Core, - DataFrame.Operations.Join, - DataFrame.Operations.Merge, - DataFrame.Operations.Subset, - DataFrame.Operations.Sorting, - DataFrame.Operations.Statistics, - DataFrame.Operations.Transformations, - DataFrame.Operations.Typing, - DataFrame.Operations.Aggregation, - DataFrame.Display.Terminal.Plot, - DataFrame.IO.CSV, - DataFrame.IO.Parquet, - DataFrame.IO.Parquet.Binary - DataFrame.IO.Parquet.Dictionary - DataFrame.IO.Parquet.Levels - DataFrame.IO.Parquet.Thrift - DataFrame.IO.Parquet.ColumnStatistics, - DataFrame.IO.Parquet.Compression, - DataFrame.IO.Parquet.Encoding, - DataFrame.IO.Parquet.Page, - DataFrame.IO.Parquet.Time, - DataFrame.IO.Parquet.Types, - DataFrame.Lazy.IO.CSV, - DataFrame.Lazy.Internal.DataFrame - build-depends: base >= 4.17.2.0 && < 4.22, + DataFrame.Lazy, + DataFrame.Functions, + DataFrame.Display.Web.Plot, + DataFrame.Display.Web.ChartJs, + DataFrame.Internal.Types, + DataFrame.Internal.Expression, + DataFrame.Internal.Parsing, + DataFrame.Internal.Column, + DataFrame.Internal.Statistics, + DataFrame.Display.Terminal.PrettyPrint, + DataFrame.Display.Terminal.Colours, + DataFrame.Internal.DataFrame, + DataFrame.Internal.Row, + DataFrame.Internal.Schema, + DataFrame.Errors, + DataFrame.Operations.Core, + DataFrame.Operations.Join, + DataFrame.Operations.Merge, + DataFrame.Operations.Subset, + DataFrame.Operations.Sorting, + DataFrame.Operations.Statistics, + DataFrame.Operations.Transformations, + DataFrame.Operations.Typing, + DataFrame.Operations.Aggregation, + DataFrame.Display.Terminal.Plot, + DataFrame.IO.CSV, + DataFrame.IO.Parquet, + DataFrame.IO.Parquet.Binary + DataFrame.IO.Parquet.Dictionary + DataFrame.IO.Parquet.Levels + DataFrame.IO.Parquet.Thrift + DataFrame.IO.Parquet.ColumnStatistics, + DataFrame.IO.Parquet.Compression, + DataFrame.IO.Parquet.Encoding, + DataFrame.IO.Parquet.Page, + DataFrame.IO.Parquet.Time, + DataFrame.IO.Parquet.Types, + DataFrame.Lazy.IO.CSV, + DataFrame.Lazy.Internal.DataFrame + build-depends: base > 4 && <5, array ^>= 0.5, - attoparsec >= 0.12 && <= 0.14.4, - bytestring >= 0.11 && <= 0.12.2.0, + attoparsec >= 0.12 && < 0.15, + bytestring >= 0.11 && < 0.13, bytestring-lexing >= 0.5 && < 0.6, containers >= 0.6.7 && < 0.8, - directory >= 1.3.0.0 && <= 1.3.9.0, - filepath >= 1.0.0.0 && <= 1.5.4.0, + directory >= 1.3.0.0 && < 2, granite ^>= 0.2, - hashable >= 1.2 && <= 1.5.0.0, + hashable >= 1.2 && < 2, + process ^>= 1.6, snappy-hs ^>= 0.1, - statistics >= 0.16.2.1 && <= 0.16.3.0, - template-haskell >= 2.0 && <= 2.30, - text >= 2.0 && <= 2.1.2, - time >= 1.12 && <= 1.14, + random >= 1 && < 2, + template-haskell >= 2.0 && < 3, + text >= 2.0 && < 3, + time >= 1.12 && < 2, vector ^>= 0.13, vector-algorithms ^>= 0.9, - zstd >= 0.1.2.0 && <= 0.1.3.0 + zstd >= 0.1.2.0 && < 0.2, hs-source-dirs: src default-language: Haskell2010 executable dataframe - main-is: Main.hs - other-modules: DataFrame, - DataFrame.Internal.Types, - DataFrame.Internal.Expression, - DataFrame.Internal.Parsing, - DataFrame.Internal.Column, - DataFrame.Display.Terminal.PrettyPrint, - DataFrame.Display.Terminal.Colours, - DataFrame.Internal.DataFrame, - DataFrame.Internal.Row, - DataFrame.Errors, - DataFrame.Operations.Core, - DataFrame.Operations.Subset, - DataFrame.Operations.Sorting, - DataFrame.Operations.Statistics, - DataFrame.Operations.Transformations, - DataFrame.Operations.Typing, - DataFrame.Operations.Aggregation, - DataFrame.Display.Terminal.Plot, - DataFrame.IO.CSV, - DataFrame.IO.Parquet, - DataFrame.IO.Parquet.Binary - DataFrame.IO.Parquet.Dictionary - DataFrame.IO.Parquet.Levels - DataFrame.IO.Parquet.Thrift - DataFrame.IO.Parquet.ColumnStatistics, - DataFrame.IO.Parquet.Compression, - DataFrame.IO.Parquet.Encoding, - DataFrame.IO.Parquet.Page, - DataFrame.IO.Parquet.Time, - DataFrame.IO.Parquet.Types, - DataFrame.Operations.Join, - DataFrame.Operations.Merge, - DataFrame.Lazy.IO.CSV, - DataFrame.Functions - build-depends: base >= 4.17.2.0 && < 4.22, - array ^>= 0.5, - attoparsec >= 0.12 && <= 0.14.4, - bytestring >= 0.11 && <= 0.12.2.0, - bytestring-lexing >= 0.5 && < 0.6, - containers >= 0.6.7 && < 0.8, - directory >= 1.3.0.0 && <= 1.3.9.0, - granite ^>= 0.2, - hashable >= 1.2 && <= 1.5.0.0, - random >= 1 && <= 1.3.1, - snappy-hs ^>= 0.1, - statistics >= 0.16.2.1 && <= 0.16.3.0, - template-haskell >= 2.0 && <= 2.30, - text >= 2.0 && <= 2.1.2, - time >= 1.12 && <= 1.14, + import: warnings + main-is: Main.hs + build-depends: base > 4 && < 5, + dataframe ^>= 0.3, + random >= 1 && < 2, + time >= 1.12 && < 2, + text, vector ^>= 0.13, - vector-algorithms ^>= 0.9, - zstd >= 0.1.2.0 && <= 0.1.3.0 - hs-source-dirs: app, - src + hs-source-dirs: app default-language: Haskell2010 + ghc-options: -rtsopts -threaded -with-rtsopts=-N benchmark dataframe-benchmark - type: exitcode-stdio-1.0 - main-is: Main.hs + import: warnings + type: exitcode-stdio-1.0 + main-is: Main.hs hs-source-dirs: benchmark - build-depends: base >= 4.17.2.0 && < 4.22, - criterion >= 1 && <= 1.6.4.0, - process >= 1.6, - text >= 2.0 && <= 2.1.2, - time >= 1.12, - random >= 1 && <= 1.3.1, - vector ^>= 0.13, - dataframe + build-depends: base > 4 && < 5, + criterion >= 1 && < 2, + process >= 1.6 && < 2, + dataframe ^>= 0.3 default-language: Haskell2010 + ghc-options: + -O2 + -threaded + -rtsopts + -with-rtsopts=-N test-suite tests + import: warnings type: exitcode-stdio-1.0 main-is: Main.hs other-modules: Assertions, @@ -164,67 +130,18 @@ Operations.Filter, Operations.GroupBy, Operations.InsertColumn, - Operations.Sort, - Operations.Statistics, + Operations.Sort, + Operations.Statistics, Operations.Take, - Parquet, - - -- Dataframe modules. - DataFrame, - DataFrame.Internal.Types, - DataFrame.Internal.Expression, - DataFrame.Internal.Parsing, - DataFrame.Internal.Column, - DataFrame.Display.Terminal.PrettyPrint, - DataFrame.Display.Terminal.Colours, - DataFrame.Internal.DataFrame, - DataFrame.Internal.Row, - DataFrame.Errors, - DataFrame.Operations.Core, - DataFrame.Operations.Subset, - DataFrame.Operations.Sorting, - DataFrame.Operations.Statistics, - DataFrame.Operations.Transformations, - DataFrame.Operations.Typing, - DataFrame.Operations.Aggregation, - DataFrame.Display.Terminal.Plot, - DataFrame.IO.CSV, - DataFrame.IO.Parquet, - DataFrame.IO.Parquet.Binary - DataFrame.IO.Parquet.Dictionary - DataFrame.IO.Parquet.Levels - DataFrame.IO.Parquet.Thrift - DataFrame.IO.Parquet.ColumnStatistics, - DataFrame.IO.Parquet.Compression, - DataFrame.IO.Parquet.Encoding, - DataFrame.IO.Parquet.Page, - DataFrame.IO.Parquet.Time, - DataFrame.IO.Parquet.Types, - DataFrame.Operations.Join, - DataFrame.Operations.Merge, - DataFrame.Lazy.IO.CSV, - DataFrame.Functions - build-depends: base >= 4.17.2.0 && < 4.22, - array ^>= 0.5, - attoparsec >= 0.12 && <= 0.14.4, - bytestring >= 0.11 && <= 0.12.2.0, - bytestring-lexing >= 0.5 && < 0.6, - containers >= 0.6.7 && < 0.8, - directory >= 1.3.0.0 && <= 1.3.9.0, - granite ^>= 0.2, - hashable >= 1.2 && <= 1.5.0.0, + Parquet + build-depends: base > 4 && < 5, + dataframe ^>= 0.3, HUnit ^>= 1.6, - random >= 1, - random-shuffle >= 0.0.4, - random >= 1 && <= 1.3.1, - snappy-hs ^>= 0.1, - statistics >= 0.16.2.1 && <= 0.16.3.0, - template-haskell >= 2.0 && <= 2.30, - text >= 2.0 && <= 2.1.2, - time >= 1.12 && <= 1.14, - vector ^>= 0.13, - vector-algorithms ^>= 0.9, - zstd >= 0.1.2.0 && <= 0.1.3.0 - hs-source-dirs: tests, - src + random >= 1 && < 2, + random-shuffle >= 0.0.4 && < 1, + random >= 1 && < 2, + text >= 2.0 && < 3, + time >= 1.12 && < 2, + vector ^>= 0.13 + hs-source-dirs: tests default-language: Haskell2010 diff --git a/src/DataFrame.hs b/src/DataFrame.hs --- a/src/DataFrame.hs +++ b/src/DataFrame.hs @@ -1,5 +1,3 @@ -{-# LANGUAGE OverloadedStrings #-} - {- | Module : DataFrame Copyright : (c) 2025 @@ -14,6 +12,7 @@ can get productive fast in GHCi, IHaskell, or scripts. __Naming convention__ + * Use the @D.@ (\"DataFrame\") prefix for core table operations. * Use the @F.@ (\"Functions\") prefix for the expression DSL (columns, math, aggregations). @@ -49,17 +48,17 @@ 9 | longitude | 20640 | 0 | 0 | 844 | Double -- 2) Project & filter -ghci> let df1 = df1 = D.filter @Text "ocean_proximity" (== "ISLAND") df0 D.|> D.select ["median_house_value", "median_income", "ocean_proximity"] +ghci> df1 = D.filter \@Text "ocean_proximity" (== \"ISLAND\") df0 D.|> D.select ["median_house_value", "median_income", "ocean_proximity"] -- 3) Add a derived column using the expression DSL -- (col types are explicit via TypeApplications) -ghci> df2 = D.derive "rooms_per_household" (F.col @Double "total_rooms" / F.col @Double "households") df0 +ghci> df2 = D.derive "rooms_per_household" (F.col \@Double "total_rooms" / F.col \@Double "households") df0 -- 4) Group + aggregate ghci> let grouped = D.groupBy ["ocean_proximity"] df0 ghci> let summary = D.aggregate - [ F.maximum (F.col @Double "median_house_value") `F.as` "max_house_value"] + [ F.maximum (F.col \@Double "median_house_value") \`F.as\` "max_house_value"] grouped ghci> D.take 5 summary ----------------------------------------- @@ -80,6 +79,7 @@ __I/O__ * @D.readCsv :: FilePath -> IO DataFrame@ + * @D.readTsv :: FilePath -> IO DataFrame@ * @D.writeCsv :: FilePath -> DataFrame -> IO ()@ * @D.readParquet :: FilePath -> IO DataFrame@ @@ -92,32 +92,32 @@ __Row ops__ - * @D.filter :: Columnable a => Text -> (a -> Bool) -> DataFrame -> DataFrame@ - * @D.sortBy :: SortOrder -> [Text] -> DataFrame -> DataFrame@ + * @D.filter :: Columnable a => Text -> (a -> Bool) -> DataFrame -> DataFrame@ + * @D.sortBy :: SortOrder -> [Text] -> DataFrame -> DataFrame@ __Column ops__ - * @D.select :: [Text] -> DataFrame -> DataFrame@ - * @D.exclude :: [Text] -> DataFrame -> DataFrame@ - * @D.rename :: [(Text,Text)] -> DataFrame -> DataFrame@ + * @D.select :: [Text] -> DataFrame -> DataFrame@ + * @D.exclude :: [Text] -> DataFrame -> DataFrame@ + * @D.rename :: [(Text,Text)] -> DataFrame -> DataFrame@ * @D.derive :: Text -> D.Expr a -> DataFrame -> DataFrame@ __Group & aggregate__ - * @D.groupBy :: [Text] -> DataFrame -> GroupedDataFrame@ + * @D.groupBy :: [Text] -> DataFrame -> GroupedDataFrame@ * @D.aggregate :: [(Text, F.UExpr)] -> GroupedDataFrame -> DataFrame@ __Joins__ - * @D.innerJoin / D.leftJoin / D.rightJoin / D.fullJoin@ + * @D.innerJoin \/ D.leftJoin \/ D.rightJoin \/ D.fullJoin@ == Expression DSL (F.*) at a glance Columns (typed): @ -F.col @Text "ocean_proximity" -F.col @Double "total_rooms" -F.lit @Double 1.0 +F.col \@Text "ocean_proximity" +F.col \@Double "total_rooms" +F.lit \@Double 1.0 @ Math & comparisons (overloaded by type): @@ -127,14 +127,14 @@ (F.eq), (F.gt), (F.geq), (F.lt), (F.leq) @ -Aggregations (for 'D.aggregate'): +Aggregations (for D.'aggregate'): @ -F.count @a (F.col @a "c") -F.sum @Double (F.col @Double "x") -F.mean @Double (F.col @Double "x") -F.min @t (F.col @t "x") -F.max @t (F.col @t "x") +F.count \@a (F.col \@a "c") +F.sum \@Double (F.col \@Double "x") +F.mean \@Double (F.col \@Double "x") +F.min \@t (F.col \@t "x") +F.max \@t (F.col \@t "x") @ == REPL power-tool: ':exposeColumns' @@ -202,13 +202,15 @@ import DataFrame.Display.Terminal.Plot as Plot import DataFrame.Errors as Errors -import DataFrame.IO.CSV as CSV (ReadOptions (..), defaultOptions, readCsv, readSeparated, readTsv) +import DataFrame.IO.CSV as CSV (ReadOptions (..), defaultOptions, readCsv, readSeparated, readTsv, writeCsv, writeSeparated) import DataFrame.IO.Parquet as Parquet (readParquet) import DataFrame.Internal.Column as Column ( Column, fromList, fromUnboxedVector, fromVector, + hasMissing, + isNumeric, toList, toVector, ) @@ -217,6 +219,7 @@ GroupedDataFrame, columnAsVector, empty, + toMarkdownTable, toMatrix, ) import DataFrame.Internal.Expression as Expression (Expr) @@ -238,6 +241,12 @@ variance, ) import DataFrame.Operations.Subset as Subset ( + SelectionCriteria, + byIndexRange, + byName, + byNameProperty, + byNameRange, + byProperty, cube, drop, dropLast, @@ -246,20 +255,16 @@ filterAllJust, filterBy, filterJust, + filterNothing, filterWhere, range, select, selectBy, - selectIntRange, - selectRange, take, takeLast, ) import DataFrame.Operations.Transformations as Transformations -import Data.Function -import Data.List -import qualified Data.Text as T -import qualified Data.Text.IO as T +import Data.Function ((&)) (|>) = (&) diff --git a/src/DataFrame/Display/Terminal/Plot.hs b/src/DataFrame/Display/Terminal/Plot.hs --- a/src/DataFrame/Display/Terminal/Plot.hs +++ b/src/DataFrame/Display/Terminal/Plot.hs @@ -11,7 +11,6 @@ import Control.Monad import qualified Data.List as L import qualified Data.Map as M -import Data.Maybe (catMaybes, fromMaybe) import qualified Data.Text as T import qualified Data.Text.IO as T import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) @@ -22,9 +21,9 @@ import GHC.Stack (HasCallStack) import Type.Reflection (typeRep) -import DataFrame.Internal.Column (Column (..), Columnable) +import DataFrame.Internal.Column (Column (..), isNumeric) import qualified DataFrame.Internal.Column as D -import DataFrame.Internal.DataFrame (DataFrame (..)) +import DataFrame.Internal.DataFrame (DataFrame (..), getColumn) import DataFrame.Operations.Core import qualified DataFrame.Operations.Subset as D import Granite @@ -53,7 +52,7 @@ } plotHistogram :: (HasCallStack) => T.Text -> DataFrame -> IO () -plotHistogram colName df = plotHistogramWith colName (defaultPlotConfig Histogram) df +plotHistogram colName = plotHistogramWith colName (defaultPlotConfig Histogram) plotHistogramWith :: (HasCallStack) => T.Text -> PlotConfig -> DataFrame -> IO () plotHistogramWith colName config df = do @@ -62,7 +61,7 @@ T.putStrLn $ histogram (bins 30 minVal maxVal) values (plotSettings config) plotScatter :: (HasCallStack) => T.Text -> T.Text -> DataFrame -> IO () -plotScatter xCol yCol df = plotScatterWith xCol yCol (defaultPlotConfig Scatter) df +plotScatter xCol yCol = plotScatterWith xCol yCol (defaultPlotConfig Scatter) plotScatterWith :: (HasCallStack) => T.Text -> T.Text -> PlotConfig -> DataFrame -> IO () plotScatterWith xCol yCol config df = do @@ -72,7 +71,7 @@ T.putStrLn $ scatter [(xCol <> " vs " <> yCol, points)] (plotSettings config) plotScatterBy :: (HasCallStack) => T.Text -> T.Text -> T.Text -> DataFrame -> IO () -plotScatterBy xCol yCol grouping df = plotScatterByWith xCol yCol grouping (defaultPlotConfig Scatter) df +plotScatterBy xCol yCol grouping = plotScatterByWith xCol yCol grouping (defaultPlotConfig Scatter) plotScatterByWith :: (HasCallStack) => T.Text -> T.Text -> T.Text -> PlotConfig -> DataFrame -> IO () plotScatterByWith xCol yCol grouping config df = do @@ -87,7 +86,7 @@ T.putStrLn $ scatter xs (plotSettings config) plotLines :: (HasCallStack) => T.Text -> [T.Text] -> DataFrame -> IO () -plotLines xAxis colNames df = plotLinesWith xAxis colNames (defaultPlotConfig Line) df +plotLines xAxis colNames = plotLinesWith xAxis colNames (defaultPlotConfig Line) plotLinesWith :: (HasCallStack) => T.Text -> [T.Text] -> PlotConfig -> DataFrame -> IO () plotLinesWith xAxis colNames config df = do @@ -98,7 +97,7 @@ T.putStrLn $ lineGraph seriesData (plotSettings config) plotBoxPlots :: (HasCallStack) => [T.Text] -> DataFrame -> IO () -plotBoxPlots colNames df = plotBoxPlotsWith colNames (defaultPlotConfig BoxPlot) df +plotBoxPlots colNames = plotBoxPlotsWith colNames (defaultPlotConfig BoxPlot) plotBoxPlotsWith :: (HasCallStack) => [T.Text] -> PlotConfig -> DataFrame -> IO () plotBoxPlotsWith colNames config df = do @@ -108,8 +107,7 @@ T.putStrLn $ boxPlot boxData (plotSettings config) plotStackedBars :: (HasCallStack) => T.Text -> [T.Text] -> DataFrame -> IO () -plotStackedBars categoryCol valueColumns df = - plotStackedBarsWith categoryCol valueColumns (defaultPlotConfig StackedBar) df +plotStackedBars categoryCol valueColumns = plotStackedBarsWith categoryCol valueColumns (defaultPlotConfig StackedBar) plotStackedBarsWith :: (HasCallStack) => T.Text -> [T.Text] -> PlotConfig -> DataFrame -> IO () plotStackedBarsWith categoryCol valueColumns config df = do @@ -127,39 +125,16 @@ T.putStrLn $ stackedBars stackData (plotSettings config) plotHeatmap :: (HasCallStack) => DataFrame -> IO () -plotHeatmap df = plotHeatmapWith (defaultPlotConfig Heatmap) df +plotHeatmap = plotHeatmapWith (defaultPlotConfig Heatmap) plotHeatmapWith :: (HasCallStack) => PlotConfig -> DataFrame -> IO () plotHeatmapWith config df = do let numericCols = filter (isNumericColumn df) (columnNames df) - matrix = map (\col -> extractNumericColumn col df) numericCols + matrix = map (`extractNumericColumn` df) numericCols T.putStrLn $ heatmap matrix (plotSettings config) isNumericColumn :: DataFrame -> T.Text -> Bool -isNumericColumn df colName = - case M.lookup colName (columnIndices df) of - Nothing -> False - Just idx -> - let col = columns df V.! idx - in case col of - BoxedColumn (vec :: V.Vector a) -> - case testEquality (typeRep @a) (typeRep @Double) of - Just _ -> True - Nothing -> case testEquality (typeRep @a) (typeRep @Int) of - Just _ -> True - Nothing -> case testEquality (typeRep @a) (typeRep @Float) of - Just _ -> True - Nothing -> False - UnboxedColumn (vec :: VU.Vector a) -> - case testEquality (typeRep @a) (typeRep @Double) of - Just _ -> True - Nothing -> case testEquality (typeRep @a) (typeRep @Int) of - Just _ -> True - Nothing -> case testEquality (typeRep @a) (typeRep @Float) of - Just _ -> True - Nothing -> False - -- Haven't dealt with optionals yet. - _ -> False +isNumericColumn df colName = maybe False isNumeric (getColumn colName df) plotAllHistograms :: (HasCallStack) => DataFrame -> IO () plotAllHistograms df = do @@ -198,7 +173,7 @@ in covXY / (stdX * stdY) plotBars :: (HasCallStack) => T.Text -> DataFrame -> IO () -plotBars colName df = plotBarsWith colName Nothing (defaultPlotConfig Bar) df +plotBars colName = plotBarsWith colName Nothing (defaultPlotConfig Bar) plotBarsWith :: (HasCallStack) => T.Text -> Maybe T.Text -> PlotConfig -> DataFrame -> IO () plotBarsWith colName groupByCol config df = @@ -226,7 +201,7 @@ T.putStrLn $ bars (zip labels values) (plotSettings config) plotBarsTopN :: (HasCallStack) => Int -> T.Text -> DataFrame -> IO () -plotBarsTopN n colName df = plotBarsTopNWith n colName (defaultPlotConfig Bar) df +plotBarsTopN n colName = plotBarsTopNWith n colName (defaultPlotConfig Bar) plotBarsTopNWith :: (HasCallStack) => Int -> T.Text -> PlotConfig -> DataFrame -> IO () plotBarsTopNWith n colName config df = do @@ -247,9 +222,9 @@ plotGroupedBarsWithN :: (HasCallStack) => Int -> T.Text -> T.Text -> PlotConfig -> DataFrame -> IO () plotGroupedBarsWithN n groupCol valCol config df = do - let isNumeric = isNumericColumnCheck valCol df + let colIsNumeric = isNumericColumnCheck valCol df - if isNumeric + if colIsNumeric then do let groups = extractStringColumn groupCol df values = extractNumericColumn valCol df @@ -269,7 +244,7 @@ T.putStrLn $ bars finalCounts (plotSettings config) plotValueCounts :: (HasCallStack) => T.Text -> DataFrame -> IO () -plotValueCounts colName df = plotValueCountsWith colName 10 (defaultPlotConfig Bar) df +plotValueCounts colName = plotValueCountsWith colName 10 (defaultPlotConfig Bar) plotValueCountsWith :: (HasCallStack) => T.Text -> Int -> PlotConfig -> DataFrame -> IO () plotValueCountsWith colName maxBars config df = do @@ -355,7 +330,9 @@ UnboxedColumn vec -> let counts = countValuesUnboxed vec in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts] - _ -> Nothing + OptionalColumn vec -> + let counts = countValues vec + in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts] where countValues :: (Ord a, Show a) => V.Vector a -> [(a, Int)] countValues vec = M.toList $ V.foldr' (\x acc -> M.insertWith (+) x 1 acc) M.empty vec @@ -364,27 +341,7 @@ countValuesUnboxed vec = M.toList $ VU.foldr' (\x acc -> M.insertWith (+) x 1 acc) M.empty vec isNumericColumnCheck :: T.Text -> DataFrame -> Bool -isNumericColumnCheck colName df = - case M.lookup colName (columnIndices df) of - Nothing -> False - Just idx -> - let col = columns df V.! idx - in case col of - BoxedColumn (vec :: V.Vector a) -> isNumericType @a - UnboxedColumn (vec :: VU.Vector a) -> isNumericType @a - _ -> False - -isNumericType :: forall a. (Typeable a) => Bool -isNumericType = - case testEquality (typeRep @a) (typeRep @Double) of - Just _ -> True - Nothing -> case testEquality (typeRep @a) (typeRep @Int) of - Just _ -> True - Nothing -> case testEquality (typeRep @a) (typeRep @Float) of - Just _ -> True - Nothing -> case testEquality (typeRep @a) (typeRep @Integer) of - Just _ -> True - Nothing -> False +isNumericColumnCheck colName df = isNumericColumn df colName extractStringColumn :: (HasCallStack) => T.Text -> DataFrame -> [T.Text] extractStringColumn colName df = @@ -442,7 +399,7 @@ in result plotPie :: (HasCallStack) => T.Text -> Maybe T.Text -> DataFrame -> IO () -plotPie valCol labelCol df = plotPieWith valCol labelCol (defaultPlotConfig Pie) df +plotPie valCol labelCol = plotPieWith valCol labelCol (defaultPlotConfig Pie) plotPieWith :: (HasCallStack) => T.Text -> Maybe T.Text -> PlotConfig -> DataFrame -> IO () plotPieWith valCol labelCol config df = do @@ -487,7 +444,7 @@ in result plotPieWithPercentages :: (HasCallStack) => T.Text -> DataFrame -> IO () -plotPieWithPercentages colName df = plotPieWithPercentagesConfig colName (defaultPlotConfig Pie) df +plotPieWithPercentages colName = plotPieWithPercentagesConfig colName (defaultPlotConfig Pie) plotPieWithPercentagesConfig :: (HasCallStack) => T.Text -> PlotConfig -> DataFrame -> IO () plotPieWithPercentagesConfig colName config df = do @@ -513,7 +470,7 @@ T.putStrLn $ pie grouped (plotSettings config) plotPieTopN :: (HasCallStack) => Int -> T.Text -> DataFrame -> IO () -plotPieTopN n colName df = plotPieTopNWith n colName (defaultPlotConfig Pie) df +plotPieTopN n colName = plotPieTopNWith n colName (defaultPlotConfig Pie) plotPieTopNWith :: (HasCallStack) => Int -> T.Text -> PlotConfig -> DataFrame -> IO () plotPieTopNWith n colName config df = do @@ -554,13 +511,13 @@ Nothing -> plotPie colName Nothing df plotPieGrouped :: (HasCallStack) => T.Text -> T.Text -> DataFrame -> IO () -plotPieGrouped groupCol valCol df = plotPieGroupedWith groupCol valCol (defaultPlotConfig Pie) df +plotPieGrouped groupCol valCol = plotPieGroupedWith groupCol valCol (defaultPlotConfig Pie) plotPieGroupedWith :: (HasCallStack) => T.Text -> T.Text -> PlotConfig -> DataFrame -> IO () plotPieGroupedWith groupCol valCol config df = do - let isNumeric = isNumericColumnCheck valCol df + let colIsNumeric = isNumericColumnCheck valCol df - if isNumeric + if colIsNumeric then do let groups = extractStringColumn groupCol df values = extractNumericColumn valCol df @@ -576,14 +533,13 @@ T.putStrLn $ pie finalCounts (plotSettings config) plotPieComparison :: (HasCallStack) => [T.Text] -> DataFrame -> IO () -plotPieComparison cols df = do - forM_ cols $ \col -> do - let counts = getCategoricalCounts col df - case counts of - Just c -> when (length c > 1 && length c <= 20) $ do - putStrLn $ "\n=== " ++ T.unpack col ++ " Distribution ===" - smartPlotPie col df - Nothing -> return () +plotPieComparison cols df = forM_ cols $ \col -> do + let counts = getCategoricalCounts col df + case counts of + Just c -> when (length c > 1 && length c <= 20) $ do + putStrLn $ "\n=== " ++ T.unpack col ++ " Distribution ===" + smartPlotPie col df + Nothing -> return () plotBinaryPie :: (HasCallStack) => T.Text -> DataFrame -> IO () plotBinaryPie colName df = do @@ -608,7 +564,7 @@ Nothing -> error $ "Column " ++ T.unpack colName ++ " is not categorical" plotMarketShare :: (HasCallStack) => T.Text -> DataFrame -> IO () -plotMarketShare colName df = plotMarketShareWith colName (defaultPlotConfig Pie) df +plotMarketShare colName = plotMarketShareWith colName (defaultPlotConfig Pie) plotMarketShareWith :: (HasCallStack) => T.Text -> PlotConfig -> DataFrame -> IO () plotMarketShareWith colName config df = do diff --git a/src/DataFrame/Display/Web/ChartJs.hs b/src/DataFrame/Display/Web/ChartJs.hs new file mode 100644 --- /dev/null +++ b/src/DataFrame/Display/Web/ChartJs.hs @@ -0,0 +1,8 @@ +{-# LANGUAGE OverloadedStrings #-} + +module DataFrame.Display.Web.ChartJs where + +import qualified Data.Text as T + +minifiedChartJs :: T.Text +minifiedChartJs = "!function(t,e){\"object\"==typeof exports&&\"undefined\"!=typeof module?module.exports=e(function(){try{return require(\"moment\")}catch(t){}}()):\"function\"==typeof define&&define.amd?define([\"require\"],(function(t){return e(function(){try{return t(\"moment\")}catch(t){}}())})):(t=t||self).Chart=e(t.moment)}(this,(function(t){\"use strict\";t=t&&t.hasOwnProperty(\"default\")?t.default:t;var e={aliceblue:[240,248,255],antiquewhite:[250,235,215],aqua:[0,255,255],aquamarine:[127,255,212],azure:[240,255,255],beige:[245,245,220],bisque:[255,228,196],black:[0,0,0],blanchedalmond:[255,235,205],blue:[0,0,255],blueviolet:[138,43,226],brown:[165,42,42],burlywood:[222,184,135],cadetblue:[95,158,160],chartreuse:[127,255,0],chocolate:[210,105,30],coral:[255,127,80],cornflowerblue:[100,149,237],cornsilk:[255,248,220],crimson:[220,20,60],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgoldenrod:[184,134,11],darkgray:[169,169,169],darkgreen:[0,100,0],darkgrey:[169,169,169],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkseagreen:[143,188,143],darkslateblue:[72,61,139],darkslategray:[47,79,79],darkslategrey:[47,79,79],darkturquoise:[0,206,209],darkviolet:[148,0,211],deeppink:[255,20,147],deepskyblue:[0,191,255],dimgray:[105,105,105],dimgrey:[105,105,105],dodgerblue:[30,144,255],firebrick:[178,34,34],floralwhite:[255,250,240],forestgreen:[34,139,34],fuchsia:[255,0,255],gainsboro:[220,220,220],ghostwhite:[248,248,255],gold:[255,215,0],goldenrod:[218,165,32],gray:[128,128,128],green:[0,128,0],greenyellow:[173,255,47],grey:[128,128,128],honeydew:[240,255,240],hotpink:[255,105,180],indianred:[205,92,92],indigo:[75,0,130],ivory:[255,255,240],khaki:[240,230,140],lavender:[230,230,250],lavenderblush:[255,240,245],lawngreen:[124,252,0],lemonchiffon:[255,250,205],lightblue:[173,216,230],lightcoral:[240,128,128],lightcyan:[224,255,255],lightgoldenrodyellow:[250,250,210],lightgray:[211,211,211],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightsalmon:[255,160,122],lightseagreen:[32,178,170],lightskyblue:[135,206,250],lightslategray:[119,136,153],lightslategrey:[119,136,153],lightsteelblue:[176,196,222],lightyellow:[255,255,224],lime:[0,255,0],limegreen:[50,205,50],linen:[250,240,230],magenta:[255,0,255],maroon:[128,0,0],mediumaquamarine:[102,205,170],mediumblue:[0,0,205],mediumorchid:[186,85,211],mediumpurple:[147,112,219],mediumseagreen:[60,179,113],mediumslateblue:[123,104,238],mediumspringgreen:[0,250,154],mediumturquoise:[72,209,204],mediumvioletred:[199,21,133],midnightblue:[25,25,112],mintcream:[245,255,250],mistyrose:[255,228,225],moccasin:[255,228,181],navajowhite:[255,222,173],navy:[0,0,128],oldlace:[253,245,230],olive:[128,128,0],olivedrab:[107,142,35],orange:[255,165,0],orangered:[255,69,0],orchid:[218,112,214],palegoldenrod:[238,232,170],palegreen:[152,251,152],paleturquoise:[175,238,238],palevioletred:[219,112,147],papayawhip:[255,239,213],peachpuff:[255,218,185],peru:[205,133,63],pink:[255,192,203],plum:[221,160,221],powderblue:[176,224,230],purple:[128,0,128],rebeccapurple:[102,51,153],red:[255,0,0],rosybrown:[188,143,143],royalblue:[65,105,225],saddlebrown:[139,69,19],salmon:[250,128,114],sandybrown:[244,164,96],seagreen:[46,139,87],seashell:[255,245,238],sienna:[160,82,45],silver:[192,192,192],skyblue:[135,206,235],slateblue:[106,90,205],slategray:[112,128,144],slategrey:[112,128,144],snow:[255,250,250],springgreen:[0,255,127],steelblue:[70,130,180],tan:[210,180,140],teal:[0,128,128],thistle:[216,191,216],tomato:[255,99,71],turquoise:[64,224,208],violet:[238,130,238],wheat:[245,222,179],white:[255,255,255],whitesmoke:[245,245,245],yellow:[255,255,0],yellowgreen:[154,205,50]},n=function(t,e){return t(e={exports:{}},e.exports),e.exports}((function(t){var n={};for(var i in e)e.hasOwnProperty(i)&&(n[e[i]]=i);var a=t.exports={rgb:{channels:3,labels:\"rgb\"},hsl:{channels:3,labels:\"hsl\"},hsv:{channels:3,labels:\"hsv\"},hwb:{channels:3,labels:\"hwb\"},cmyk:{channels:4,labels:\"cmyk\"},xyz:{channels:3,labels:\"xyz\"},lab:{channels:3,labels:\"lab\"},lch:{channels:3,labels:\"lch\"},hex:{channels:1,labels:[\"hex\"]},keyword:{channels:1,labels:[\"keyword\"]},ansi16:{channels:1,labels:[\"ansi16\"]},ansi256:{channels:1,labels:[\"ansi256\"]},hcg:{channels:3,labels:[\"h\",\"c\",\"g\"]},apple:{channels:3,labels:[\"r16\",\"g16\",\"b16\"]},gray:{channels:1,labels:[\"gray\"]}};for(var r in a)if(a.hasOwnProperty(r)){if(!(\"channels\"in a[r]))throw new Error(\"missing channels property: \"+r);if(!(\"labels\"in a[r]))throw new Error(\"missing channel labels property: \"+r);if(a[r].labels.length!==a[r].channels)throw new Error(\"channel and label counts mismatch: \"+r);var o=a[r].channels,l=a[r].labels;delete a[r].channels,delete a[r].labels,Object.defineProperty(a[r],\"channels\",{value:o}),Object.defineProperty(a[r],\"labels\",{value:l})}a.rgb.hsl=function(t){var e,n,i=t[0]/255,a=t[1]/255,r=t[2]/255,o=Math.min(i,a,r),l=Math.max(i,a,r),s=l-o;return l===o?e=0:i===l?e=(a-r)/s:a===l?e=2+(r-i)/s:r===l&&(e=4+(i-a)/s),(e=Math.min(60*e,360))<0&&(e+=360),n=(o+l)/2,[e,100*(l===o?0:n<=.5?s/(l+o):s/(2-l-o)),100*n]},a.rgb.hsv=function(t){var e,n,i,a,r,o=t[0]/255,l=t[1]/255,s=t[2]/255,u=Math.max(o,l,s),d=u-Math.min(o,l,s),c=function(t){return(u-t)/6/d+.5};return 0===d?a=r=0:(r=d/u,e=c(o),n=c(l),i=c(s),o===u?a=i-n:l===u?a=1/3+e-i:s===u&&(a=2/3+n-e),a<0?a+=1:a>1&&(a-=1)),[360*a,100*r,100*u]},a.rgb.hwb=function(t){var e=t[0],n=t[1],i=t[2];return[a.rgb.hsl(t)[0],100*(1/255*Math.min(e,Math.min(n,i))),100*(i=1-1/255*Math.max(e,Math.max(n,i)))]},a.rgb.cmyk=function(t){var e,n=t[0]/255,i=t[1]/255,a=t[2]/255;return[100*((1-n-(e=Math.min(1-n,1-i,1-a)))/(1-e)||0),100*((1-i-e)/(1-e)||0),100*((1-a-e)/(1-e)||0),100*e]},a.rgb.keyword=function(t){var i=n[t];if(i)return i;var a,r,o,l=1/0;for(var s in e)if(e.hasOwnProperty(s)){var u=e[s],d=(r=t,o=u,Math.pow(r[0]-o[0],2)+Math.pow(r[1]-o[1],2)+Math.pow(r[2]-o[2],2));d.04045?Math.pow((e+.055)/1.055,2.4):e/12.92)+.3576*(n=n>.04045?Math.pow((n+.055)/1.055,2.4):n/12.92)+.1805*(i=i>.04045?Math.pow((i+.055)/1.055,2.4):i/12.92)),100*(.2126*e+.7152*n+.0722*i),100*(.0193*e+.1192*n+.9505*i)]},a.rgb.lab=function(t){var e=a.rgb.xyz(t),n=e[0],i=e[1],r=e[2];return i/=100,r/=108.883,n=(n/=95.047)>.008856?Math.pow(n,1/3):7.787*n+16/116,[116*(i=i>.008856?Math.pow(i,1/3):7.787*i+16/116)-16,500*(n-i),200*(i-(r=r>.008856?Math.pow(r,1/3):7.787*r+16/116))]},a.hsl.rgb=function(t){var e,n,i,a,r,o=t[0]/360,l=t[1]/100,s=t[2]/100;if(0===l)return[r=255*s,r,r];e=2*s-(n=s<.5?s*(1+l):s+l-s*l),a=[0,0,0];for(var u=0;u<3;u++)(i=o+1/3*-(u-1))<0&&i++,i>1&&i--,r=6*i<1?e+6*(n-e)*i:2*i<1?n:3*i<2?e+(n-e)*(2/3-i)*6:e,a[u]=255*r;return a},a.hsl.hsv=function(t){var e=t[0],n=t[1]/100,i=t[2]/100,a=n,r=Math.max(i,.01);return n*=(i*=2)<=1?i:2-i,a*=r<=1?r:2-r,[e,100*(0===i?2*a/(r+a):2*n/(i+n)),100*((i+n)/2)]},a.hsv.rgb=function(t){var e=t[0]/60,n=t[1]/100,i=t[2]/100,a=Math.floor(e)%6,r=e-Math.floor(e),o=255*i*(1-n),l=255*i*(1-n*r),s=255*i*(1-n*(1-r));switch(i*=255,a){case 0:return[i,s,o];case 1:return[l,i,o];case 2:return[o,i,s];case 3:return[o,l,i];case 4:return[s,o,i];case 5:return[i,o,l]}},a.hsv.hsl=function(t){var e,n,i,a=t[0],r=t[1]/100,o=t[2]/100,l=Math.max(o,.01);return i=(2-r)*o,n=r*l,[a,100*(n=(n/=(e=(2-r)*l)<=1?e:2-e)||0),100*(i/=2)]},a.hwb.rgb=function(t){var e,n,i,a,r,o,l,s=t[0]/360,u=t[1]/100,d=t[2]/100,c=u+d;switch(c>1&&(u/=c,d/=c),i=6*s-(e=Math.floor(6*s)),0!=(1&e)&&(i=1-i),a=u+i*((n=1-d)-u),e){default:case 6:case 0:r=n,o=a,l=u;break;case 1:r=a,o=n,l=u;break;case 2:r=u,o=n,l=a;break;case 3:r=u,o=a,l=n;break;case 4:r=a,o=u,l=n;break;case 5:r=n,o=u,l=a}return[255*r,255*o,255*l]},a.cmyk.rgb=function(t){var e=t[0]/100,n=t[1]/100,i=t[2]/100,a=t[3]/100;return[255*(1-Math.min(1,e*(1-a)+a)),255*(1-Math.min(1,n*(1-a)+a)),255*(1-Math.min(1,i*(1-a)+a))]},a.xyz.rgb=function(t){var e,n,i,a=t[0]/100,r=t[1]/100,o=t[2]/100;return n=-.9689*a+1.8758*r+.0415*o,i=.0557*a+-.204*r+1.057*o,e=(e=3.2406*a+-1.5372*r+-.4986*o)>.0031308?1.055*Math.pow(e,1/2.4)-.055:12.92*e,n=n>.0031308?1.055*Math.pow(n,1/2.4)-.055:12.92*n,i=i>.0031308?1.055*Math.pow(i,1/2.4)-.055:12.92*i,[255*(e=Math.min(Math.max(0,e),1)),255*(n=Math.min(Math.max(0,n),1)),255*(i=Math.min(Math.max(0,i),1))]},a.xyz.lab=function(t){var e=t[0],n=t[1],i=t[2];return n/=100,i/=108.883,e=(e/=95.047)>.008856?Math.pow(e,1/3):7.787*e+16/116,[116*(n=n>.008856?Math.pow(n,1/3):7.787*n+16/116)-16,500*(e-n),200*(n-(i=i>.008856?Math.pow(i,1/3):7.787*i+16/116))]},a.lab.xyz=function(t){var e,n,i,a=t[0];e=t[1]/500+(n=(a+16)/116),i=n-t[2]/200;var r=Math.pow(n,3),o=Math.pow(e,3),l=Math.pow(i,3);return n=r>.008856?r:(n-16/116)/7.787,e=o>.008856?o:(e-16/116)/7.787,i=l>.008856?l:(i-16/116)/7.787,[e*=95.047,n*=100,i*=108.883]},a.lab.lch=function(t){var e,n=t[0],i=t[1],a=t[2];return(e=360*Math.atan2(a,i)/2/Math.PI)<0&&(e+=360),[n,Math.sqrt(i*i+a*a),e]},a.lch.lab=function(t){var e,n=t[0],i=t[1];return e=t[2]/360*2*Math.PI,[n,i*Math.cos(e),i*Math.sin(e)]},a.rgb.ansi16=function(t){var e=t[0],n=t[1],i=t[2],r=1 in arguments?arguments[1]:a.rgb.hsv(t)[2];if(0===(r=Math.round(r/50)))return 30;var o=30+(Math.round(i/255)<<2|Math.round(n/255)<<1|Math.round(e/255));return 2===r&&(o+=60),o},a.hsv.ansi16=function(t){return a.rgb.ansi16(a.hsv.rgb(t),t[2])},a.rgb.ansi256=function(t){var e=t[0],n=t[1],i=t[2];return e===n&&n===i?e<8?16:e>248?231:Math.round((e-8)/247*24)+232:16+36*Math.round(e/255*5)+6*Math.round(n/255*5)+Math.round(i/255*5)},a.ansi16.rgb=function(t){var e=t%10;if(0===e||7===e)return t>50&&(e+=3.5),[e=e/10.5*255,e,e];var n=.5*(1+~~(t>50));return[(1&e)*n*255,(e>>1&1)*n*255,(e>>2&1)*n*255]},a.ansi256.rgb=function(t){if(t>=232){var e=10*(t-232)+8;return[e,e,e]}var n;return t-=16,[Math.floor(t/36)/5*255,Math.floor((n=t%36)/6)/5*255,n%6/5*255]},a.rgb.hex=function(t){var e=(((255&Math.round(t[0]))<<16)+((255&Math.round(t[1]))<<8)+(255&Math.round(t[2]))).toString(16).toUpperCase();return\"000000\".substring(e.length)+e},a.hex.rgb=function(t){var e=t.toString(16).match(/[a-f0-9]{6}|[a-f0-9]{3}/i);if(!e)return[0,0,0];var n=e[0];3===e[0].length&&(n=n.split(\"\").map((function(t){return t+t})).join(\"\"));var i=parseInt(n,16);return[i>>16&255,i>>8&255,255&i]},a.rgb.hcg=function(t){var e,n=t[0]/255,i=t[1]/255,a=t[2]/255,r=Math.max(Math.max(n,i),a),o=Math.min(Math.min(n,i),a),l=r-o;return e=l<=0?0:r===n?(i-a)/l%6:r===i?2+(a-n)/l:4+(n-i)/l+4,e/=6,[360*(e%=1),100*l,100*(l<1?o/(1-l):0)]},a.hsl.hcg=function(t){var e=t[1]/100,n=t[2]/100,i=1,a=0;return(i=n<.5?2*e*n:2*e*(1-n))<1&&(a=(n-.5*i)/(1-i)),[t[0],100*i,100*a]},a.hsv.hcg=function(t){var e=t[1]/100,n=t[2]/100,i=e*n,a=0;return i<1&&(a=(n-i)/(1-i)),[t[0],100*i,100*a]},a.hcg.rgb=function(t){var e=t[0]/360,n=t[1]/100,i=t[2]/100;if(0===n)return[255*i,255*i,255*i];var a,r=[0,0,0],o=e%1*6,l=o%1,s=1-l;switch(Math.floor(o)){case 0:r[0]=1,r[1]=l,r[2]=0;break;case 1:r[0]=s,r[1]=1,r[2]=0;break;case 2:r[0]=0,r[1]=1,r[2]=l;break;case 3:r[0]=0,r[1]=s,r[2]=1;break;case 4:r[0]=l,r[1]=0,r[2]=1;break;default:r[0]=1,r[1]=0,r[2]=s}return a=(1-n)*i,[255*(n*r[0]+a),255*(n*r[1]+a),255*(n*r[2]+a)]},a.hcg.hsv=function(t){var e=t[1]/100,n=e+t[2]/100*(1-e),i=0;return n>0&&(i=e/n),[t[0],100*i,100*n]},a.hcg.hsl=function(t){var e=t[1]/100,n=t[2]/100*(1-e)+.5*e,i=0;return n>0&&n<.5?i=e/(2*n):n>=.5&&n<1&&(i=e/(2*(1-n))),[t[0],100*i,100*n]},a.hcg.hwb=function(t){var e=t[1]/100,n=e+t[2]/100*(1-e);return[t[0],100*(n-e),100*(1-n)]},a.hwb.hcg=function(t){var e=t[1]/100,n=1-t[2]/100,i=n-e,a=0;return i<1&&(a=(n-i)/(1-i)),[t[0],100*i,100*a]},a.apple.rgb=function(t){return[t[0]/65535*255,t[1]/65535*255,t[2]/65535*255]},a.rgb.apple=function(t){return[t[0]/255*65535,t[1]/255*65535,t[2]/255*65535]},a.gray.rgb=function(t){return[t[0]/100*255,t[0]/100*255,t[0]/100*255]},a.gray.hsl=a.gray.hsv=function(t){return[0,0,t[0]]},a.gray.hwb=function(t){return[0,100,t[0]]},a.gray.cmyk=function(t){return[0,0,0,t[0]]},a.gray.lab=function(t){return[t[0],0,0]},a.gray.hex=function(t){var e=255&Math.round(t[0]/100*255),n=((e<<16)+(e<<8)+e).toString(16).toUpperCase();return\"000000\".substring(n.length)+n},a.rgb.gray=function(t){return[(t[0]+t[1]+t[2])/3/255*100]}}));n.rgb,n.hsl,n.hsv,n.hwb,n.cmyk,n.xyz,n.lab,n.lch,n.hex,n.keyword,n.ansi16,n.ansi256,n.hcg,n.apple,n.gray;function i(t){var e=function(){for(var t={},e=Object.keys(n),i=e.length,a=0;a1&&(e=Array.prototype.slice.call(arguments));var n=t(e);if(\"object\"==typeof n)for(var i=n.length,a=0;a1&&(e=Array.prototype.slice.call(arguments)),t(e))};return\"conversion\"in t&&(e.conversion=t.conversion),e}(i)}))}));var l=o,s={aliceblue:[240,248,255],antiquewhite:[250,235,215],aqua:[0,255,255],aquamarine:[127,255,212],azure:[240,255,255],beige:[245,245,220],bisque:[255,228,196],black:[0,0,0],blanchedalmond:[255,235,205],blue:[0,0,255],blueviolet:[138,43,226],brown:[165,42,42],burlywood:[222,184,135],cadetblue:[95,158,160],chartreuse:[127,255,0],chocolate:[210,105,30],coral:[255,127,80],cornflowerblue:[100,149,237],cornsilk:[255,248,220],crimson:[220,20,60],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgoldenrod:[184,134,11],darkgray:[169,169,169],darkgreen:[0,100,0],darkgrey:[169,169,169],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkseagreen:[143,188,143],darkslateblue:[72,61,139],darkslategray:[47,79,79],darkslategrey:[47,79,79],darkturquoise:[0,206,209],darkviolet:[148,0,211],deeppink:[255,20,147],deepskyblue:[0,191,255],dimgray:[105,105,105],dimgrey:[105,105,105],dodgerblue:[30,144,255],firebrick:[178,34,34],floralwhite:[255,250,240],forestgreen:[34,139,34],fuchsia:[255,0,255],gainsboro:[220,220,220],ghostwhite:[248,248,255],gold:[255,215,0],goldenrod:[218,165,32],gray:[128,128,128],green:[0,128,0],greenyellow:[173,255,47],grey:[128,128,128],honeydew:[240,255,240],hotpink:[255,105,180],indianred:[205,92,92],indigo:[75,0,130],ivory:[255,255,240],khaki:[240,230,140],lavender:[230,230,250],lavenderblush:[255,240,245],lawngreen:[124,252,0],lemonchiffon:[255,250,205],lightblue:[173,216,230],lightcoral:[240,128,128],lightcyan:[224,255,255],lightgoldenrodyellow:[250,250,210],lightgray:[211,211,211],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightsalmon:[255,160,122],lightseagreen:[32,178,170],lightskyblue:[135,206,250],lightslategray:[119,136,153],lightslategrey:[119,136,153],lightsteelblue:[176,196,222],lightyellow:[255,255,224],lime:[0,255,0],limegreen:[50,205,50],linen:[250,240,230],magenta:[255,0,255],maroon:[128,0,0],mediumaquamarine:[102,205,170],mediumblue:[0,0,205],mediumorchid:[186,85,211],mediumpurple:[147,112,219],mediumseagreen:[60,179,113],mediumslateblue:[123,104,238],mediumspringgreen:[0,250,154],mediumturquoise:[72,209,204],mediumvioletred:[199,21,133],midnightblue:[25,25,112],mintcream:[245,255,250],mistyrose:[255,228,225],moccasin:[255,228,181],navajowhite:[255,222,173],navy:[0,0,128],oldlace:[253,245,230],olive:[128,128,0],olivedrab:[107,142,35],orange:[255,165,0],orangered:[255,69,0],orchid:[218,112,214],palegoldenrod:[238,232,170],palegreen:[152,251,152],paleturquoise:[175,238,238],palevioletred:[219,112,147],papayawhip:[255,239,213],peachpuff:[255,218,185],peru:[205,133,63],pink:[255,192,203],plum:[221,160,221],powderblue:[176,224,230],purple:[128,0,128],rebeccapurple:[102,51,153],red:[255,0,0],rosybrown:[188,143,143],royalblue:[65,105,225],saddlebrown:[139,69,19],salmon:[250,128,114],sandybrown:[244,164,96],seagreen:[46,139,87],seashell:[255,245,238],sienna:[160,82,45],silver:[192,192,192],skyblue:[135,206,235],slateblue:[106,90,205],slategray:[112,128,144],slategrey:[112,128,144],snow:[255,250,250],springgreen:[0,255,127],steelblue:[70,130,180],tan:[210,180,140],teal:[0,128,128],thistle:[216,191,216],tomato:[255,99,71],turquoise:[64,224,208],violet:[238,130,238],wheat:[245,222,179],white:[255,255,255],whitesmoke:[245,245,245],yellow:[255,255,0],yellowgreen:[154,205,50]},u={getRgba:d,getHsla:c,getRgb:function(t){var e=d(t);return e&&e.slice(0,3)},getHsl:function(t){var e=c(t);return e&&e.slice(0,3)},getHwb:h,getAlpha:function(t){var e=d(t);if(e)return e[3];if(e=c(t))return e[3];if(e=h(t))return e[3]},hexString:function(t,e){e=void 0!==e&&3===t.length?e:t[3];return\"#\"+v(t[0])+v(t[1])+v(t[2])+(e>=0&&e<1?v(Math.round(255*e)):\"\")},rgbString:function(t,e){if(e<1||t[3]&&t[3]<1)return f(t,e);return\"rgb(\"+t[0]+\", \"+t[1]+\", \"+t[2]+\")\"},rgbaString:f,percentString:function(t,e){if(e<1||t[3]&&t[3]<1)return g(t,e);var n=Math.round(t[0]/255*100),i=Math.round(t[1]/255*100),a=Math.round(t[2]/255*100);return\"rgb(\"+n+\"%, \"+i+\"%, \"+a+\"%)\"},percentaString:g,hslString:function(t,e){if(e<1||t[3]&&t[3]<1)return p(t,e);return\"hsl(\"+t[0]+\", \"+t[1]+\"%, \"+t[2]+\"%)\"},hslaString:p,hwbString:function(t,e){void 0===e&&(e=void 0!==t[3]?t[3]:1);return\"hwb(\"+t[0]+\", \"+t[1]+\"%, \"+t[2]+\"%\"+(void 0!==e&&1!==e?\", \"+e:\"\")+\")\"},keyword:function(t){return b[t.slice(0,3)]}};function d(t){if(t){var e=[0,0,0],n=1,i=t.match(/^#([a-fA-F0-9]{3,4})$/i),a=\"\";if(i){a=(i=i[1])[3];for(var r=0;rn?(e+.05)/(n+.05):(n+.05)/(e+.05)},level:function(t){var e=this.contrast(t);return e>=7.1?\"AAA\":e>=4.5?\"AA\":\"\"},dark:function(){var t=this.values.rgb;return(299*t[0]+587*t[1]+114*t[2])/1e3<128},light:function(){return!this.dark()},negate:function(){for(var t=[],e=0;e<3;e++)t[e]=255-this.values.rgb[e];return this.setValues(\"rgb\",t),this},lighten:function(t){var e=this.values.hsl;return e[2]+=e[2]*t,this.setValues(\"hsl\",e),this},darken:function(t){var e=this.values.hsl;return e[2]-=e[2]*t,this.setValues(\"hsl\",e),this},saturate:function(t){var e=this.values.hsl;return e[1]+=e[1]*t,this.setValues(\"hsl\",e),this},desaturate:function(t){var e=this.values.hsl;return e[1]-=e[1]*t,this.setValues(\"hsl\",e),this},whiten:function(t){var e=this.values.hwb;return e[1]+=e[1]*t,this.setValues(\"hwb\",e),this},blacken:function(t){var e=this.values.hwb;return e[2]+=e[2]*t,this.setValues(\"hwb\",e),this},greyscale:function(){var t=this.values.rgb,e=.3*t[0]+.59*t[1]+.11*t[2];return this.setValues(\"rgb\",[e,e,e]),this},clearer:function(t){var e=this.values.alpha;return this.setValues(\"alpha\",e-e*t),this},opaquer:function(t){var e=this.values.alpha;return this.setValues(\"alpha\",e+e*t),this},rotate:function(t){var e=this.values.hsl,n=(e[0]+t)%360;return e[0]=n<0?360+n:n,this.setValues(\"hsl\",e),this},mix:function(t,e){var n=this,i=t,a=void 0===e?.5:e,r=2*a-1,o=n.alpha()-i.alpha(),l=((r*o==-1?r:(r+o)/(1+r*o))+1)/2,s=1-l;return this.rgb(l*n.red()+s*i.red(),l*n.green()+s*i.green(),l*n.blue()+s*i.blue()).alpha(n.alpha()*a+i.alpha()*(1-a))},toJSON:function(){return this.rgb()},clone:function(){var t,e,n=new y,i=this.values,a=n.values;for(var r in i)i.hasOwnProperty(r)&&(t=i[r],\"[object Array]\"===(e={}.toString.call(t))?a[r]=t.slice(0):\"[object Number]\"===e?a[r]=t:console.error(\"unexpected color value:\",t));return n}},y.prototype.spaces={rgb:[\"red\",\"green\",\"blue\"],hsl:[\"hue\",\"saturation\",\"lightness\"],hsv:[\"hue\",\"saturation\",\"value\"],hwb:[\"hue\",\"whiteness\",\"blackness\"],cmyk:[\"cyan\",\"magenta\",\"yellow\",\"black\"]},y.prototype.maxes={rgb:[255,255,255],hsl:[360,100,100],hsv:[360,100,100],hwb:[360,100,100],cmyk:[100,100,100,100]},y.prototype.getValues=function(t){for(var e=this.values,n={},i=0;i=0;a--)e.call(n,t[a],a);else for(a=0;a=1?t:-(Math.sqrt(1-t*t)-1)},easeOutCirc:function(t){return Math.sqrt(1-(t-=1)*t)},easeInOutCirc:function(t){return(t/=.5)<1?-.5*(Math.sqrt(1-t*t)-1):.5*(Math.sqrt(1-(t-=2)*t)+1)},easeInElastic:function(t){var e=1.70158,n=0,i=1;return 0===t?0:1===t?1:(n||(n=.3),i<1?(i=1,e=n/4):e=n/(2*Math.PI)*Math.asin(1/i),-i*Math.pow(2,10*(t-=1))*Math.sin((t-e)*(2*Math.PI)/n))},easeOutElastic:function(t){var e=1.70158,n=0,i=1;return 0===t?0:1===t?1:(n||(n=.3),i<1?(i=1,e=n/4):e=n/(2*Math.PI)*Math.asin(1/i),i*Math.pow(2,-10*t)*Math.sin((t-e)*(2*Math.PI)/n)+1)},easeInOutElastic:function(t){var e=1.70158,n=0,i=1;return 0===t?0:2==(t/=.5)?1:(n||(n=.45),i<1?(i=1,e=n/4):e=n/(2*Math.PI)*Math.asin(1/i),t<1?i*Math.pow(2,10*(t-=1))*Math.sin((t-e)*(2*Math.PI)/n)*-.5:i*Math.pow(2,-10*(t-=1))*Math.sin((t-e)*(2*Math.PI)/n)*.5+1)},easeInBack:function(t){var e=1.70158;return t*t*((e+1)*t-e)},easeOutBack:function(t){var e=1.70158;return(t-=1)*t*((e+1)*t+e)+1},easeInOutBack:function(t){var e=1.70158;return(t/=.5)<1?t*t*((1+(e*=1.525))*t-e)*.5:.5*((t-=2)*t*((1+(e*=1.525))*t+e)+2)},easeInBounce:function(t){return 1-C.easeOutBounce(1-t)},easeOutBounce:function(t){return t<1/2.75?7.5625*t*t:t<2/2.75?7.5625*(t-=1.5/2.75)*t+.75:t<2.5/2.75?7.5625*(t-=2.25/2.75)*t+.9375:7.5625*(t-=2.625/2.75)*t+.984375},easeInOutBounce:function(t){return t<.5?.5*C.easeInBounce(2*t):.5*C.easeOutBounce(2*t-1)+.5}},P={effects:C};S.easingEffects=C;var A=Math.PI,D=A/180,T=2*A,I=A/2,F=A/4,O=2*A/3,L={clear:function(t){t.ctx.clearRect(0,0,t.width,t.height)},roundedRect:function(t,e,n,i,a,r){if(r){var o=Math.min(r,a/2,i/2),l=e+o,s=n+o,u=e+i-o,d=n+a-o;t.moveTo(e,s),le.left-n&&t.xe.top-n&&t.y0&&t.requestAnimationFrame()},advance:function(){for(var t,e,n,i,a=this.animations,r=0;r=n?(j.callback(t.onAnimationComplete,[t],e),e.animating=!1,a.splice(r,1)):++r}},tt=j.options.resolve,et=[\"push\",\"pop\",\"shift\",\"splice\",\"unshift\"];function nt(t,e){var n=t._chartjs;if(n){var i=n.listeners,a=i.indexOf(e);-1!==a&&i.splice(a,1),i.length>0||(et.forEach((function(e){delete t[e]})),delete t._chartjs)}}var it=function(t,e){this.initialize(t,e)};j.extend(it.prototype,{datasetElementType:null,dataElementType:null,_datasetElementOptions:[\"backgroundColor\",\"borderCapStyle\",\"borderColor\",\"borderDash\",\"borderDashOffset\",\"borderJoinStyle\",\"borderWidth\"],_dataElementOptions:[\"backgroundColor\",\"borderColor\",\"borderWidth\",\"pointStyle\"],initialize:function(t,e){var n=this;n.chart=t,n.index=e,n.linkScales(),n.addElements(),n._type=n.getMeta().type},updateIndex:function(t){this.index=t},linkScales:function(){var t=this,e=t.getMeta(),n=t.chart,i=n.scales,a=t.getDataset(),r=n.options.scales;null!==e.xAxisID&&e.xAxisID in i&&!a.xAxisID||(e.xAxisID=a.xAxisID||r.xAxes[0].id),null!==e.yAxisID&&e.yAxisID in i&&!a.yAxisID||(e.yAxisID=a.yAxisID||r.yAxes[0].id)},getDataset:function(){return this.chart.data.datasets[this.index]},getMeta:function(){return this.chart.getDatasetMeta(this.index)},getScaleForId:function(t){return this.chart.scales[t]},_getValueScaleId:function(){return this.getMeta().yAxisID},_getIndexScaleId:function(){return this.getMeta().xAxisID},_getValueScale:function(){return this.getScaleForId(this._getValueScaleId())},_getIndexScale:function(){return this.getScaleForId(this._getIndexScaleId())},reset:function(){this._update(!0)},destroy:function(){this._data&&nt(this._data,this)},createMetaDataset:function(){var t=this,e=t.datasetElementType;return e&&new e({_chart:t.chart,_datasetIndex:t.index})},createMetaData:function(t){var e=this,n=e.dataElementType;return n&&new n({_chart:e.chart,_datasetIndex:e.index,_index:t})},addElements:function(){var t,e,n=this,i=n.getMeta(),a=n.getDataset().data||[],r=i.data;for(t=0,e=a.length;ti&&t.insertElements(i,a-i)},insertElements:function(t,e){for(var n=0;na?(r=a/e.innerRadius,t.arc(o,l,e.innerRadius-a,i+r,n-r,!0)):t.arc(o,l,a,i+Math.PI/2,n-Math.PI/2),t.closePath(),t.clip()}function lt(t,e,n){var i=\"inner\"===e.borderAlign;i?(t.lineWidth=2*e.borderWidth,t.lineJoin=\"round\"):(t.lineWidth=e.borderWidth,t.lineJoin=\"bevel\"),n.fullCircles&&function(t,e,n,i){var a,r=n.endAngle;for(i&&(n.endAngle=n.startAngle+rt,ot(t,n),n.endAngle=r,n.endAngle===n.startAngle&&n.fullCircles&&(n.endAngle+=rt,n.fullCircles--)),t.beginPath(),t.arc(n.x,n.y,n.innerRadius,n.startAngle+rt,n.startAngle,!0),a=0;al;)a-=rt;for(;a=o&&a<=l,u=r>=n.innerRadius&&r<=n.outerRadius;return s&&u}return!1},getCenterPoint:function(){var t=this._view,e=(t.startAngle+t.endAngle)/2,n=(t.innerRadius+t.outerRadius)/2;return{x:t.x+Math.cos(e)*n,y:t.y+Math.sin(e)*n}},getArea:function(){var t=this._view;return Math.PI*((t.endAngle-t.startAngle)/(2*Math.PI))*(Math.pow(t.outerRadius,2)-Math.pow(t.innerRadius,2))},tooltipPosition:function(){var t=this._view,e=t.startAngle+(t.endAngle-t.startAngle)/2,n=(t.outerRadius-t.innerRadius)/2+t.innerRadius;return{x:t.x+Math.cos(e)*n,y:t.y+Math.sin(e)*n}},draw:function(){var t,e=this._chart.ctx,n=this._view,i=\"inner\"===n.borderAlign?.33:0,a={x:n.x,y:n.y,innerRadius:n.innerRadius,outerRadius:Math.max(n.outerRadius-i,0),pixelMargin:i,startAngle:n.startAngle,endAngle:n.endAngle,fullCircles:Math.floor(n.circumference/rt)};if(e.save(),e.fillStyle=n.backgroundColor,e.strokeStyle=n.borderColor,a.fullCircles){for(a.endAngle=a.startAngle+rt,e.beginPath(),e.arc(a.x,a.y,a.outerRadius,a.startAngle,a.endAngle),e.arc(a.x,a.y,a.innerRadius,a.endAngle,a.startAngle,!0),e.closePath(),t=0;tt.x&&(e=xt(e,\"left\",\"right\")):t.basen?n:i,r:s.right||a<0?0:a>e?e:a,b:s.bottom||r<0?0:r>n?n:r,l:s.left||o<0?0:o>e?e:o}}function _t(t,e,n){var i=null===e,a=null===n,r=!(!t||i&&a)&&bt(t);return r&&(i||e>=r.left&&e<=r.right)&&(a||n>=r.top&&n<=r.bottom)}N._set(\"global\",{elements:{rectangle:{backgroundColor:mt,borderColor:mt,borderSkipped:\"bottom\",borderWidth:0}}});var kt=Z.extend({_type:\"rectangle\",draw:function(){var t=this._chart.ctx,e=this._view,n=function(t){var e=bt(t),n=e.right-e.left,i=e.bottom-e.top,a=yt(t,n/2,i/2);return{outer:{x:e.left,y:e.top,w:n,h:i},inner:{x:e.left+a.l,y:e.top+a.t,w:n-a.l-a.r,h:i-a.t-a.b}}}(e),i=n.outer,a=n.inner;t.fillStyle=e.backgroundColor,t.fillRect(i.x,i.y,i.w,i.h),i.w===a.w&&i.h===a.h||(t.save(),t.beginPath(),t.rect(i.x,i.y,i.w,i.h),t.clip(),t.fillStyle=e.borderColor,t.rect(a.x,a.y,a.w,a.h),t.fill(\"evenodd\"),t.restore())},height:function(){var t=this._view;return t.base-t.y},inRange:function(t,e){return _t(this._view,t,e)},inLabelRange:function(t,e){var n=this._view;return vt(n)?_t(n,t,null):_t(n,null,e)},inXRange:function(t){return _t(this._view,t,null)},inYRange:function(t){return _t(this._view,null,t)},getCenterPoint:function(){var t,e,n=this._view;return vt(n)?(t=n.x,e=(n.y+n.base)/2):(t=(n.x+n.base)/2,e=n.y),{x:t,y:e}},getArea:function(){var t=this._view;return vt(t)?t.width*Math.abs(t.y-t.base):t.height*Math.abs(t.x-t.base)},tooltipPosition:function(){var t=this._view;return{x:t.x,y:t.y}}}),wt={},Mt=st,St=ct,Ct=pt,Pt=kt;wt.Arc=Mt,wt.Line=St,wt.Point=Ct,wt.Rectangle=Pt;var At=j._deprecated,Dt=j.valueOrDefault;function Tt(t,e,n){var i,a,r=n.barThickness,o=e.stackCount,l=e.pixels[t],s=j.isNullOrUndef(r)?function(t,e){var n,i,a,r,o=t._length;for(a=1,r=e.length;a0?Math.min(o,Math.abs(i-n)):o,n=i;return o}(e.scale,e.pixels):-1;return j.isNullOrUndef(r)?(i=s*n.categoryPercentage,a=n.barPercentage):(i=r*o,a=1),{chunk:i/o,ratio:a,start:l-i/2}}N._set(\"bar\",{hover:{mode:\"label\"},scales:{xAxes:[{type:\"category\",offset:!0,gridLines:{offsetGridLines:!0}}],yAxes:[{type:\"linear\"}]}}),N._set(\"global\",{datasets:{bar:{categoryPercentage:.8,barPercentage:.9}}});var It=at.extend({dataElementType:wt.Rectangle,_dataElementOptions:[\"backgroundColor\",\"borderColor\",\"borderSkipped\",\"borderWidth\",\"barPercentage\",\"barThickness\",\"categoryPercentage\",\"maxBarThickness\",\"minBarLength\"],initialize:function(){var t,e,n=this;at.prototype.initialize.apply(n,arguments),(t=n.getMeta()).stack=n.getDataset().stack,t.bar=!0,e=n._getIndexScale().options,At(\"bar chart\",e.barPercentage,\"scales.[x/y]Axes.barPercentage\",\"dataset.barPercentage\"),At(\"bar chart\",e.barThickness,\"scales.[x/y]Axes.barThickness\",\"dataset.barThickness\"),At(\"bar chart\",e.categoryPercentage,\"scales.[x/y]Axes.categoryPercentage\",\"dataset.categoryPercentage\"),At(\"bar chart\",n._getValueScale().options.minBarLength,\"scales.[x/y]Axes.minBarLength\",\"dataset.minBarLength\"),At(\"bar chart\",e.maxBarThickness,\"scales.[x/y]Axes.maxBarThickness\",\"dataset.maxBarThickness\")},update:function(t){var e,n,i=this,a=i.getMeta().data;for(i._ruler=i.getRuler(),e=0,n=a.length;e=0&&m.min>=0?m.min:m.max,_=void 0===m.start?m.end:m.max>=0&&m.min>=0?m.max-m.min:m.min-m.max,k=p.length;if(b||void 0===b&&void 0!==x)for(i=0;i=0&&u.max>=0?u.max:u.min,(m.min<0&&r<0||m.max>=0&&r>0)&&(y+=r));return o=h.getPixelForValue(y),s=(l=h.getPixelForValue(y+_))-o,void 0!==v&&Math.abs(s)=0&&!f||_<0&&f?o-v:o+v),{size:s,base:o,head:l,center:l+s/2}},calculateBarIndexPixels:function(t,e,n,i){var a=\"flex\"===i.barThickness?function(t,e,n){var i,a=e.pixels,r=a[t],o=t>0?a[t-1]:null,l=t=zt?-Nt:b<-zt?Nt:0)+m,y=Math.cos(b),_=Math.sin(b),k=Math.cos(x),w=Math.sin(x),M=b<=0&&x>=0||x>=Nt,S=b<=Bt&&x>=Bt||x>=Nt+Bt,C=b<=-Bt&&x>=-Bt||x>=zt+Bt,P=b===-zt||x>=zt?-1:Math.min(y,y*p,k,k*p),A=C?-1:Math.min(_,_*p,w,w*p),D=M?1:Math.max(y,y*p,k,k*p),T=S?1:Math.max(_,_*p,w,w*p);u=(D-P)/2,d=(T-A)/2,c=-(D+P)/2,h=-(T+A)/2}for(i=0,a=g.length;i0&&!isNaN(t)?Nt*(Math.abs(t)/e):0},getMaxBorderWidth:function(t){var e,n,i,a,r,o,l,s,u=0,d=this.chart;if(!t)for(e=0,n=d.data.datasets.length;e(u=l>u?l:u)?s:u);return u},setHoverStyle:function(t){var e=t._model,n=t._options,i=j.getHoverColor;t.$previousStyle={backgroundColor:e.backgroundColor,borderColor:e.borderColor,borderWidth:e.borderWidth},e.backgroundColor=Rt(n.hoverBackgroundColor,i(n.backgroundColor)),e.borderColor=Rt(n.hoverBorderColor,i(n.borderColor)),e.borderWidth=Rt(n.hoverBorderWidth,n.borderWidth)},_getRingWeightOffset:function(t){for(var e=0,n=0;n0&&jt(s[t-1]._model,l)&&(n.controlPointPreviousX=u(n.controlPointPreviousX,l.left,l.right),n.controlPointPreviousY=u(n.controlPointPreviousY,l.top,l.bottom)),t0&&(r=t.getDatasetMeta(r[0]._datasetIndex).data),r},\"x-axis\":function(t,e){return ie(t,e,{intersect:!1})},point:function(t,e){return te(t,Jt(e,t))},nearest:function(t,e,n){var i=Jt(e,t);n.axis=n.axis||\"xy\";var a=ne(n.axis);return ee(t,i,n.intersect,a)},x:function(t,e,n){var i=Jt(e,t),a=[],r=!1;return Qt(t,(function(t){t.inXRange(i.x)&&a.push(t),t.inRange(i.x,i.y)&&(r=!0)})),n.intersect&&!r&&(a=[]),a},y:function(t,e,n){var i=Jt(e,t),a=[],r=!1;return Qt(t,(function(t){t.inYRange(i.y)&&a.push(t),t.inRange(i.x,i.y)&&(r=!0)})),n.intersect&&!r&&(a=[]),a}}},re=j.extend;function oe(t,e){return j.where(t,(function(t){return t.pos===e}))}function le(t,e){return t.sort((function(t,n){var i=e?n:t,a=e?t:n;return i.weight===a.weight?i.index-a.index:i.weight-a.weight}))}function se(t,e,n,i){return Math.max(t[n],e[n])+Math.max(t[i],e[i])}function ue(t,e,n){var i,a,r=n.box,o=t.maxPadding;if(n.size&&(t[n.pos]-=n.size),n.size=n.horizontal?r.height:r.width,t[n.pos]+=n.size,r.getPadding){var l=r.getPadding();o.top=Math.max(o.top,l.top),o.left=Math.max(o.left,l.left),o.bottom=Math.max(o.bottom,l.bottom),o.right=Math.max(o.right,l.right)}if(i=e.outerWidth-se(o,t,\"left\",\"right\"),a=e.outerHeight-se(o,t,\"top\",\"bottom\"),i!==t.w||a!==t.h){t.w=i,t.h=a;var s=n.horizontal?[i,t.w]:[a,t.h];return!(s[0]===s[1]||isNaN(s[0])&&isNaN(s[1]))}}function de(t,e){var n=e.maxPadding;function i(t){var i={left:0,top:0,right:0,bottom:0};return t.forEach((function(t){i[t]=Math.max(e[t],n[t])})),i}return i(t?[\"left\",\"right\"]:[\"top\",\"bottom\"])}function ce(t,e,n){var i,a,r,o,l,s,u=[];for(i=0,a=t.length;i div {\\r\\n\\tposition: absolute;\\r\\n\\twidth: 1000000px;\\r\\n\\theight: 1000000px;\\r\\n\\tleft: 0;\\r\\n\\ttop: 0;\\r\\n}\\r\\n\\r\\n.chartjs-size-monitor-shrink > div {\\r\\n\\tposition: absolute;\\r\\n\\twidth: 200%;\\r\\n\\theight: 200%;\\r\\n\\tleft: 0;\\r\\n\\ttop: 0;\\r\\n}\\r\\n\"}))&&fe.default||fe,me=\"$chartjs\",ve=\"chartjs-\",be=ve+\"size-monitor\",xe=ve+\"render-monitor\",ye=ve+\"render-animation\",_e=[\"animationstart\",\"webkitAnimationStart\"],ke={touchstart:\"mousedown\",touchmove:\"mousemove\",touchend:\"mouseup\",pointerenter:\"mouseenter\",pointerdown:\"mousedown\",pointermove:\"mousemove\",pointerup:\"mouseup\",pointerleave:\"mouseout\",pointerout:\"mouseout\"};function we(t,e){var n=j.getStyle(t,e),i=n&&n.match(/^(\\d+)(\\.\\d+)?px$/);return i?Number(i[1]):void 0}var Me=function(){var t=!1;try{var e=Object.defineProperty({},\"passive\",{get:function(){t=!0}});window.addEventListener(\"e\",null,e)}catch(t){}return t}(),Se=!!Me&&{passive:!0};function Ce(t,e,n){t.addEventListener(e,n,Se)}function Pe(t,e,n){t.removeEventListener(e,n,Se)}function Ae(t,e,n,i,a){return{type:t,chart:e,native:a||null,x:void 0!==n?n:null,y:void 0!==i?i:null}}function De(t){var e=document.createElement(\"div\");return e.className=t||\"\",e}function Te(t,e,n){var i,a,r,o,l=t[me]||(t[me]={}),s=l.resizer=function(t){var e=1e6,n=De(be),i=De(be+\"-expand\"),a=De(be+\"-shrink\");i.appendChild(De()),a.appendChild(De()),n.appendChild(i),n.appendChild(a),n._reset=function(){i.scrollLeft=e,i.scrollTop=e,a.scrollLeft=e,a.scrollTop=e};var r=function(){n._reset(),t()};return Ce(i,\"scroll\",r.bind(i,\"expand\")),Ce(a,\"scroll\",r.bind(a,\"shrink\")),n}((i=function(){if(l.resizer){var i=n.options.maintainAspectRatio&&t.parentNode,a=i?i.clientWidth:0;e(Ae(\"resize\",n)),i&&i.clientWidth0){var r=t[0];r.label?n=r.label:r.xLabel?n=r.xLabel:a>0&&r.index-1?t.split(\"\\n\"):t}function He(t){var e=N.global;return{xPadding:t.xPadding,yPadding:t.yPadding,xAlign:t.xAlign,yAlign:t.yAlign,rtl:t.rtl,textDirection:t.textDirection,bodyFontColor:t.bodyFontColor,_bodyFontFamily:Ne(t.bodyFontFamily,e.defaultFontFamily),_bodyFontStyle:Ne(t.bodyFontStyle,e.defaultFontStyle),_bodyAlign:t.bodyAlign,bodyFontSize:Ne(t.bodyFontSize,e.defaultFontSize),bodySpacing:t.bodySpacing,titleFontColor:t.titleFontColor,_titleFontFamily:Ne(t.titleFontFamily,e.defaultFontFamily),_titleFontStyle:Ne(t.titleFontStyle,e.defaultFontStyle),titleFontSize:Ne(t.titleFontSize,e.defaultFontSize),_titleAlign:t.titleAlign,titleSpacing:t.titleSpacing,titleMarginBottom:t.titleMarginBottom,footerFontColor:t.footerFontColor,_footerFontFamily:Ne(t.footerFontFamily,e.defaultFontFamily),_footerFontStyle:Ne(t.footerFontStyle,e.defaultFontStyle),footerFontSize:Ne(t.footerFontSize,e.defaultFontSize),_footerAlign:t.footerAlign,footerSpacing:t.footerSpacing,footerMarginTop:t.footerMarginTop,caretSize:t.caretSize,cornerRadius:t.cornerRadius,backgroundColor:t.backgroundColor,opacity:0,legendColorBackground:t.multiKeyBackground,displayColors:t.displayColors,borderColor:t.borderColor,borderWidth:t.borderWidth}}function je(t,e){return\"center\"===e?t.x+t.width/2:\"right\"===e?t.x+t.width-t.xPadding:t.x+t.xPadding}function qe(t){return We([],Ve(t))}var Ue=Z.extend({initialize:function(){this._model=He(this._options),this._lastActive=[]},getTitle:function(){var t=this,e=t._options.callbacks,n=e.beforeTitle.apply(t,arguments),i=e.title.apply(t,arguments),a=e.afterTitle.apply(t,arguments),r=[];return r=We(r,Ve(n)),r=We(r,Ve(i)),r=We(r,Ve(a))},getBeforeBody:function(){return qe(this._options.callbacks.beforeBody.apply(this,arguments))},getBody:function(t,e){var n=this,i=n._options.callbacks,a=[];return j.each(t,(function(t){var r={before:[],lines:[],after:[]};We(r.before,Ve(i.beforeLabel.call(n,t,e))),We(r.lines,i.label.call(n,t,e)),We(r.after,Ve(i.afterLabel.call(n,t,e))),a.push(r)})),a},getAfterBody:function(){return qe(this._options.callbacks.afterBody.apply(this,arguments))},getFooter:function(){var t=this,e=t._options.callbacks,n=e.beforeFooter.apply(t,arguments),i=e.footer.apply(t,arguments),a=e.afterFooter.apply(t,arguments),r=[];return r=We(r,Ve(n)),r=We(r,Ve(i)),r=We(r,Ve(a))},update:function(t){var e,n,i,a,r,o,l,s,u,d,c=this,h=c._options,f=c._model,g=c._model=He(h),p=c._active,m=c._data,v={xAlign:f.xAlign,yAlign:f.yAlign},b={x:f.x,y:f.y},x={width:f.width,height:f.height},y={x:f.caretX,y:f.caretY};if(p.length){g.opacity=1;var _=[],k=[];y=Ee[h.position].call(c,p,c._eventPosition);var w=[];for(e=0,n=p.length;es.height-e.height&&(c=\"bottom\");var h=(u.left+u.right)/2,f=(u.top+u.bottom)/2;\"center\"===c?(n=function(t){return t<=h},i=function(t){return t>h}):(n=function(t){return t<=e.width/2},i=function(t){return t>=s.width-e.width/2}),a=function(t){return t+e.width+l.caretSize+l.caretPadding>s.width},r=function(t){return t-e.width-l.caretSize-l.caretPadding<0},o=function(t){return t<=f?\"top\":\"bottom\"},n(l.x)?(d=\"left\",a(l.x)&&(d=\"center\",c=o(l.y))):i(l.x)&&(d=\"right\",r(l.x)&&(d=\"center\",c=o(l.y)));var g=t._options;return{xAlign:g.xAlign?g.xAlign:d,yAlign:g.yAlign?g.yAlign:c}}(this,x),b=function(t,e,n,i){var a=t.x,r=t.y,o=t.caretSize,l=t.caretPadding,s=t.cornerRadius,u=n.xAlign,d=n.yAlign,c=o+l,h=s+l;return\"right\"===u?a-=e.width:\"center\"===u&&((a-=e.width/2)+e.width>i.width&&(a=i.width-e.width),a<0&&(a=0)),\"top\"===d?r+=c:r-=\"bottom\"===d?e.height+c:e.height/2,\"center\"===d?\"left\"===u?a+=c:\"right\"===u&&(a-=c):\"left\"===u?a-=h:\"right\"===u&&(a+=h),{x:a,y:r}}(g,x,v,c._chart)}else g.opacity=0;return g.xAlign=v.xAlign,g.yAlign=v.yAlign,g.x=b.x,g.y=b.y,g.width=x.width,g.height=x.height,g.caretX=y.x,g.caretY=y.y,c._model=g,t&&h.custom&&h.custom.call(c,g),c},drawCaret:function(t,e){var n=this._chart.ctx,i=this._view,a=this.getCaretPosition(t,e,i);n.lineTo(a.x1,a.y1),n.lineTo(a.x2,a.y2),n.lineTo(a.x3,a.y3)},getCaretPosition:function(t,e,n){var i,a,r,o,l,s,u=n.caretSize,d=n.cornerRadius,c=n.xAlign,h=n.yAlign,f=t.x,g=t.y,p=e.width,m=e.height;if(\"center\"===h)l=g+m/2,\"left\"===c?(a=(i=f)-u,r=i,o=l+u,s=l-u):(a=(i=f+p)+u,r=i,o=l-u,s=l+u);else if(\"left\"===c?(i=(a=f+d+u)-u,r=a+u):\"right\"===c?(i=(a=f+p-d-u)-u,r=a+u):(i=(a=n.caretX)-u,r=a+u),\"top\"===h)l=(o=g)-u,s=o;else{l=(o=g+m)+u,s=o;var v=r;r=i,i=v}return{x1:i,x2:a,x3:r,y1:o,y2:l,y3:s}},drawTitle:function(t,e,n){var i,a,r,o=e.title,l=o.length;if(l){var s=Be(e.rtl,e.x,e.width);for(t.x=je(e,e._titleAlign),n.textAlign=s.textAlign(e._titleAlign),n.textBaseline=\"middle\",i=e.titleFontSize,a=e.titleSpacing,n.fillStyle=e.titleFontColor,n.font=j.fontString(i,e._titleFontStyle,e._titleFontFamily),r=0;r0&&n.stroke()},draw:function(){var t=this._chart.ctx,e=this._view;if(0!==e.opacity){var n={width:e.width,height:e.height},i={x:e.x,y:e.y},a=Math.abs(e.opacity<.001)?0:e.opacity,r=e.title.length||e.beforeBody.length||e.body.length||e.afterBody.length||e.footer.length;this._options.enabled&&r&&(t.save(),t.globalAlpha=a,this.drawBackground(i,e,t,n),i.y+=e.yPadding,j.rtl.overrideTextDirection(t,e.textDirection),this.drawTitle(i,e,t),this.drawBody(i,e,t),this.drawFooter(i,e,t),j.rtl.restoreTextDirection(t,e.textDirection),t.restore())}},handleEvent:function(t){var e,n=this,i=n._options;return n._lastActive=n._lastActive||[],\"mouseout\"===t.type?n._active=[]:(n._active=n._chart.getElementsAtEventForMode(t,i.mode,i),i.reverse&&n._active.reverse()),(e=!j.arrayEquals(n._active,n._lastActive))&&(n._lastActive=n._active,(i.enabled||i.custom)&&(n._eventPosition={x:t.x,y:t.y},n.update(!0),n.pivot())),e}}),Ye=Ee,Ge=Ue;Ge.positioners=Ye;var Xe=j.valueOrDefault;function Ke(){return j.merge(Object.create(null),[].slice.call(arguments),{merger:function(t,e,n,i){if(\"xAxes\"===t||\"yAxes\"===t){var a,r,o,l=n[t].length;for(e[t]||(e[t]=[]),a=0;a=e[t].length&&e[t].push({}),!e[t][a].type||o.type&&o.type!==e[t][a].type?j.merge(e[t][a],[ze.getScaleDefaults(r),o]):j.merge(e[t][a],o)}else j._merger(t,e,n,i)}})}function Ze(){return j.merge(Object.create(null),[].slice.call(arguments),{merger:function(t,e,n,i){var a=e[t]||Object.create(null),r=n[t];\"scales\"===t?e[t]=Ke(a,r):\"scale\"===t?e[t]=j.merge(a,[ze.getScaleDefaults(r.type),r]):j._merger(t,e,n,i)}})}function $e(t,e,n){var i,a=function(t){return t.id===i};do{i=e+n++}while(j.findIndex(t,a)>=0);return i}function Je(t){return\"top\"===t||\"bottom\"===t}function Qe(t,e){return function(n,i){return n[t]===i[t]?n[e]-i[e]:n[t]-i[t]}}N._set(\"global\",{elements:{},events:[\"mousemove\",\"mouseout\",\"click\",\"touchstart\",\"touchmove\"],hover:{onHover:null,mode:\"nearest\",intersect:!0,animationDuration:400},onClick:null,maintainAspectRatio:!0,responsive:!0,responsiveAnimationDuration:0});var tn=function(t,e){return this.construct(t,e),this};j.extend(tn.prototype,{construct:function(t,e){var n=this;e=function(t){var e=(t=t||Object.create(null)).data=t.data||{};return e.datasets=e.datasets||[],e.labels=e.labels||[],t.options=Ze(N.global,N[t.type],t.options||{}),t}(e);var i=Le.acquireContext(t,e),a=i&&i.canvas,r=a&&a.height,o=a&&a.width;n.id=j.uid(),n.ctx=i,n.canvas=a,n.config=e,n.width=o,n.height=r,n.aspectRatio=r?o/r:null,n.options=e.options,n._bufferedRender=!1,n._layers=[],n.chart=n,n.controller=n,tn.instances[n.id]=n,Object.defineProperty(n,\"data\",{get:function(){return n.config.data},set:function(t){n.config.data=t}}),i&&a?(n.initialize(),n.update()):console.error(\"Failed to create chart: can't acquire context from the given item\")},initialize:function(){var t=this;return Re.notify(t,\"beforeInit\"),j.retinaScale(t,t.options.devicePixelRatio),t.bindEvents(),t.options.responsive&&t.resize(!0),t.initToolTip(),Re.notify(t,\"afterInit\"),t},clear:function(){return j.canvas.clear(this),this},stop:function(){return Q.cancelAnimation(this),this},resize:function(t){var e=this,n=e.options,i=e.canvas,a=n.maintainAspectRatio&&e.aspectRatio||null,r=Math.max(0,Math.floor(j.getMaximumWidth(i))),o=Math.max(0,Math.floor(a?r/a:j.getMaximumHeight(i)));if((e.width!==r||e.height!==o)&&(i.width=e.width=r,i.height=e.height=o,i.style.width=r+\"px\",i.style.height=o+\"px\",j.retinaScale(e,n.devicePixelRatio),!t)){var l={width:r,height:o};Re.notify(e,\"resize\",[l]),n.onResize&&n.onResize(e,l),e.stop(),e.update({duration:n.responsiveAnimationDuration})}},ensureScalesHaveIDs:function(){var t=this.options,e=t.scales||{},n=t.scale;j.each(e.xAxes,(function(t,n){t.id||(t.id=$e(e.xAxes,\"x-axis-\",n))})),j.each(e.yAxes,(function(t,n){t.id||(t.id=$e(e.yAxes,\"y-axis-\",n))})),n&&(n.id=n.id||\"scale\")},buildOrUpdateScales:function(){var t=this,e=t.options,n=t.scales||{},i=[],a=Object.keys(n).reduce((function(t,e){return t[e]=!1,t}),{});e.scales&&(i=i.concat((e.scales.xAxes||[]).map((function(t){return{options:t,dtype:\"category\",dposition:\"bottom\"}})),(e.scales.yAxes||[]).map((function(t){return{options:t,dtype:\"linear\",dposition:\"left\"}})))),e.scale&&i.push({options:e.scale,dtype:\"radialLinear\",isDefault:!0,dposition:\"chartArea\"}),j.each(i,(function(e){var i=e.options,r=i.id,o=Xe(i.type,e.dtype);Je(i.position)!==Je(e.dposition)&&(i.position=e.dposition),a[r]=!0;var l=null;if(r in n&&n[r].type===o)(l=n[r]).options=i,l.ctx=t.ctx,l.chart=t;else{var s=ze.getScaleConstructor(o);if(!s)return;l=new s({id:r,type:o,options:i,ctx:t.ctx,chart:t}),n[l.id]=l}l.mergeTicksOptions(),e.isDefault&&(t.scale=l)})),j.each(a,(function(t,e){t||delete n[e]})),t.scales=n,ze.addScalesToLayout(this)},buildOrUpdateControllers:function(){var t,e,n=this,i=[],a=n.data.datasets;for(t=0,e=a.length;t=0;--n)i.drawDataset(e[n],t);Re.notify(i,\"afterDatasetsDraw\",[t])}},drawDataset:function(t,e){var n={meta:t,index:t.index,easingValue:e};!1!==Re.notify(this,\"beforeDatasetDraw\",[n])&&(t.controller.draw(e),Re.notify(this,\"afterDatasetDraw\",[n]))},_drawTooltip:function(t){var e=this,n=e.tooltip,i={tooltip:n,easingValue:t};!1!==Re.notify(e,\"beforeTooltipDraw\",[i])&&(n.draw(),Re.notify(e,\"afterTooltipDraw\",[i]))},getElementAtEvent:function(t){return ae.modes.single(this,t)},getElementsAtEvent:function(t){return ae.modes.label(this,t,{intersect:!0})},getElementsAtXAxis:function(t){return ae.modes[\"x-axis\"](this,t,{intersect:!0})},getElementsAtEventForMode:function(t,e,n){var i=ae.modes[e];return\"function\"==typeof i?i(this,t,n):[]},getDatasetAtEvent:function(t){return ae.modes.dataset(this,t,{intersect:!0})},getDatasetMeta:function(t){var e=this,n=e.data.datasets[t];n._meta||(n._meta={});var i=n._meta[e.id];return i||(i=n._meta[e.id]={type:null,data:[],dataset:null,controller:null,hidden:null,xAxisID:null,yAxisID:null,order:n.order||0,index:t}),i},getVisibleDatasetCount:function(){for(var t=0,e=0,n=this.data.datasets.length;e3?n[2]-n[1]:n[1]-n[0];Math.abs(i)>1&&t!==Math.floor(t)&&(i=t-Math.floor(t));var a=j.log10(Math.abs(i)),r=\"\";if(0!==t)if(Math.max(Math.abs(n[0]),Math.abs(n[n.length-1]))<1e-4){var o=j.log10(Math.abs(t)),l=Math.floor(o)-Math.floor(a);l=Math.max(Math.min(l,20),0),r=t.toExponential(l)}else{var s=-1*Math.floor(a);s=Math.max(Math.min(s,20),0),r=t.toFixed(s)}else r=\"0\";return r},logarithmic:function(t,e,n){var i=t/Math.pow(10,Math.floor(j.log10(t)));return 0===t?\"0\":1===i||2===i||5===i||0===e||e===n.length-1?t.toExponential():\"\"}}},ln=j.isArray,sn=j.isNullOrUndef,un=j.valueOrDefault,dn=j.valueAtIndexOrDefault;function cn(t,e,n){var i,a=t.getTicks().length,r=Math.min(e,a-1),o=t.getPixelForTick(r),l=t._startPixel,s=t._endPixel,u=1e-6;if(!(n&&(i=1===a?Math.max(o-l,s-o):0===e?(t.getPixelForTick(1)-o)/2:(o-t.getPixelForTick(r-1))/2,(o+=rs+u)))return o}function hn(t,e,n,i){var a,r,o,l,s,u,d,c,h,f,g,p,m,v=n.length,b=[],x=[],y=[],_=0,k=0;for(a=0;ae){for(n=0;n=h||d<=1||!l.isHorizontal()?l.labelRotation=c:(e=(t=l._getLabelSizes()).widest.width,n=t.highest.height-t.highest.offset,i=Math.min(l.maxWidth,l.chart.width-e),e+6>(a=s.offset?l.maxWidth/d:i/(d-1))&&(a=i/(d-(s.offset?.5:1)),r=l.maxHeight-fn(s.gridLines)-u.padding-gn(s.scaleLabel),o=Math.sqrt(e*e+n*n),f=j.toDegrees(Math.min(Math.asin(Math.min((t.highest.height+6)/a,1)),Math.asin(Math.min(r/o,1))-Math.asin(n/o))),f=Math.max(c,Math.min(h,f))),l.labelRotation=f)},afterCalculateTickRotation:function(){j.callback(this.options.afterCalculateTickRotation,[this])},beforeFit:function(){j.callback(this.options.beforeFit,[this])},fit:function(){var t=this,e=t.minSize={width:0,height:0},n=t.chart,i=t.options,a=i.ticks,r=i.scaleLabel,o=i.gridLines,l=t._isVisible(),s=\"bottom\"===i.position,u=t.isHorizontal();if(u?e.width=t.maxWidth:l&&(e.width=fn(o)+gn(r)),u?l&&(e.height=fn(o)+gn(r)):e.height=t.maxHeight,a.display&&l){var d=mn(a),c=t._getLabelSizes(),h=c.first,f=c.last,g=c.widest,p=c.highest,m=.4*d.minor.lineHeight,v=a.padding;if(u){var b=0!==t.labelRotation,x=j.toRadians(t.labelRotation),y=Math.cos(x),_=Math.sin(x),k=_*g.width+y*(p.height-(b?p.offset:0))+(b?0:m);e.height=Math.min(t.maxHeight,e.height+k+v);var w,M,S=t.getPixelForTick(0)-t.left,C=t.right-t.getPixelForTick(t.getTicks().length-1);b?(w=s?y*h.width+_*h.offset:_*(h.height-h.offset),M=s?_*(f.height-f.offset):y*f.width+_*f.offset):(w=h.width/2,M=f.width/2),t.paddingLeft=Math.max((w-S)*t.width/(t.width-S),0)+3,t.paddingRight=Math.max((M-C)*t.width/(t.width-C),0)+3}else{var P=a.mirror?0:g.width+v+m;e.width=Math.min(t.maxWidth,e.width+P),t.paddingTop=h.height/2,t.paddingBottom=f.height/2}}t.handleMargins(),u?(t.width=t._length=n.width-t.margins.left-t.margins.right,t.height=e.height):(t.width=e.width,t.height=t._length=n.height-t.margins.top-t.margins.bottom)},handleMargins:function(){var t=this;t.margins&&(t.margins.left=Math.max(t.paddingLeft,t.margins.left),t.margins.top=Math.max(t.paddingTop,t.margins.top),t.margins.right=Math.max(t.paddingRight,t.margins.right),t.margins.bottom=Math.max(t.paddingBottom,t.margins.bottom))},afterFit:function(){j.callback(this.options.afterFit,[this])},isHorizontal:function(){var t=this.options.position;return\"top\"===t||\"bottom\"===t},isFullWidth:function(){return this.options.fullWidth},getRightValue:function(t){if(sn(t))return NaN;if((\"number\"==typeof t||t instanceof Number)&&!isFinite(t))return NaN;if(t)if(this.isHorizontal()){if(void 0!==t.x)return this.getRightValue(t.x)}else if(void 0!==t.y)return this.getRightValue(t.y);return t},_convertTicksToLabels:function(t){var e,n,i,a=this;for(a.ticks=t.map((function(t){return t.value})),a.beforeTickToLabelConversion(),e=a.convertTicksToLabels(t)||a.ticks,a.afterTickToLabelConversion(),n=0,i=t.length;ni-1?null:e.getPixelForDecimal(t*a+(n?a/2:0))},getPixelForDecimal:function(t){var e=this;return e._reversePixels&&(t=1-t),e._startPixel+t*e._length},getDecimalForPixel:function(t){var e=(t-this._startPixel)/this._length;return this._reversePixels?1-e:e},getBasePixel:function(){return this.getPixelForValue(this.getBaseValue())},getBaseValue:function(){var t=this,e=t.min,n=t.max;return t.beginAtZero?0:e<0&&n<0?n:e>0&&n>0?e:0},_autoSkip:function(t){var e,n,i,a,r=this,o=r.options.ticks,l=r._length,s=o.maxTicksLimit||l/r._tickSize()+1,u=o.major.enabled?function(t){var e,n,i=[];for(e=0,n=t.length;es)return function(t,e,n){var i,a,r=0,o=e[0];for(n=Math.ceil(n),i=0;iu)return r;return Math.max(u,1)}(u,t,0,s),d>0){for(e=0,n=d-1;e1?(h-c)/(d-1):null,bn(t,i,j.isNullOrUndef(a)?0:c-a,c),bn(t,i,h,j.isNullOrUndef(a)?t.length:h+a),vn(t)}return bn(t,i),vn(t)},_tickSize:function(){var t=this,e=t.options.ticks,n=j.toRadians(t.labelRotation),i=Math.abs(Math.cos(n)),a=Math.abs(Math.sin(n)),r=t._getLabelSizes(),o=e.autoSkipPadding||0,l=r?r.widest.width+o:0,s=r?r.highest.height+o:0;return t.isHorizontal()?s*i>l*a?l/i:s/a:s*a=0&&(o=t),void 0!==r&&(t=n.indexOf(r))>=0&&(l=t),e.minIndex=o,e.maxIndex=l,e.min=n[o],e.max=n[l]},buildTicks:function(){var t=this,e=t._getLabels(),n=t.minIndex,i=t.maxIndex;t.ticks=0===n&&i===e.length-1?e:e.slice(n,i+1)},getLabelForIndex:function(t,e){var n=this,i=n.chart;return i.getDatasetMeta(e).controller._getValueScaleId()===n.id?n.getRightValue(i.data.datasets[e].data[t]):n._getLabels()[t]},_configure:function(){var t=this,e=t.options.offset,n=t.ticks;yn.prototype._configure.call(t),t.isHorizontal()||(t._reversePixels=!t._reversePixels),n&&(t._startValue=t.minIndex-(e?.5:0),t._valueRange=Math.max(n.length-(e?0:1),1))},getPixelForValue:function(t,e,n){var i,a,r,o=this;return _n(e)||_n(n)||(t=o.chart.data.datasets[n].data[e]),_n(t)||(i=o.isHorizontal()?t.x:t.y),(void 0!==i||void 0!==t&&isNaN(e))&&(a=o._getLabels(),t=j.valueOrDefault(i,t),e=-1!==(r=a.indexOf(t))?r:e,isNaN(e)&&(e=t)),o.getPixelForDecimal((e-o._startValue)/o._valueRange)},getPixelForTick:function(t){var e=this.ticks;return t<0||t>e.length-1?null:this.getPixelForValue(e[t],t+this.minIndex)},getValueForPixel:function(t){var e=this,n=Math.round(e._startValue+e.getDecimalForPixel(t)*e._valueRange);return Math.min(Math.max(n,0),e.ticks.length-1)},getBasePixel:function(){return this.bottom}}),wn={position:\"bottom\"};kn._defaults=wn;var Mn=j.noop,Sn=j.isNullOrUndef;var Cn=yn.extend({getRightValue:function(t){return\"string\"==typeof t?+t:yn.prototype.getRightValue.call(this,t)},handleTickRangeOptions:function(){var t=this,e=t.options.ticks;if(e.beginAtZero){var n=j.sign(t.min),i=j.sign(t.max);n<0&&i<0?t.max=0:n>0&&i>0&&(t.min=0)}var a=void 0!==e.min||void 0!==e.suggestedMin,r=void 0!==e.max||void 0!==e.suggestedMax;void 0!==e.min?t.min=e.min:void 0!==e.suggestedMin&&(null===t.min?t.min=e.suggestedMin:t.min=Math.min(t.min,e.suggestedMin)),void 0!==e.max?t.max=e.max:void 0!==e.suggestedMax&&(null===t.max?t.max=e.suggestedMax:t.max=Math.max(t.max,e.suggestedMax)),a!==r&&t.min>=t.max&&(a?t.max=t.min+1:t.min=t.max-1),t.min===t.max&&(t.max++,e.beginAtZero||t.min--)},getTickLimit:function(){var t,e=this,n=e.options.ticks,i=n.stepSize,a=n.maxTicksLimit;return i?t=Math.ceil(e.max/i)-Math.floor(e.min/i)+1:(t=e._computeTickLimit(),a=a||11),a&&(t=Math.min(a,t)),t},_computeTickLimit:function(){return Number.POSITIVE_INFINITY},handleDirectionalChanges:Mn,buildTicks:function(){var t=this,e=t.options.ticks,n=t.getTickLimit(),i={maxTicks:n=Math.max(2,n),min:e.min,max:e.max,precision:e.precision,stepSize:j.valueOrDefault(e.fixedStepSize,e.stepSize)},a=t.ticks=function(t,e){var n,i,a,r,o=[],l=t.stepSize,s=l||1,u=t.maxTicks-1,d=t.min,c=t.max,h=t.precision,f=e.min,g=e.max,p=j.niceNum((g-f)/u/s)*s;if(p<1e-14&&Sn(d)&&Sn(c))return[f,g];(r=Math.ceil(g/p)-Math.floor(f/p))>u&&(p=j.niceNum(r*p/u/s)*s),l||Sn(h)?n=Math.pow(10,j._decimalPlaces(p)):(n=Math.pow(10,h),p=Math.ceil(p*n)/n),i=Math.floor(f/p)*p,a=Math.ceil(g/p)*p,l&&(!Sn(d)&&j.almostWhole(d/p,p/1e3)&&(i=d),!Sn(c)&&j.almostWhole(c/p,p/1e3)&&(a=c)),r=(a-i)/p,r=j.almostEquals(r,Math.round(r),p/1e3)?Math.round(r):Math.ceil(r),i=Math.round(i*n)/n,a=Math.round(a*n)/n,o.push(Sn(d)?i:d);for(var m=1;me.length-1?null:this.getPixelForValue(e[t])}}),In=Pn;Tn._defaults=In;var Fn=j.valueOrDefault,On=j.math.log10;var Ln={position:\"left\",ticks:{callback:on.formatters.logarithmic}};function Rn(t,e){return j.isFinite(t)&&t>=0?t:e}var zn=yn.extend({determineDataLimits:function(){var t,e,n,i,a,r,o=this,l=o.options,s=o.chart,u=s.data.datasets,d=o.isHorizontal();function c(t){return d?t.xAxisID===o.id:t.yAxisID===o.id}o.min=Number.POSITIVE_INFINITY,o.max=Number.NEGATIVE_INFINITY,o.minNotZero=Number.POSITIVE_INFINITY;var h=l.stacked;if(void 0===h)for(t=0;t0){var e=j.min(t),n=j.max(t);o.min=Math.min(o.min,e),o.max=Math.max(o.max,n)}}))}else for(t=0;t0?t.minNotZero=t.min:t.max<1?t.minNotZero=Math.pow(10,Math.floor(On(t.max))):t.minNotZero=1)},buildTicks:function(){var t=this,e=t.options.ticks,n=!t.isHorizontal(),i={min:Rn(e.min),max:Rn(e.max)},a=t.ticks=function(t,e){var n,i,a=[],r=Fn(t.min,Math.pow(10,Math.floor(On(e.min)))),o=Math.floor(On(e.max)),l=Math.ceil(e.max/Math.pow(10,o));0===r?(n=Math.floor(On(e.minNotZero)),i=Math.floor(e.minNotZero/Math.pow(10,n)),a.push(r),r=i*Math.pow(10,n)):(n=Math.floor(On(r)),i=Math.floor(r/Math.pow(10,n)));var s=n<0?Math.pow(10,Math.abs(n)):1;do{a.push(r),10==++i&&(i=1,s=++n>=0?1:s),r=Math.round(i*Math.pow(10,n)*s)/s}while(ne.length-1?null:this.getPixelForValue(e[t])},_getFirstTickValue:function(t){var e=Math.floor(On(t));return Math.floor(t/Math.pow(10,e))*Math.pow(10,e)},_configure:function(){var t=this,e=t.min,n=0;yn.prototype._configure.call(t),0===e&&(e=t._getFirstTickValue(t.minNotZero),n=Fn(t.options.ticks.fontSize,N.global.defaultFontSize)/t._length),t._startValue=On(e),t._valueOffset=n,t._valueRange=(On(t.max)-On(e))/(1-n)},getPixelForValue:function(t){var e=this,n=0;return(t=+e.getRightValue(t))>e.min&&t>0&&(n=(On(t)-e._startValue)/e._valueRange+e._valueOffset),e.getPixelForDecimal(n)},getValueForPixel:function(t){var e=this,n=e.getDecimalForPixel(t);return 0===n&&0===e.min?0:Math.pow(10,e._startValue+(n-e._valueOffset)*e._valueRange)}}),Nn=Ln;zn._defaults=Nn;var Bn=j.valueOrDefault,En=j.valueAtIndexOrDefault,Wn=j.options.resolve,Vn={display:!0,animate:!0,position:\"chartArea\",angleLines:{display:!0,color:\"rgba(0,0,0,0.1)\",lineWidth:1,borderDash:[],borderDashOffset:0},gridLines:{circular:!1},ticks:{showLabelBackdrop:!0,backdropColor:\"rgba(255,255,255,0.75)\",backdropPaddingY:2,backdropPaddingX:2,callback:on.formatters.linear},pointLabels:{display:!0,fontSize:10,callback:function(t){return t}}};function Hn(t){var e=t.ticks;return e.display&&t.display?Bn(e.fontSize,N.global.defaultFontSize)+2*e.backdropPaddingY:0}function jn(t,e,n,i,a){return t===i||t===a?{start:e-n/2,end:e+n/2}:ta?{start:e-n,end:e}:{start:e,end:e+n}}function qn(t){return 0===t||180===t?\"center\":t<180?\"left\":\"right\"}function Un(t,e,n,i){var a,r,o=n.y+i/2;if(j.isArray(e))for(a=0,r=e.length;a270||t<90)&&(n.y-=e.h)}function Gn(t){return j.isNumber(t)?t:0}var Xn=Cn.extend({setDimensions:function(){var t=this;t.width=t.maxWidth,t.height=t.maxHeight,t.paddingTop=Hn(t.options)/2,t.xCenter=Math.floor(t.width/2),t.yCenter=Math.floor((t.height-t.paddingTop)/2),t.drawingArea=Math.min(t.height-t.paddingTop,t.width)/2},determineDataLimits:function(){var t=this,e=t.chart,n=Number.POSITIVE_INFINITY,i=Number.NEGATIVE_INFINITY;j.each(e.data.datasets,(function(a,r){if(e.isDatasetVisible(r)){var o=e.getDatasetMeta(r);j.each(a.data,(function(e,a){var r=+t.getRightValue(e);isNaN(r)||o.data[a].hidden||(n=Math.min(r,n),i=Math.max(r,i))}))}})),t.min=n===Number.POSITIVE_INFINITY?0:n,t.max=i===Number.NEGATIVE_INFINITY?0:i,t.handleTickRangeOptions()},_computeTickLimit:function(){return Math.ceil(this.drawingArea/Hn(this.options))},convertTicksToLabels:function(){var t=this;Cn.prototype.convertTicksToLabels.call(t),t.pointLabels=t.chart.data.labels.map((function(){var e=j.callback(t.options.pointLabels.callback,arguments,t);return e||0===e?e:\"\"}))},getLabelForIndex:function(t,e){return+this.getRightValue(this.chart.data.datasets[e].data[t])},fit:function(){var t=this,e=t.options;e.display&&e.pointLabels.display?function(t){var e,n,i,a=j.options._parseFont(t.options.pointLabels),r={l:0,r:t.width,t:0,b:t.height-t.paddingTop},o={};t.ctx.font=a.string,t._pointLabelSizes=[];var l,s,u,d=t.chart.data.labels.length;for(e=0;er.r&&(r.r=f.end,o.r=c),g.startr.b&&(r.b=g.end,o.b=c)}t.setReductions(t.drawingArea,r,o)}(t):t.setCenterPoint(0,0,0,0)},setReductions:function(t,e,n){var i=this,a=e.l/Math.sin(n.l),r=Math.max(e.r-i.width,0)/Math.sin(n.r),o=-e.t/Math.cos(n.t),l=-Math.max(e.b-(i.height-i.paddingTop),0)/Math.cos(n.b);a=Gn(a),r=Gn(r),o=Gn(o),l=Gn(l),i.drawingArea=Math.min(Math.floor(t-(a+r)/2),Math.floor(t-(o+l)/2)),i.setCenterPoint(a,r,o,l)},setCenterPoint:function(t,e,n,i){var a=this,r=a.width-e-a.drawingArea,o=t+a.drawingArea,l=n+a.drawingArea,s=a.height-a.paddingTop-i-a.drawingArea;a.xCenter=Math.floor((o+r)/2+a.left),a.yCenter=Math.floor((l+s)/2+a.top+a.paddingTop)},getIndexAngle:function(t){var e=this.chart,n=(t*(360/e.data.labels.length)+((e.options||{}).startAngle||0))%360;return(n<0?n+360:n)*Math.PI*2/360},getDistanceFromCenterForValue:function(t){var e=this;if(j.isNullOrUndef(t))return NaN;var n=e.drawingArea/(e.max-e.min);return e.options.ticks.reverse?(e.max-t)*n:(t-e.min)*n},getPointPosition:function(t,e){var n=this,i=n.getIndexAngle(t)-Math.PI/2;return{x:Math.cos(i)*e+n.xCenter,y:Math.sin(i)*e+n.yCenter}},getPointPositionForValue:function(t,e){return this.getPointPosition(t,this.getDistanceFromCenterForValue(e))},getBasePosition:function(t){var e=this,n=e.min,i=e.max;return e.getPointPositionForValue(t||0,e.beginAtZero?0:n<0&&i<0?i:n>0&&i>0?n:0)},_drawGrid:function(){var t,e,n,i=this,a=i.ctx,r=i.options,o=r.gridLines,l=r.angleLines,s=Bn(l.lineWidth,o.lineWidth),u=Bn(l.color,o.color);if(r.pointLabels.display&&function(t){var e=t.ctx,n=t.options,i=n.pointLabels,a=Hn(n),r=t.getDistanceFromCenterForValue(n.ticks.reverse?t.min:t.max),o=j.options._parseFont(i);e.save(),e.font=o.string,e.textBaseline=\"middle\";for(var l=t.chart.data.labels.length-1;l>=0;l--){var s=0===l?a/2:0,u=t.getPointPosition(l,r+s+5),d=En(i.fontColor,l,N.global.defaultFontColor);e.fillStyle=d;var c=t.getIndexAngle(l),h=j.toDegrees(c);e.textAlign=qn(h),Yn(h,t._pointLabelSizes[l],u),Un(e,t.pointLabels[l],u,o.lineHeight)}e.restore()}(i),o.display&&j.each(i.ticks,(function(t,n){0!==n&&(e=i.getDistanceFromCenterForValue(i.ticksAsNumbers[n]),function(t,e,n,i){var a,r=t.ctx,o=e.circular,l=t.chart.data.labels.length,s=En(e.color,i-1),u=En(e.lineWidth,i-1);if((o||l)&&s&&u){if(r.save(),r.strokeStyle=s,r.lineWidth=u,r.setLineDash&&(r.setLineDash(e.borderDash||[]),r.lineDashOffset=e.borderDashOffset||0),r.beginPath(),o)r.arc(t.xCenter,t.yCenter,n,0,2*Math.PI);else{a=t.getPointPosition(0,n),r.moveTo(a.x,a.y);for(var d=1;d=0;t--)e=i.getDistanceFromCenterForValue(r.ticks.reverse?i.min:i.max),n=i.getPointPosition(t,e),a.beginPath(),a.moveTo(i.xCenter,i.yCenter),a.lineTo(n.x,n.y),a.stroke();a.restore()}},_drawLabels:function(){var t=this,e=t.ctx,n=t.options.ticks;if(n.display){var i,a,r=t.getIndexAngle(0),o=j.options._parseFont(n),l=Bn(n.fontColor,N.global.defaultFontColor);e.save(),e.font=o.string,e.translate(t.xCenter,t.yCenter),e.rotate(r),e.textAlign=\"center\",e.textBaseline=\"middle\",j.each(t.ticks,(function(r,s){(0!==s||n.reverse)&&(i=t.getDistanceFromCenterForValue(t.ticksAsNumbers[s]),n.showLabelBackdrop&&(a=e.measureText(r).width,e.fillStyle=n.backdropColor,e.fillRect(-a/2-n.backdropPaddingX,-i-o.size/2-n.backdropPaddingY,a+2*n.backdropPaddingX,o.size+2*n.backdropPaddingY)),e.fillStyle=l,e.fillText(r,0,-i))})),e.restore()}},_drawTitle:j.noop}),Kn=Vn;Xn._defaults=Kn;var Zn=j._deprecated,$n=j.options.resolve,Jn=j.valueOrDefault,Qn=Number.MIN_SAFE_INTEGER||-9007199254740991,ti=Number.MAX_SAFE_INTEGER||9007199254740991,ei={millisecond:{common:!0,size:1,steps:1e3},second:{common:!0,size:1e3,steps:60},minute:{common:!0,size:6e4,steps:60},hour:{common:!0,size:36e5,steps:24},day:{common:!0,size:864e5,steps:30},week:{common:!1,size:6048e5,steps:4},month:{common:!0,size:2628e6,steps:12},quarter:{common:!1,size:7884e6,steps:4},year:{common:!0,size:3154e7}},ni=Object.keys(ei);function ii(t,e){return t-e}function ai(t){return j.valueOrDefault(t.time.min,t.ticks.min)}function ri(t){return j.valueOrDefault(t.time.max,t.ticks.max)}function oi(t,e,n,i){var a=function(t,e,n){for(var i,a,r,o=0,l=t.length-1;o>=0&&o<=l;){if(a=t[(i=o+l>>1)-1]||null,r=t[i],!a)return{lo:null,hi:r};if(r[e]n))return{lo:a,hi:r};l=i-1}}return{lo:r,hi:null}}(t,e,n),r=a.lo?a.hi?a.lo:t[t.length-2]:t[0],o=a.lo?a.hi?a.hi:t[t.length-1]:t[1],l=o[e]-r[e],s=l?(n-r[e])/l:0,u=(o[i]-r[i])*s;return r[i]+u}function li(t,e){var n=t._adapter,i=t.options.time,a=i.parser,r=a||i.format,o=e;return\"function\"==typeof a&&(o=a(o)),j.isFinite(o)||(o=\"string\"==typeof r?n.parse(o,r):n.parse(o)),null!==o?+o:(a||\"function\"!=typeof r||(o=r(e),j.isFinite(o)||(o=n.parse(o))),o)}function si(t,e){if(j.isNullOrUndef(e))return null;var n=t.options.time,i=li(t,t.getRightValue(e));return null===i||n.round&&(i=+t._adapter.startOf(i,n.round)),i}function ui(t,e,n,i){var a,r,o,l=ni.length;for(a=ni.indexOf(t);a=0&&(e[r].major=!0);return e}(t,r,o,n):r}var ci=yn.extend({initialize:function(){this.mergeTicksOptions(),yn.prototype.initialize.call(this)},update:function(){var t=this,e=t.options,n=e.time||(e.time={}),i=t._adapter=new rn._date(e.adapters.date);return Zn(\"time scale\",n.format,\"time.format\",\"time.parser\"),Zn(\"time scale\",n.min,\"time.min\",\"ticks.min\"),Zn(\"time scale\",n.max,\"time.max\",\"ticks.max\"),j.mergeIf(n.displayFormats,i.formats()),yn.prototype.update.apply(t,arguments)},getRightValue:function(t){return t&&void 0!==t.t&&(t=t.t),yn.prototype.getRightValue.call(this,t)},determineDataLimits:function(){var t,e,n,i,a,r,o,l=this,s=l.chart,u=l._adapter,d=l.options,c=d.time.unit||\"day\",h=ti,f=Qn,g=[],p=[],m=[],v=l._getLabels();for(t=0,n=v.length;t1?function(t){var e,n,i,a={},r=[];for(e=0,n=t.length;e1e5*u)throw e+\" and \"+n+\" are too far apart with stepSize of \"+u+\" \"+s;for(a=c;a=a&&n<=r&&d.push(n);return i.min=a,i.max=r,i._unit=s.unit||(l.autoSkip?ui(s.minUnit,i.min,i.max,c):function(t,e,n,i,a){var r,o;for(r=ni.length-1;r>=ni.indexOf(n);r--)if(o=ni[r],ei[o].common&&t._adapter.diff(a,i,o)>=e-1)return o;return ni[n?ni.indexOf(n):0]}(i,d.length,s.minUnit,i.min,i.max)),i._majorUnit=l.major.enabled&&\"year\"!==i._unit?function(t){for(var e=ni.indexOf(t)+1,n=ni.length;ee&&l=0&&t0?l:1}}),hi={position:\"bottom\",distribution:\"linear\",bounds:\"data\",adapters:{},time:{parser:!1,unit:!1,round:!1,displayFormat:!1,isoWeekday:!1,minUnit:\"millisecond\",displayFormats:{}},ticks:{autoSkip:!1,source:\"auto\",major:{enabled:!1}}};ci._defaults=hi;var fi={category:kn,linear:Tn,logarithmic:zn,radialLinear:Xn,time:ci},gi={datetime:\"MMM D, YYYY, h:mm:ss a\",millisecond:\"h:mm:ss.SSS a\",second:\"h:mm:ss a\",minute:\"h:mm a\",hour:\"hA\",day:\"MMM D\",week:\"ll\",month:\"MMM YYYY\",quarter:\"[Q]Q - YYYY\",year:\"YYYY\"};rn._date.override(\"function\"==typeof t?{_id:\"moment\",formats:function(){return gi},parse:function(e,n){return\"string\"==typeof e&&\"string\"==typeof n?e=t(e,n):e instanceof t||(e=t(e)),e.isValid()?e.valueOf():null},format:function(e,n){return t(e).format(n)},add:function(e,n,i){return t(e).add(n,i).valueOf()},diff:function(e,n,i){return t(e).diff(t(n),i)},startOf:function(e,n,i){return e=t(e),\"isoWeek\"===n?e.isoWeekday(i).valueOf():e.startOf(n).valueOf()},endOf:function(e,n){return t(e).endOf(n).valueOf()},_create:function(e){return t(e)}}:{}),N._set(\"global\",{plugins:{filler:{propagate:!0}}});var pi={dataset:function(t){var e=t.fill,n=t.chart,i=n.getDatasetMeta(e),a=i&&n.isDatasetVisible(e)&&i.dataset._children||[],r=a.length||0;return r?function(t,e){return e=n)&&i;switch(r){case\"bottom\":return\"start\";case\"top\":return\"end\";case\"zero\":return\"origin\";case\"origin\":case\"start\":case\"end\":return r;default:return!1}}function vi(t){return(t.el._scale||{}).getPointPositionForValue?function(t){var e,n,i,a,r,o=t.el._scale,l=o.options,s=o.chart.data.labels.length,u=t.fill,d=[];if(!s)return null;for(e=l.ticks.reverse?o.max:o.min,n=l.ticks.reverse?o.min:o.max,i=o.getPointPositionForValue(0,e),a=0;a0;--r)j.canvas.lineTo(t,n[r],n[r-1],!0);else for(o=n[0].cx,l=n[0].cy,s=Math.sqrt(Math.pow(n[0].x-o,2)+Math.pow(n[0].y-l,2)),r=a-1;r>0;--r)t.arc(o,l,s,n[r].angle,n[r-1].angle,!0)}}function ki(t,e,n,i,a,r){var o,l,s,u,d,c,h,f,g=e.length,p=i.spanGaps,m=[],v=[],b=0,x=0;for(t.beginPath(),o=0,l=g;o=0;--n)(e=s[n].$filler)&&e.visible&&(a=(i=e.el)._view,r=i._children||[],o=e.mapper,l=a.backgroundColor||N.global.defaultColor,o&&l&&r.length&&(j.canvas.clipArea(u,t.chartArea),ki(u,r,o,a,l,i._loop),j.canvas.unclipArea(u)))}},Mi=j.rtl.getRtlAdapter,Si=j.noop,Ci=j.valueOrDefault;function Pi(t,e){return t.usePointStyle&&t.boxWidth>e?e:t.boxWidth}N._set(\"global\",{legend:{display:!0,position:\"top\",align:\"center\",fullWidth:!0,reverse:!1,weight:1e3,onClick:function(t,e){var n=e.datasetIndex,i=this.chart,a=i.getDatasetMeta(n);a.hidden=null===a.hidden?!i.data.datasets[n].hidden:null,i.update()},onHover:null,onLeave:null,labels:{boxWidth:40,padding:10,generateLabels:function(t){var e=t.data.datasets,n=t.options.legend||{},i=n.labels&&n.labels.usePointStyle;return t._getSortedDatasetMetas().map((function(n){var a=n.controller.getStyle(i?0:void 0);return{text:e[n.index].label,fillStyle:a.backgroundColor,hidden:!t.isDatasetVisible(n.index),lineCap:a.borderCapStyle,lineDash:a.borderDash,lineDashOffset:a.borderDashOffset,lineJoin:a.borderJoinStyle,lineWidth:a.borderWidth,strokeStyle:a.borderColor,pointStyle:a.pointStyle,rotation:a.rotation,datasetIndex:n.index}}),this)}}},legendCallback:function(t){var e,n,i,a=document.createElement(\"ul\"),r=t.data.datasets;for(a.setAttribute(\"class\",t.id+\"-legend\"),e=0,n=r.length;es.width)&&(c+=o+n.padding,d[d.length-(e>0?0:1)]=0),l[e]={left:0,top:0,width:i,height:o},d[d.length-1]+=i+n.padding})),s.height+=c}else{var h=n.padding,f=t.columnWidths=[],g=t.columnHeights=[],p=n.padding,m=0,v=0;j.each(t.legendItems,(function(t,e){var i=Pi(n,o)+o/2+a.measureText(t.text).width;e>0&&v+o+2*h>s.height&&(p+=m+n.padding,f.push(m),g.push(v),m=0,v=0),m=Math.max(m,i),v+=o+h,l[e]={left:0,top:0,width:i,height:o}})),p+=m,f.push(m),g.push(v),s.width+=p}t.width=s.width,t.height=s.height}else t.width=s.width=t.height=s.height=0},afterFit:Si,isHorizontal:function(){return\"top\"===this.options.position||\"bottom\"===this.options.position},draw:function(){var t=this,e=t.options,n=e.labels,i=N.global,a=i.defaultColor,r=i.elements.line,o=t.height,l=t.columnHeights,s=t.width,u=t.lineWidths;if(e.display){var d,c=Mi(e.rtl,t.left,t.minSize.width),h=t.ctx,f=Ci(n.fontColor,i.defaultFontColor),g=j.options._parseFont(n),p=g.size;h.textAlign=c.textAlign(\"left\"),h.textBaseline=\"middle\",h.lineWidth=.5,h.strokeStyle=f,h.fillStyle=f,h.font=g.string;var m=Pi(n,p),v=t.legendHitBoxes,b=function(t,i){switch(e.align){case\"start\":return n.padding;case\"end\":return t-i;default:return(t-i+n.padding)/2}},x=t.isHorizontal();d=x?{x:t.left+b(s,u[0]),y:t.top+n.padding,line:0}:{x:t.left+n.padding,y:t.top+b(o,l[0]),line:0},j.rtl.overrideTextDirection(t.ctx,e.textDirection);var y=p+n.padding;j.each(t.legendItems,(function(e,i){var f=h.measureText(e.text).width,g=m+p/2+f,_=d.x,k=d.y;c.setWidth(t.minSize.width),x?i>0&&_+g+n.padding>t.left+t.minSize.width&&(k=d.y+=y,d.line++,_=d.x=t.left+b(s,u[d.line])):i>0&&k+y>t.top+t.minSize.height&&(_=d.x=_+t.columnWidths[d.line]+n.padding,d.line++,k=d.y=t.top+b(o,l[d.line]));var w=c.x(_);!function(t,e,i){if(!(isNaN(m)||m<=0)){h.save();var o=Ci(i.lineWidth,r.borderWidth);if(h.fillStyle=Ci(i.fillStyle,a),h.lineCap=Ci(i.lineCap,r.borderCapStyle),h.lineDashOffset=Ci(i.lineDashOffset,r.borderDashOffset),h.lineJoin=Ci(i.lineJoin,r.borderJoinStyle),h.lineWidth=o,h.strokeStyle=Ci(i.strokeStyle,a),h.setLineDash&&h.setLineDash(Ci(i.lineDash,r.borderDash)),n&&n.usePointStyle){var l=m*Math.SQRT2/2,s=c.xPlus(t,m/2),u=e+p/2;j.canvas.drawPoint(h,i.pointStyle,l,s,u,i.rotation)}else h.fillRect(c.leftForLtr(t,m),e,m,p),0!==o&&h.strokeRect(c.leftForLtr(t,m),e,m,p);h.restore()}}(w,k,e),v[i].left=c.leftForLtr(w,v[i].width),v[i].top=k,function(t,e,n,i){var a=p/2,r=c.xPlus(t,m+a),o=e+a;h.fillText(n.text,r,o),n.hidden&&(h.beginPath(),h.lineWidth=2,h.moveTo(r,o),h.lineTo(c.xPlus(r,i),o),h.stroke())}(w,k,e,f),x?d.x+=g+n.padding:d.y+=y})),j.rtl.restoreTextDirection(t.ctx,e.textDirection)}},_getLegendItemAt:function(t,e){var n,i,a,r=this;if(t>=r.left&&t<=r.right&&e>=r.top&&e<=r.bottom)for(a=r.legendHitBoxes,n=0;n=(i=a[n]).left&&t<=i.left+i.width&&e>=i.top&&e<=i.top+i.height)return r.legendItems[n];return null},handleEvent:function(t){var e,n=this,i=n.options,a=\"mouseup\"===t.type?\"click\":t.type;if(\"mousemove\"===a){if(!i.onHover&&!i.onLeave)return}else{if(\"click\"!==a)return;if(!i.onClick)return}e=n._getLegendItemAt(t.x,t.y),\"click\"===a?e&&i.onClick&&i.onClick.call(n,t.native,e):(i.onLeave&&e!==n._hoveredItem&&(n._hoveredItem&&i.onLeave.call(n,t.native,n._hoveredItem),n._hoveredItem=e),i.onHover&&e&&i.onHover.call(n,t.native,e))}});function Di(t,e){var n=new Ai({ctx:t.ctx,options:e,chart:t});ge.configure(t,n,e),ge.addBox(t,n),t.legend=n}var Ti={id:\"legend\",_element:Ai,beforeInit:function(t){var e=t.options.legend;e&&Di(t,e)},beforeUpdate:function(t){var e=t.options.legend,n=t.legend;e?(j.mergeIf(e,N.global.legend),n?(ge.configure(t,n,e),n.options=e):Di(t,e)):n&&(ge.removeBox(t,n),delete t.legend)},afterEvent:function(t,e){var n=t.legend;n&&n.handleEvent(e)}},Ii=j.noop;N._set(\"global\",{title:{display:!1,fontStyle:\"bold\",fullWidth:!0,padding:10,position:\"top\",text:\"\",weight:2e3}});var Fi=Z.extend({initialize:function(t){j.extend(this,t),this.legendHitBoxes=[]},beforeUpdate:Ii,update:function(t,e,n){var i=this;return i.beforeUpdate(),i.maxWidth=t,i.maxHeight=e,i.margins=n,i.beforeSetDimensions(),i.setDimensions(),i.afterSetDimensions(),i.beforeBuildLabels(),i.buildLabels(),i.afterBuildLabels(),i.beforeFit(),i.fit(),i.afterFit(),i.afterUpdate(),i.minSize},afterUpdate:Ii,beforeSetDimensions:Ii,setDimensions:function(){var t=this;t.isHorizontal()?(t.width=t.maxWidth,t.left=0,t.right=t.width):(t.height=t.maxHeight,t.top=0,t.bottom=t.height),t.paddingLeft=0,t.paddingTop=0,t.paddingRight=0,t.paddingBottom=0,t.minSize={width:0,height:0}},afterSetDimensions:Ii,beforeBuildLabels:Ii,buildLabels:Ii,afterBuildLabels:Ii,beforeFit:Ii,fit:function(){var t,e=this,n=e.options,i=e.minSize={},a=e.isHorizontal();n.display?(t=(j.isArray(n.text)?n.text.length:1)*j.options._parseFont(n).lineHeight+2*n.padding,e.width=i.width=a?e.maxWidth:t,e.height=i.height=a?t:e.maxHeight):e.width=i.width=e.height=i.height=0},afterFit:Ii,isHorizontal:function(){var t=this.options.position;return\"top\"===t||\"bottom\"===t},draw:function(){var t=this,e=t.ctx,n=t.options;if(n.display){var i,a,r,o=j.options._parseFont(n),l=o.lineHeight,s=l/2+n.padding,u=0,d=t.top,c=t.left,h=t.bottom,f=t.right;e.fillStyle=j.valueOrDefault(n.fontColor,N.global.defaultFontColor),e.font=o.string,t.isHorizontal()?(a=c+(f-c)/2,r=d+s,i=f-c):(a=\"left\"===n.position?c+s:f-s,r=d+(h-d)/2,i=h-d,u=Math.PI*(\"left\"===n.position?-.5:.5)),e.save(),e.translate(a,r),e.rotate(u),e.textAlign=\"center\",e.textBaseline=\"middle\";var g=n.text;if(j.isArray(g))for(var p=0,m=0;m=0;i--){var a=t[i];if(e(a))return a}},j.isNumber=function(t){return!isNaN(parseFloat(t))&&isFinite(t)},j.almostEquals=function(t,e,n){return Math.abs(t-e)=t},j.max=function(t){return t.reduce((function(t,e){return isNaN(e)?t:Math.max(t,e)}),Number.NEGATIVE_INFINITY)},j.min=function(t){return t.reduce((function(t,e){return isNaN(e)?t:Math.min(t,e)}),Number.POSITIVE_INFINITY)},j.sign=Math.sign?function(t){return Math.sign(t)}:function(t){return 0===(t=+t)||isNaN(t)?t:t>0?1:-1},j.toRadians=function(t){return t*(Math.PI/180)},j.toDegrees=function(t){return t*(180/Math.PI)},j._decimalPlaces=function(t){if(j.isFinite(t)){for(var e=1,n=0;Math.round(t*e)/e!==t;)e*=10,n++;return n}},j.getAngleFromPoint=function(t,e){var n=e.x-t.x,i=e.y-t.y,a=Math.sqrt(n*n+i*i),r=Math.atan2(i,n);return r<-.5*Math.PI&&(r+=2*Math.PI),{angle:r,distance:a}},j.distanceBetweenPoints=function(t,e){return Math.sqrt(Math.pow(e.x-t.x,2)+Math.pow(e.y-t.y,2))},j.aliasPixel=function(t){return t%2==0?0:.5},j._alignPixel=function(t,e,n){var i=t.currentDevicePixelRatio,a=n/2;return Math.round((e-a)*i)/i+a},j.splineCurve=function(t,e,n,i){var a=t.skip?e:t,r=e,o=n.skip?e:n,l=Math.sqrt(Math.pow(r.x-a.x,2)+Math.pow(r.y-a.y,2)),s=Math.sqrt(Math.pow(o.x-r.x,2)+Math.pow(o.y-r.y,2)),u=l/(l+s),d=s/(l+s),c=i*(u=isNaN(u)?0:u),h=i*(d=isNaN(d)?0:d);return{previous:{x:r.x-c*(o.x-a.x),y:r.y-c*(o.y-a.y)},next:{x:r.x+h*(o.x-a.x),y:r.y+h*(o.y-a.y)}}},j.EPSILON=Number.EPSILON||1e-14,j.splineCurveMonotone=function(t){var e,n,i,a,r,o,l,s,u,d=(t||[]).map((function(t){return{model:t._model,deltaK:0,mK:0}})),c=d.length;for(e=0;e0?d[e-1]:null,(a=e0?d[e-1]:null,a=e=t.length-1?t[0]:t[e+1]:e>=t.length-1?t[t.length-1]:t[e+1]},j.previousItem=function(t,e,n){return n?e<=0?t[t.length-1]:t[e-1]:e<=0?t[0]:t[e-1]},j.niceNum=function(t,e){var n=Math.floor(j.log10(t)),i=t/Math.pow(10,n);return(e?i<1.5?1:i<3?2:i<7?5:10:i<=1?1:i<=2?2:i<=5?5:10)*Math.pow(10,n)},j.requestAnimFrame=\"undefined\"==typeof window?function(t){t()}:window.requestAnimationFrame||window.webkitRequestAnimationFrame||window.mozRequestAnimationFrame||window.oRequestAnimationFrame||window.msRequestAnimationFrame||function(t){return window.setTimeout(t,1e3/60)},j.getRelativePosition=function(t,e){var n,i,a=t.originalEvent||t,r=t.target||t.srcElement,o=r.getBoundingClientRect(),l=a.touches;l&&l.length>0?(n=l[0].clientX,i=l[0].clientY):(n=a.clientX,i=a.clientY);var s=parseFloat(j.getStyle(r,\"padding-left\")),u=parseFloat(j.getStyle(r,\"padding-top\")),d=parseFloat(j.getStyle(r,\"padding-right\")),c=parseFloat(j.getStyle(r,\"padding-bottom\")),h=o.right-o.left-s-d,f=o.bottom-o.top-u-c;return{x:n=Math.round((n-o.left-s)/h*r.width/e.currentDevicePixelRatio),y:i=Math.round((i-o.top-u)/f*r.height/e.currentDevicePixelRatio)}},j.getConstraintWidth=function(t){return n(t,\"max-width\",\"clientWidth\")},j.getConstraintHeight=function(t){return n(t,\"max-height\",\"clientHeight\")},j._calculatePadding=function(t,e,n){return(e=j.getStyle(t,e)).indexOf(\"%\")>-1?n*parseInt(e,10)/100:parseInt(e,10)},j._getParentNode=function(t){var e=t.parentNode;return e&&\"[object ShadowRoot]\"===e.toString()&&(e=e.host),e},j.getMaximumWidth=function(t){var e=j._getParentNode(t);if(!e)return t.clientWidth;var n=e.clientWidth,i=n-j._calculatePadding(e,\"padding-left\",n)-j._calculatePadding(e,\"padding-right\",n),a=j.getConstraintWidth(t);return isNaN(a)?i:Math.min(i,a)},j.getMaximumHeight=function(t){var e=j._getParentNode(t);if(!e)return t.clientHeight;var n=e.clientHeight,i=n-j._calculatePadding(e,\"padding-top\",n)-j._calculatePadding(e,\"padding-bottom\",n),a=j.getConstraintHeight(t);return isNaN(a)?i:Math.min(i,a)},j.getStyle=function(t,e){return t.currentStyle?t.currentStyle[e]:document.defaultView.getComputedStyle(t,null).getPropertyValue(e)},j.retinaScale=function(t,e){var n=t.currentDevicePixelRatio=e||\"undefined\"!=typeof window&&window.devicePixelRatio||1;if(1!==n){var i=t.canvas,a=t.height,r=t.width;i.height=a*n,i.width=r*n,t.ctx.scale(n,n),i.style.height||i.style.width||(i.style.height=a+\"px\",i.style.width=r+\"px\")}},j.fontString=function(t,e,n){return e+\" \"+t+\"px \"+n},j.longestText=function(t,e,n,i){var a=(i=i||{}).data=i.data||{},r=i.garbageCollect=i.garbageCollect||[];i.font!==e&&(a=i.data={},r=i.garbageCollect=[],i.font=e),t.font=e;var o,l,s,u,d,c=0,h=n.length;for(o=0;on.length){for(o=0;oi&&(i=r),i},j.numberOfLabelLines=function(t){var e=1;return j.each(t,(function(t){j.isArray(t)&&t.length>e&&(e=t.length)})),e},j.color=_?function(t){return t instanceof CanvasGradient&&(t=N.global.defaultColor),_(t)}:function(t){return console.error(\"Color.js not found!\"),t},j.getHoverColor=function(t){return t instanceof CanvasPattern||t instanceof CanvasGradient?t:j.color(t).saturate(.5).darken(.1).rgbString()}}(),en._adapters=rn,en.Animation=J,en.animationService=Q,en.controllers=$t,en.DatasetController=at,en.defaults=N,en.Element=Z,en.elements=wt,en.Interaction=ae,en.layouts=ge,en.platform=Le,en.plugins=Re,en.Scale=yn,en.scaleService=ze,en.Ticks=on,en.Tooltip=Ge,en.helpers.each(fi,(function(t,e){en.scaleService.registerScaleType(e,t,t._defaults)})),Li)Li.hasOwnProperty(Bi)&&en.plugins.register(Li[Bi]);en.platform.initialize();var Ei=en;return\"undefined\"!=typeof window&&(window.Chart=en),en.Chart=en,en.Legend=Li.legend._element,en.Title=Li.title._element,en.pluginService=en.plugins,en.PluginBase=en.Element.extend({}),en.canvasHelpers=en.helpers.canvas,en.layoutService=en.layouts,en.LinearScaleBase=Cn,en.helpers.each([\"Bar\",\"Bubble\",\"Doughnut\",\"Line\",\"PolarArea\",\"Radar\",\"Scatter\"],(function(t){en[t]=function(e,n){return new en(e,en.helpers.merge(n||{},{type:t.charAt(0).toLowerCase()+t.slice(1)}))}})),Ei}));" diff --git a/src/DataFrame/Display/Web/Plot.hs b/src/DataFrame/Display/Web/Plot.hs new file mode 100644 --- /dev/null +++ b/src/DataFrame/Display/Web/Plot.hs @@ -0,0 +1,977 @@ +{-# LANGUAGE AllowAmbiguousTypes #-} +{-# LANGUAGE ExplicitNamespaces #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} + +module DataFrame.Display.Web.Plot where + +import Control.Monad +import Data.Char +import qualified Data.List as L +import qualified Data.Map as M +import Data.Maybe (fromMaybe) +import qualified Data.Text as T +import qualified Data.Text.IO as T +import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) +import Data.Typeable (Typeable) +import qualified Data.Vector as V +import qualified Data.Vector.Generic as VG +import qualified Data.Vector.Unboxed as VU +import GHC.Stack (HasCallStack) +import System.Random (newStdGen, randomRs) +import Type.Reflection (typeRep) + +import DataFrame.Display.Web.ChartJs +import DataFrame.Internal.Column (Column (..), isNumeric) +import qualified DataFrame.Internal.Column as D +import DataFrame.Internal.DataFrame (DataFrame (..), getColumn) +import DataFrame.Operations.Core +import qualified DataFrame.Operations.Subset as D +import System.Directory +import System.Info +import System.Process ( + StdStream (NoStream), + createProcess, + proc, + std_err, + std_in, + std_out, + waitForProcess, + ) + +newtype HtmlPlot = HtmlPlot T.Text deriving (Show) + +data PlotConfig = PlotConfig + { plotType :: PlotType + , plotTitle :: T.Text + , plotWidth :: Int + , plotHeight :: Int + , plotFile :: Maybe FilePath + } + +data PlotType + = Histogram + | Scatter + | Line + | Bar + | BoxPlot + | Pie + | StackedBar + | Heatmap + deriving (Eq, Show) + +defaultPlotConfig :: PlotType -> PlotConfig +defaultPlotConfig ptype = + PlotConfig + { plotType = ptype + , plotTitle = "" + , plotWidth = 600 + , plotHeight = 400 + , plotFile = Nothing + } + +generateChartId :: IO T.Text +generateChartId = do + gen <- newStdGen + let randomWords = filter (\c -> c `elem` ([49 .. 57] ++ [65 .. 90] ++ [97 .. 122])) (take 64 (randomRs (49, 126) gen :: [Int])) + return $ "chart_" <> (T.pack $ (map chr randomWords)) + +wrapInHTML :: T.Text -> T.Text -> Int -> Int -> T.Text +wrapInHTML chartId content width height = + T.concat + [ "\n" + , "\n" + , "\n" + ] + +plotHistogram :: (HasCallStack) => T.Text -> DataFrame -> IO HtmlPlot +plotHistogram colName = plotHistogramWith colName (defaultPlotConfig Histogram) + +plotHistogramWith :: (HasCallStack) => T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot +plotHistogramWith colName config df = do + chartId <- generateChartId + let values = extractNumericColumn colName df + (minVal, maxVal) = if null values then (0, 1) else (minimum values, maximum values) + numBins = 30 + binWidth = (maxVal - minVal) / fromIntegral numBins + bins = [minVal + fromIntegral i * binWidth | i <- [0 .. numBins - 1]] + counts = calculateHistogram values bins binWidth + + labels = T.intercalate "," ["\"" <> T.pack (show (round b :: Int)) <> "\"" | b <- bins] + dataPoints = T.intercalate "," [T.pack (show c) | c <- counts] + + chartTitle = if T.null (plotTitle config) then "Histogram of " <> colName else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"bar\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [{\n" + , " label: \"" + , colName + , "\",\n" + , " data: [" + , dataPoints + , "],\n" + , " backgroundColor: \"rgba(75, 192, 192, 0.6)\",\n" + , " borderColor: \"rgba(75, 192, 192, 1)\",\n" + , " borderWidth: 1\n" + , " }]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" },\n" + , " scales: {\n" + , " yAxes: [{ ticks: { beginAtZero: true } }]\n" + , " }\n" + , " }\n" + , "});" + ] + + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + +calculateHistogram :: [Double] -> [Double] -> Double -> [Int] +calculateHistogram values bins binWidth = + let countBin b = length [v | v <- values, v >= b && v < b + binWidth] + in map countBin bins + +plotScatter :: (HasCallStack) => T.Text -> T.Text -> DataFrame -> IO HtmlPlot +plotScatter xCol yCol = plotScatterWith xCol yCol (defaultPlotConfig Scatter) + +plotScatterWith :: (HasCallStack) => T.Text -> T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot +plotScatterWith xCol yCol config df = do + chartId <- generateChartId + let xVals = extractNumericColumn xCol df + yVals = extractNumericColumn yCol df + points = zip xVals yVals + + dataPoints = T.intercalate "," ["{x:" <> T.pack (show x) <> ", y:" <> T.pack (show y) <> "}" | (x, y) <- points] + chartTitle = if T.null (plotTitle config) then xCol <> " vs " <> yCol else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"scatter\",\n" + , " data: {\n" + , " datasets: [{\n" + , " label: \"" + , chartTitle + , "\",\n" + , " data: [" + , dataPoints + , "],\n" + , " pointRadius: 4,\n" + , " pointBackgroundColor: \"rgb(75, 192, 192)\"\n" + , " }]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" },\n" + , " scales: {\n" + , " xAxes: [{ scaleLabel: { display: true, labelString: \"" + , xCol + , "\" } }],\n" + , " yAxes: [{ scaleLabel: { display: true, labelString: \"" + , yCol + , "\" } }]\n" + , " }\n" + , " }\n" + , "});" + ] + + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + +plotScatterBy :: (HasCallStack) => T.Text -> T.Text -> T.Text -> DataFrame -> IO HtmlPlot +plotScatterBy xCol yCol grouping = plotScatterByWith xCol yCol grouping (defaultPlotConfig Scatter) + +plotScatterByWith :: (HasCallStack) => T.Text -> T.Text -> T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot +plotScatterByWith xCol yCol grouping config df = do + chartId <- generateChartId + let vals = extractStringColumn grouping df + df' = insertColumn grouping (D.fromList vals) df + uniqueVals = L.nub vals + + colors = + cycle + [ "rgb(255, 99, 132)" + , "rgb(54, 162, 235)" + , "rgb(255, 206, 86)" + , "rgb(75, 192, 192)" + , "rgb(153, 102, 255)" + , "rgb(255, 159, 64)" + ] + + datasets <- forM (zip uniqueVals colors) $ \(val, color) -> do + let filtered = D.filter grouping (== val) df' + xVals = extractNumericColumn xCol filtered + yVals = extractNumericColumn yCol filtered + points = zip xVals yVals + dataPoints = T.intercalate "," ["{x:" <> T.pack (show x) <> ", y:" <> T.pack (show y) <> "}" | (x, y) <- points] + return $ + T.concat + [ " {\n" + , " label: \"" + , val + , "\",\n" + , " data: [" + , dataPoints + , "],\n" + , " pointRadius: 4,\n" + , " pointBackgroundColor: \"" + , color + , "\"\n" + , " }" + ] + + let datasetsStr = T.intercalate ",\n" datasets + chartTitle = if T.null (plotTitle config) then xCol <> " vs " <> yCol <> " by " <> grouping else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"scatter\",\n" + , " data: {\n" + , " datasets: [\n" + , datasetsStr + , "\n ]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" },\n" + , " scales: {\n" + , " xAxes: [{ scaleLabel: { display: true, labelString: \"" + , xCol + , "\" } }],\n" + , " yAxes: [{ scaleLabel: { display: true, labelString: \"" + , yCol + , "\" } }]\n" + , " }\n" + , " }\n" + , "});" + ] + + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + +plotLines :: (HasCallStack) => T.Text -> [T.Text] -> DataFrame -> IO HtmlPlot +plotLines xAxis colNames = plotLinesWith xAxis colNames (defaultPlotConfig Line) + +plotLinesWith :: (HasCallStack) => T.Text -> [T.Text] -> PlotConfig -> DataFrame -> IO HtmlPlot +plotLinesWith xAxis colNames config df = do + chartId <- generateChartId + let xValues = extractNumericColumn xAxis df + labels = T.intercalate "," [T.pack (show x) | x <- xValues] + + colors = + cycle + [ "rgb(255, 99, 132)" + , "rgb(54, 162, 235)" + , "rgb(255, 206, 86)" + , "rgb(75, 192, 192)" + , "rgb(153, 102, 255)" + , "rgb(255, 159, 64)" + ] + + datasets <- forM (zip colNames colors) $ \(col, color) -> do + let values = extractNumericColumn col df + dataPoints = T.intercalate "," [T.pack (show v) | v <- values] + return $ + T.concat + [ " {\n" + , " label: \"" + , col + , "\",\n" + , " data: [" + , dataPoints + , "],\n" + , " fill: false,\n" + , " borderColor: \"" + , color + , "\",\n" + , " tension: 0.1\n" + , " }" + ] + + let datasetsStr = T.intercalate ",\n" datasets + chartTitle = if T.null (plotTitle config) then "Line Chart" else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"line\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [\n" + , datasetsStr + , "\n ]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" },\n" + , " scales: {\n" + , " xAxes: [{ scaleLabel: { display: true, labelString: \"" + , xAxis + , "\" } }]\n" + , " }\n" + , " }\n" + , "});" + ] + + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + +plotBars :: (HasCallStack) => T.Text -> DataFrame -> IO HtmlPlot +plotBars colName = plotBarsWith colName Nothing (defaultPlotConfig Bar) + +plotBarsWith :: (HasCallStack) => T.Text -> Maybe T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot +plotBarsWith colName groupByCol config df = + case groupByCol of + Nothing -> plotSingleBars colName config df + Just grpCol -> plotGroupedBarsWith grpCol colName config df + +plotSingleBars :: (HasCallStack) => T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot +plotSingleBars colName config df = do + chartId <- generateChartId + let barData = getCategoricalCounts colName df + case barData of + Just counts -> do + let grouped = groupWithOther 10 counts + labels = T.intercalate "," ["\"" <> label <> "\"" | (label, _) <- grouped] + dataPoints = T.intercalate "," [T.pack (show val) | (_, val) <- grouped] + chartTitle = if T.null (plotTitle config) then colName else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"bar\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [{\n" + , " label: \"Count\",\n" + , " data: [" + , dataPoints + , "],\n" + , " backgroundColor: \"rgba(54, 162, 235, 0.6)\",\n" + , " borderColor: \"rgba(54, 162, 235, 1)\",\n" + , " borderWidth: 1\n" + , " }]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" },\n" + , " scales: {\n" + , " yAxes: [{ ticks: { beginAtZero: true } }]\n" + , " }\n" + , " }\n" + , "});" + ] + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + Nothing -> do + let values = extractNumericColumn colName df + labels' = + if length values > 20 + then take 20 ["Item " <> T.pack (show i) | i <- [1 ..]] + else ["Item " <> T.pack (show i) | i <- [1 .. length values]] + vals = if length values > 20 then take 20 values else values + labels = T.intercalate "," ["\"" <> label <> "\"" | label <- labels'] + dataPoints = T.intercalate "," [T.pack (show val) | val <- vals] + chartTitle = if T.null (plotTitle config) then colName else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"bar\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [{\n" + , " label: \"Value\",\n" + , " data: [" + , dataPoints + , "],\n" + , " backgroundColor: \"rgba(54, 162, 235, 0.6)\",\n" + , " borderColor: \"rgba(54, 162, 235, 1)\",\n" + , " borderWidth: 1\n" + , " }]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" },\n" + , " scales: {\n" + , " yAxes: [{ ticks: { beginAtZero: true } }]\n" + , " }\n" + , " }\n" + , "});" + ] + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + +plotPie :: (HasCallStack) => T.Text -> Maybe T.Text -> DataFrame -> IO HtmlPlot +plotPie valCol labelCol = plotPieWith valCol labelCol (defaultPlotConfig Pie) + +plotPieWith :: (HasCallStack) => T.Text -> Maybe T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot +plotPieWith valCol labelCol config df = do + chartId <- generateChartId + let categoricalData = getCategoricalCounts valCol df + case categoricalData of + Just counts -> do + let grouped = groupWithOtherForPie 8 counts + labels = T.intercalate "," ["\"" <> label <> "\"" | (label, _) <- grouped] + dataPoints = T.intercalate "," [T.pack (show val) | (_, val) <- grouped] + colors = T.intercalate "," ["\"" <> c <> "\"" | c <- take (length grouped) pieColors] + chartTitle = if T.null (plotTitle config) then valCol else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"pie\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [{\n" + , " data: [" + , dataPoints + , "],\n" + , " backgroundColor: [" + , colors + , "]\n" + , " }]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" }\n" + , " }\n" + , "});" + ] + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + Nothing -> do + let values = extractNumericColumn valCol df + labels' = case labelCol of + Nothing -> map (\i -> "Item " <> T.pack (show i)) [1 .. length values] + Just lCol -> extractStringColumn lCol df + pieData = zip labels' values + grouped = + if length pieData > 10 + then groupWithOtherForPie 8 pieData + else pieData + labels = T.intercalate "," ["\"" <> label <> "\"" | (label, _) <- grouped] + dataPoints = T.intercalate "," [T.pack (show val) | (_, val) <- grouped] + colors = T.intercalate "," ["\"" <> c <> "\"" | c <- take (length grouped) pieColors] + chartTitle = if T.null (plotTitle config) then valCol else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"pie\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [{\n" + , " data: [" + , dataPoints + , "],\n" + , " backgroundColor: [" + , colors + , "]\n" + , " }]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" }\n" + , " }\n" + , "});" + ] + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + +pieColors :: [T.Text] +pieColors = + [ "rgb(255, 99, 132)" + , "rgb(54, 162, 235)" + , "rgb(255, 206, 86)" + , "rgb(75, 192, 192)" + , "rgb(153, 102, 255)" + , "rgb(255, 159, 64)" + , "rgb(201, 203, 207)" + , "rgb(255, 99, 71)" + , "rgb(60, 179, 113)" + , "rgb(238, 130, 238)" + ] + +plotStackedBars :: (HasCallStack) => T.Text -> [T.Text] -> DataFrame -> IO HtmlPlot +plotStackedBars categoryCol valueColumns = plotStackedBarsWith categoryCol valueColumns (defaultPlotConfig StackedBar) + +plotStackedBarsWith :: (HasCallStack) => T.Text -> [T.Text] -> PlotConfig -> DataFrame -> IO HtmlPlot +plotStackedBarsWith categoryCol valueColumns config df = do + chartId <- generateChartId + let categories = extractStringColumn categoryCol df + uniqueCategories = L.nub categories + + colors = + cycle + [ "rgb(255, 99, 132)" + , "rgb(54, 162, 235)" + , "rgb(255, 206, 86)" + , "rgb(75, 192, 192)" + , "rgb(153, 102, 255)" + , "rgb(255, 159, 64)" + ] + + datasets <- forM (zip valueColumns colors) $ \(col, color) -> do + dataVals <- forM uniqueCategories $ \cat -> do + let indices = [i | (i, c) <- zip [0 ..] categories, c == cat] + allValues = extractNumericColumn col df + values = [allValues !! i | i <- indices, i < length allValues] + return $ sum values + let dataPoints = T.intercalate "," [T.pack (show v) | v <- dataVals] + return $ + T.concat + [ " {\n" + , " label: \"" + , col + , "\",\n" + , " data: [" + , dataPoints + , "],\n" + , " backgroundColor: \"" + , color + , "\"\n" + , " }" + ] + + let datasetsStr = T.intercalate ",\n" datasets + labels = T.intercalate "," ["\"" <> cat <> "\"" | cat <- uniqueCategories] + chartTitle = if T.null (plotTitle config) then "Stacked Bar Chart" else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"bar\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [\n" + , datasetsStr + , "\n ]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" },\n" + , " scales: {\n" + , " xAxes: [{ stacked: true }],\n" + , " yAxes: [{ stacked: true, ticks: { beginAtZero: true } }]\n" + , " }\n" + , " }\n" + , "});" + ] + + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + +plotBoxPlots :: (HasCallStack) => [T.Text] -> DataFrame -> IO HtmlPlot +plotBoxPlots colNames = plotBoxPlotsWith colNames (defaultPlotConfig BoxPlot) + +plotBoxPlotsWith :: (HasCallStack) => [T.Text] -> PlotConfig -> DataFrame -> IO HtmlPlot +plotBoxPlotsWith colNames config df = do + chartId <- generateChartId + boxData <- forM colNames $ \col -> do + let values = extractNumericColumn col df + sorted = L.sort values + n = length values + q1 = sorted !! (n `div` 4) + median = sorted !! (n `div` 2) + q3 = sorted !! (3 * n `div` 4) + minVal = minimum values + maxVal = maximum values + return (col, minVal, q1, median, q3, maxVal) + + let labels = T.intercalate "," ["\"" <> col <> "\"" | (col, _, _, _, _, _) <- boxData] + medians = T.intercalate "," [T.pack (show med) | (_, _, _, med, _, _) <- boxData] + mins = T.intercalate "," [T.pack (show minV) | (_, minV, _, _, _, _) <- boxData] + maxs = T.intercalate "," [T.pack (show maxV) | (_, _, _, _, _, maxV) <- boxData] + chartTitle = if T.null (plotTitle config) then "Box Plot" else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"bar\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [{\n" + , " label: \"Median\",\n" + , " data: [" + , medians + , "],\n" + , " backgroundColor: \"rgba(75, 192, 192, 0.6)\",\n" + , " borderColor: \"rgba(75, 192, 192, 1)\",\n" + , " borderWidth: 1\n" + , " }]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , " (showing medians)\" },\n" + , " scales: {\n" + , " yAxes: [{ ticks: { beginAtZero: true } }]\n" + , " }\n" + , " }\n" + , "});" + ] + + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + +plotGroupedBarsWith :: (HasCallStack) => T.Text -> T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot +plotGroupedBarsWith = plotGroupedBarsWithN 10 + +plotGroupedBarsWithN :: (HasCallStack) => Int -> T.Text -> T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot +plotGroupedBarsWithN n groupCol valCol config df = do + chartId <- generateChartId + let colIsNumeric = isNumericColumnCheck valCol df + + if colIsNumeric + then do + let groups = extractStringColumn groupCol df + values = extractNumericColumn valCol df + grouped = M.toList $ M.fromListWith (+) (zip groups values) + finalGroups = groupWithOther n grouped + labels = T.intercalate "," ["\"" <> label <> "\"" | (label, _) <- finalGroups] + dataPoints = T.intercalate "," [T.pack (show val) | (_, val) <- finalGroups] + chartTitle = if T.null (plotTitle config) then groupCol <> " by " <> valCol else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"bar\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [{\n" + , " label: \"" + , valCol + , "\",\n" + , " data: [" + , dataPoints + , "],\n" + , " backgroundColor: \"rgba(54, 162, 235, 0.6)\",\n" + , " borderColor: \"rgba(54, 162, 235, 1)\",\n" + , " borderWidth: 1\n" + , " }]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" },\n" + , " scales: {\n" + , " yAxes: [{ ticks: { beginAtZero: true } }]\n" + , " }\n" + , " }\n" + , "});" + ] + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + else do + let groups = extractStringColumn groupCol df + vals = extractStringColumn valCol df + pairs = zip groups vals + counts = + M.toList $ + M.fromListWith + (+) + [(g <> " - " <> v, 1) | (g, v) <- pairs] + finalCounts = groupWithOther n [(k, fromIntegral v) | (k, v) <- counts] + labels = T.intercalate "," ["\"" <> label <> "\"" | (label, _) <- finalCounts] + dataPoints = T.intercalate "," [T.pack (show val) | (_, val) <- finalCounts] + chartTitle = if T.null (plotTitle config) then groupCol <> " by " <> valCol else plotTitle config + + jsCode = + T.concat + [ "new Chart(\"" + , chartId + , "\", {\n" + , " type: \"bar\",\n" + , " data: {\n" + , " labels: [" + , labels + , "],\n" + , " datasets: [{\n" + , " label: \"Count\",\n" + , " data: [" + , dataPoints + , "],\n" + , " backgroundColor: \"rgba(54, 162, 235, 0.6)\",\n" + , " borderColor: \"rgba(54, 162, 235, 1)\",\n" + , " borderWidth: 1\n" + , " }]\n" + , " },\n" + , " options: {\n" + , " title: { display: true, text: \"" + , chartTitle + , "\" },\n" + , " scales: {\n" + , " yAxes: [{ ticks: { beginAtZero: true } }]\n" + , " }\n" + , " }\n" + , "});" + ] + return $ HtmlPlot $ wrapInHTML chartId jsCode (plotWidth config) (plotHeight config) + +-- TODO: Move these helpers to a common module. + +isNumericColumn :: DataFrame -> T.Text -> Bool +isNumericColumn df colName = maybe False isNumeric (getColumn colName df) + +isNumericColumnCheck :: T.Text -> DataFrame -> Bool +isNumericColumnCheck colName df = isNumericColumn df colName + +extractStringColumn :: (HasCallStack) => T.Text -> DataFrame -> [T.Text] +extractStringColumn colName df = + case M.lookup colName (columnIndices df) of + Nothing -> error $ "Column " ++ T.unpack colName ++ " not found" + Just idx -> + let col = columns df V.! idx + in case col of + BoxedColumn (vec :: V.Vector a) -> case testEquality (typeRep @a) (typeRep @T.Text) of + Just Refl -> V.toList vec + Nothing -> V.toList $ V.map (T.pack . show) vec + UnboxedColumn vec -> V.toList $ VG.map (T.pack . show) (VG.convert vec) + OptionalColumn (vec :: V.Vector (Maybe a)) -> case testEquality (typeRep @a) (typeRep @T.Text) of + Nothing -> V.toList $ V.map (T.pack . show) vec + Just Refl -> V.toList $ V.map (fromMaybe "Nothing" . fmap ("Just " <>)) vec + +extractNumericColumn :: (HasCallStack) => T.Text -> DataFrame -> [Double] +extractNumericColumn colName df = + case M.lookup colName (columnIndices df) of + Nothing -> error $ "Column " ++ T.unpack colName ++ " not found" + Just idx -> + let col = columns df V.! idx + in case col of + BoxedColumn vec -> vectorToDoubles vec + UnboxedColumn vec -> unboxedVectorToDoubles vec + _ -> [] + +vectorToDoubles :: forall a. (Typeable a, Show a) => V.Vector a -> [Double] +vectorToDoubles vec = + case testEquality (typeRep @a) (typeRep @Double) of + Just Refl -> V.toList vec + Nothing -> case testEquality (typeRep @a) (typeRep @Int) of + Just Refl -> V.toList $ V.map fromIntegral vec + Nothing -> case testEquality (typeRep @a) (typeRep @Integer) of + Just Refl -> V.toList $ V.map fromIntegral vec + Nothing -> case testEquality (typeRep @a) (typeRep @Float) of + Just Refl -> V.toList $ V.map realToFrac vec + Nothing -> error $ "Column is not numeric (type: " ++ show (typeRep @a) ++ ")" + +unboxedVectorToDoubles :: forall a. (Typeable a, VU.Unbox a, Show a) => VU.Vector a -> [Double] +unboxedVectorToDoubles vec = + case testEquality (typeRep @a) (typeRep @Double) of + Just Refl -> VU.toList vec + Nothing -> case testEquality (typeRep @a) (typeRep @Int) of + Just Refl -> VU.toList $ VU.map fromIntegral vec + Nothing -> case testEquality (typeRep @a) (typeRep @Float) of + Just Refl -> VU.toList $ VU.map realToFrac vec + Nothing -> error $ "Column is not numeric (type: " ++ show (typeRep @a) ++ ")" + +getCategoricalCounts :: (HasCallStack) => T.Text -> DataFrame -> Maybe [(T.Text, Double)] +getCategoricalCounts colName df = + case M.lookup colName (columnIndices df) of + Nothing -> error $ "Column " ++ T.unpack colName ++ " not found" + Just idx -> + let col = columns df V.! idx + in case col of + BoxedColumn (vec :: V.Vector a) -> + let counts = countValues vec + in case testEquality (typeRep @a) (typeRep @T.Text) of + Nothing -> Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts] + Just Refl -> Just [(k, fromIntegral v) | (k, v) <- counts] + UnboxedColumn vec -> + let counts = countValuesUnboxed vec + in Just [(T.pack (show k), fromIntegral v) | (k, v) <- counts] + OptionalColumn (vec :: V.Vector (Maybe a)) -> + let counts = countValues vec + in case testEquality (typeRep @a) (typeRep @T.Text) of + Nothing -> Just [((T.pack . show) k, fromIntegral v) | (k, v) <- counts] + Just Refl -> Just [(fromMaybe "Nothing" (fmap ("Just " <>) k), fromIntegral v) | (k, v) <- counts] + where + countValues :: (Ord a, Show a) => V.Vector a -> [(a, Int)] + countValues vec = M.toList $ V.foldr' (\x acc -> M.insertWith (+) x 1 acc) M.empty vec + + countValuesUnboxed :: (Ord a, Show a, VU.Unbox a) => VU.Vector a -> [(a, Int)] + countValuesUnboxed vec = M.toList $ VU.foldr' (\x acc -> M.insertWith (+) x 1 acc) M.empty vec + +groupWithOther :: Int -> [(T.Text, Double)] -> [(T.Text, Double)] +groupWithOther n items = + let sorted = L.sortOn (negate . snd) items + (topN, rest) = splitAt n sorted + otherSum = sum (map snd rest) + result = + if null rest || otherSum == 0 + then topN + else topN ++ [("Other (" <> T.pack (show (length rest)) <> " items)", otherSum)] + in result + +groupWithOtherForPie :: Int -> [(T.Text, Double)] -> [(T.Text, Double)] +groupWithOtherForPie n items = + let total = sum (map snd items) + sorted = L.sortOn (negate . snd) items + (topN, rest) = splitAt n sorted + otherSum = sum (map snd rest) + otherPct = round (100 * otherSum / total) :: Int + result = + if null rest || otherSum == 0 + then topN + else + topN + ++ [ + ( "Other (" + <> T.pack (show (length rest)) + <> " items, " + <> T.pack (show otherPct) + <> "%)" + , otherSum + ) + ] + in result + +plotBarsTopN :: (HasCallStack) => Int -> T.Text -> DataFrame -> IO HtmlPlot +plotBarsTopN n colName = plotBarsTopNWith n colName (defaultPlotConfig Bar) + +plotBarsTopNWith :: (HasCallStack) => Int -> T.Text -> PlotConfig -> DataFrame -> IO HtmlPlot +plotBarsTopNWith n colName config df = do + let config' = config{plotTitle = plotTitle config <> " (Top " <> T.pack (show n) <> ")"} + plotBarsWith colName Nothing config' df + +plotValueCounts :: (HasCallStack) => T.Text -> DataFrame -> IO HtmlPlot +plotValueCounts colName = plotValueCountsWith colName 10 (defaultPlotConfig Bar) + +plotValueCountsWith :: (HasCallStack) => T.Text -> Int -> PlotConfig -> DataFrame -> IO HtmlPlot +plotValueCountsWith colName maxBars config df = do + let config' = config{plotTitle = "Value counts for " <> colName} + plotBarsTopNWith maxBars colName config' df + +plotAllHistograms :: (HasCallStack) => DataFrame -> IO HtmlPlot +plotAllHistograms df = do + let numericCols = filter (isNumericColumn df) (columnNames df) + xs <- forM numericCols $ \col -> do + T.putStrLn $ "" + plotHistogram col df + let allPlots = L.foldl' (\acc (HtmlPlot contents) -> acc <> "\n" <> (T.replace minifiedChartJs "" contents)) "" xs + return (HtmlPlot (T.concat ["\n\n", allPlots])) + +plotCategoricalSummary :: (HasCallStack) => DataFrame -> IO HtmlPlot +plotCategoricalSummary df = do + let cols = columnNames df + xs <- forM cols $ \col -> do + let counts = getCategoricalCounts col df + case counts of + Just c -> do + if (length c > 1) + then + ( do + let numUnique = length c + putStrLn $ "\n" + if numUnique > 15 then plotBarsTopN 10 col df else plotBars col df + ) + else return (HtmlPlot "") + Nothing -> return (HtmlPlot "") + let allPlots = L.foldl' (\acc (HtmlPlot contents) -> acc <> "\n" <> contents) "" xs + return (HtmlPlot allPlots) + +plotBarsWithPercentages :: (HasCallStack) => T.Text -> DataFrame -> IO HtmlPlot +plotBarsWithPercentages colName df = do + let config = (defaultPlotConfig Bar){plotTitle = "Distribution of " <> colName} + plotBarsWith colName Nothing config df + +smartPlotBars :: (HasCallStack) => T.Text -> DataFrame -> IO HtmlPlot +smartPlotBars colName df = do + let counts = getCategoricalCounts colName df + case counts of + Just c -> do + let numUnique = length c + config = (defaultPlotConfig Bar){plotTitle = colName <> " (" <> T.pack (show numUnique) <> " unique values)"} + if numUnique <= 12 + then plotBarsWith colName Nothing config df + else plotBarsTopNWith 10 colName config df + Nothing -> plotBars colName df + +showInDefaultBrowser :: HtmlPlot -> IO () +showInDefaultBrowser (HtmlPlot p) = do + plotId <- generateChartId + home <- getHomeDirectory + let operatingSystem = os + let path = "plot-" <> (T.unpack plotId) <> ".html" + + let fullPath = + if operatingSystem == "mingw32" + then home <> "\\" <> path + else home <> "/" <> path + putStr "Saving plot to: " + putStrLn fullPath + T.writeFile fullPath p + if operatingSystem == "mingw32" + then openFileSilently "start" fullPath + else openFileSilently "xdg-open" fullPath + pure () + +openFileSilently :: FilePath -> FilePath -> IO () +openFileSilently program path = do + (_, _, _, ph) <- + createProcess + (proc program [path]) + { std_in = NoStream + , std_out = NoStream + , std_err = NoStream + } + void (waitForProcess ph) diff --git a/src/DataFrame/Errors.hs b/src/DataFrame/Errors.hs --- a/src/DataFrame/Errors.hs +++ b/src/DataFrame/Errors.hs @@ -7,10 +7,10 @@ module DataFrame.Errors where import qualified Data.Text as T +import qualified Data.Vector.Unboxed as VU import Control.Exception import Data.Array -import Data.Either import Data.Typeable (Typeable) import DataFrame.Display.Terminal.Colours import Type.Reflection (TypeRep) @@ -30,6 +30,8 @@ DataFrameException ColumnNotFoundException :: T.Text -> T.Text -> [T.Text] -> DataFrameException EmptyDataSetException :: T.Text -> DataFrameException + WrongQuantileNumberException :: Int -> DataFrameException + WrongQuantileIndexException :: VU.Vector Int -> Int -> DataFrameException deriving (Exception) instance Show DataFrameException where @@ -41,6 +43,8 @@ addCallPointInfo (errorColumnName context) (callingFunctionName context) errorString show (ColumnNotFoundException columnName callPoint availableColumns) = columnNotFound columnName callPoint availableColumns show (EmptyDataSetException callPoint) = emptyDataSetError callPoint + show (WrongQuantileNumberException q) = wrongQuantileNumberError q + show (WrongQuantileIndexException qs q) = wrongQuantileIndexError qs q columnNotFound :: T.Text -> T.Text -> [T.Text] -> String columnNotFound name callPoint columns = @@ -68,6 +72,24 @@ red "\n\n[ERROR] " ++ T.unpack callPoint ++ " cannot be called on empty data sets" + +wrongQuantileNumberError :: Int -> String +wrongQuantileNumberError q = + red "\n\n[ERROR] " + ++ "Quantile number q should satisfy " + ++ "q >= 2, but here q is " + ++ show q + +wrongQuantileIndexError :: VU.Vector Int -> Int -> String +wrongQuantileIndexError qs q = + red "\n\n[ERROR] " + ++ "For quantile number q, " + ++ "each quantile index i " + ++ "should satisfy 0 <= i <= q, " + ++ "but here q is " + ++ show q + ++ " and indexes are " + ++ show qs addCallPointInfo :: Maybe String -> Maybe String -> String -> String addCallPointInfo (Just name) (Just cp) err = diff --git a/src/DataFrame/Functions.hs b/src/DataFrame/Functions.hs --- a/src/DataFrame/Functions.hs +++ b/src/DataFrame/Functions.hs @@ -16,76 +16,225 @@ import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (DataFrame (..), unsafeGetColumn) -import DataFrame.Internal.Expression (Expr (..), UExpr (..)) +import DataFrame.Internal.Expression (Expr (..), UExpr (..), eSize, interpret) +import DataFrame.Internal.Statistics +import DataFrame.Operations.Subset (exclude) import Control.Monad import qualified Data.Char as Char import Data.Function import qualified Data.List as L import qualified Data.Map as M +import qualified Data.Set as S import qualified Data.Text as T -import qualified Data.Vector as VB import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU -import Debug.Trace (traceShow) +import Debug.Trace ( traceShow ) import Language.Haskell.TH import qualified Language.Haskell.TH.Syntax as TH import Type.Reflection (typeRep) +import Prelude hiding (sum, minimum, maximum) +import Data.Type.Equality +name :: (Show a) => Expr a -> T.Text +name (Col n) = n +name other = error $ "You must call `name` on a column reference. Not the expression: " ++ show other + col :: (Columnable a) => T.Text -> Expr a col = Col as :: (Columnable a) => Expr a -> T.Text -> (T.Text, UExpr) as expr name = (name, Wrap expr) +ifThenElse :: (Columnable a) => Expr Bool -> Expr a -> Expr a -> Expr a +ifThenElse = If + lit :: (Columnable a) => a -> Expr a lit = Lit lift :: (Columnable a, Columnable b) => (a -> b) -> Expr a -> Expr b -lift = Apply "udf" +lift = UnaryOp "udf" lift2 :: (Columnable c, Columnable b, Columnable a) => (c -> b -> a) -> Expr c -> Expr b -> Expr a -lift2 = BinOp "udf" +lift2 = BinaryOp "udf" -eq :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool -eq = BinOp "eq" (==) +(==) :: (Columnable a, Eq a) => Expr a -> Expr a -> Expr Bool +(==) = BinaryOp "eq" (Prelude.==) -lt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool -lt = BinOp "lt" (<) +(<) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool +(<) = BinaryOp "lt" (Prelude.<) -gt :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool -gt = BinOp "gt" (>) +(>) :: (Columnable a, Ord a) => Expr a -> Expr a -> Expr Bool +(>) = BinaryOp "gt" (Prelude.>) -leq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool -leq = BinOp "leq" (<=) +(<=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool +(<=) = BinaryOp "leq" (Prelude.<=) -geq :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool -geq = BinOp "geq" (>=) +(>=) :: (Columnable a, Ord a, Eq a) => Expr a -> Expr a -> Expr Bool +(>=) = BinaryOp "geq" (Prelude.>=) +and :: Expr Bool -> Expr Bool -> Expr Bool +and = BinaryOp "and" (&&) + +or :: Expr Bool -> Expr Bool -> Expr Bool +or = BinaryOp "or" (||) + +not :: Expr Bool -> Expr Bool +not = UnaryOp "not" Prelude.not + count :: (Columnable a) => Expr a -> Expr Int -count (Col name) = GeneralAggregate name "count" VG.length -count _ = error "Argument can only be a column reference not an unevaluated expression" +count expr = AggFold expr "foldUdf" 0 (\acc _ -> acc + 1) -minimum :: (Columnable a) => Expr a -> Expr a -minimum (Col name) = ReductionAggregate name "minimum" min +minimum :: (Columnable a, Ord a) => Expr a -> Expr a +minimum expr = AggReduce expr "minimum" min -maximum :: (Columnable a) => Expr a -> Expr a -maximum (Col name) = ReductionAggregate name "maximum" max +maximum :: (Columnable a, Ord a) => Expr a -> Expr a +maximum expr = AggReduce expr "maximum" max sum :: forall a. (Columnable a, Num a, VU.Unbox a) => Expr a -> Expr a -sum (Col name) = NumericAggregate name "sum" VG.sum +sum expr = AggNumericVector expr "sum" VG.sum -mean :: (Columnable a, Num a, VU.Unbox a) => Expr a -> Expr Double -mean (Col name) = - let - mean' samp = - let - (!total, !n) = VG.foldl' (\(!total, !n) v -> (total + v, n + 1)) (0 :: Double, 0 :: Int) samp - in - total / fromIntegral n - in - NumericAggregate name "mean" mean' +mean :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double +mean expr = AggNumericVector expr "mean" mean' +standardDeviation :: (Columnable a, Real a, VU.Unbox a) => Expr a -> Expr Double +standardDeviation expr = AggNumericVector expr "stddev" (sqrt . variance') + +zScore :: Expr Double -> Expr Double +zScore c = (c - mean c) / standardDeviation c + +reduce :: forall a b. (Columnable a, Columnable b) => Expr b -> a -> (a -> b -> a) -> Expr a +reduce expr = AggFold expr "foldUdf" + +generatePrograms :: [Expr Double] -> [Expr Double] -> [Expr Double] +generatePrograms vars existingPrograms = + existingPrograms ++ + [ transform p + | p <- existingPrograms + , transform <- [zScore, abs, log . (+ Lit 1), exp] + ] ++ + [ p + q + | (i, p) <- zip [0..] existingPrograms + , (j, q) <- zip [0..] existingPrograms + , i Prelude.>= j + ] ++ + [ p - q + | p <- existingPrograms + , q <- existingPrograms + ] ++ + [ p * q + | (i, p) <- zip [0..] existingPrograms + , (j, q) <- zip [0..] existingPrograms + , i Prelude.>= j + ] ++ + [ p / q + | p <- existingPrograms + , q <- existingPrograms + ] ++ + [ t p + | v <- vars + , p <- existingPrograms + , t <- transformsWithVar v + ] + where + transformsWithVar v = + [ (v +) + , (* v) + , (/ v) + , (v /) + , \v' -> v' - v + , (v -) + ] + +-- | Deduplicate programs pick the least smallest one by size. +deduplicate :: DataFrame + -> [Expr Double] + -> [Expr Double] +deduplicate df = go S.empty . L.sortBy (\e1 e2 -> compare (eSize e1) (eSize e2)) + where + go _ [] = [] + go seen (x : xs) + | hasInvalid = go seen xs + | S.member res seen = go seen xs + | otherwise = x : go (S.insert res seen) xs + where + res = interpret df x + infinity = read @Double "Infinity" + hasInvalid = case res of + (TColumn (UnboxedColumn (col :: VU.Vector a))) -> case testEquality (typeRep @Double) (typeRep @a) of + Just Refl -> VU.any (\n -> isNaN n || isInfinite n) col + Nothing -> False + _ -> False + +-- | Checks if two programs generate the same outputs given all the same inputs. +equivalent :: DataFrame -> Expr Double -> Expr Double -> Bool +equivalent df p1 p2 = interpret df p1 Prelude.== interpret df p2 + +synthesizeFeatureExpr :: + -- | Target expression + T.Text -> + -- | Depth of search (Roughly, how many terms in the final expression) + Int -> + -- | Beam size - the number of candidate expressions to consider at a time. + Int -> + DataFrame -> + Either String (Expr Double) +synthesizeFeatureExpr target d b df = let + df' = exclude [target] df + names = (map fst . L.sortBy (compare `on` snd) . M.toList . columnIndices) df' + variables = map col names + in case beamSearch df' b (interpret df (Col target)) variables d of + Nothing -> Left "No programs found" + Just p -> Right p + +beamSearch :: + DataFrame -> + -- | Beam size + Int -> + -- | Examples + TypedColumn Double -> + -- | Programs + [Expr Double] -> + -- | Search depth + Int -> + Maybe (Expr Double) +beamSearch df b outputs programs d + | d Prelude.== 0 = case ps of + [] -> Nothing + (x:_) -> Just x + | otherwise = + case findFirst ps of + Just p -> Just p + Nothing -> beamSearch df b outputs (generatePrograms (map col names) ps) (d - 1) + where + ps = pickTopN df outputs b $ deduplicate df programs + names = (map fst . L.sortBy (compare `on` snd) . M.toList . columnIndices) df + findFirst [] = Nothing + findFirst (p : ps') + | satisfiesExamples df outputs p = Just p + | otherwise = findFirst ps' + +pickTopN :: DataFrame + -> TypedColumn Double + -> Int + -> [Expr Double] + -> [Expr Double] +pickTopN df (TColumn col) n ps = let + l = VU.convert (toVector @Double col) + ordered = take n (L.sortBy (\e e' -> let + c1 = abs <$> correlation' l (asDoubleVector e') + c2 = abs <$> correlation' l (asDoubleVector e) + in if maybe False isInfinite c1 || maybe False isInfinite c2 then LT else compare c1 c2) ps) + asDoubleVector e = let + (TColumn col') = interpret df e + in VU.convert (toVector @Double col') + in ordered + +satisfiesExamples :: DataFrame -> TypedColumn Double -> Expr Double -> Bool +satisfiesExamples df col expr = let + result = interpret df expr + in result Prelude.== col + -- See Section 2.4 of the Haskell Report https://www.haskell.org/definition/haskell2010.pdf isReservedId :: T.Text -> Bool isReservedId t = case t of @@ -124,7 +273,7 @@ Nothing -> False isHaskellIdentifier :: T.Text -> Bool -isHaskellIdentifier t = not (isVarId t) || isReservedId t +isHaskellIdentifier t = Prelude.not (isVarId t) || isReservedId t sanitize :: T.Text -> T.Text sanitize t @@ -133,10 +282,10 @@ | otherwise = t' where isValid = - not (isHaskellIdentifier t) + Prelude.not (isHaskellIdentifier t) && isVarId t && T.all Char.isAlphaNum t - t' = T.map replaceInvalidCharacters . T.filter (not . parentheses) $ t + t' = T.map replaceInvalidCharacters . T.filter (Prelude.not . parentheses) $ t replaceInvalidCharacters c | Char.isUpper c = Char.toLower c | Char.isSpace c = '_' @@ -169,19 +318,19 @@ lhs <- typeFromString [t1] rhs <- typeFromString [t2] return (AppT (AppT outer lhs) rhs) -typeFromString s = fail $ "Unsupported type: " ++ (unwords s) +typeFromString s = fail $ "Unsupported type: " ++ unwords s declareColumns :: DataFrame -> DecsQ declareColumns df = let names = (map fst . L.sortBy (compare `on` snd) . M.toList . columnIndices) df types = map (columnTypeString . (`unsafeGetColumn` df)) names - specs = zipWith (\name type_ -> (sanitize name, type_)) names types + specs = zipWith (\name type_ -> (name, sanitize name, type_)) names types in - fmap concat $ forM specs $ \(nm, tyStr) -> do - traceShow nm (pure ()) + fmap concat $ forM specs $ \(raw, nm, tyStr) -> do ty <- typeFromString (words tyStr) + traceShow (nm <> " :: Expr " <> T.pack tyStr) (pure ()) let n = mkName (T.unpack nm) sig <- sigD n [t|Expr $(pure ty)|] - val <- valD (varP n) (normalB [|col $(TH.lift nm)|]) [] + val <- valD (varP n) (normalB [|col $(TH.lift raw)|]) [] pure [sig, val] diff --git a/src/DataFrame/IO/CSV.hs b/src/DataFrame/IO/CSV.hs --- a/src/DataFrame/IO/CSV.hs +++ b/src/DataFrame/IO/CSV.hs @@ -23,25 +23,20 @@ import qualified Data.Vector.Unboxed as VU import qualified Data.Vector.Unboxed.Mutable as VUM -import Control.Applicative (many, (*>), (<$>), (<*), (<*>), (<|>)) -import Control.Monad (forM_, unless, void, when, zipWithM_) -import Control.Monad.ST (runST) +import Control.Applicative (many, (<|>)) +import Control.Monad (forM_, unless, zipWithM_) import Data.Attoparsec.ByteString.Char8 hiding (endOfLine) import Data.Bits (shiftL) import Data.Char import Data.Either import Data.Function (on) +import Data.Functor import Data.IORef import Data.Maybe -import Data.Type.Equality ( - TestEquality (testEquality), - type (:~:) (Refl), - ) -import DataFrame.Internal.Column (Column (..), MutableColumn (..), columnLength, freezeColumn', writeColumn) +import Data.Type.Equality (TestEquality (testEquality)) +import DataFrame.Internal.Column (Column (..), columnLength) import DataFrame.Internal.DataFrame (DataFrame (..)) import DataFrame.Internal.Parsing -import DataFrame.Operations.Typing -import GHC.IO.Handle (Handle) import System.IO import Type.Reflection import Prelude hiding (concat, takeWhile) @@ -70,7 +65,7 @@ , inferTypes :: Bool -- ^ Whether to try and infer types. (default: True) , safeRead :: Bool - -- ^ Whether to partially parse values into `Maybe`/Either`. (default: True) + -- ^ Whether to partially parse values into `Maybe`/`Either`. (default: True) , chunkSize :: Int -- ^ Default chunk size (in bytes) for csv reader. (default: 512'000) } @@ -148,64 +143,63 @@ ==== __Example__ @ -ghci> D.readCsv "./data/taxi.csv" df +ghci> D.readCsv ".\/data\/taxi.csv" @ -} -readCsv :: String -> IO DataFrame +readCsv :: FilePath -> IO DataFrame readCsv = readSeparated ',' defaultOptions {- | Read TSV (tab separated) file from path and load it into a dataframe. ==== __Example__ @ -ghci> D.readTsv "./data/taxi.tsv" df +ghci> D.readTsv ".\/data\/taxi.tsv" @ -} -readTsv :: String -> IO DataFrame +readTsv :: FilePath -> IO DataFrame readTsv = readSeparated '\t' defaultOptions {- | Read text file with specified delimiter into a dataframe. ==== __Example__ @ -ghci> D.readSeparated ';' D.defaultOptions "./data/taxi.txt" df +ghci> D.readSeparated ';' D.defaultOptions ".\/data\/taxi.txt" @ -} -readSeparated :: Char -> ReadOptions -> String -> IO DataFrame -readSeparated !sep !opts !path = do - withFile path ReadMode $ \handle -> do - hSetBuffering handle (BlockBuffering (Just (chunkSize opts))) +readSeparated :: Char -> ReadOptions -> FilePath -> IO DataFrame +readSeparated !sep !opts !path = withFile path ReadMode $ \handle -> do + hSetBuffering handle (BlockBuffering (Just (chunkSize opts))) - firstLine <- C8.hGetLine handle - let firstRow = parseLine sep firstLine - columnNames = - if hasHeader opts - then map (T.filter (/= '\"') . TE.decodeUtf8Lenient) firstRow - else map (T.singleton . intToDigit) [0 .. length firstRow - 1] + firstLine <- C8.hGetLine handle + let firstRow = parseLine sep firstLine + columnNames = + if hasHeader opts + then map (T.filter (/= '\"') . TE.decodeUtf8Lenient) firstRow + else map (T.singleton . intToDigit) [0 .. length firstRow - 1] - unless (hasHeader opts) $ hSeek handle AbsoluteSeek 0 + unless (hasHeader opts) $ hSeek handle AbsoluteSeek 0 - dataLine <- C8.hGetLine handle - let dataRow = parseLine sep dataLine - growingCols <- initializeColumns dataRow opts + dataLine <- C8.hGetLine handle + let dataRow = parseLine sep dataLine + growingCols <- initializeColumns dataRow opts - processRow 0 dataRow growingCols + processRow 0 dataRow growingCols - processFile handle sep growingCols (chunkSize opts) 1 + processFile handle sep growingCols (chunkSize opts) 1 - frozenCols <- V.fromList <$> mapM freezeGrowingColumn growingCols + frozenCols <- V.fromList <$> mapM freezeGrowingColumn growingCols - let numRows = maybe 0 columnLength (frozenCols V.!? 0) + let numRows = maybe 0 columnLength (frozenCols V.!? 0) - return $ - DataFrame - { columns = frozenCols - , columnIndices = M.fromList (zip columnNames [0 ..]) - , dataframeDimensions = (numRows, V.length frozenCols) - } + return $ + DataFrame + { columns = frozenCols + , columnIndices = M.fromList (zip columnNames [0 ..]) + , dataframeDimensions = (numRows, V.length frozenCols) + } initializeColumns :: [BS.ByteString] -> ReadOptions -> IO [GrowingColumn] initializeColumns row opts = mapM initColumn row @@ -261,7 +255,7 @@ processFile :: Handle -> Char -> [GrowingColumn] -> Int -> Int -> IO () processFile !handle !sep !cols !chunk r = do - let go remain !rowIdx = do + let go remain !rowIdx = parseWith (C8.hGetNonBlocking handle chunk) (parseRow sep) remain >>= \case Fail unconsumed ctx er -> do erpos <- hTell handle @@ -276,8 +270,7 @@ fail "Partial handler is called" Done (unconsumed :: C8.ByteString) (row :: [C8.ByteString]) -> do processRow rowIdx row cols - unless (row == [] || unconsumed == mempty) $ go unconsumed $! rowIdx + 1 - return () + unless (null row || unconsumed == mempty) $ go unconsumed $! rowIdx + 1 go "" r parseLine :: Char -> BS.ByteString -> [BS.ByteString] @@ -306,8 +299,8 @@ where parseQuotedContents = mconcat <$> many quotedChar quotedChar = - (Builder.byteString <$> takeWhile1 (/= '"')) - <|> (char '"' *> char '"' *> pure (Builder.char8 '"')) + Builder.byteString <$> takeWhile1 (/= '"') + <|> ((char '"' *> char '"') Data.Functor.$> Builder.char8 '"') "quoted field content" endOfLine :: Parser () @@ -328,7 +321,7 @@ if i `elem` nulls then VM.write mvec i Nothing else VM.write mvec i (Just (vec VU.! i)) - BoxedColumn <$> V.freeze mvec + OptionalColumn <$> V.freeze mvec freezeGrowingColumn (GrowingDouble gv nullsRef) = do vec <- freezeGrowingUnboxedVector gv nulls <- readIORef nullsRef @@ -341,7 +334,7 @@ if i `elem` nulls then VM.write mvec i Nothing else VM.write mvec i (Just (vec VU.! i)) - BoxedColumn <$> V.freeze mvec + OptionalColumn <$> V.freeze mvec freezeGrowingColumn (GrowingText gv nullsRef) = do vec <- freezeGrowingVector gv nulls <- readIORef nullsRef @@ -354,16 +347,16 @@ if i `elem` nulls then VM.write mvec i Nothing else VM.write mvec i (Just (vec V.! i)) - BoxedColumn <$> V.freeze mvec + OptionalColumn <$> V.freeze mvec -writeCsv :: String -> DataFrame -> IO () +writeCsv :: FilePath -> DataFrame -> IO () writeCsv = writeSeparated ',' writeSeparated :: -- | Separator Char -> -- | Path to write to - String -> + FilePath -> DataFrame -> IO () writeSeparated c filepath df = withFile filepath WriteMode $ \handle -> do @@ -387,7 +380,7 @@ App t1 t2 -> case eqTypeRep t1 (typeRep @Maybe) of Just HRefl -> case testEquality t2 (typeRep @T.Text) of Just Refl -> fromMaybe "null" e - Nothing -> (fromOptional . (T.pack . show)) e + Nothing -> (fromOptional . T.pack . show) e where fromOptional s | T.isPrefixOf "Just " s = T.drop (T.length "Just ") s diff --git a/src/DataFrame/IO/Parquet.hs b/src/DataFrame/IO/Parquet.hs --- a/src/DataFrame/IO/Parquet.hs +++ b/src/DataFrame/IO/Parquet.hs @@ -1,27 +1,24 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TypeApplications #-} -module DataFrame.IO.Parquet ( - readParquet, -) where +module DataFrame.IO.Parquet where import Control.Monad +import Data.Bits import qualified Data.ByteString as BSO import Data.Char -import Data.Foldable import Data.IORef -import Data.List +import Data.Int +import qualified Data.List as L import qualified Data.Map as M import Data.Maybe import qualified Data.Text as T +import Data.Word import qualified DataFrame.Internal.Column as DI import DataFrame.Internal.DataFrame (DataFrame) -import qualified DataFrame.Internal.DataFrame as DI import qualified DataFrame.Operations.Core as DI -import Foreign -import System.IO -import DataFrame.IO.Parquet.Binary import DataFrame.IO.Parquet.Dictionary import DataFrame.IO.Parquet.Levels import DataFrame.IO.Parquet.Page @@ -32,22 +29,19 @@ ==== __Example__ @ -ghci> D.readParquet "./data/mtcars.parquet" df - +ghci> D.readParquet ".\/data\/mtcars.parquet" @ -} -readParquet :: String -> IO DataFrame -readParquet path = withBinaryFile path ReadMode $ \handle -> do - (size, magicString) <- readMetadataSizeFromFooter handle - when (magicString /= "PAR1") $ error "Invalid Parquet file" - - fileMetadata <- readMetadata handle size - +readParquet :: FilePath -> IO DataFrame +readParquet path = do + fileMetadata <- readMetadataFromPath path let columnPaths = getColumnPaths (drop 1 $ schema fileMetadata) let columnNames = map fst columnPaths colMap <- newIORef (M.empty :: M.Map T.Text DI.Column) + contents <- BSO.readFile path + let schemaElements = schema fileMetadata let getTypeLength :: [String] -> Maybe Int32 getTypeLength path = findTypeLength schemaElements path 0 @@ -79,7 +73,7 @@ else colDataPageOffset let colLength = columnTotalCompressedSize metadata - columnBytes <- readBytes handle colStart colLength + let columnBytes = map (BSO.index contents . fromIntegral) [colStart .. (colStart + colLength - 1)] pages <- readAllPages (columnCodec metadata) columnBytes @@ -88,7 +82,7 @@ then getTypeLength colPath else Nothing - let primaryEncoding = fromMaybe EPLAIN (fmap fst (uncons (columnEncodings metadata))) + let primaryEncoding = fromMaybe EPLAIN (fmap fst (L.uncons (columnEncodings metadata))) let schemaTail = drop 1 (schema fileMetadata) let colPath = columnPathInSchema (columnMetaData colChunk) @@ -105,20 +99,22 @@ pure $ DI.fromNamedColumns orderedColumns -readMetadataSizeFromFooter :: Handle -> IO (Integer, BSO.ByteString) -readMetadataSizeFromFooter handle = do - footerOffSet <- numBytesInFile handle - buf <- mallocBytes 8 :: IO (Ptr Word8) - hSeek handle AbsoluteSeek (fromIntegral $ footerOffSet - 8) - _ <- hGetBuf handle buf 8 - - sizeBytes <- mapM (\i -> fromIntegral <$> (peekElemOff buf i :: IO Word8) :: IO Int32) [0 .. 3] - let size = fromIntegral $ foldl' (.|.) 0 $ zipWith shift sizeBytes [0, 8, 16, 24] +readMetadataFromPath path = do + contents <- BSO.readFile path + let (size, magicString) = contents `seq` readMetadataSizeFromFooter contents + when (magicString /= "PAR1") $ error "Invalid Parquet file" + readMetadata contents size - magicStringBytes <- mapM (\i -> peekElemOff buf i :: IO Word8) [4 .. 7] - let magicString = BSO.pack magicStringBytes - free buf - return (size, magicString) +readMetadataSizeFromFooter :: BSO.ByteString -> (Int, BSO.ByteString) +readMetadataSizeFromFooter contents = + let + footerOffSet = BSO.length contents - 8 + sizeBytes = map (fromIntegral @Word8 @Int32 . BSO.index contents) [footerOffSet .. footerOffSet + 3] + size = fromIntegral $ L.foldl' (.|.) 0 $ zipWith shift sizeBytes [0, 8, 16, 24] + magicStringBytes = map (BSO.index contents) [footerOffSet + 4 .. footerOffSet + 7] + magicString = BSO.pack magicStringBytes + in + (size, magicString) getColumnPaths :: [SchemaElement] -> [(T.Text, Int)] getColumnPaths schema = extractLeafPaths schema 0 [] @@ -247,4 +243,4 @@ case cols of [] -> pure $ DI.fromList ([] :: [Maybe Int]) - (c : cs) -> pure $ foldl' (\l r -> fromMaybe (error "concat failed") (DI.concatColumns l r)) c cs + (c : cs) -> pure $ L.foldl' (\l r -> fromMaybe (error "concat failed") (DI.concatColumns l r)) c cs diff --git a/src/DataFrame/IO/Parquet/Binary.hs b/src/DataFrame/IO/Parquet/Binary.hs --- a/src/DataFrame/IO/Parquet/Binary.hs +++ b/src/DataFrame/IO/Parquet/Binary.hs @@ -2,22 +2,21 @@ module DataFrame.IO.Parquet.Binary where -import Control.Monad -import Data.Bits +import Control.Monad +import Data.Bits import qualified Data.ByteString as BS -import Data.Char -import Data.IORef -import Data.Word -import Foreign -import System.IO +import Data.Char +import Data.IORef +import Data.Int +import Data.Word littleEndianWord32 :: [Word8] -> Word32 littleEndianWord32 bytes - | length bytes >= 4 = foldr (.|.) 0 (zipWith (\b i -> (fromIntegral b) `shiftL` i) (take 4 bytes) [0, 8, 16, 24]) + | length bytes >= 4 = foldr (.|.) 0 (zipWith (\b i -> fromIntegral b `shiftL` i) (take 4 bytes) [0, 8, 16, 24]) | otherwise = littleEndianWord32 (take 4 $ bytes ++ repeat 0) littleEndianWord64 :: [Word8] -> Word64 -littleEndianWord64 bytes = foldr (.|.) 0 (zipWith (\b i -> (fromIntegral b) `shiftL` i) (take 8 bytes) [0, 8 ..]) +littleEndianWord64 bytes = foldr (.|.) 0 (zipWith (\b i -> fromIntegral b `shiftL` i) (take 8 bytes) [0, 8 ..]) littleEndianInt32 :: [Word8] -> Int32 littleEndianInt32 = fromIntegral . littleEndianWord32 @@ -33,8 +32,8 @@ where loop bs x _ 10 = (x, bs) loop (b : bs) x s i - | b < 0x80 = (x .|. ((fromIntegral b) `shiftL` s), bs) - | otherwise = loop bs (x .|. (fromIntegral ((b .&. 0x7f) `shiftL` s))) (s + 7) (i + 1) + | b < 0x80 = (x .|. (fromIntegral b) `shiftL` s, bs) + | otherwise = loop bs (x .|. fromIntegral ((b .&. 0x7f) `shiftL` s)) (s + 7) (i + 1) readVarIntFromBytes :: (Integral a) => [Word8] -> (a, [Word8]) readVarIntFromBytes bs = (fromIntegral n, rem) @@ -42,8 +41,8 @@ (n, rem) = loop 0 0 bs loop _ result [] = (result, []) loop shift result (x : xs) = - let res = result .|. ((fromIntegral (x .&. 0x7f) :: Integer) `shiftL` shift) - in if (x .&. 0x80) /= 0x80 then (res, xs) else loop (shift + 7) res xs + let res = result .|. (fromIntegral (x .&. 0x7f) :: Integer) `shiftL` shift + in if x .&. 0x80 /= 0x80 then (res, xs) else loop (shift + 7) res xs readIntFromBytes :: (Integral a) => [Word8] -> (a, [Word8]) readIntFromBytes bs = @@ -58,75 +57,60 @@ u = fromIntegral n :: Word32 in ((fromIntegral (u `shiftR` 1) :: Int32) .^. (-(n .&. 1)), rem) -readAndAdvance :: IORef Int -> Ptr b -> IO Word8 +readAndAdvance :: IORef Int -> BS.ByteString -> IO Word8 readAndAdvance bufferPos buffer = do pos <- readIORef bufferPos - b <- peekByteOff buffer pos :: IO Word8 + let b = BS.index buffer pos modifyIORef bufferPos (+ 1) return b -readVarIntFromBuffer :: (Integral a) => Ptr b -> IORef Int -> IO a +readVarIntFromBuffer :: (Integral a) => BS.ByteString -> IORef Int -> IO a readVarIntFromBuffer buf bufferPos = do start <- readIORef bufferPos let loop i shift result = do b <- readAndAdvance bufferPos buf - let res = result .|. ((fromIntegral (b .&. 0x7f) :: Integer) `shiftL` shift) - if (b .&. 0x80) /= 0x80 + let res = result .|. (fromIntegral (b .&. 0x7f) :: Integer) `shiftL` shift + if b .&. 0x80 /= 0x80 then return res else loop (i + 1) (shift + 7) res fromIntegral <$> loop start 0 0 -readIntFromBuffer :: (Integral a) => Ptr b -> IORef Int -> IO a +readIntFromBuffer :: (Integral a) => BS.ByteString -> IORef Int -> IO a readIntFromBuffer buf bufferPos = do n <- readVarIntFromBuffer buf bufferPos let u = fromIntegral n :: Word32 return $ fromIntegral $ (fromIntegral (u `shiftR` 1) :: Int32) .^. (-(n .&. 1)) -readInt32FromBuffer :: Ptr b -> IORef Int -> IO Int32 +readInt32FromBuffer :: BS.ByteString -> IORef Int -> IO Int32 readInt32FromBuffer buf bufferPos = do n <- (fromIntegral <$> readVarIntFromBuffer @Int64 buf bufferPos) :: IO Int32 let u = fromIntegral n :: Word32 return $ (fromIntegral (u `shiftR` 1) :: Int32) .^. (-(n .&. 1)) -readString :: Ptr Word8 -> IORef Int -> IO String +readString :: BS.ByteString -> IORef Int -> IO String readString buf pos = do nameSize <- readVarIntFromBuffer @Int buf pos map (chr . fromIntegral) <$> replicateM nameSize (readAndAdvance pos buf) -readBytes :: Handle -> Int64 -> Int64 -> IO [Word8] -readBytes h colStart colLen = do - hSeek h AbsoluteSeek (fromIntegral colStart) - bs <- BS.hGet h (fromIntegral colLen) - if BS.length bs /= (fromIntegral colLen) - then ioError (userError ("short read: wanted " - ++ show colLen ++ " bytes, got " - ++ show (BS.length bs))) - else pure (BS.unpack bs) - -numBytesInFile :: Handle -> IO Integer -numBytesInFile handle = do - hSeek handle SeekFromEnd 0 - hTell handle - readByteStringFromBytes :: [Word8] -> ([Word8], [Word8]) readByteStringFromBytes xs = let (size, rem) = readVarIntFromBytes @Int xs in - (take size rem, drop size rem) + splitAt size rem -readByteString :: Ptr Word8 -> IORef Int -> IO [Word8] +readByteString :: BS.ByteString -> IORef Int -> IO [Word8] readByteString buf pos = do size <- readVarIntFromBuffer @Int buf pos replicateM size (readAndAdvance pos buf) -readByteString' :: Ptr Word8 -> Int64 -> IO [Word8] +readByteString' :: BS.ByteString -> Int64 -> IO [Word8] readByteString' buf size = mapM (`readSingleByte` buf) [0 .. (size - 1)] -readSingleByte :: Int64 -> Ptr b -> IO Word8 -readSingleByte pos buffer = peekByteOff buffer (fromIntegral pos) +readSingleByte :: Int64 -> BS.ByteString -> IO Word8 +readSingleByte pos buffer = return $ BS.index buffer (fromIntegral pos) -readNoAdvance :: IORef Int -> Ptr b -> IO Word8 +readNoAdvance :: IORef Int -> BS.ByteString -> IO Word8 readNoAdvance bufferPos buffer = do pos <- readIORef bufferPos - peekByteOff buffer pos :: IO Word8 + return $ BS.index buffer pos diff --git a/src/DataFrame/IO/Parquet/Dictionary.hs b/src/DataFrame/IO/Parquet/Dictionary.hs --- a/src/DataFrame/IO/Parquet/Dictionary.hs +++ b/src/DataFrame/IO/Parquet/Dictionary.hs @@ -3,6 +3,7 @@ module DataFrame.IO.Parquet.Dictionary where import Control.Monad +import Data.Bits import Data.Char import Data.Int import Data.Maybe @@ -15,10 +16,7 @@ import DataFrame.IO.Parquet.Time import DataFrame.IO.Parquet.Types import qualified DataFrame.Internal.Column as DI -import Foreign import GHC.Float -import GHC.IO (unsafePerformIO) -import Text.Printf dictCardinality :: DictVals -> Int dictCardinality (DBool ds) = length ds diff --git a/src/DataFrame/IO/Parquet/Encoding.hs b/src/DataFrame/IO/Parquet/Encoding.hs --- a/src/DataFrame/IO/Parquet/Encoding.hs +++ b/src/DataFrame/IO/Parquet/Encoding.hs @@ -1,8 +1,7 @@ module DataFrame.IO.Parquet.Encoding where import Data.Bits -import Data.Foldable -import Data.List (mapAccumL) +import Data.List (foldl') import Data.Word import DataFrame.IO.Parquet.Binary @@ -35,10 +34,10 @@ | length bitsLeft < bw = [] | otherwise = let (this, bitsLeft') = splitAt bw bitsLeft - in toN this : extractValues (n - 1) bitsLeft' + in toN this : extractValues (n - 1) bitsLeft' vals = extractValues count bits - in (map fromIntegral vals, rest) + in (map fromIntegral vals, rest) decodeRLEBitPackedHybrid :: Int -> Int -> [Word8] -> ([Word32], [Word8]) decodeRLEBitPackedHybrid bw need bs @@ -49,7 +48,7 @@ mask = if bw == 32 then maxBound else (1 `shiftL` bw) - 1 go 0 rest acc = (reverse acc, rest) go n rest acc - | null rest = error $ "Premature end of stream: need " ++ show n ++ " more values" + | null rest = (reverse acc, rest) | otherwise = let (hdr64, afterHdr) = readUVarInt rest isPacked = (hdr64 .&. 1) == 1 diff --git a/src/DataFrame/IO/Parquet/Page.hs b/src/DataFrame/IO/Parquet/Page.hs --- a/src/DataFrame/IO/Parquet/Page.hs +++ b/src/DataFrame/IO/Parquet/Page.hs @@ -4,25 +4,16 @@ module DataFrame.IO.Parquet.Page where import Codec.Compression.Zstd.Streaming -import Control.Monad import Data.Bits import qualified Data.ByteString as BSO -import Data.Char -import Data.Foldable import Data.Int -import Data.List import Data.Maybe -import qualified Data.Text as T import Data.Word import DataFrame.IO.Parquet.Binary -import DataFrame.IO.Parquet.Dictionary -import DataFrame.IO.Parquet.Levels import DataFrame.IO.Parquet.Thrift import DataFrame.IO.Parquet.Types -import qualified DataFrame.Internal.Column as DI import GHC.Float -import qualified Snappy as Snappy -import Text.Printf +import qualified Snappy isDataPage :: Page -> Bool isDataPage page = case pageTypeHeader (pageHeader page) of diff --git a/src/DataFrame/IO/Parquet/Thrift.hs b/src/DataFrame/IO/Parquet/Thrift.hs --- a/src/DataFrame/IO/Parquet/Thrift.hs +++ b/src/DataFrame/IO/Parquet/Thrift.hs @@ -5,6 +5,7 @@ import Control.Monad import Data.Bits +import qualified Data.ByteString as BS import Data.Char import Data.IORef import Data.Int @@ -14,8 +15,6 @@ import Data.Word import DataFrame.IO.Parquet.Binary import DataFrame.IO.Parquet.Types -import Foreign -import System.IO data SchemaElement = SchemaElement { elementName :: T.Text @@ -163,7 +162,7 @@ , (compactStruct, STRUCT) ] -readField :: Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO (Maybe (TType, Int16)) +readField :: BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO (Maybe (TType, Int16)) readField buf pos lastFieldId fieldStack = do t <- readAndAdvance pos buf if t .&. 0x0f == 0 @@ -177,7 +176,7 @@ let elemType = toTType (t .&. 0x0f) pure $ Just (elemType, identifier) -skipToStructEnd :: Ptr Word8 -> IORef Int -> IO () +skipToStructEnd :: BS.ByteString -> IORef Int -> IO () skipToStructEnd buf pos = do t <- readAndAdvance pos buf if t .&. 0x0f == 0 @@ -192,7 +191,7 @@ skipFieldData elemType buf pos skipToStructEnd buf pos -skipFieldData :: TType -> Ptr Word8 -> IORef Int -> IO () +skipFieldData :: TType -> BS.ByteString -> IORef Int -> IO () skipFieldData fieldType buf pos = case fieldType of BOOL -> return () I32 -> readIntFromBuffer @Int32 buf pos >> pure () @@ -203,29 +202,24 @@ STRUCT -> skipToStructEnd buf pos _ -> return () -skipList :: Ptr Word8 -> IORef Int -> IO () +skipList :: BS.ByteString -> IORef Int -> IO () skipList buf pos = do sizeAndType <- readAndAdvance pos buf let sizeOnly = fromIntegral ((sizeAndType `shiftR` 4) .&. 0x0f) :: Int let elemType = toTType sizeAndType replicateM_ sizeOnly (skipFieldData elemType buf pos) -readMetadata :: Handle -> Integer -> IO FileMetadata -readMetadata handle size = do - metaDataBuf <- mallocBytes (fromIntegral size) :: IO (Ptr Word8) - footerOffSet <- numBytesInFile handle - - hSeek handle AbsoluteSeek (fromIntegral $! footerOffSet - footerSize - size) - - metadataBytesRead <- hGetBuf handle metaDataBuf (fromIntegral size) +readMetadata :: BS.ByteString -> Int -> IO FileMetadata +readMetadata contents size = do + let metadataStartPos = BS.length contents - footerSize - size + let metadataBytes = BS.pack $ map (BS.index contents) [metadataStartPos .. (metadataStartPos + size - 1)] let lastFieldId = 0 let fieldStack = [] bufferPos <- newIORef (0 :: Int) - metadata <- readFileMetaData defaultMetadata metaDataBuf bufferPos lastFieldId fieldStack - free metaDataBuf + metadata <- readFileMetaData defaultMetadata metadataBytes bufferPos lastFieldId fieldStack return metadata -readFileMetaData :: FileMetadata -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO FileMetadata +readFileMetaData :: FileMetadata -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO FileMetadata readFileMetaData metadata metaDataBuf bufferPos lastFieldId fieldStack = do fieldContents <- readField metaDataBuf bufferPos lastFieldId fieldStack case fieldContents of @@ -288,7 +282,7 @@ readFileMetaData (metadata{footerSigningKeyMetadata = footerSigningKeyMetadata}) metaDataBuf bufferPos identifier fieldStack n -> return $ error $ "UNIMPLEMENTED " ++ show n -readSchemaElement :: SchemaElement -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO SchemaElement +readSchemaElement :: SchemaElement -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO SchemaElement readSchemaElement schemaElement buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -333,7 +327,7 @@ skipFieldData elemType buf pos readSchemaElement schemaElement buf pos identifier fieldStack -readRowGroup :: RowGroup -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO RowGroup +readRowGroup :: RowGroup -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO RowGroup readRowGroup r buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -364,7 +358,7 @@ readRowGroup (r{ordinal = ordinal}) buf pos identifier fieldStack _ -> error $ "Unknown row group field: " ++ show identifier -readColumnChunk :: ColumnChunk -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO ColumnChunk +readColumnChunk :: ColumnChunk -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ColumnChunk readColumnChunk c buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -394,7 +388,7 @@ readColumnChunk (c{columnChunkColumnIndexLength = columnChunkColumnIndexLength}) buf pos identifier fieldStack _ -> return c -readColumnMetadata :: ColumnMetaData -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO ColumnMetaData +readColumnMetadata :: ColumnMetaData -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ColumnMetaData readColumnMetadata cm buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -463,7 +457,7 @@ 17 -> return $ error "UNIMPLEMENTED" _ -> return cm -readEncryptionAlgorithm :: Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO EncryptionAlgorithm +readEncryptionAlgorithm :: BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO EncryptionAlgorithm readEncryptionAlgorithm buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -475,7 +469,7 @@ readAesGcmCtrV1 (AesGcmCtrV1{aadPrefix = [], aadFileUnique = [], supplyAadPrefix = False}) buf pos 0 [] n -> return ENCRYPTION_ALGORITHM_UNKNOWN -readColumnOrder :: Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO ColumnOrder +readColumnOrder :: BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ColumnOrder readColumnOrder buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -486,7 +480,7 @@ return TYPE_ORDER _ -> return COLUMN_ORDER_UNKNOWN -readAesGcmCtrV1 :: EncryptionAlgorithm -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO EncryptionAlgorithm +readAesGcmCtrV1 :: EncryptionAlgorithm -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO EncryptionAlgorithm readAesGcmCtrV1 v@(AesGcmCtrV1 aadPrefix aadFileUnique supplyAadPrefix) buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -503,7 +497,7 @@ readAesGcmCtrV1 (v{supplyAadPrefix = supplyAadPrefix == compactBooleanTrue}) buf pos lastFieldId fieldStack _ -> return ENCRYPTION_ALGORITHM_UNKNOWN -readAesGcmV1 :: EncryptionAlgorithm -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO EncryptionAlgorithm +readAesGcmV1 :: EncryptionAlgorithm -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO EncryptionAlgorithm readAesGcmV1 v@(AesGcmV1 aadPrefix aadFileUnique supplyAadPrefix) buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -520,7 +514,7 @@ readAesGcmV1 (v{supplyAadPrefix = supplyAadPrefix == compactBooleanTrue}) buf pos lastFieldId fieldStack _ -> return ENCRYPTION_ALGORITHM_UNKNOWN -readTypeOrder :: Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO ColumnOrder +readTypeOrder :: BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ColumnOrder readTypeOrder buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -530,7 +524,7 @@ then return TYPE_ORDER else readTypeOrder buf pos identifier fieldStack -readKeyValue :: KeyValue -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO KeyValue +readKeyValue :: KeyValue -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO KeyValue readKeyValue kv buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -544,7 +538,7 @@ readKeyValue (kv{value = v}) buf pos identifier fieldStack _ -> return kv -readPageEncodingStats :: PageEncodingStats -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO PageEncodingStats +readPageEncodingStats :: PageEncodingStats -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO PageEncodingStats readPageEncodingStats pes buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -561,10 +555,10 @@ readPageEncodingStats (pes{pagesWithEncoding = encodedCount}) buf pos identifier [] _ -> pure pes -readParquetEncoding :: Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO ParquetEncoding +readParquetEncoding :: BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ParquetEncoding readParquetEncoding buf pos lastFieldId fieldStack = parquetEncodingFromInt <$> readInt32FromBuffer buf pos -readStatistics :: ColumnStatistics -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO ColumnStatistics +readStatistics :: ColumnStatistics -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO ColumnStatistics readStatistics cs buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -596,7 +590,7 @@ readStatistics (cs{isColumnMinValueExact = isMinValueExact == compactBooleanTrue}) buf pos identifier fieldStack _ -> pure cs -readSizeStatistics :: SizeStatistics -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO SizeStatistics +readSizeStatistics :: SizeStatistics -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO SizeStatistics readSizeStatistics ss buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -619,7 +613,7 @@ readSizeStatistics (ss{definitionLevelHistogram = definitionLevelHistogram}) buf pos identifier fieldStack _ -> pure ss -footerSize :: Integer +footerSize :: Int footerSize = 8 toIntegralType :: Int32 -> TType @@ -630,7 +624,7 @@ | n == 6 = STRING | otherwise = STRING -readLogicalType :: Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO LogicalType +readLogicalType :: BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO LogicalType readLogicalType buf pos lastFieldId fieldStack = do t <- readAndAdvance pos buf if t .&. 0x0f == 0 @@ -693,7 +687,7 @@ return GeographyType{crs = "", algorithm = SPHERICAL} _ -> return LOGICAL_TYPE_UNKNOWN -readIntType :: LogicalType -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO LogicalType +readIntType :: LogicalType -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO LogicalType readIntType v@(IntType bitWidth intIsSigned) buf pos lastFieldId fieldStack = do t <- readAndAdvance pos buf if t .&. 0x0f == 0 @@ -714,7 +708,7 @@ readIntType (v{intIsSigned = isSigned}) buf pos identifier fieldStack _ -> error $ "UNKNOWN field ID for IntType: " ++ show identifier -readDecimalType :: LogicalType -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO LogicalType +readDecimalType :: LogicalType -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO LogicalType readDecimalType v@(DecimalType p s) buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -728,7 +722,7 @@ readDecimalType (v{decimalTypePrecision = p'}) buf pos lastFieldId fieldStack _ -> error $ "UNKNOWN field ID for DecimalType" ++ show identifier -readTimeType :: LogicalType -> Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO LogicalType +readTimeType :: LogicalType -> BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO LogicalType readTimeType v@(TimeType _ _) buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of @@ -756,7 +750,7 @@ readTimeType (v{unit = u}) buf pos lastFieldId fieldStack _ -> error $ "UNKNOWN field ID for TimestampType" ++ show identifier -readUnit :: Ptr Word8 -> IORef Int -> Int16 -> [Int16] -> IO TimeUnit +readUnit :: BS.ByteString -> IORef Int -> Int16 -> [Int16] -> IO TimeUnit readUnit buf pos lastFieldId fieldStack = do fieldContents <- readField buf pos lastFieldId fieldStack case fieldContents of diff --git a/src/DataFrame/IO/Parquet/Time.hs b/src/DataFrame/IO/Parquet/Time.hs --- a/src/DataFrame/IO/Parquet/Time.hs +++ b/src/DataFrame/IO/Parquet/Time.hs @@ -2,7 +2,6 @@ module DataFrame.IO.Parquet.Time where -import Data.Int import Data.Time import Data.Word diff --git a/src/DataFrame/Internal/Column.hs b/src/DataFrame/Internal/Column.hs --- a/src/DataFrame/Internal/Column.hs +++ b/src/DataFrame/Internal/Column.hs @@ -17,8 +17,6 @@ module DataFrame.Internal.Column where -import qualified Data.ByteString.Char8 as C -import qualified Data.List as L import qualified Data.Set as S import qualified Data.Text as T import qualified Data.Vector as VB @@ -30,19 +28,12 @@ import Control.Exception (throw) import Control.Monad.ST (runST) -import Data.Int -import Data.Kind (Constraint, Type) import Data.Maybe -import Data.Proxy -import Data.Text.Encoding (decodeUtf8Lenient) -import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) -import Data.Typeable (Typeable, cast) -import Data.Word +import Data.Type.Equality (TestEquality (..)) import DataFrame.Errors import DataFrame.Internal.Parsing import DataFrame.Internal.Types import Type.Reflection -import Unsafe.Coerce (unsafeCoerce) {- | Our representation of a column is a GADT that can store data based on the underlying data. @@ -64,17 +55,28 @@ data TypedColumn a where TColumn :: (Columnable a) => Column -> TypedColumn a +instance (Eq a) => Eq (TypedColumn a) where + (==) (TColumn a) (TColumn b) = a == b + +instance (Ord a) => Ord (TypedColumn a) where + (compare) (TColumn a) (TColumn b) = compare a b + -- | Gets the underlying value from a TypedColumn. unwrapTypedColumn :: TypedColumn a -> Column unwrapTypedColumn (TColumn value) = value -{- | An internal function that checks if a column can -be used in missing value operations. --} -isOptional :: Column -> Bool -isOptional (OptionalColumn column) = True -isOptional _ = False +-- | Checks if a column contains missing values. +hasMissing :: Column -> Bool +hasMissing (OptionalColumn column) = True +hasMissing _ = False +-- | Checks if a column contains numeric values. +isNumeric :: Column -> Bool +isNumeric (UnboxedColumn (vec :: VU.Vector a)) = case sNumeric @a of + STrue -> True + _ -> False +isNumeric _ = False + -- | An internal/debugging function to get the column type of a column. columnVersionString :: Column -> String columnVersionString column = case column of @@ -92,6 +94,7 @@ OptionalColumn (column :: VB.Vector a) -> show (typeRep @a) instance (Show a) => Show (TypedColumn a) where + show :: Show a => TypedColumn a -> String show (TColumn col) = show col instance Show Column where @@ -116,6 +119,22 @@ Just Refl -> a == b (==) _ _ = False +instance Ord Column where + (<=) :: Column -> Column -> Bool + (<=) (BoxedColumn (a :: VB.Vector t1)) (BoxedColumn (b :: VB.Vector t2)) = + case testEquality (typeRep @t1) (typeRep @t2) of + Nothing -> False + Just Refl -> a <= b + (<=) (OptionalColumn (a :: VB.Vector t1)) (OptionalColumn (b :: VB.Vector t2)) = + case testEquality (typeRep @t1) (typeRep @t2) of + Nothing -> False + Just Refl -> a <= b + (<=) (UnboxedColumn (a :: VU.Vector t1)) (UnboxedColumn (b :: VU.Vector t2)) = + case testEquality (typeRep @t1) (typeRep @t2) of + Nothing -> False + Just Refl -> a <= b + (<=) _ _ = False + {- | A class for converting a vector to a column of the appropriate type. Given each Rep we tell the `toColumnRep` function which Column type to pick. -} @@ -123,18 +142,30 @@ toColumnRep :: VB.Vector a -> Column -- | Constraint synonym for what we can put into columns. -type Columnable a = (Columnable' a, ColumnifyRep (KindOf a) a, UnboxIf a, SBoolI (Unboxable a), SBoolI (Numeric a)) +type Columnable a = + ( Columnable' a + , ColumnifyRep (KindOf a) a + , UnboxIf a + , IntegralIf a + , FloatingIf a + , SBoolI (Unboxable a) + , SBoolI (Numeric a) + , SBoolI (IntegralTypes a) + , SBoolI (FloatingTypes a) + ) instance (Columnable a, VU.Unbox a) => ColumnifyRep 'RUnboxed a where + toColumnRep :: (Columnable a, VUM.Unbox a) => VB.Vector a -> Column toColumnRep = UnboxedColumn . VU.convert instance (Columnable a) => ColumnifyRep 'RBoxed a where + toColumnRep :: Columnable a => VB.Vector a -> Column toColumnRep = BoxedColumn instance @@ -224,7 +255,7 @@ numElements :: Column -> Int numElements (BoxedColumn xs) = VG.length xs numElements (UnboxedColumn xs) = VG.length xs -numElements (OptionalColumn xs) = VG.foldl' (\acc x -> acc + (fromEnum (isJust x))) 0 xs +numElements (OptionalColumn xs) = VG.foldl' (\acc x -> acc + fromEnum (isJust x)) 0 xs {-# INLINE numElements #-} -- | O(n) Takes the first n values of a column. diff --git a/src/DataFrame/Internal/DataFrame.hs b/src/DataFrame/Internal/DataFrame.hs --- a/src/DataFrame/Internal/DataFrame.hs +++ b/src/DataFrame/Internal/DataFrame.hs @@ -14,11 +14,11 @@ import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU -import Control.Monad (join) +import Control.Exception (throw) import Data.Function (on) import Data.List (sortBy, transpose, (\\)) -import Data.Maybe (isJust) import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) +import DataFrame.Errors (DataFrameException(..)) import DataFrame.Display.Terminal.PrettyPrint import DataFrame.Internal.Column import Text.Printf @@ -58,13 +58,17 @@ instance Eq DataFrame where (==) :: DataFrame -> DataFrame -> Bool a == b = - map fst (M.toList $ columnIndices a) == map fst (M.toList $ columnIndices b) + M.keys (columnIndices a) == M.keys (columnIndices b) && foldr (\(name, index) acc -> acc && (columns a V.!? index == (columns b V.!? (columnIndices b M.! name)))) True (M.toList $ columnIndices a) instance Show DataFrame where show :: DataFrame -> String show d = T.unpack (asText d False) +-- | For showing the dataframe as markdown in notebooks. +toMarkdownTable :: DataFrame -> T.Text +toMarkdownTable df = asText df True + asText :: DataFrame -> Bool -> T.Text asText d properMarkdown = let header = "index" : map fst (sortBy (compare `on` snd) $ M.toList (columnIndices d)) @@ -83,6 +87,7 @@ Nothing -> V.map (T.pack . show) column get (Just (UnboxedColumn column)) = V.map (T.pack . show) (V.convert column) get (Just (OptionalColumn column)) = V.map (T.pack . show) column + get Nothing = V.empty getTextColumnFromFrame df (i, name) = if i == 0 then V.fromList (map (T.pack . show) [0 .. (fst (dataframeDimensions df) - 1)]) @@ -107,7 +112,9 @@ columns df V.!? i unsafeGetColumn :: T.Text -> DataFrame -> Column -unsafeGetColumn name df = columns df V.! (columnIndices df M.! name) +unsafeGetColumn name df = case getColumn name df of + Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df) + Just col -> col null :: DataFrame -> Bool null df = V.null (columns df) diff --git a/src/DataFrame/Internal/Expression.hs b/src/DataFrame/Internal/Expression.hs --- a/src/DataFrame/Internal/Expression.hs +++ b/src/DataFrame/Internal/Expression.hs @@ -14,7 +14,6 @@ module DataFrame.Internal.Expression where import Control.Exception (throw) -import Data.Data (Typeable) import qualified Data.Map as M import Data.Maybe (fromMaybe) import qualified Data.Text as T @@ -26,12 +25,17 @@ import DataFrame.Internal.Column import DataFrame.Internal.DataFrame import DataFrame.Internal.Types -import Type.Reflection (typeRep) +import Type.Reflection (typeRep, Typeable) data Expr a where Col :: (Columnable a) => T.Text -> Expr a Lit :: (Columnable a) => a -> Expr a - Apply :: + If :: (Columnable a) + => Expr Bool + -> Expr a + -> Expr a + -> Expr a + UnaryOp :: ( Columnable a , Columnable b ) => @@ -39,7 +43,7 @@ (b -> a) -> Expr b -> Expr a - BinOp :: + BinaryOp :: ( Columnable c , Columnable b , Columnable a @@ -49,19 +53,22 @@ Expr c -> Expr b -> Expr a - GeneralAggregate :: - (Columnable a) => - T.Text -> -- Column name + AggVector :: + ( VG.Vector v b + , Typeable v + , Columnable a + , Columnable b) => + Expr b -> T.Text -> -- Operation name - (forall v b. (VG.Vector v b, Columnable b) => v b -> a) -> + (v b -> a) -> Expr a - ReductionAggregate :: + AggReduce :: (Columnable a) => - T.Text -> -- Column name + Expr a -> T.Text -> -- Operation name (forall a. (Columnable a) => a -> a -> a) -> Expr a - NumericAggregate :: + AggNumericVector :: ( Columnable a , Columnable b , VU.Unbox a @@ -69,10 +76,18 @@ , Num a , Num b ) => - T.Text -> -- Column name + Expr b -> T.Text -> -- Operation name (VU.Vector b -> a) -> Expr a + AggFold :: + forall a b. + (Columnable a, Columnable b) => + Expr b -> + T.Text -> -- Operation name + a -> + (a -> b -> a) -> + Expr a data UExpr where Wrap :: (Columnable a) => Expr a -> UExpr @@ -80,115 +95,156 @@ interpret :: forall a. (Columnable a) => DataFrame -> Expr a -> TypedColumn a interpret df (Lit value) = TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value interpret df (Col name) = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df) Just col -> TColumn col -interpret df (Apply _ (f :: c -> d) value) = +interpret df (If cond l r) = let + (TColumn conditions) = interpret @Bool df cond + (TColumn left) = interpret @a df l + (TColumn right) = interpret @a df r + in + TColumn $ fromMaybe (error "zipWithColumns returned nothing") $ zipWithColumns (\(c :: Bool) (l' :: a, r' :: a)-> if c then l' else r') conditions (zipColumns left right) +interpret df (UnaryOp _ (f :: c -> d) value) = + let (TColumn value') = interpret @c df value in -- TODO: Handle this gracefully. TColumn $ fromMaybe (error "mapColumn returned nothing") (mapColumn f value') -interpret df (BinOp _ (f :: c -> d -> e) (Lit left) right) = +interpret df (BinaryOp _ (f :: c -> d -> e) (Lit left) right) = let (TColumn right') = interpret @d df right in TColumn $ fromMaybe (error "mapColumn returned nothing") (mapColumn (f left) right') -interpret df (BinOp _ (f :: c -> d -> e) left (Lit right)) = +interpret df (BinaryOp _ (f :: c -> d -> e) left (Lit right)) = let (TColumn left') = interpret @c df left in - TColumn $ fromMaybe (error "mapColumn returned nothing") (mapColumn (flip f right) left') -interpret df (BinOp _ (f :: c -> d -> e) left right) = + TColumn $ fromMaybe (error "mapColumn returned nothing") (mapColumn (`f` right) left') +interpret df (BinaryOp _ (f :: c -> d -> e) left right) = let (TColumn left') = interpret @c df left (TColumn right') = interpret @d df right in TColumn $ fromMaybe (error "mapColumn returned nothing") (zipWithColumns f left' right') -interpret df (ReductionAggregate name op (f :: forall a. (Columnable a) => a -> a -> a)) = +interpret df (AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) = let - (TColumn column) = interpret @a df (Col name) + (TColumn column) = interpret @a df expr in case headColumn @a column of Nothing -> error "Invalid operation" Just h -> case ifoldlColumn (\acc _ v -> f acc v) h column of Nothing -> error "Invalid operation" Just value -> TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value +interpret df (AggNumericVector expr op (f :: VU.Vector b -> c)) = + let + (TColumn column) = interpret @b df expr + in + case column of + (UnboxedColumn (v :: VU.Vector d)) -> case testEquality (typeRep @d) (typeRep @b) of + Just Refl -> TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) (f v) + Nothing -> error "Invalid operation" + _ -> error "Invalid operation" +interpret df (AggFold expr op start (f :: (a -> b -> a))) = + let + (TColumn column) = interpret @b df expr + in + case headColumn @a column of + Nothing -> error "Invalid operation" + Just h -> case ifoldlColumn (\acc _ v -> f acc v) h column of + Nothing -> error "Invalid operation" + Just value -> TColumn $ fromVector $ V.replicate (fst $ dataframeDimensions df) value interpret _ expr = error ("Invalid operation for dataframe: " ++ show expr) interpretAggregation :: forall a. (Columnable a) => GroupedDataFrame -> Expr a -> TypedColumn a interpretAggregation gdf (Lit value) = TColumn $ fromVector $ V.replicate (VG.length (offsets gdf) - 1) value interpretAggregation gdf@(Grouped df names indices os) (Col name) = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException name "" (M.keys $ columnIndices df) Just col -> TColumn $ atIndicesStable (VG.map (indices `VG.unsafeIndex`) (VG.init os)) col -interpretAggregation gdf (Apply _ (f :: c -> d) expr) = +interpretAggregation gdf (UnaryOp _ (f :: c -> d) expr) = let (TColumn value) = interpretAggregation @c gdf expr in case mapColumn f value of Nothing -> error "Type error in interpretation" Just col -> TColumn col -interpretAggregation gdf (BinOp _ (f :: c -> d -> e) left right) = +interpretAggregation gdf (If cond l r) = let + (TColumn conditions) = interpretAggregation @Bool gdf cond + (TColumn left) = interpretAggregation @a gdf l + (TColumn right) = interpretAggregation @a gdf r + in + TColumn $ fromMaybe (error "zipWithColumns returned nothing") $ zipWithColumns (\(c :: Bool) (l' :: a, r' :: a)-> if c then l' else r') conditions (zipColumns left right) +interpretAggregation gdf (BinaryOp _ (f :: c -> d -> e) left right) = + let (TColumn left') = interpretAggregation @c gdf left (TColumn right') = interpretAggregation @d gdf right in case zipWithColumns f left' right' of Nothing -> error "Type error in binary operation" Just col -> TColumn col -interpretAggregation gdf@(Grouped df names indices os) (GeneralAggregate name op (f :: forall v b. (VG.Vector v b, Columnable b) => v b -> c)) = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "" (map fst $ M.toList $ columnIndices df) - Just (BoxedColumn col) -> - TColumn $ - fromVector $ - V.generate - (VG.length os - 1) - ( \i -> - f - ( V.generate - (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) - (\j -> col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))) +interpretAggregation gdf@(Grouped df names indices os) (AggVector expr op (f :: v b -> c)) = + case unwrapTypedColumn (interpretAggregation @b gdf expr) of + BoxedColumn (col :: V.Vector d) -> case testEquality (typeRep @b) (typeRep @d) of + Nothing -> error "Type mismatch" + Just Refl -> case testEquality (typeRep @v) (typeRep @V.Vector) of + Nothing -> error "Container mismatch" + Just Refl -> TColumn $ + fromVector $ + V.generate + (VG.length os - 1) + ( \i -> + f + ( V.generate + (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) + (\j -> col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))) + ) ) - ) - Just (UnboxedColumn col) -> case sUnbox @c of - SFalse -> - TColumn $ - fromVector $ - V.generate - (VG.length os - 1) - ( \i -> - f - ( VU.generate - (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) - (\j -> col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))) - ) - ) - STrue -> - TColumn $ - fromUnboxedVector $ - VU.generate - (VG.length os - 1) - ( \i -> - f - ( VU.generate - (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) - (\j -> col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))) - ) - ) - Just (OptionalColumn col) -> - TColumn $ - fromVector $ - V.generate - (VG.length os - 1) - ( \i -> - f - ( V.generate - (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) - (\j -> col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))) + UnboxedColumn (col :: VU.Vector d) -> case testEquality (typeRep @b) (typeRep @d) of + Just Refl -> case testEquality (typeRep @v) (typeRep @VU.Vector) of + Nothing -> error "Container mismatch" + Just Refl -> + case sUnbox @c of + SFalse -> + TColumn $ + fromVector $ + V.generate + (VG.length os - 1) + ( \i -> + f + ( VU.generate + (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) + (\j -> col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))) + ) + ) + STrue -> + TColumn $ + fromUnboxedVector $ + VU.generate + (VG.length os - 1) + ( \i -> + f + ( VU.generate + (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) + (\j -> col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))) + ) + ) + OptionalColumn (col :: V.Vector d) -> case testEquality (typeRep @b) (typeRep @d) of + Nothing -> error "Type mismatch" + Just Refl -> case testEquality (typeRep @v) (typeRep @V.Vector) of + Nothing -> error "Container mismatch" + Just Refl -> TColumn $ + fromVector $ + V.generate + (VG.length os - 1) + ( \i -> + f + ( V.generate + (os `VG.unsafeIndex` (i + 1) - (os `VG.unsafeIndex` i)) + (\j -> col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))) + ) ) - ) -interpretAggregation gdf@(Grouped df names indices os) (ReductionAggregate name op (f :: forall a. (Columnable a) => a -> a -> a)) = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "" (map fst $ M.toList $ columnIndices df) - Just (BoxedColumn col) -> TColumn $ +interpretAggregation gdf@(Grouped df names indices os) (AggReduce expr op (f :: forall a. (Columnable a) => a -> a -> a)) = case unwrapTypedColumn (interpretAggregation @a gdf expr) of + BoxedColumn col -> TColumn $ fromVector $ VG.generate (VG.length os - 1) $ \g -> let !start = os `VG.unsafeIndex` g @@ -201,7 +257,7 @@ | otherwise = let !x = col `VG.unsafeIndex` (indices `VG.unsafeIndex` j) in go (f acc x) (j + 1) e - Just (UnboxedColumn col) -> case sUnbox @a of + UnboxedColumn col -> case sUnbox @a of SFalse -> TColumn $ fromVector $ VG.generate (VG.length os - 1) $ \g -> @@ -216,7 +272,7 @@ let !x = col `VG.unsafeIndex` (indices `VG.unsafeIndex` j) in go (f acc x) (j + 1) e STrue -> TColumn $ - fromVector $ + fromUnboxedVector $ VG.generate (VG.length os - 1) $ \g -> let !start = os `VG.unsafeIndex` g !end = os `VG.unsafeIndex` (g + 1) @@ -228,7 +284,7 @@ | otherwise = let !x = col `VG.unsafeIndex` (indices `VG.unsafeIndex` j) in go (f acc x) (j + 1) e - Just (OptionalColumn col) -> TColumn $ + OptionalColumn col -> TColumn $ fromVector $ VG.generate (VG.length os - 1) $ \g -> let !start = os `VG.unsafeIndex` g @@ -241,9 +297,68 @@ | otherwise = let !x = col `VG.unsafeIndex` (indices `VG.unsafeIndex` j) in go (f acc x) (j + 1) e -interpretAggregation gdf@(Grouped df names indices os) (NumericAggregate name op (f :: VU.Vector b -> c)) = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "" (map fst $ M.toList $ columnIndices df) - Just (UnboxedColumn (col :: VU.Vector d)) -> case testEquality (typeRep @b) (typeRep @d) of +interpretAggregation gdf@(Grouped df names indices os) (AggFold expr op s (f :: (a -> b -> a))) = case unwrapTypedColumn (interpretAggregation @b gdf expr) of + BoxedColumn (col :: V.Vector c) -> case testEquality (typeRep @b) (typeRep @c) of + Nothing -> error "Type mismatch" + Just Refl -> TColumn $ + fromVector $ + VG.generate (VG.length os - 1) $ \g -> + let !start = os `VG.unsafeIndex` g + !end = os `VG.unsafeIndex` (g + 1) + in go s start end + where + {-# INLINE go #-} + go !acc j e + | j == e = acc + | otherwise = + let !x = col `VG.unsafeIndex` (indices `VG.unsafeIndex` j) + in go (f acc x) (j + 1) e + UnboxedColumn (col :: VU.Vector c) -> case testEquality (typeRep @b) (typeRep @c) of + Nothing -> error "Type mismatch" + Just Refl -> case sUnbox @a of + SFalse -> TColumn $ + fromVector $ + VG.generate (VG.length os - 1) $ \g -> + let !start = os `VG.unsafeIndex` g + !end = os `VG.unsafeIndex` (g + 1) + in go s start end + where + {-# INLINE go #-} + go !acc j e + | j == e = acc + | otherwise = + let !x = col `VG.unsafeIndex` (indices `VG.unsafeIndex` j) + in go (f acc x) (j + 1) e + STrue -> TColumn $ + fromUnboxedVector $ + VG.generate (VG.length os - 1) $ \g -> + let !start = os `VG.unsafeIndex` g + !end = os `VG.unsafeIndex` (g + 1) + in go s start end + where + {-# INLINE go #-} + go !acc j e + | j == e = acc + | otherwise = + let !x = col `VG.unsafeIndex` (indices `VG.unsafeIndex` j) + in go (f acc x) (j + 1) e + OptionalColumn (col :: V.Vector c) -> case testEquality (typeRep @b) (typeRep @c) of + Nothing -> error "Type mismatch" + Just Refl -> TColumn $ + fromVector $ + VG.generate (VG.length os - 1) $ \g -> + let !start = os `VG.unsafeIndex` g + !end = os `VG.unsafeIndex` (g + 1) + in go s start end + where + {-# INLINE go #-} + go !acc j e + | j == e = acc + | otherwise = + let !x = col `VG.unsafeIndex` (indices `VG.unsafeIndex` j) + in go (f acc x) (j + 1) e +interpretAggregation gdf@(Grouped df names indices os) (AggNumericVector expr op (f :: VU.Vector b -> c)) = case unwrapTypedColumn (interpretAggregation @b gdf expr) of + UnboxedColumn (col :: VU.Vector d) -> case testEquality (typeRep @b) (typeRep @d) of Nothing -> case testEquality (typeRep @d) (typeRep @Int) of Just Refl -> case sUnbox @c of SFalse -> @@ -271,7 +386,7 @@ ) ) Just Refl -> case sNumeric @d of - SFalse -> error $ "Cannot apply numeric aggregation to non-numeric column: " ++ (T.unpack name) + SFalse -> error $ "Cannot apply numeric aggregation to non-numeric column: " ++ show expr STrue -> case sUnbox @c of SFalse -> TColumn $ @@ -297,65 +412,93 @@ (\j -> col `VG.unsafeIndex` (indices `VG.unsafeIndex` (j + (os `VG.unsafeIndex` i)))) ) ) - _ -> error $ "Cannot apply numeric aggregation to non-numeric column: " ++ (T.unpack name) + _ -> error $ "Cannot apply numeric aggregation to non-numeric: " ++ show expr instance (Num a, Columnable a) => Num (Expr a) where (+) :: Expr a -> Expr a -> Expr a - (+) = BinOp "add" (+) + (+) = BinaryOp "add" (+) (*) :: Expr a -> Expr a -> Expr a - (*) = BinOp "mult" (*) + (*) = BinaryOp "mult" (*) fromInteger :: Integer -> Expr a fromInteger = Lit . fromInteger negate :: Expr a -> Expr a - negate = Apply "negate" negate + negate = UnaryOp "negate" negate abs :: (Num a) => Expr a -> Expr a - abs = Apply "abs" abs + abs = UnaryOp "abs" abs signum :: (Num a) => Expr a -> Expr a - signum = Apply "signum" signum + signum = UnaryOp "signum" signum +add :: (Num a, Columnable a) => Expr a -> Expr a -> Expr a +add = (+) + +sub :: (Num a, Columnable a) => Expr a -> Expr a -> Expr a +sub = (-) + +mult :: (Num a, Columnable a) => Expr a -> Expr a -> Expr a +mult = (*) + instance (Fractional a, Columnable a) => Fractional (Expr a) where fromRational :: (Fractional a, Columnable a) => Rational -> Expr a fromRational = Lit . fromRational (/) :: (Fractional a, Columnable a) => Expr a -> Expr a -> Expr a - (/) = BinOp "divide" (/) + (/) = BinaryOp "divide" (/) +divide :: (Fractional a, Columnable a) => Expr a -> Expr a -> Expr a +divide = (/) + instance (Floating a, Columnable a) => Floating (Expr a) where pi :: (Floating a, Columnable a) => Expr a pi = Lit pi exp :: (Floating a, Columnable a) => Expr a -> Expr a - exp = Apply "exp" exp + exp = UnaryOp "exp" exp log :: (Floating a, Columnable a) => Expr a -> Expr a - log = Apply "log" log + log = UnaryOp "log" log sin :: (Floating a, Columnable a) => Expr a -> Expr a - sin = Apply "sin" sin + sin = UnaryOp "sin" sin cos :: (Floating a, Columnable a) => Expr a -> Expr a - cos = Apply "cos" cos + cos = UnaryOp "cos" cos asin :: (Floating a, Columnable a) => Expr a -> Expr a - asin = Apply "asin" asin + asin = UnaryOp "asin" asin acos :: (Floating a, Columnable a) => Expr a -> Expr a - acos = Apply "acos" acos + acos = UnaryOp "acos" acos atan :: (Floating a, Columnable a) => Expr a -> Expr a - atan = Apply "atan" atan + atan = UnaryOp "atan" atan sinh :: (Floating a, Columnable a) => Expr a -> Expr a - sinh = Apply "sinh" sinh + sinh = UnaryOp "sinh" sinh cosh :: (Floating a, Columnable a) => Expr a -> Expr a - cosh = Apply "cosh" cosh + cosh = UnaryOp "cosh" cosh asinh :: (Floating a, Columnable a) => Expr a -> Expr a - asinh = Apply "asinh" sinh + asinh = UnaryOp "asinh" sinh acosh :: (Floating a, Columnable a) => Expr a -> Expr a - acosh = Apply "acosh" acosh + acosh = UnaryOp "acosh" acosh atanh :: (Floating a, Columnable a) => Expr a -> Expr a - atanh = Apply "atanh" atanh + atanh = UnaryOp "atanh" atanh instance (Show a) => Show (Expr a) where show :: forall a. (Show a) => Expr a -> String - show (Col name) = "col@" ++ show (typeRep @a) ++ "(" ++ T.unpack name ++ ")" + show (Col name) = "(col @" ++ show (typeRep @a) ++ " " ++ show name ++ ")" show (Lit value) = show value - show (Apply name f value) = T.unpack name ++ "(" ++ show value ++ ")" - show (BinOp name f a b) = T.unpack name ++ "(" ++ show a ++ ", " ++ show b ++ ")" + show (If cond l r) = "(ifThenElse " ++ show l ++ " " ++ show r ++ ")" + show (UnaryOp name f value) = "(" ++ T.unpack name ++ " " ++ show value ++ ")" + show (BinaryOp name f a b) = "(" ++ T.unpack name ++ " " ++ show a ++ " " ++ show b ++ ")" + show (AggNumericVector expr op _) = "(" ++ T.unpack op ++ " " ++ show expr ++ ")" + show (AggVector expr op _) = "(" ++ T.unpack op ++ " " ++ show expr ++ ")" + show (AggReduce expr op _) = "(" ++ T.unpack op ++ " " ++ show expr ++ ")" + show (AggFold expr op _ _ ) = "(" ++ T.unpack op ++ " " ++ show expr ++ ")" + +eSize :: Expr a -> Int +eSize (Col _) = 1 +eSize (Lit _) = 1 +eSize (If c l r) = 1 + eSize c + eSize l + eSize r +eSize (UnaryOp _ _ e) = 1 + eSize e +eSize (BinaryOp _ _ l r) = 1 + eSize l + eSize r +eSize (AggNumericVector expr op _) = eSize expr + 1 +eSize (AggVector expr op _) = eSize expr + 1 +eSize (AggReduce expr op _) = eSize expr + 1 +eSize (AggFold expr op _ _ ) = eSize expr + 1 diff --git a/src/DataFrame/Internal/Row.hs b/src/DataFrame/Internal/Row.hs --- a/src/DataFrame/Internal/Row.hs +++ b/src/DataFrame/Internal/Row.hs @@ -22,15 +22,14 @@ import Data.Function (on) import Data.Maybe (fromMaybe) import Data.Type.Equality (TestEquality (..)) -import Data.Typeable (Typeable, type (:~:) (..)) -import Data.Word (Word16, Word32, Word64, Word8) +import Data.Typeable (type (:~:) (..)) import DataFrame.Errors (DataFrameException (..)) import DataFrame.Internal.Column import DataFrame.Internal.DataFrame import DataFrame.Internal.Types import Text.ParserCombinators.ReadPrec (ReadPrec) import Text.Read (Lexeme (Ident), lexP, parens, readListPrec, readListPrecDefault, readPrec) -import Type.Reflection (TypeRep, typeOf, typeRep) +import Type.Reflection (typeOf, typeRep) data Any where Value :: (Columnable' a) => a -> Any @@ -90,7 +89,7 @@ mkRowFromArgs names df i = V.map get (V.fromList names) where get name = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "[INTERNAL] mkRowFromArgs" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException name "[INTERNAL] mkRowFromArgs" (M.keys $ columnIndices df) Just (BoxedColumn column) -> toAny (column V.! i) Just (UnboxedColumn column) -> toAny (column VU.! i) Just (OptionalColumn column) -> toAny (column V.! i) diff --git a/src/DataFrame/Internal/Schema.hs b/src/DataFrame/Internal/Schema.hs --- a/src/DataFrame/Internal/Schema.hs +++ b/src/DataFrame/Internal/Schema.hs @@ -13,7 +13,7 @@ import qualified Data.Text as T import Data.Maybe -import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) +import Data.Type.Equality (TestEquality (..)) import DataFrame.Internal.Column import Type.Reflection (typeRep) diff --git a/src/DataFrame/Internal/Statistics.hs b/src/DataFrame/Internal/Statistics.hs new file mode 100644 --- /dev/null +++ b/src/DataFrame/Internal/Statistics.hs @@ -0,0 +1,132 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE OverloadedStrings #-} + +module DataFrame.Internal.Statistics where + +import qualified Data.Vector.Algorithms.Intro as VA +import qualified Data.Vector.Unboxed as VU +import qualified Data.Vector.Unboxed.Mutable as VUM + +import Control.Exception (throw) +import Control.Monad.ST (runST) +import DataFrame.Errors (DataFrameException (..)) + +mean' :: (Real a, VU.Unbox a) => VU.Vector a -> Double +mean' samp = VU.sum (VU.map realToFrac samp) / fromIntegral (VU.length samp) +{-# INLINE mean' #-} + +median' :: VU.Vector Double -> Double +median' samp + | VU.null samp = throw $ EmptyDataSetException "median" + | otherwise = runST $ do + mutableSamp <- VU.thaw samp + VA.sort mutableSamp + let len = VU.length samp + middleIndex = len `div` 2 + middleElement <- VUM.read mutableSamp middleIndex + if odd len + then pure middleElement + else do + prev <- VUM.read mutableSamp (middleIndex - 1) + pure ((middleElement + prev) / 2) +{-# INLINE median' #-} + +-- accumulator: count, mean, m2 +data VarAcc = VarAcc !Int !Double !Double deriving (Show) + +varianceStep :: Real a => VarAcc -> a -> VarAcc +varianceStep (VarAcc !n !mean !m2) !x = + let !n' = n + 1 + !delta = realToFrac x - mean + !mean' = mean + delta / fromIntegral n' + !m2' = m2 + delta * (realToFrac x - mean') + in VarAcc n' mean' m2' +{-# INLINE varianceStep #-} + +computeVariance :: VarAcc -> Double +computeVariance (VarAcc !n _ !m2) + | n < 2 = 0 -- or error "variance of <2 samples" + | otherwise = m2 / fromIntegral (n - 1) +{-# INLINE computeVariance #-} + +variance' :: (Real a, VU.Unbox a) => VU.Vector a -> Double +variance' = computeVariance . VU.foldl' varianceStep (VarAcc 0 0 0) +{-# INLINE variance' #-} + +-- accumulator: count, mean, m2, m3 +data SkewAcc = SkewAcc !Int !Double !Double !Double deriving (Show) + +skewnessStep :: SkewAcc -> Double -> SkewAcc +skewnessStep (SkewAcc !n !mean !m2 !m3) !x = + let !n' = n + 1 + !k = fromIntegral n' + !delta = x - mean + !mean' = mean + delta / k + !m2' = m2 + (delta ^ 2 * (k - 1)) / k + !m3' = m3 + (delta ^ 3 * (k - 1) * (k - 2)) / k ^ 2 - (3 * delta * m2) / k + in SkewAcc n' mean' m2' m3' +{-# INLINE skewnessStep #-} + +computeSkewness :: SkewAcc -> Double +computeSkewness (SkewAcc n _ m2 m3) + | n < 3 = 0 -- or error "skewness of <3 samples" + | otherwise = (sqrt (fromIntegral n - 1) * m3) / sqrt (m2 ^ 3) +{-# INLINE computeSkewness #-} + +skewness' :: VU.Vector Double -> Double +skewness' = computeSkewness . VU.foldl' skewnessStep (SkewAcc 0 0 0 0) +{-# INLINE skewness' #-} + +correlation' :: VU.Vector Double -> VU.Vector Double -> Maybe Double +correlation' xs ys + | VU.length xs /= VU.length ys = Nothing + | nI < 2 = Nothing + | otherwise = + let !nf = fromIntegral nI + (!sumX, !sumY, !sumSquaredX, !sumSquaredY, !sumXY) = go 0 0 0 0 0 0 + !num = nf * sumXY - sumX * sumY + !den = sqrt ((nf * sumSquaredX - sumX * sumX) * (nf * sumSquaredY - sumY * sumY)) + in pure (num / den) + where + !nI = VU.length xs + go !i !sumX !sumY !sumSquaredX !sumSquaredY !sumXY + | i < nI = + let !x = VU.unsafeIndex xs i + !y = VU.unsafeIndex ys i + !sumX' = sumX + x + !sumY' = sumY + y + !sumSquaredX' = sumSquaredX + x * x + !sumSquaredY' = sumSquaredY + y * y + !sumXY' = sumXY + x * y + in go (i + 1) sumX' sumY' sumSquaredX' sumSquaredY' sumXY' + | otherwise = (sumX, sumY, sumSquaredX, sumSquaredY, sumXY) +{-# INLINE correlation' #-} + +quantiles' :: VU.Vector Int -> Int -> VU.Vector Double -> VU.Vector Double +quantiles' qs q samp + | VU.null samp = throw $ EmptyDataSetException "quantiles" + | q < 2 = throw $ WrongQuantileNumberException q + | VU.any (\i -> i < 0 || i > q) qs = throw $ WrongQuantileIndexException qs q + | otherwise = runST $ do + let !n = VU.length samp + mutableSamp <- VU.thaw samp + VA.sort mutableSamp + VU.mapM (\i -> do + let !p = fromIntegral i / fromIntegral q + !position = p * fromIntegral (n - 1) + !index = floor position + !f = position - fromIntegral index + x <- VUM.read mutableSamp index + if f == 0 + then return x + else do + y <- VUM.read mutableSamp (index + 1) + return $ (1 - f) * x + f * y + ) qs +{-# INLINE quantiles' #-} + +interQuartileRange' :: VU.Vector Double -> Double +interQuartileRange' samp = + let quartiles = quantiles' (VU.fromList [1, 3]) 4 samp + in quartiles VU.! 1 - quartiles VU.! 0 +{-# INLINE interQuartileRange' #-} diff --git a/src/DataFrame/Internal/Types.hs b/src/DataFrame/Internal/Types.hs --- a/src/DataFrame/Internal/Types.hs +++ b/src/DataFrame/Internal/Types.hs @@ -15,13 +15,9 @@ import Data.Int (Int16, Int32, Int64, Int8) import Data.Kind (Constraint, Type) -import Data.Maybe (fromMaybe) -import Data.Type.Equality (TestEquality (..)) -import Data.Typeable (Typeable, type (:~:) (..)) -import qualified Data.Vector as VB +import Data.Typeable (Typeable) import qualified Data.Vector.Unboxed as VU import Data.Word (Word16, Word32, Word64, Word8) -import Type.Reflection (TypeRep, typeOf, typeRep) type Columnable' a = (Typeable a, Show a, Ord a, Eq a, Read a) @@ -56,8 +52,8 @@ Unboxable Float = 'True Unboxable _ = 'False --- | All unboxable types (according to the `vector` package). type family Numeric (a :: Type) :: Bool where + Numeric Integer = 'True Numeric Int = 'True Numeric Int8 = 'True Numeric Int16 = 'True @@ -101,3 +97,32 @@ When 'False c = () -- empty constraint type UnboxIf a = When (Unboxable a) (VU.Unbox a) + +type family IntegralTypes (a :: Type) :: Bool where + IntegralTypes Integer = 'True + IntegralTypes Int = 'True + IntegralTypes Int8 = 'True + IntegralTypes Int16 = 'True + IntegralTypes Int32 = 'True + IntegralTypes Int64 = 'True + IntegralTypes Word = 'True + IntegralTypes Word8 = 'True + IntegralTypes Word16 = 'True + IntegralTypes Word32 = 'True + IntegralTypes Word64 = 'True + IntegralTypes _ = 'False + +sIntegral :: forall a. (SBoolI (IntegralTypes a)) => SBool (IntegralTypes a) +sIntegral = sbool @(IntegralTypes a) + +type IntegralIf a = When (IntegralTypes a) (Integral a) + +type family FloatingTypes (a :: Type) :: Bool where + FloatingTypes Float = 'True + FloatingTypes Double = 'True + FloatingTypes _ = 'False + +sFloating :: forall a. (SBoolI (FloatingTypes a)) => SBool (FloatingTypes a) +sFloating = sbool @(FloatingTypes a) + +type FloatingIf a = When (FloatingTypes a) (Real a, Fractional a) diff --git a/src/DataFrame/Lazy/IO/CSV.hs b/src/DataFrame/Lazy/IO/CSV.hs --- a/src/DataFrame/Lazy/IO/CSV.hs +++ b/src/DataFrame/Lazy/IO/CSV.hs @@ -8,37 +8,28 @@ module DataFrame.Lazy.IO.CSV where -import qualified Data.ByteString.Char8 as C import qualified Data.List as L import qualified Data.Map as M import qualified Data.Set as S import qualified Data.Text as T import qualified Data.Text.IO as TIO -import qualified Data.Text.Lazy as TL -import qualified Data.Text.Lazy.IO as TLIO import qualified Data.Vector as V import qualified Data.Vector.Mutable as VM import qualified Data.Vector.Unboxed as VU import qualified Data.Vector.Unboxed.Mutable as VUM -import Control.Applicative (many, (*>), (<$>), (<*), (<*>), (<|>)) -import Control.Monad (forM_, replicateM_, unless, void, when, zipWithM_) +import Control.Applicative (many, (<|>)) +import Control.Monad (forM_, unless, when, zipWithM_) import Data.Attoparsec.Text import Data.Char import Data.Foldable (fold) import Data.Function (on) import Data.IORef import Data.Maybe -import Data.Text.Encoding (decodeUtf8Lenient) -import Data.Type.Equality ( - TestEquality (testEquality), - type (:~:) (Refl), - ) +import Data.Type.Equality (TestEquality (testEquality)) import DataFrame.Internal.Column (Column (..), MutableColumn (..), columnLength, freezeColumn', writeColumn) import DataFrame.Internal.DataFrame (DataFrame (..)) import DataFrame.Internal.Parsing -import DataFrame.Operations.Typing -import GHC.IO.Handle (Handle) import System.IO import Type.Reflection import Prelude hiding (concat, takeWhile) @@ -65,18 +56,18 @@ Note this file stores intermediate temporary files while converting the CSV from a row to a columnar format. -} -readCsv :: String -> IO DataFrame +readCsv :: FilePath -> IO DataFrame readCsv path = fst <$> readSeparated ',' defaultOptions path {- | Reads a tab separated file from the given path. Note this file stores intermediate temporary files while converting the CSV from a row to a columnar format. -} -readTsv :: String -> IO DataFrame +readTsv :: FilePath -> IO DataFrame readTsv path = fst <$> readSeparated '\t' defaultOptions path -- | Reads a character separated file into a dataframe using mutable vectors. -readSeparated :: Char -> ReadOptions -> String -> IO (DataFrame, (Integer, T.Text, Int)) +readSeparated :: Char -> ReadOptions -> FilePath -> IO (DataFrame, (Integer, T.Text, Int)) readSeparated c opts path = do totalRows <- case totalRows opts of Nothing -> countRows c path >>= \total -> if hasHeader opts then return (total - 1) else return total @@ -289,14 +280,14 @@ go (n + 1) unconsumed h {-# INLINE countRows #-} -writeCsv :: String -> DataFrame -> IO () +writeCsv :: FilePath -> DataFrame -> IO () writeCsv = writeSeparated ',' writeSeparated :: -- | Separator Char -> -- | Path to write to - String -> + FilePath -> DataFrame -> IO () writeSeparated c filepath df = withFile filepath WriteMode $ \handle -> do diff --git a/src/DataFrame/Lazy/Internal/DataFrame.hs b/src/DataFrame/Lazy/Internal/DataFrame.hs --- a/src/DataFrame/Lazy/Internal/DataFrame.hs +++ b/src/DataFrame/Lazy/Internal/DataFrame.hs @@ -8,22 +8,16 @@ module DataFrame.Lazy.Internal.DataFrame where -import Control.Monad (foldM, forM) -import Data.IORef -import Data.Kind +import Control.Monad (foldM) import qualified Data.List as L -import qualified Data.Map as M import qualified Data.Text as T -import qualified Data.Vector as V import qualified DataFrame.Internal.Column as C import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Internal.Expression as E import qualified DataFrame.Lazy.IO.CSV as D -import qualified DataFrame.Operations.Core as D -import DataFrame.Operations.Merge +import DataFrame.Operations.Merge () import qualified DataFrame.Operations.Subset as D import qualified DataFrame.Operations.Transformations as D -import System.FilePath data LazyOperation where Derive :: (C.Columnable a) => T.Text -> E.Expr a -> LazyOperation diff --git a/src/DataFrame/Operations/Aggregation.hs b/src/DataFrame/Operations/Aggregation.hs --- a/src/DataFrame/Operations/Aggregation.hs +++ b/src/DataFrame/Operations/Aggregation.hs @@ -8,44 +8,30 @@ module DataFrame.Operations.Aggregation where -import qualified Data.Set as S -import qualified DataFrame.Functions as F - import qualified Data.List as L import qualified Data.Map as M -import qualified Data.Map.Strict as MS import qualified Data.Text as T import qualified Data.Vector as V import qualified Data.Vector.Algorithms.Merge as VA import qualified Data.Vector.Generic as VG -import qualified Data.Vector.Mutable as VM import qualified Data.Vector.Unboxed as VU import Control.Exception (throw) -import Control.Monad (foldM_) import Control.Monad.ST (runST) -import Data.Function ((&)) import Data.Hashable -import Data.List ((\\)) -import Data.Maybe import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) import DataFrame.Errors import DataFrame.Internal.Column ( Column (..), Columnable, atIndicesStable, - columnVersionString, - fromVector, getIndices, getIndicesUnboxed, unwrapTypedColumn, ) -import DataFrame.Internal.DataFrame (DataFrame (..), GroupedDataFrame (..), empty, getColumn, unsafeGetColumn) +import DataFrame.Internal.DataFrame (DataFrame (..), GroupedDataFrame (..)) import DataFrame.Internal.Expression -import DataFrame.Internal.Parsing -import DataFrame.Internal.Types import DataFrame.Operations.Core -import DataFrame.Operations.Merge import DataFrame.Operations.Subset import Type.Reflection (typeOf, typeRep) @@ -113,7 +99,7 @@ let vs = indices `getIndicesUnboxed` column in insertUnboxedVector name vs acc -{- | Aggregate a grouped dataframe using the expressions give. +{- | Aggregate a grouped dataframe using the expressions given. All ungrouped columns will be dropped. -} aggregate :: [(T.Text, UExpr)] -> GroupedDataFrame -> DataFrame diff --git a/src/DataFrame/Operations/Core.hs b/src/DataFrame/Operations/Core.hs --- a/src/DataFrame/Operations/Core.hs +++ b/src/DataFrame/Operations/Core.hs @@ -21,10 +21,10 @@ import Data.Either import Data.Function (on, (&)) import Data.Maybe -import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) +import Data.Type.Equality (TestEquality (..)) import DataFrame.Errors import DataFrame.Internal.Column (Column (..), Columnable, columnLength, columnTypeString, expandColumn, fromList, fromVector) -import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn, null) +import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn) import DataFrame.Internal.Parsing (isNullish) import Type.Reflection import Prelude hiding (null) @@ -111,7 +111,7 @@ T.Text -> -- | Data to add to column V.Vector a -> - -- | DataFrame to add to column + -- | DataFrame to add the column to DataFrame -> DataFrame insertVectorWithDefault defaultValue name xs d = @@ -131,7 +131,7 @@ T.Text -> -- | Unboxed vector to add to column VU.Vector a -> - -- | DataFrame to add to column + -- | DataFrame to add the column to DataFrame -> DataFrame insertUnboxedVector name xs = insertColumn name (UnboxedColumn xs) @@ -165,7 +165,7 @@ T.Text -> -- | Column to add Column -> - -- | DataFrame to add to column + -- | DataFrame to add the column to DataFrame -> DataFrame insertColumn name column d = @@ -206,7 +206,7 @@ @ -} cloneColumn :: T.Text -> T.Text -> DataFrame -> DataFrame -cloneColumn original new df = fromMaybe (throw $ ColumnNotFoundException original "cloneColumn" (map fst $ M.toList $ columnIndices df)) $ do +cloneColumn original new df = fromMaybe (throw $ ColumnNotFoundException original "cloneColumn" (M.keys $ columnIndices df)) $ do column <- getColumn original df return $ insertColumn new column df @@ -291,7 +291,7 @@ renameMany replacements df = fold (uncurry rename) replacements df renameSafe :: T.Text -> T.Text -> DataFrame -> Either DataFrameException DataFrame -renameSafe orig new df = fromMaybe (Left $ ColumnNotFoundException orig "rename" (map fst $ M.toList $ columnIndices df)) $ do +renameSafe orig new df = fromMaybe (Left $ ColumnNotFoundException orig "rename" (M.keys $ columnIndices df)) $ do columnIndex <- M.lookup orig (columnIndices df) let origRemoved = M.delete orig (columnIndices df) let newAdded = M.insert new columnIndex origRemoved @@ -462,7 +462,7 @@ -} valueCounts :: forall a. (Columnable a) => T.Text -> DataFrame -> [(a, Int)] valueCounts columnName df = case getColumn columnName df of - Nothing -> throw $ ColumnNotFoundException columnName "valueCounts" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException columnName "valueCounts" (M.keys $ columnIndices df) Just (BoxedColumn (column' :: V.Vector c)) -> let column = V.foldl' (\m v -> MS.insertWith (+) v (1 :: Int) m) M.empty column' @@ -513,7 +513,7 @@ Just Refl -> M.toAscList column {- | A left fold for dataframes that takes the dataframe as the last object. -this makes it easier to chain operations. +This makes it easier to chain operations. ==== __Example__ @ diff --git a/src/DataFrame/Operations/Merge.hs b/src/DataFrame/Operations/Merge.hs --- a/src/DataFrame/Operations/Merge.hs +++ b/src/DataFrame/Operations/Merge.hs @@ -4,7 +4,6 @@ import qualified Data.List as L import qualified Data.Text as T -import qualified Data.Vector as V import qualified DataFrame.Internal.Column as D import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Operations.Core as D diff --git a/src/DataFrame/Operations/Sorting.hs b/src/DataFrame/Operations/Sorting.hs --- a/src/DataFrame/Operations/Sorting.hs +++ b/src/DataFrame/Operations/Sorting.hs @@ -9,16 +9,16 @@ import Control.Exception (throw) import DataFrame.Errors (DataFrameException (..)) import DataFrame.Internal.Column -import DataFrame.Internal.DataFrame (DataFrame (..), getColumn) +import DataFrame.Internal.DataFrame (DataFrame (..)) import DataFrame.Internal.Row import DataFrame.Operations.Core --- | Sort order taken as a parameter by the sortby function. +-- | Sort order taken as a parameter by the 'sortBy' function. data SortOrder = Ascending | Descending deriving (Eq) {- | O(k log n) Sorts the dataframe by a given row. -> sortBy "Age" df +> sortBy Ascending ["Age"] df -} sortBy :: SortOrder -> @@ -29,9 +29,6 @@ | any (`notElem` columnNames df) names = throw $ ColumnNotFoundException (T.pack $ show $ names L.\\ columnNames df) "sortBy" (columnNames df) | otherwise = let - -- TODO: Remove the SortOrder defintion from operations so we can share it between here and internal and - -- we don't have to do this Bool mapping. indexes = sortedIndexes' (order == Ascending) (toRowVector names df) - pick idxs col = atIndicesStable idxs col in - df{columns = V.map (pick indexes) (columns df)} + df{columns = V.map (atIndicesStable indexes) (columns df)} diff --git a/src/DataFrame/Operations/Statistics.hs b/src/DataFrame/Operations/Statistics.hs --- a/src/DataFrame/Operations/Statistics.hs +++ b/src/DataFrame/Operations/Statistics.hs @@ -1,4 +1,3 @@ -{-# LANGUAGE BangPatterns #-} {-# LANGUAGE ExplicitNamespaces #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE GADTs #-} @@ -9,34 +8,27 @@ module DataFrame.Operations.Statistics where -import Data.Bifunctor (second) import qualified Data.List as L import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Vector as V -import qualified Data.Vector.Algorithms.Intro as VA import qualified Data.Vector.Generic as VG import qualified Data.Vector.Unboxed as VU -import qualified Data.Vector.Unboxed.Mutable as VUM -import qualified Statistics.Quantile as SS -import qualified Statistics.Sample as SS import Prelude as P import Control.Exception (throw) -import Control.Monad.ST (runST) -import qualified Data.Bifunctor as Data -import Data.Foldable (asum) import Data.Function ((&)) import Data.Maybe (fromMaybe, isJust) import Data.Type.Equality (TestEquality (testEquality), type (:~:) (Refl)) import DataFrame.Errors (DataFrameException (..)) import DataFrame.Internal.Column -import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn, unsafeGetColumn) +import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn) import DataFrame.Internal.Row (showValue, toAny) +import DataFrame.Internal.Statistics +import DataFrame.Internal.Types import DataFrame.Operations.Core import DataFrame.Operations.Subset (filterJust) -import GHC.Float (int2Double) import Text.Printf (printf) import Type.Reflection (typeRep) @@ -45,7 +37,7 @@ __Examples:__ @ -ghci> df <- D.readCsv "./data/housing.csv" +ghci> df <- D.readCsv ".\/data\/housing.csv" ghci> D.frequencies "ocean_proximity" df @@ -69,7 +61,7 @@ freqs col = L.foldl' (\d (col, k) -> insertVector (showValue @a col) (V.fromList [toAny k, calculatePercentage (counts @a) k]) d) initDf counts in case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "frequencies" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException name "frequencies" (M.keys $ columnIndices df) Just ((BoxedColumn (column :: V.Vector a))) -> freqs column Just ((OptionalColumn (column :: V.Vector a))) -> freqs column Just ((UnboxedColumn (column :: VU.Vector a))) -> freqs column @@ -96,7 +88,7 @@ -- | Calculates the inter-quartile range of a given column as a standalone value. interQuartileRange :: T.Text -> DataFrame -> Maybe Double -interQuartileRange = applyStatistic (SS.midspread SS.medianUnbiased 4) +interQuartileRange = applyStatistic interQuartileRange' -- | Calculates the Pearson's correlation coefficient between two given columns as a standalone value. correlation :: T.Text -> T.Text -> DataFrame -> Maybe Double @@ -109,50 +101,37 @@ _getColumnAsDouble name df = case getColumn name df of Just (UnboxedColumn (f :: VU.Vector a)) -> case testEquality (typeRep @a) (typeRep @Double) of Just Refl -> Just f - Nothing -> case testEquality (typeRep @a) (typeRep @Int) of - Just Refl -> Just $ VU.map fromIntegral f - Nothing -> case testEquality (typeRep @a) (typeRep @Float) of - Just Refl -> Just $ VU.map realToFrac f - Nothing -> Nothing + Nothing -> case sIntegral @a of + STrue -> Just (VU.map fromIntegral f) + SFalse -> case sFloating @a of + STrue -> Just (VU.map realToFrac f) + SFalse -> Nothing + Nothing -> throw $ ColumnNotFoundException name "applyStatistic" (M.keys $ columnIndices df) _ -> Nothing {-# INLINE _getColumnAsDouble #-} -- | Calculates the sum of a given column as a standalone value. sum :: forall a. (Columnable a, Num a, VU.Unbox a) => T.Text -> DataFrame -> Maybe a sum name df = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "sum" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException name "sum" (M.keys $ columnIndices df) Just ((UnboxedColumn (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @a) of Just Refl -> Just $ VG.sum column Nothing -> Nothing applyStatistic :: (VU.Vector Double -> Double) -> T.Text -> DataFrame -> Maybe Double -applyStatistic f name df = case getColumn name (filterJust name df) of - Nothing -> throw $ ColumnNotFoundException name "applyStatistic" (map fst $ M.toList $ columnIndices df) - Just column@(UnboxedColumn (col :: VU.Vector a)) -> case testEquality (typeRep @a) (typeRep @Double) of - Just Refl -> - let - res = (f col) - in - if isNaN res then Nothing else pure res - Nothing -> do - col' <- _getColumnAsDouble name df - let res = (f col') +applyStatistic f name df = apply =<< _getColumnAsDouble name (filterJust name df) + where + apply col = + let + res = f col + in if isNaN res then Nothing else pure res - _ -> Nothing {-# INLINE applyStatistic #-} applyStatistics :: (VU.Vector Double -> VU.Vector Double) -> T.Text -> DataFrame -> Maybe (VU.Vector Double) -applyStatistics f name df = case getColumn name (filterJust name df) of - Just ((UnboxedColumn (column :: VU.Vector a'))) -> case testEquality (typeRep @a') (typeRep @Int) of - Just Refl -> Just $! f (VU.map fromIntegral column) - Nothing -> case testEquality (typeRep @a') (typeRep @Double) of - Just Refl -> Just $! f column - Nothing -> case testEquality (typeRep @a') (typeRep @Float) of - Just Refl -> Just $! f (VG.map realToFrac column) - Nothing -> Nothing - _ -> Nothing +applyStatistics f name df = fmap f (_getColumnAsDouble name (filterJust name df)) --- | Descriprive statistics of the numeric columns. +-- | Descriptive statistics of the numeric columns. summarize :: DataFrame -> DataFrame summarize df = fold columnStats (columnNames df) (fromNamedColumns [("Statistic", fromList ["Count" :: T.Text, "Mean", "Minimum", "25%", "Median", "75%", "Max", "StdDev", "IQR", "Skewness"])]) where @@ -160,7 +139,7 @@ stats name = let count = fromIntegral . numElements <$> getColumn name df - quantiles = applyStatistics (SS.quantilesVec SS.medianUnbiased (VU.fromList [0, 1, 2, 3, 4]) 4) name df + quantiles = applyStatistics (quantiles' (VU.fromList [0, 1, 2, 3, 4]) 4) name df min' = flip (VG.!) 0 <$> quantiles quartile1 = flip (VG.!) 1 <$> quantiles median' = flip (VG.!) 2 <$> quantiles @@ -182,100 +161,9 @@ -- | Round a @Double@ to Specified Precision roundTo :: Int -> Double -> Double -roundTo n x = fromInteger (round $ x * (10 ^ n)) / (10.0 ^^ n) +roundTo n x = fromInteger (round $ x * 10 ^ n) / 10.0 ^^ n toPct2dp :: Double -> String toPct2dp x | x < 0.00005 = "<0.01%" | otherwise = printf "%.2f%%" (x * 100) - -mean' :: VU.Vector Double -> Double -mean' samp = VU.sum samp / fromIntegral (VU.length samp) -{-# INLINE mean #-} - -median' :: VU.Vector Double -> Double -median' samp - | VU.null samp = throw $ EmptyDataSetException "median" - | otherwise = runST $ do - mutableSamp <- VU.thaw samp - VA.sort mutableSamp - let len = VU.length samp - middleIndex = len `div` 2 - middleElement <- VUM.read mutableSamp middleIndex - if odd len - then pure middleElement - else do - prev <- VUM.read mutableSamp (middleIndex - 1) - pure ((middleElement + prev) / 2) -{-# INLINE median' #-} - --- accumulator: count, mean, m2 -data VarAcc = VarAcc !Int !Double !Double deriving (Show) - -varianceStep :: VarAcc -> Double -> VarAcc -varianceStep (VarAcc !n !mean !m2) !x = - let !n' = n + 1 - !delta = x - mean - !mean' = mean + delta / fromIntegral n' - !m2' = m2 + delta * (x - mean') - in VarAcc n' mean' m2' -{-# INLINE varianceStep #-} - -computeVariance :: VarAcc -> Double -computeVariance (VarAcc !n _ !m2) - | n < 2 = 0 -- or error "variance of <2 samples" - | otherwise = m2 / fromIntegral (n - 1) -{-# INLINE computeVariance #-} - -variance' :: VU.Vector Double -> Double -variance' = computeVariance . VU.foldl' varianceStep (VarAcc 0 0 0) -{-# INLINE variance' #-} - --- accumulator: count, mean, m2, m3 -data SkewAcc = SkewAcc !Int !Double !Double !Double deriving (Show) - -skewnessStep :: SkewAcc -> Double -> SkewAcc -skewnessStep (SkewAcc !n !mean !m2 !m3) !x = - let !n' = n + 1 - !k = fromIntegral n' - !delta = x - mean - !mean' = mean + delta / k - !m2' = m2 + (delta ^ 2 * (k - 1)) / k - !m3' = m3 + (delta ^ 3 * (k - 1) * (k - 2)) / k ^ 2 - (3 * delta * m2) / k - in SkewAcc n' mean' m2' m3' -{-# INLINE skewnessStep #-} - -computeSkewness :: SkewAcc -> Double -computeSkewness (SkewAcc n _ m2 m3) - | n < 3 = 0 -- or error "skewness of <3 samples" - | otherwise = (sqrt (fromIntegral n - 1) * m3) / sqrt (m2 ^ 3) -{-# INLINE computeSkewness #-} - -skewness' :: VU.Vector Double -> Double -skewness' = computeSkewness . VU.foldl' skewnessStep (SkewAcc 0 0 0 0) -{-# INLINE skewness' #-} - -correlation' :: VU.Vector Double -> VU.Vector Double -> Maybe Double -correlation' xs ys - | VU.length xs /= VU.length ys = Nothing - | nI < 2 = Nothing - | otherwise = - let !nf = fromIntegral nI - (!sumX, !sumY, !sumSquaredX, !sumSquaredY, !sumXY) = go 0 0 0 0 0 0 - !num = nf * sumXY - sumX * sumY - !den = sqrt ((nf * sumSquaredX - sumX * sumX) * (nf * sumSquaredY - sumY * sumY)) - in pure (num / den) - where - !nI = VU.length xs - go !i !sumX !sumY !sumSquaredX !sumSquaredY !sumXY - | i < nI = - let !x = VU.unsafeIndex xs i - !y = VU.unsafeIndex ys i - !sumX' = sumX + x - !sumY' = sumY + y - !sumSquaredX' = sumSquaredX + x * x - !sumSquaredY' = sumSquaredY + y * y - !sumXY' = sumXY + x * y - in go (i + 1) sumX' sumY' sumSquaredX' sumSquaredY' sumXY' - | otherwise = (sumX, sumY, sumSquaredX, sumSquaredY, sumXY) -{-# INLINE correlation' #-} diff --git a/src/DataFrame/Operations/Subset.hs b/src/DataFrame/Operations/Subset.hs --- a/src/DataFrame/Operations/Subset.hs +++ b/src/DataFrame/Operations/Subset.hs @@ -11,7 +11,6 @@ import qualified Data.List as L import qualified Data.Map as M -import qualified Data.Set as S import qualified Data.Text as T import qualified Data.Vector as V import qualified Data.Vector.Generic as VG @@ -22,13 +21,12 @@ import Control.Exception (throw) import Control.Monad.ST import Data.Function ((&)) -import Data.Maybe (fromJust, fromMaybe, isJust) -import Data.Type.Equality (TestEquality (..), type (:~:) (Refl)) +import Data.Maybe (fromJust, fromMaybe, isJust, isNothing) +import Data.Type.Equality (TestEquality (..)) import DataFrame.Errors (DataFrameException (..), TypeErrorContext (..)) import DataFrame.Internal.Column import DataFrame.Internal.DataFrame (DataFrame (..), empty, getColumn) import DataFrame.Internal.Expression -import DataFrame.Internal.Row (Any, mkRowFromArgs, toAny) import DataFrame.Operations.Core import DataFrame.Operations.Transformations (apply) import Type.Reflection @@ -74,7 +72,7 @@ {- | O(n * k) Filter rows by a given condition. -filter "x" even df +> filter "x" even df -} filter :: forall a. @@ -87,7 +85,7 @@ DataFrame -> DataFrame filter filterColumnName condition df = case getColumn filterColumnName df of - Nothing -> throw $ ColumnNotFoundException filterColumnName "filter" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException filterColumnName "filter" (M.keys $ columnIndices df) Just (BoxedColumn (column :: V.Vector b)) -> filterByVector filterColumnName column condition df Just (OptionalColumn (column :: V.Vector b)) -> filterByVector filterColumnName column condition df Just (UnboxedColumn (column :: VU.Vector b)) -> filterByVector filterColumnName column condition df @@ -147,23 +145,32 @@ (TColumn col) = interpret @Bool df expr (Just indexes) = findIndices (== True) col c' = snd $ dataframeDimensions df - pick idxs col = atIndicesStable idxs col in - df{columns = V.map (pick indexes) (columns df), dataframeDimensions = (VU.length indexes, c')} + df{columns = V.map (atIndicesStable indexes) (columns df), dataframeDimensions = (VU.length indexes, c')} {- | O(k) removes all rows with `Nothing` in a given column from the dataframe. -> filterJust df +> filterJust "col" df -} filterJust :: T.Text -> DataFrame -> DataFrame filterJust name df = case getColumn name df of - Nothing -> throw $ ColumnNotFoundException name "filterJust" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException name "filterJust" (M.keys $ columnIndices df) Just column@(OptionalColumn (col :: V.Vector (Maybe a))) -> filter @(Maybe a) name isJust df & apply @(Maybe a) fromJust name Just column -> df +{- | O(k) returns all rows with `Nothing` in a give column. + +> filterNothing "col" df +-} +filterNothing :: T.Text -> DataFrame -> DataFrame +filterNothing name df = case getColumn name df of + Nothing -> throw $ ColumnNotFoundException name "filterNothing" (M.keys $ columnIndices df) + Just (OptionalColumn (col :: V.Vector (Maybe a))) -> filter @(Maybe a) name isNothing df + _ -> df + {- | O(n * k) removes all rows with `Nothing` from the dataframe. -> filterJust df +> filterAllJust df -} filterAllJust :: DataFrame -> DataFrame filterAllJust df = foldr filterJust df (columnNames df) @@ -174,7 +181,7 @@ > cube (10, 5) df -} cube :: (Int, Int) -> DataFrame -> DataFrame -cube (length, width) = take length . selectIntRange (0, width - 1) +cube (length, width) = take length . selectBy [ColumnIndexRange (0, width - 1)] {- | O(n) Selects a number of columns in a given dataframe. @@ -193,17 +200,59 @@ col <- getColumn k df pure $ insertColumn k col d --- | O(n) select columns by index range of column names. -selectIntRange :: (Int, Int) -> DataFrame -> DataFrame -selectIntRange (from, to) df = select (Prelude.take (to - from + 1) $ Prelude.drop from (columnNames df)) df +data SelectionCriteria + = ColumnProperty (Column -> Bool) + | ColumnNameProperty (T.Text -> Bool) + | ColumnTextRange (T.Text, T.Text) + | ColumnIndexRange (Int, Int) + | ColumnName T.Text --- | O(n) select columns by index range of column names. -selectRange :: (T.Text, T.Text) -> DataFrame -> DataFrame -selectRange (from, to) df = select (reverse $ Prelude.dropWhile (to /=) $ reverse $ dropWhile (from /=) (columnNames df)) df +-- | Criteria for selecting a column by name. +-- +-- > selectBy [byName "Age"] df +-- +-- equivalent to: +-- +-- > select ["Age"] df +byName :: T.Text -> SelectionCriteria +byName = ColumnName +-- | Criteria for selecting columns whose property satisfies given predicate. +-- +-- > selectBy [byProperty isNumeric] df +byProperty :: (Column -> Bool) -> SelectionCriteria +byProperty = ColumnProperty + +-- | Criteria for selecting columns whose name satisfies given predicate. +-- +-- > selectBy [byNameProperty (T.isPrefixOf "weight")] df +byNameProperty :: (T.Text -> Bool) -> SelectionCriteria +byNameProperty = ColumnNameProperty + +-- | Criteria for selecting columns whose names are in the given lexicographic range (inclusive). +-- +-- > selectBy [byNameRange ("a", "c")] df +byNameRange :: (T.Text, T.Text) -> SelectionCriteria +byNameRange = ColumnTextRange + +-- | Criteria for selecting columns whose indices are in the given (inclusive) range. +-- +-- > selectBy [byIndexRange (0, 5)] df +byIndexRange :: (Int, Int) -> SelectionCriteria +byIndexRange = ColumnIndexRange + -- | O(n) select columns by column predicate name. -selectBy :: (T.Text -> Bool) -> DataFrame -> DataFrame -selectBy f df = select (L.filter f (columnNames df)) df +selectBy :: [SelectionCriteria] -> DataFrame -> DataFrame +selectBy xs df = select columnsWithProperties df + where + columnsWithProperties = L.foldl' columnWithProperty [] xs + columnWithProperty acc (ColumnName name) = acc ++ [name] + columnWithProperty acc (ColumnNameProperty f) = acc ++ L.filter f (columnNames df) + columnWithProperty acc (ColumnTextRange (from, to)) = acc ++ reverse (Prelude.dropWhile (to /=) $ reverse $ dropWhile (from /=) (columnNames df)) + columnWithProperty acc (ColumnIndexRange (from, to)) = acc ++ Prelude.take (to - from + 1) (Prelude.drop from (columnNames df)) + columnWithProperty acc (ColumnProperty f) = acc ++ map fst (L.filter (\(k, v) -> v `elem` ixs) (M.toAscList (columnIndices df))) + where + ixs = V.ifoldl' (\acc i c -> if f c then i : acc else acc) [] (columns df) {- | O(n) inverse of select diff --git a/src/DataFrame/Operations/Transformations.hs b/src/DataFrame/Operations/Transformations.hs --- a/src/DataFrame/Operations/Transformations.hs +++ b/src/DataFrame/Operations/Transformations.hs @@ -10,18 +10,15 @@ import qualified Data.Map as M import qualified Data.Text as T import qualified Data.Vector as V -import qualified Data.Vector.Generic as VG -import qualified Data.Vector.Unboxed as VU import Control.Exception (throw) import Data.Maybe import DataFrame.Errors (DataFrameException (..), TypeErrorContext (..)) -import DataFrame.Internal.Column (Column (..), Columnable, TypedColumn (TColumn), columnTypeString, ifoldrColumn, imapColumn, mapColumn, unwrapTypedColumn) +import DataFrame.Internal.Column (Column (..), Columnable, columnTypeString, ifoldrColumn, imapColumn, mapColumn, unwrapTypedColumn) import DataFrame.Internal.DataFrame (DataFrame (..), getColumn) import DataFrame.Internal.Expression -import DataFrame.Internal.Row (Any, mkRowFromArgs, toAny) import DataFrame.Operations.Core -import Type.Reflection (TypeRep, typeOf, typeRep) +import Type.Reflection (TypeRep, typeRep) -- | O(k) Apply a function to a given column in a dataframe. apply :: @@ -50,7 +47,7 @@ DataFrame -> Either DataFrameException DataFrame safeApply f columnName d = case getColumn columnName d of - Nothing -> Left $ ColumnNotFoundException columnName "apply" (map fst $ M.toList $ columnIndices d) + Nothing -> Left $ ColumnNotFoundException columnName "apply" (M.keys $ columnIndices d) Just column -> case mapColumn f column of Nothing -> Left $ @@ -86,10 +83,9 @@ -- | O(k) Convenience function that applies to an int column. applyInt :: (Columnable b) => - {- | Column name - | function to apply - -} + -- | function to apply (Int -> b) -> + -- | Column name T.Text -> -- | DataFrame to apply operation to DataFrame -> @@ -99,10 +95,9 @@ -- | O(k) Convenience function that applies to an double column. applyDouble :: (Columnable b) => - {- | Column name - | function to apply - -} + -- | function to apply (Double -> b) -> + -- | Column name T.Text -> -- | DataFrame to apply operation to DataFrame -> @@ -112,19 +107,19 @@ {- | O(k * n) Apply a function to a column only if there is another column value that matches the given criterion. -> applyWhere "Age" (<20) "Generation" (const "Gen-Z") +> applyWhere (<20) "Age" (const "Gen-Z") "Generation" df -} applyWhere :: forall a b. (Columnable a, Columnable b) => - (a -> Bool) -> -- Filter condition - T.Text -> -- Criterion Column - (b -> b) -> -- function to apply - T.Text -> -- Column name - DataFrame -> -- DataFrame to apply operation to + (a -> Bool) -> -- ^ Filter condition + T.Text -> -- ^ Criterion Column + (b -> b) -> -- ^ function to apply + T.Text -> -- ^ Column name + DataFrame -> -- ^ DataFrame to apply operation to DataFrame applyWhere condition filterColumnName f columnName df = case getColumn filterColumnName df of - Nothing -> throw $ ColumnNotFoundException filterColumnName "applyWhere" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException filterColumnName "applyWhere" (M.keys $ columnIndices df) Just column -> case ifoldrColumn (\i val acc -> if condition val then V.cons i acc else acc) V.empty column of Nothing -> throw $ @@ -155,7 +150,7 @@ DataFrame -> DataFrame applyAtIndex i f columnName df = case getColumn columnName df of - Nothing -> throw $ ColumnNotFoundException columnName "applyAtIndex" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException columnName "applyAtIndex" (M.keys $ columnIndices df) Just column -> case imapColumn (\index value -> if index == i then f value else value) column of Nothing -> throw $ @@ -178,9 +173,9 @@ DataFrame -> DataFrame impute columnName value df = case getColumn columnName df of - Nothing -> throw $ ColumnNotFoundException columnName "impute" (map fst $ M.toList $ columnIndices df) + Nothing -> throw $ ColumnNotFoundException columnName "impute" (M.keys $ columnIndices df) Just (OptionalColumn _) -> case safeApply (fromMaybe value) columnName df of Left (TypeMismatchException context) -> throw $ TypeMismatchException (context{callingFunctionName = Just "impute"}) Left exception -> throw exception Right res -> res - _ -> error "Cannot impute to a non-Empty column" + _ -> error $ "Cannot impute to a non-Empty column: " ++ T.unpack columnName diff --git a/src/DataFrame/Operations/Typing.hs b/src/DataFrame/Operations/Typing.hs --- a/src/DataFrame/Operations/Typing.hs +++ b/src/DataFrame/Operations/Typing.hs @@ -6,7 +6,6 @@ module DataFrame.Operations.Typing where -import qualified Data.Set as S import qualified Data.Text as T import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU diff --git a/tests/Functions.hs b/tests/Functions.hs --- a/tests/Functions.hs +++ b/tests/Functions.hs @@ -2,8 +2,6 @@ module Functions where -import Control.Exception -import qualified Data.Text as T import DataFrame.Functions (sanitize) import Test.HUnit diff --git a/tests/Main.hs b/tests/Main.hs --- a/tests/Main.hs +++ b/tests/Main.hs @@ -3,23 +3,15 @@ module Main where -import qualified Data.List as L import qualified Data.Text as T import qualified Data.Vector as V -import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D -import qualified DataFrame.Internal.Column as D import qualified DataFrame.Internal.Column as DI -import qualified DataFrame.Internal.DataFrame as D -import qualified DataFrame.Internal.DataFrame as DI import qualified DataFrame.Operations.Typing as D import qualified System.Exit as Exit -import Control.Exception import Data.Time import Test.HUnit - -import Assertions import qualified Functions import qualified Operations.Apply diff --git a/tests/Operations/Apply.hs b/tests/Operations/Apply.hs --- a/tests/Operations/Apply.hs +++ b/tests/Operations/Apply.hs @@ -9,7 +9,6 @@ import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D import qualified DataFrame as DE -import qualified DataFrame.Internal.Column as D import qualified DataFrame.Internal.Column as DI import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Internal.DataFrame as DI diff --git a/tests/Operations/Derive.hs b/tests/Operations/Derive.hs --- a/tests/Operations/Derive.hs +++ b/tests/Operations/Derive.hs @@ -6,17 +6,12 @@ import qualified Data.Text as T import qualified Data.Vector as V -import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D import qualified DataFrame.Functions as F -import qualified DataFrame.Internal.Column as D import qualified DataFrame.Internal.Column as DI -import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Internal.DataFrame as DI -import Assertions import Test.HUnit -import Type.Reflection (typeRep) values :: [(T.Text, DI.Column)] values = diff --git a/tests/Operations/Filter.hs b/tests/Operations/Filter.hs --- a/tests/Operations/Filter.hs +++ b/tests/Operations/Filter.hs @@ -4,13 +4,8 @@ module Operations.Filter where import qualified Data.Text as T -import qualified Data.Vector as V -import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D -import qualified DataFrame.Internal.Column as D -import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Internal.Column as DI -import qualified DataFrame.Internal.DataFrame as DI import Assertions import Test.HUnit diff --git a/tests/Operations/GroupBy.hs b/tests/Operations/GroupBy.hs --- a/tests/Operations/GroupBy.hs +++ b/tests/Operations/GroupBy.hs @@ -3,13 +3,10 @@ module Operations.GroupBy where import qualified Data.Text as T -import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D -import qualified DataFrame.Internal.Column as D import qualified DataFrame.Internal.Column as DI import qualified DataFrame.Internal.DataFrame as D -import qualified DataFrame.Internal.DataFrame as DI import Assertions import Test.HUnit diff --git a/tests/Operations/InsertColumn.hs b/tests/Operations/InsertColumn.hs --- a/tests/Operations/InsertColumn.hs +++ b/tests/Operations/InsertColumn.hs @@ -7,12 +7,9 @@ import qualified Data.Vector as V import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D -import qualified DataFrame.Internal.Column as D import qualified DataFrame.Internal.Column as DI -import qualified DataFrame.Internal.DataFrame as D import qualified DataFrame.Internal.DataFrame as DI -import Assertions import Test.HUnit testData :: D.DataFrame diff --git a/tests/Operations/Sort.hs b/tests/Operations/Sort.hs --- a/tests/Operations/Sort.hs +++ b/tests/Operations/Sort.hs @@ -3,16 +3,10 @@ module Operations.Sort where import Assertions -import Control.Monad import Data.Char import qualified Data.Text as T -import qualified Data.Vector as V -import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D -import qualified DataFrame.Internal.Column as D import qualified DataFrame.Internal.Column as DI -import qualified DataFrame.Internal.DataFrame as D -import qualified DataFrame.Internal.DataFrame as DI import System.Random import System.Random.Shuffle (shuffle') import Test.HUnit diff --git a/tests/Operations/Statistics.hs b/tests/Operations/Statistics.hs --- a/tests/Operations/Statistics.hs +++ b/tests/Operations/Statistics.hs @@ -1,14 +1,12 @@ +{-# LANGUAGE NumericUnderscores #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE TypeApplications #-} -{-# LANGUAGE NumericUnderscores #-} module Operations.Statistics where import qualified Data.Vector.Unboxed as VU import qualified DataFrame as D -import qualified DataFrame.Internal.Column as D -import qualified DataFrame.Internal.DataFrame as D -import qualified DataFrame.Operations.Statistics as D +import qualified DataFrame.Internal.Statistics as D import Assertions import Test.HUnit @@ -74,6 +72,91 @@ 0 ) +twoQuantileOfOddLengthDataSet :: Test +twoQuantileOfOddLengthDataSet = + TestCase + ( assertEqual + "2-quantile of an odd length data set" + (D.quantiles' (VU.fromList [0, 1, 2]) 2 (VU.fromList [179.94, 231.94, 839.06, 534.23, 248.94])) + (VU.fromList [179.94, 248.94, 839.06]) + ) + +twoQuantileOfEvenLengthDataSet :: Test +twoQuantileOfEvenLengthDataSet = + TestCase + ( assertEqual + "2-quantile of an even length data set" + (D.quantiles' (VU.fromList [0, 1, 2]) 2 (VU.fromList [179.94, 231.94, 839.06, 534.23, 248.94, 276.37])) + (VU.fromList [179.94, 262.655, 839.06]) + ) + +quartilesOfOddLengthDataSet :: Test +quartilesOfOddLengthDataSet = + TestCase + ( assertEqual + "Quartiles of an odd length data set" + (D.quantiles' (VU.fromList [0, 1, 2, 3, 4]) 4 (VU.fromList [3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20])) + (VU.fromList [3, 7.5, 9, 14, 20]) + ) + +quartilesOfEvenLengthDataSet :: Test +quartilesOfEvenLengthDataSet = + TestCase + ( assertEqual + "Quartiles of an even length data set" + (D.quantiles' (VU.fromList [0, 1, 2, 3, 4]) 4 (VU.fromList [3, 6, 7, 8, 8, 10, 13, 15, 16, 20])) + (VU.fromList [3, 7.25, 9, 14.5, 20]) + ) + +deciles :: Test +deciles = + TestCase + ( assertEqual + "Deciles" + ( D.quantiles' + (VU.fromList [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + 10 + (VU.fromList [4, 7, 3, 1, 11, 6, 2, 9, 8, 10, 5]) + ) + (VU.fromList [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + ) + +interQuartileRangeOfOddLengthDataSet :: Test +interQuartileRangeOfOddLengthDataSet = + TestCase + ( assertEqual + "Inter quartile range of an odd length data set" + (D.interQuartileRange' (VU.fromList [3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20])) + 6.5 + ) + +interQuartileRangeOfEvenLengthDataSet :: Test +interQuartileRangeOfEvenLengthDataSet = + TestCase + ( assertEqual + "Inter quartile range of an even length data set" + (D.interQuartileRange' (VU.fromList [3, 6, 7, 8, 8, 10, 13, 15, 16, 20])) + 7.25 + ) + +wrongQuantileNumber :: Test +wrongQuantileNumber = + TestCase + ( assertExpectException + "[Error Case]" + (D.wrongQuantileNumberError 1) + (print $ D.quantiles' (VU.fromList [0]) 1 (VU.fromList [1, 2, 3, 4, 5])) + ) + +wrongQuantileIndex :: Test +wrongQuantileIndex = + TestCase + ( assertExpectException + "[Error Case]" + (D.wrongQuantileIndexError (VU.fromList [5]) 4) + (print $ D.quantiles' (VU.fromList [5]) 4 (VU.fromList [1, 2, 3, 4, 5])) + ) + tests :: [Test] tests = [ TestLabel "medianOfOddLengthDataSet" medianOfOddLengthDataSet @@ -83,4 +166,13 @@ , TestLabel "skewnessOfSymmetricDataSet" skewnessOfSymmetricDataSet , TestLabel "skewnessOfSimpleDataSet" skewnessOfSimpleDataSet , TestLabel "skewnessOfEmptyDataSet" skewnessOfEmptyDataSet + , TestLabel "twoQuantileOfOddLengthDataSet" twoQuantileOfOddLengthDataSet + , TestLabel "twoQuantileOfEvenLengthDataSet" twoQuantileOfEvenLengthDataSet + , TestLabel "quartilesOfOddLengthDataSet" quartilesOfOddLengthDataSet + , TestLabel "quartilesOfEvenLengthDataSet" quartilesOfEvenLengthDataSet + , TestLabel "deciles" deciles + , TestLabel "interQuartileRangeOfOddLengthDataSet" interQuartileRangeOfOddLengthDataSet + , TestLabel "interQuartileRangeOfEvenLengthDataSet" interQuartileRangeOfEvenLengthDataSet + , TestLabel "wrongQuantileNumber" wrongQuantileNumber + , TestLabel "wrongQuantileIndex" wrongQuantileIndex ] diff --git a/tests/Operations/Take.hs b/tests/Operations/Take.hs --- a/tests/Operations/Take.hs +++ b/tests/Operations/Take.hs @@ -3,10 +3,8 @@ module Operations.Take where import qualified DataFrame as D -import qualified DataFrame.Internal.Column as D import qualified DataFrame.Internal.Column as DI import qualified DataFrame.Internal.DataFrame as D -import qualified DataFrame.Internal.DataFrame as DI import Test.HUnit diff --git a/tests/Parquet.hs b/tests/Parquet.hs --- a/tests/Parquet.hs +++ b/tests/Parquet.hs @@ -4,11 +4,9 @@ import qualified DataFrame as D -import Assertions import Data.Int import Data.Text (Text) import Data.Time -import Data.Time.Calendar import GHC.IO (unsafePerformIO) import Test.HUnit