datasets 0.1.0.1 → 0.2
raw patch · 12 files changed
+202/−182 lines, 12 filesdep +aesondep +file-embeddep +timedep ~base
Dependencies added: aeson, file-embed, time
Dependency ranges changed: base
Files
- datafiles/iris.data +0/−151
- datasets.cabal +24/−9
- src/Numeric/Datasets.hs +51/−13
- src/Numeric/Datasets/Abalone.hs +1/−1
- src/Numeric/Datasets/Adult.hs +2/−2
- src/Numeric/Datasets/BostonHousing.hs +60/−0
- src/Numeric/Datasets/BreastCancerWisconsin.hs +1/−1
- src/Numeric/Datasets/Car.hs +1/−1
- src/Numeric/Datasets/Iris.hs +6/−3
- src/Numeric/Datasets/Michelson.hs +21/−0
- src/Numeric/Datasets/Nightingale.hs +34/−0
- src/Numeric/Datasets/Wine.hs +1/−1
− datafiles/iris.data
@@ -1,151 +0,0 @@-5.1,3.5,1.4,0.2,Iris-setosa-4.9,3.0,1.4,0.2,Iris-setosa-4.7,3.2,1.3,0.2,Iris-setosa-4.6,3.1,1.5,0.2,Iris-setosa-5.0,3.6,1.4,0.2,Iris-setosa-5.4,3.9,1.7,0.4,Iris-setosa-4.6,3.4,1.4,0.3,Iris-setosa-5.0,3.4,1.5,0.2,Iris-setosa-4.4,2.9,1.4,0.2,Iris-setosa-4.9,3.1,1.5,0.1,Iris-setosa-5.4,3.7,1.5,0.2,Iris-setosa-4.8,3.4,1.6,0.2,Iris-setosa-4.8,3.0,1.4,0.1,Iris-setosa-4.3,3.0,1.1,0.1,Iris-setosa-5.8,4.0,1.2,0.2,Iris-setosa-5.7,4.4,1.5,0.4,Iris-setosa-5.4,3.9,1.3,0.4,Iris-setosa-5.1,3.5,1.4,0.3,Iris-setosa-5.7,3.8,1.7,0.3,Iris-setosa-5.1,3.8,1.5,0.3,Iris-setosa-5.4,3.4,1.7,0.2,Iris-setosa-5.1,3.7,1.5,0.4,Iris-setosa-4.6,3.6,1.0,0.2,Iris-setosa-5.1,3.3,1.7,0.5,Iris-setosa-4.8,3.4,1.9,0.2,Iris-setosa-5.0,3.0,1.6,0.2,Iris-setosa-5.0,3.4,1.6,0.4,Iris-setosa-5.2,3.5,1.5,0.2,Iris-setosa-5.2,3.4,1.4,0.2,Iris-setosa-4.7,3.2,1.6,0.2,Iris-setosa-4.8,3.1,1.6,0.2,Iris-setosa-5.4,3.4,1.5,0.4,Iris-setosa-5.2,4.1,1.5,0.1,Iris-setosa-5.5,4.2,1.4,0.2,Iris-setosa-4.9,3.1,1.5,0.1,Iris-setosa-5.0,3.2,1.2,0.2,Iris-setosa-5.5,3.5,1.3,0.2,Iris-setosa-4.9,3.1,1.5,0.1,Iris-setosa-4.4,3.0,1.3,0.2,Iris-setosa-5.1,3.4,1.5,0.2,Iris-setosa-5.0,3.5,1.3,0.3,Iris-setosa-4.5,2.3,1.3,0.3,Iris-setosa-4.4,3.2,1.3,0.2,Iris-setosa-5.0,3.5,1.6,0.6,Iris-setosa-5.1,3.8,1.9,0.4,Iris-setosa-4.8,3.0,1.4,0.3,Iris-setosa-5.1,3.8,1.6,0.2,Iris-setosa-4.6,3.2,1.4,0.2,Iris-setosa-5.3,3.7,1.5,0.2,Iris-setosa-5.0,3.3,1.4,0.2,Iris-setosa-7.0,3.2,4.7,1.4,Iris-versicolor-6.4,3.2,4.5,1.5,Iris-versicolor-6.9,3.1,4.9,1.5,Iris-versicolor-5.5,2.3,4.0,1.3,Iris-versicolor-6.5,2.8,4.6,1.5,Iris-versicolor-5.7,2.8,4.5,1.3,Iris-versicolor-6.3,3.3,4.7,1.6,Iris-versicolor-4.9,2.4,3.3,1.0,Iris-versicolor-6.6,2.9,4.6,1.3,Iris-versicolor-5.2,2.7,3.9,1.4,Iris-versicolor-5.0,2.0,3.5,1.0,Iris-versicolor-5.9,3.0,4.2,1.5,Iris-versicolor-6.0,2.2,4.0,1.0,Iris-versicolor-6.1,2.9,4.7,1.4,Iris-versicolor-5.6,2.9,3.6,1.3,Iris-versicolor-6.7,3.1,4.4,1.4,Iris-versicolor-5.6,3.0,4.5,1.5,Iris-versicolor-5.8,2.7,4.1,1.0,Iris-versicolor-6.2,2.2,4.5,1.5,Iris-versicolor-5.6,2.5,3.9,1.1,Iris-versicolor-5.9,3.2,4.8,1.8,Iris-versicolor-6.1,2.8,4.0,1.3,Iris-versicolor-6.3,2.5,4.9,1.5,Iris-versicolor-6.1,2.8,4.7,1.2,Iris-versicolor-6.4,2.9,4.3,1.3,Iris-versicolor-6.6,3.0,4.4,1.4,Iris-versicolor-6.8,2.8,4.8,1.4,Iris-versicolor-6.7,3.0,5.0,1.7,Iris-versicolor-6.0,2.9,4.5,1.5,Iris-versicolor-5.7,2.6,3.5,1.0,Iris-versicolor-5.5,2.4,3.8,1.1,Iris-versicolor-5.5,2.4,3.7,1.0,Iris-versicolor-5.8,2.7,3.9,1.2,Iris-versicolor-6.0,2.7,5.1,1.6,Iris-versicolor-5.4,3.0,4.5,1.5,Iris-versicolor-6.0,3.4,4.5,1.6,Iris-versicolor-6.7,3.1,4.7,1.5,Iris-versicolor-6.3,2.3,4.4,1.3,Iris-versicolor-5.6,3.0,4.1,1.3,Iris-versicolor-5.5,2.5,4.0,1.3,Iris-versicolor-5.5,2.6,4.4,1.2,Iris-versicolor-6.1,3.0,4.6,1.4,Iris-versicolor-5.8,2.6,4.0,1.2,Iris-versicolor-5.0,2.3,3.3,1.0,Iris-versicolor-5.6,2.7,4.2,1.3,Iris-versicolor-5.7,3.0,4.2,1.2,Iris-versicolor-5.7,2.9,4.2,1.3,Iris-versicolor-6.2,2.9,4.3,1.3,Iris-versicolor-5.1,2.5,3.0,1.1,Iris-versicolor-5.7,2.8,4.1,1.3,Iris-versicolor-6.3,3.3,6.0,2.5,Iris-virginica-5.8,2.7,5.1,1.9,Iris-virginica-7.1,3.0,5.9,2.1,Iris-virginica-6.3,2.9,5.6,1.8,Iris-virginica-6.5,3.0,5.8,2.2,Iris-virginica-7.6,3.0,6.6,2.1,Iris-virginica-4.9,2.5,4.5,1.7,Iris-virginica-7.3,2.9,6.3,1.8,Iris-virginica-6.7,2.5,5.8,1.8,Iris-virginica-7.2,3.6,6.1,2.5,Iris-virginica-6.5,3.2,5.1,2.0,Iris-virginica-6.4,2.7,5.3,1.9,Iris-virginica-6.8,3.0,5.5,2.1,Iris-virginica-5.7,2.5,5.0,2.0,Iris-virginica-5.8,2.8,5.1,2.4,Iris-virginica-6.4,3.2,5.3,2.3,Iris-virginica-6.5,3.0,5.5,1.8,Iris-virginica-7.7,3.8,6.7,2.2,Iris-virginica-7.7,2.6,6.9,2.3,Iris-virginica-6.0,2.2,5.0,1.5,Iris-virginica-6.9,3.2,5.7,2.3,Iris-virginica-5.6,2.8,4.9,2.0,Iris-virginica-7.7,2.8,6.7,2.0,Iris-virginica-6.3,2.7,4.9,1.8,Iris-virginica-6.7,3.3,5.7,2.1,Iris-virginica-7.2,3.2,6.0,1.8,Iris-virginica-6.2,2.8,4.8,1.8,Iris-virginica-6.1,3.0,4.9,1.8,Iris-virginica-6.4,2.8,5.6,2.1,Iris-virginica-7.2,3.0,5.8,1.6,Iris-virginica-7.4,2.8,6.1,1.9,Iris-virginica-7.9,3.8,6.4,2.0,Iris-virginica-6.4,2.8,5.6,2.2,Iris-virginica-6.3,2.8,5.1,1.5,Iris-virginica-6.1,2.6,5.6,1.4,Iris-virginica-7.7,3.0,6.1,2.3,Iris-virginica-6.3,3.4,5.6,2.4,Iris-virginica-6.4,3.1,5.5,1.8,Iris-virginica-6.0,3.0,4.8,1.8,Iris-virginica-6.9,3.1,5.4,2.1,Iris-virginica-6.7,3.1,5.6,2.4,Iris-virginica-6.9,3.1,5.1,2.3,Iris-virginica-5.8,2.7,5.1,1.9,Iris-virginica-6.8,3.2,5.9,2.3,Iris-virginica-6.7,3.3,5.7,2.5,Iris-virginica-6.7,3.0,5.2,2.3,Iris-virginica-6.3,2.5,5.0,1.9,Iris-virginica-6.5,3.0,5.2,2.0,Iris-virginica-6.2,3.4,5.4,2.3,Iris-virginica-5.9,3.0,5.1,1.8,Iris-virginica-
datasets.cabal view
@@ -1,16 +1,30 @@ Name: datasets-Version: 0.1.0.1+Version: 0.2 Synopsis: Classical data sets for statistics and machine learning Description: Classical machine learning and statistics datasets from the UCI Machine Learning Repository and other sources. .+ The datasets package defines two different kinds of datasets:+ .+ * small data sets which are directly (or indirectly with `file-embed`)+ embedded in the package as pure values and do not require network or IO to download+ the data set. This includes Iris, Anscombe and OldFaithful.+ .+ * other data sets which need to be fetched over the network with+ `Numeric.Datasets.getDataset` and are cached in a local temporary directory.+ . > import Numeric.Datasets (getDataset) > import Numeric.Datasets.Iris (iris)+ > import Numeric.Datasets.Abalone (abalone) > > main = do- > irises <- getDataset iris- > print (length irises)- > print (head irises)+ > -- The Iris data set is embedded+ > print (length iris)+ > print (head iris)+ > -- The Abalone dataset is fetched+ > abas <- getDataset abalone+ > print (length abas)+ > print (head abas) License: MIT License-file: LICENSE@@ -22,9 +36,6 @@ bug-reports: https://github.com/glutamate/datasets/issues category: Statistics, Machine Learning, Data Mining, Data -Data-files:- datafiles/iris.data- source-repository head type: git location: https://github.com/glutamate/datasets@@ -37,15 +48,16 @@ Exposed-modules: Numeric.Datasets , Numeric.Datasets.Anscombe+ , Numeric.Datasets.BostonHousing , Numeric.Datasets.OldFaithful , Numeric.Datasets.Abalone , Numeric.Datasets.Adult , Numeric.Datasets.BreastCancerWisconsin , Numeric.Datasets.Car , Numeric.Datasets.Iris+ , Numeric.Datasets.Michelson+ , Numeric.Datasets.Nightingale , Numeric.Datasets.Wine- other-modules: Paths_datasets- Build-depends: base >= 4.6 && < 5 , cassava@@ -57,3 +69,6 @@ , vector , text , stringsearch+ , file-embed+ , aeson+ , time
src/Numeric/Datasets.hs view
@@ -1,3 +1,21 @@+{- |++The datasets package defines two different kinds of datasets:++* small data sets which are directly (or indirectly with `file-embed`)+ embedded in the package as pure values and do not require+ network or IO to download the data set.++* other data sets which need to be fetched over the network with+ `getDataset` and are cached in a local temporary directory++This module defines the `getDataset` function for fetching datasets+and utilies for defining new data sets. It is only necessary to import+this module when using fetched data sets. Embedded data sets can be+imported directly.++-}+ {-# LANGUAGE OverloadedStrings #-} module Numeric.Datasets where@@ -10,6 +28,7 @@ import Data.Monoid import qualified Data.ByteString.Lazy as BL import qualified Data.Vector as V+import qualified Data.Aeson as JSON import Data.Char (toUpper) import Text.Read (readMaybe)@@ -17,7 +36,6 @@ import qualified Data.ByteString.Lazy.Char8 as BL8 import Data.ByteString.Lazy.Search (replace) -import Paths_datasets (getDataFileName) -- * Using datasets -- |Load a dataset, using the system temporary directory as a cache@@ -32,28 +50,24 @@ -- * Defining datasets -data Source = URL String | CabalDataFile FilePath-+data Source = URL String -- |Define a dataset from a pre-processing function and a source for a CSV file csvDatasetPreprocess :: FromRecord a => (BL.ByteString -> BL.ByteString) -> Source -> Dataset a csvDatasetPreprocess preF src cacheDir = do-- let parseFile contents = do- case decode NoHeader (preF contents) of- Right theData -> return $ V.toList theData- Left err -> fail err-- getFileFromSource cacheDir src >>= parseFile+ parseCSV preF <$> getFileFromSource cacheDir src -- |Define a dataset from a source for a CSV file csvDataset :: FromRecord a => Source -> Dataset a csvDataset = csvDatasetPreprocess id +jsonDataset :: JSON.FromJSON a => Source -> Dataset a+jsonDataset src cacheDir = do+ bs <- getFileFromSource cacheDir src+ return $ parseJSON bs++-- | Get a ByteString from the specified Source getFileFromSource :: FilePath -> Source -> IO (BL.ByteString)-getFileFromSource _ (CabalDataFile fnm) = do- fullpath <- getDataFileName fnm- BL.readFile fullpath getFileFromSource cacheDir (URL url) = do createDirectoryIfMissing True cacheDir let fnm = cacheDir </> "ds" <> show (hash url)@@ -69,7 +83,19 @@ BL.writeFile fnm bs return bs +parseCSV :: FromRecord a => (BL.ByteString -> BL.ByteString) -> BL.ByteString -> [a]+parseCSV preF contents =+ case decode NoHeader (preF contents) of+ Right theData -> V.toList theData+ Left err -> error err +parseJSON :: JSON.FromJSON a => BL.ByteString -> [a]+parseJSON bs = case JSON.decode bs of+ Just theData -> theData+ Nothing -> error "failed to parse json"+++ -- * Helper functions for parsing -- |Turn dashes to CamlCase@@ -100,3 +126,15 @@ -- | Turn US-style decimals starting with a period (e.g. .2) into something Haskell can parse (e.g. 0.2) fixAmericanDecimals :: BL.ByteString -> BL.ByteString fixAmericanDecimals = replace ",." (",0."::BL.ByteString)++-- | Convert a Fixed-width format to a CSV+fixedWidthToCSV :: BL.ByteString -> BL.ByteString+fixedWidthToCSV = BL8.pack . fnl . BL8.unpack where+ f [] = []+ f (' ':cs) = ',':f (chomp cs)+ f ('\n':cs) = '\n':fnl cs+ f (c:cs) = c:f cs+ fnl cs = f (chomp cs) --newline+ chomp (' ':cs) = chomp cs+ chomp (c:cs) = c:cs+ chomp [] = []
src/Numeric/Datasets/Abalone.hs view
@@ -36,4 +36,4 @@ instance FromRecord Abalone abalone :: Dataset Abalone-abalone = csvDataset $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/abalone/abalone.data"+abalone = csvDataset $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/abalone/abalone.data"
src/Numeric/Datasets/Adult.hs view
@@ -96,7 +96,7 @@ <*> v.!9 <*> v.!10 <*> v.!11 <*> v.!12<*> v.!13<*> v.!14 adult :: Dataset Adult-adult = csvDataset $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data"+adult = csvDataset $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/adult/adult.data" adultTestSet :: Dataset Adult-adultTestSet = csvDatasetPreprocess (dropLines 1) $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test"+adultTestSet = csvDatasetPreprocess (dropLines 1) $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/adult/adult.test"
+ src/Numeric/Datasets/BostonHousing.hs view
@@ -0,0 +1,60 @@+{-# LANGUAGE DeriveGeneric, OverloadedStrings #-}++{-|++BostonHousing Data set++scikit-learn calls this "boston" and UCI calls it "Housing"++UCI ML Repository link <http://mlr.cs.umass.edu/ml/datasets/housing>++-}++module Numeric.Datasets.BostonHousing where++import Numeric.Datasets++import Data.Csv+import GHC.Generics++data BostonHousing = BostonHousing+ { crimeRate :: Double+ , zoned :: Double+ , industrial :: Double+ , charlesRiver :: Bool+ , nitricOxides :: Double+ , rooms :: Double+ , age :: Double+ , distance :: Double+ , radialHwy :: Double+ , tax :: Double+ , ptRatio :: Double+ , b :: Double+ , lowerStatus :: Double+ , medianValue :: Double+ } deriving (Show, Read, Generic)++instance FromRecord BostonHousing where+ parseRecord v = BostonHousing <$>+ v .! 0 <*>+ v .! 1 <*>+ v .! 2 <*>+ (intToBool <$> v .! 3) <*>+ v .! 4 <*>+ v .! 5 <*>+ v .! 6 <*>+ v .! 7 <*>+ v .! 8 <*>+ v .! 9 <*>+ v .! 10 <*>+ v .! 11 <*>+ v .! 12 <*>+ v .! 13+ where intToBool :: Int -> Bool+ intToBool 0 = False+ intToBool 1 = True+ intToBool _ = error "intToBool"+bostonHousing :: Dataset BostonHousing+bostonHousing = csvDatasetPreprocess+ fixedWidthToCSV+ $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/housing/housing.data"
src/Numeric/Datasets/BreastCancerWisconsin.hs view
@@ -45,7 +45,7 @@ breastCancerDatabase :: Dataset BreastCancerEntry breastCancerDatabase = csvDataset- $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data"+ $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data" data DiagnosticBreastCancer = DiagnosticBreastCancer { diagnosticID :: Int
src/Numeric/Datasets/Car.hs view
@@ -65,4 +65,4 @@ car :: Dataset Car car = csvDataset- $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/car/car.data"+ $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/car/car.data"
src/Numeric/Datasets/Iris.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE DeriveGeneric, OverloadedStrings #-}+{-# LANGUAGE DeriveGeneric, OverloadedStrings, TemplateHaskell #-} {-| @@ -14,7 +14,10 @@ import Data.Csv import GHC.Generics+import Data.FileEmbed+import Data.ByteString.Lazy (fromStrict) + data IrisClass = Setosa | Versicolor | Virginica deriving (Show, Read, Eq, Generic) @@ -34,5 +37,5 @@ instance FromRecord Iris -iris :: Dataset Iris-iris = csvDataset $ CabalDataFile "datafiles/iris.data"+iris :: [Iris]+iris = parseCSV id $ fromStrict $(embedFile "datafiles/iris.data")
+ src/Numeric/Datasets/Michelson.hs view
@@ -0,0 +1,21 @@+{-# LANGUAGE DeriveGeneric, OverloadedStrings, TemplateHaskell #-}++{-|++Michelson's speed of light dataset - five repeated measurements of the speed of light.++Data from <https://github.com/datasets-io/michelson-speed-of-light>++The embedded dataset is Copyright (c) 2015 The Compute.io Authors.++-}++module Numeric.Datasets.Michelson where++import Numeric.Datasets+import Data.FileEmbed+import Data.ByteString.Lazy (fromStrict)+++michelson :: [[Double]]+michelson = parseJSON $ fromStrict $(embedFile "datafiles/michelson.json")
+ src/Numeric/Datasets/Nightingale.hs view
@@ -0,0 +1,34 @@+{-# LANGUAGE DeriveGeneric, OverloadedStrings, TemplateHaskell #-}++{-|++Florence Nightingale's count of injuries in the Crimean War, used for her rose plots++Data from <https://github.com/datasets-io/nightingales-rose>++The embedded dataset is Copyright (c) 2015 The Compute.io Authors.++-}++module Numeric.Datasets.Nightingale where++import Numeric.Datasets+import Data.FileEmbed+import Data.ByteString.Lazy (fromStrict)+import Data.Aeson hiding (parseJSON)+import Data.Time (UTCTime)+import GHC.Generics+++data Nightingale = Nightingale+ { date :: UTCTime+ , army_size :: Int+ , disease :: Int+ , wounds :: Int+ , other :: Int+ } deriving (Show, Read, Generic)++instance FromJSON Nightingale++nightingale :: [Nightingale]+nightingale = parseJSON $ fromStrict $(embedFile "datafiles/nightingale.json")
src/Numeric/Datasets/Wine.hs view
@@ -37,4 +37,4 @@ wine :: Dataset Wine wine = csvDatasetPreprocess fixAmericanDecimals- $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data"+ $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/wine/wine.data"