diff --git a/datafiles/iris.data b/datafiles/iris.data deleted file mode 100644 --- a/datafiles/iris.data +++ /dev/null @@ -1,151 +0,0 @@ -5.1,3.5,1.4,0.2,Iris-setosa -4.9,3.0,1.4,0.2,Iris-setosa -4.7,3.2,1.3,0.2,Iris-setosa -4.6,3.1,1.5,0.2,Iris-setosa -5.0,3.6,1.4,0.2,Iris-setosa -5.4,3.9,1.7,0.4,Iris-setosa -4.6,3.4,1.4,0.3,Iris-setosa -5.0,3.4,1.5,0.2,Iris-setosa -4.4,2.9,1.4,0.2,Iris-setosa -4.9,3.1,1.5,0.1,Iris-setosa -5.4,3.7,1.5,0.2,Iris-setosa -4.8,3.4,1.6,0.2,Iris-setosa -4.8,3.0,1.4,0.1,Iris-setosa -4.3,3.0,1.1,0.1,Iris-setosa -5.8,4.0,1.2,0.2,Iris-setosa -5.7,4.4,1.5,0.4,Iris-setosa -5.4,3.9,1.3,0.4,Iris-setosa -5.1,3.5,1.4,0.3,Iris-setosa -5.7,3.8,1.7,0.3,Iris-setosa -5.1,3.8,1.5,0.3,Iris-setosa -5.4,3.4,1.7,0.2,Iris-setosa -5.1,3.7,1.5,0.4,Iris-setosa -4.6,3.6,1.0,0.2,Iris-setosa -5.1,3.3,1.7,0.5,Iris-setosa -4.8,3.4,1.9,0.2,Iris-setosa -5.0,3.0,1.6,0.2,Iris-setosa -5.0,3.4,1.6,0.4,Iris-setosa -5.2,3.5,1.5,0.2,Iris-setosa -5.2,3.4,1.4,0.2,Iris-setosa -4.7,3.2,1.6,0.2,Iris-setosa -4.8,3.1,1.6,0.2,Iris-setosa -5.4,3.4,1.5,0.4,Iris-setosa -5.2,4.1,1.5,0.1,Iris-setosa -5.5,4.2,1.4,0.2,Iris-setosa -4.9,3.1,1.5,0.1,Iris-setosa -5.0,3.2,1.2,0.2,Iris-setosa -5.5,3.5,1.3,0.2,Iris-setosa -4.9,3.1,1.5,0.1,Iris-setosa -4.4,3.0,1.3,0.2,Iris-setosa -5.1,3.4,1.5,0.2,Iris-setosa -5.0,3.5,1.3,0.3,Iris-setosa -4.5,2.3,1.3,0.3,Iris-setosa -4.4,3.2,1.3,0.2,Iris-setosa -5.0,3.5,1.6,0.6,Iris-setosa -5.1,3.8,1.9,0.4,Iris-setosa -4.8,3.0,1.4,0.3,Iris-setosa -5.1,3.8,1.6,0.2,Iris-setosa -4.6,3.2,1.4,0.2,Iris-setosa -5.3,3.7,1.5,0.2,Iris-setosa -5.0,3.3,1.4,0.2,Iris-setosa -7.0,3.2,4.7,1.4,Iris-versicolor -6.4,3.2,4.5,1.5,Iris-versicolor -6.9,3.1,4.9,1.5,Iris-versicolor -5.5,2.3,4.0,1.3,Iris-versicolor -6.5,2.8,4.6,1.5,Iris-versicolor -5.7,2.8,4.5,1.3,Iris-versicolor -6.3,3.3,4.7,1.6,Iris-versicolor -4.9,2.4,3.3,1.0,Iris-versicolor -6.6,2.9,4.6,1.3,Iris-versicolor -5.2,2.7,3.9,1.4,Iris-versicolor -5.0,2.0,3.5,1.0,Iris-versicolor -5.9,3.0,4.2,1.5,Iris-versicolor -6.0,2.2,4.0,1.0,Iris-versicolor -6.1,2.9,4.7,1.4,Iris-versicolor -5.6,2.9,3.6,1.3,Iris-versicolor -6.7,3.1,4.4,1.4,Iris-versicolor -5.6,3.0,4.5,1.5,Iris-versicolor -5.8,2.7,4.1,1.0,Iris-versicolor -6.2,2.2,4.5,1.5,Iris-versicolor -5.6,2.5,3.9,1.1,Iris-versicolor -5.9,3.2,4.8,1.8,Iris-versicolor -6.1,2.8,4.0,1.3,Iris-versicolor -6.3,2.5,4.9,1.5,Iris-versicolor -6.1,2.8,4.7,1.2,Iris-versicolor -6.4,2.9,4.3,1.3,Iris-versicolor -6.6,3.0,4.4,1.4,Iris-versicolor -6.8,2.8,4.8,1.4,Iris-versicolor -6.7,3.0,5.0,1.7,Iris-versicolor -6.0,2.9,4.5,1.5,Iris-versicolor -5.7,2.6,3.5,1.0,Iris-versicolor -5.5,2.4,3.8,1.1,Iris-versicolor -5.5,2.4,3.7,1.0,Iris-versicolor -5.8,2.7,3.9,1.2,Iris-versicolor -6.0,2.7,5.1,1.6,Iris-versicolor -5.4,3.0,4.5,1.5,Iris-versicolor -6.0,3.4,4.5,1.6,Iris-versicolor -6.7,3.1,4.7,1.5,Iris-versicolor -6.3,2.3,4.4,1.3,Iris-versicolor -5.6,3.0,4.1,1.3,Iris-versicolor -5.5,2.5,4.0,1.3,Iris-versicolor -5.5,2.6,4.4,1.2,Iris-versicolor -6.1,3.0,4.6,1.4,Iris-versicolor -5.8,2.6,4.0,1.2,Iris-versicolor -5.0,2.3,3.3,1.0,Iris-versicolor -5.6,2.7,4.2,1.3,Iris-versicolor -5.7,3.0,4.2,1.2,Iris-versicolor -5.7,2.9,4.2,1.3,Iris-versicolor -6.2,2.9,4.3,1.3,Iris-versicolor -5.1,2.5,3.0,1.1,Iris-versicolor -5.7,2.8,4.1,1.3,Iris-versicolor -6.3,3.3,6.0,2.5,Iris-virginica -5.8,2.7,5.1,1.9,Iris-virginica -7.1,3.0,5.9,2.1,Iris-virginica -6.3,2.9,5.6,1.8,Iris-virginica -6.5,3.0,5.8,2.2,Iris-virginica -7.6,3.0,6.6,2.1,Iris-virginica -4.9,2.5,4.5,1.7,Iris-virginica -7.3,2.9,6.3,1.8,Iris-virginica -6.7,2.5,5.8,1.8,Iris-virginica -7.2,3.6,6.1,2.5,Iris-virginica -6.5,3.2,5.1,2.0,Iris-virginica -6.4,2.7,5.3,1.9,Iris-virginica -6.8,3.0,5.5,2.1,Iris-virginica -5.7,2.5,5.0,2.0,Iris-virginica -5.8,2.8,5.1,2.4,Iris-virginica -6.4,3.2,5.3,2.3,Iris-virginica -6.5,3.0,5.5,1.8,Iris-virginica -7.7,3.8,6.7,2.2,Iris-virginica -7.7,2.6,6.9,2.3,Iris-virginica -6.0,2.2,5.0,1.5,Iris-virginica -6.9,3.2,5.7,2.3,Iris-virginica -5.6,2.8,4.9,2.0,Iris-virginica -7.7,2.8,6.7,2.0,Iris-virginica -6.3,2.7,4.9,1.8,Iris-virginica -6.7,3.3,5.7,2.1,Iris-virginica -7.2,3.2,6.0,1.8,Iris-virginica -6.2,2.8,4.8,1.8,Iris-virginica -6.1,3.0,4.9,1.8,Iris-virginica -6.4,2.8,5.6,2.1,Iris-virginica -7.2,3.0,5.8,1.6,Iris-virginica -7.4,2.8,6.1,1.9,Iris-virginica -7.9,3.8,6.4,2.0,Iris-virginica -6.4,2.8,5.6,2.2,Iris-virginica -6.3,2.8,5.1,1.5,Iris-virginica -6.1,2.6,5.6,1.4,Iris-virginica -7.7,3.0,6.1,2.3,Iris-virginica -6.3,3.4,5.6,2.4,Iris-virginica -6.4,3.1,5.5,1.8,Iris-virginica -6.0,3.0,4.8,1.8,Iris-virginica -6.9,3.1,5.4,2.1,Iris-virginica -6.7,3.1,5.6,2.4,Iris-virginica -6.9,3.1,5.1,2.3,Iris-virginica -5.8,2.7,5.1,1.9,Iris-virginica -6.8,3.2,5.9,2.3,Iris-virginica -6.7,3.3,5.7,2.5,Iris-virginica -6.7,3.0,5.2,2.3,Iris-virginica -6.3,2.5,5.0,1.9,Iris-virginica -6.5,3.0,5.2,2.0,Iris-virginica -6.2,3.4,5.4,2.3,Iris-virginica -5.9,3.0,5.1,1.8,Iris-virginica - diff --git a/datasets.cabal b/datasets.cabal --- a/datasets.cabal +++ b/datasets.cabal @@ -1,16 +1,30 @@ Name: datasets -Version: 0.1.0.1 +Version: 0.2 Synopsis: Classical data sets for statistics and machine learning Description: Classical machine learning and statistics datasets from the UCI Machine Learning Repository and other sources. . + The datasets package defines two different kinds of datasets: + . + * small data sets which are directly (or indirectly with `file-embed`) + embedded in the package as pure values and do not require network or IO to download + the data set. This includes Iris, Anscombe and OldFaithful. + . + * other data sets which need to be fetched over the network with + `Numeric.Datasets.getDataset` and are cached in a local temporary directory. + . > import Numeric.Datasets (getDataset) > import Numeric.Datasets.Iris (iris) + > import Numeric.Datasets.Abalone (abalone) > > main = do - > irises <- getDataset iris - > print (length irises) - > print (head irises) + > -- The Iris data set is embedded + > print (length iris) + > print (head iris) + > -- The Abalone dataset is fetched + > abas <- getDataset abalone + > print (length abas) + > print (head abas) License: MIT License-file: LICENSE @@ -22,9 +36,6 @@ bug-reports: https://github.com/glutamate/datasets/issues category: Statistics, Machine Learning, Data Mining, Data -Data-files: - datafiles/iris.data - source-repository head type: git location: https://github.com/glutamate/datasets @@ -37,15 +48,16 @@ Exposed-modules: Numeric.Datasets , Numeric.Datasets.Anscombe + , Numeric.Datasets.BostonHousing , Numeric.Datasets.OldFaithful , Numeric.Datasets.Abalone , Numeric.Datasets.Adult , Numeric.Datasets.BreastCancerWisconsin , Numeric.Datasets.Car , Numeric.Datasets.Iris + , Numeric.Datasets.Michelson + , Numeric.Datasets.Nightingale , Numeric.Datasets.Wine - other-modules: Paths_datasets - Build-depends: base >= 4.6 && < 5 , cassava @@ -57,3 +69,6 @@ , vector , text , stringsearch + , file-embed + , aeson + , time diff --git a/src/Numeric/Datasets.hs b/src/Numeric/Datasets.hs --- a/src/Numeric/Datasets.hs +++ b/src/Numeric/Datasets.hs @@ -1,3 +1,21 @@ +{- | + +The datasets package defines two different kinds of datasets: + +* small data sets which are directly (or indirectly with `file-embed`) + embedded in the package as pure values and do not require + network or IO to download the data set. + +* other data sets which need to be fetched over the network with + `getDataset` and are cached in a local temporary directory + +This module defines the `getDataset` function for fetching datasets +and utilies for defining new data sets. It is only necessary to import +this module when using fetched data sets. Embedded data sets can be +imported directly. + +-} + {-# LANGUAGE OverloadedStrings #-} module Numeric.Datasets where @@ -10,6 +28,7 @@ import Data.Monoid import qualified Data.ByteString.Lazy as BL import qualified Data.Vector as V +import qualified Data.Aeson as JSON import Data.Char (toUpper) import Text.Read (readMaybe) @@ -17,7 +36,6 @@ import qualified Data.ByteString.Lazy.Char8 as BL8 import Data.ByteString.Lazy.Search (replace) -import Paths_datasets (getDataFileName) -- * Using datasets -- |Load a dataset, using the system temporary directory as a cache @@ -32,28 +50,24 @@ -- * Defining datasets -data Source = URL String | CabalDataFile FilePath - +data Source = URL String -- |Define a dataset from a pre-processing function and a source for a CSV file csvDatasetPreprocess :: FromRecord a => (BL.ByteString -> BL.ByteString) -> Source -> Dataset a csvDatasetPreprocess preF src cacheDir = do - - let parseFile contents = do - case decode NoHeader (preF contents) of - Right theData -> return $ V.toList theData - Left err -> fail err - - getFileFromSource cacheDir src >>= parseFile + parseCSV preF <$> getFileFromSource cacheDir src -- |Define a dataset from a source for a CSV file csvDataset :: FromRecord a => Source -> Dataset a csvDataset = csvDatasetPreprocess id +jsonDataset :: JSON.FromJSON a => Source -> Dataset a +jsonDataset src cacheDir = do + bs <- getFileFromSource cacheDir src + return $ parseJSON bs + +-- | Get a ByteString from the specified Source getFileFromSource :: FilePath -> Source -> IO (BL.ByteString) -getFileFromSource _ (CabalDataFile fnm) = do - fullpath <- getDataFileName fnm - BL.readFile fullpath getFileFromSource cacheDir (URL url) = do createDirectoryIfMissing True cacheDir let fnm = cacheDir "ds" <> show (hash url) @@ -69,7 +83,19 @@ BL.writeFile fnm bs return bs +parseCSV :: FromRecord a => (BL.ByteString -> BL.ByteString) -> BL.ByteString -> [a] +parseCSV preF contents = + case decode NoHeader (preF contents) of + Right theData -> V.toList theData + Left err -> error err +parseJSON :: JSON.FromJSON a => BL.ByteString -> [a] +parseJSON bs = case JSON.decode bs of + Just theData -> theData + Nothing -> error "failed to parse json" + + + -- * Helper functions for parsing -- |Turn dashes to CamlCase @@ -100,3 +126,15 @@ -- | Turn US-style decimals starting with a period (e.g. .2) into something Haskell can parse (e.g. 0.2) fixAmericanDecimals :: BL.ByteString -> BL.ByteString fixAmericanDecimals = replace ",." (",0."::BL.ByteString) + +-- | Convert a Fixed-width format to a CSV +fixedWidthToCSV :: BL.ByteString -> BL.ByteString +fixedWidthToCSV = BL8.pack . fnl . BL8.unpack where + f [] = [] + f (' ':cs) = ',':f (chomp cs) + f ('\n':cs) = '\n':fnl cs + f (c:cs) = c:f cs + fnl cs = f (chomp cs) --newline + chomp (' ':cs) = chomp cs + chomp (c:cs) = c:cs + chomp [] = [] diff --git a/src/Numeric/Datasets/Abalone.hs b/src/Numeric/Datasets/Abalone.hs --- a/src/Numeric/Datasets/Abalone.hs +++ b/src/Numeric/Datasets/Abalone.hs @@ -36,4 +36,4 @@ instance FromRecord Abalone abalone :: Dataset Abalone -abalone = csvDataset $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/abalone/abalone.data" +abalone = csvDataset $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/abalone/abalone.data" diff --git a/src/Numeric/Datasets/Adult.hs b/src/Numeric/Datasets/Adult.hs --- a/src/Numeric/Datasets/Adult.hs +++ b/src/Numeric/Datasets/Adult.hs @@ -96,7 +96,7 @@ <*> v.!9 <*> v.!10 <*> v.!11 <*> v.!12<*> v.!13<*> v.!14 adult :: Dataset Adult -adult = csvDataset $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data" +adult = csvDataset $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/adult/adult.data" adultTestSet :: Dataset Adult -adultTestSet = csvDatasetPreprocess (dropLines 1) $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test" +adultTestSet = csvDatasetPreprocess (dropLines 1) $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/adult/adult.test" diff --git a/src/Numeric/Datasets/BostonHousing.hs b/src/Numeric/Datasets/BostonHousing.hs new file mode 100644 --- /dev/null +++ b/src/Numeric/Datasets/BostonHousing.hs @@ -0,0 +1,60 @@ +{-# LANGUAGE DeriveGeneric, OverloadedStrings #-} + +{-| + +BostonHousing Data set + +scikit-learn calls this "boston" and UCI calls it "Housing" + +UCI ML Repository link + +-} + +module Numeric.Datasets.BostonHousing where + +import Numeric.Datasets + +import Data.Csv +import GHC.Generics + +data BostonHousing = BostonHousing + { crimeRate :: Double + , zoned :: Double + , industrial :: Double + , charlesRiver :: Bool + , nitricOxides :: Double + , rooms :: Double + , age :: Double + , distance :: Double + , radialHwy :: Double + , tax :: Double + , ptRatio :: Double + , b :: Double + , lowerStatus :: Double + , medianValue :: Double + } deriving (Show, Read, Generic) + +instance FromRecord BostonHousing where + parseRecord v = BostonHousing <$> + v .! 0 <*> + v .! 1 <*> + v .! 2 <*> + (intToBool <$> v .! 3) <*> + v .! 4 <*> + v .! 5 <*> + v .! 6 <*> + v .! 7 <*> + v .! 8 <*> + v .! 9 <*> + v .! 10 <*> + v .! 11 <*> + v .! 12 <*> + v .! 13 + where intToBool :: Int -> Bool + intToBool 0 = False + intToBool 1 = True + intToBool _ = error "intToBool" +bostonHousing :: Dataset BostonHousing +bostonHousing = csvDatasetPreprocess + fixedWidthToCSV + $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/housing/housing.data" diff --git a/src/Numeric/Datasets/BreastCancerWisconsin.hs b/src/Numeric/Datasets/BreastCancerWisconsin.hs --- a/src/Numeric/Datasets/BreastCancerWisconsin.hs +++ b/src/Numeric/Datasets/BreastCancerWisconsin.hs @@ -45,7 +45,7 @@ breastCancerDatabase :: Dataset BreastCancerEntry breastCancerDatabase = csvDataset - $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data" + $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data" data DiagnosticBreastCancer = DiagnosticBreastCancer { diagnosticID :: Int diff --git a/src/Numeric/Datasets/Car.hs b/src/Numeric/Datasets/Car.hs --- a/src/Numeric/Datasets/Car.hs +++ b/src/Numeric/Datasets/Car.hs @@ -65,4 +65,4 @@ car :: Dataset Car car = csvDataset - $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/car/car.data" + $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/car/car.data" diff --git a/src/Numeric/Datasets/Iris.hs b/src/Numeric/Datasets/Iris.hs --- a/src/Numeric/Datasets/Iris.hs +++ b/src/Numeric/Datasets/Iris.hs @@ -1,4 +1,4 @@ -{-# LANGUAGE DeriveGeneric, OverloadedStrings #-} +{-# LANGUAGE DeriveGeneric, OverloadedStrings, TemplateHaskell #-} {-| @@ -14,7 +14,10 @@ import Data.Csv import GHC.Generics +import Data.FileEmbed +import Data.ByteString.Lazy (fromStrict) + data IrisClass = Setosa | Versicolor | Virginica deriving (Show, Read, Eq, Generic) @@ -34,5 +37,5 @@ instance FromRecord Iris -iris :: Dataset Iris -iris = csvDataset $ CabalDataFile "datafiles/iris.data" +iris :: [Iris] +iris = parseCSV id $ fromStrict $(embedFile "datafiles/iris.data") diff --git a/src/Numeric/Datasets/Michelson.hs b/src/Numeric/Datasets/Michelson.hs new file mode 100644 --- /dev/null +++ b/src/Numeric/Datasets/Michelson.hs @@ -0,0 +1,21 @@ +{-# LANGUAGE DeriveGeneric, OverloadedStrings, TemplateHaskell #-} + +{-| + +Michelson's speed of light dataset - five repeated measurements of the speed of light. + +Data from + +The embedded dataset is Copyright (c) 2015 The Compute.io Authors. + +-} + +module Numeric.Datasets.Michelson where + +import Numeric.Datasets +import Data.FileEmbed +import Data.ByteString.Lazy (fromStrict) + + +michelson :: [[Double]] +michelson = parseJSON $ fromStrict $(embedFile "datafiles/michelson.json") diff --git a/src/Numeric/Datasets/Nightingale.hs b/src/Numeric/Datasets/Nightingale.hs new file mode 100644 --- /dev/null +++ b/src/Numeric/Datasets/Nightingale.hs @@ -0,0 +1,34 @@ +{-# LANGUAGE DeriveGeneric, OverloadedStrings, TemplateHaskell #-} + +{-| + +Florence Nightingale's count of injuries in the Crimean War, used for her rose plots + +Data from + +The embedded dataset is Copyright (c) 2015 The Compute.io Authors. + +-} + +module Numeric.Datasets.Nightingale where + +import Numeric.Datasets +import Data.FileEmbed +import Data.ByteString.Lazy (fromStrict) +import Data.Aeson hiding (parseJSON) +import Data.Time (UTCTime) +import GHC.Generics + + +data Nightingale = Nightingale + { date :: UTCTime + , army_size :: Int + , disease :: Int + , wounds :: Int + , other :: Int + } deriving (Show, Read, Generic) + +instance FromJSON Nightingale + +nightingale :: [Nightingale] +nightingale = parseJSON $ fromStrict $(embedFile "datafiles/nightingale.json") diff --git a/src/Numeric/Datasets/Wine.hs b/src/Numeric/Datasets/Wine.hs --- a/src/Numeric/Datasets/Wine.hs +++ b/src/Numeric/Datasets/Wine.hs @@ -37,4 +37,4 @@ wine :: Dataset Wine wine = csvDatasetPreprocess fixAmericanDecimals - $ URL "http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data" + $ URL "http://mlr.cs.umass.edu/ml/machine-learning-databases/wine/wine.data"