datasets 0.2.4 → 0.2.5

raw patch · 4 files changed

+128/−278 lines, 4 filesdep ~basePVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: base

API changes (from Hackage documentation)

Files

• changelog.md +4/−0
• datasets.cabal +4/−4
• src/Numeric/Datasets/Netflix.hs +104/−46
• src/Numeric/Datasets/OldFaithful.hs +16/−228

@@ -1,3 +1,7 @@+0.2.5++* Old faithful matches R dataset+ 0.2.4  * Netflix dataset

@@ -1,5 +1,5 @@ Name:                datasets-Version:             0.2.4+Version:             0.2.5 Synopsis:            Classical data sets for statistics and machine learning Description:         Classical machine learning and statistics datasets from                      the UCI Machine Learning Repository and other sources.@@ -32,8 +32,8 @@ Maintainer:          Tom Nielsen <tanielsen@gmail.com> build-type:          Simple Cabal-Version: 	     >= 1.10-homepage:            https://github.com/filopodia/open/datasets-bug-reports:         https://github.com/filopodia/open/issues+homepage:            https://github.com/diffusionkinetics/open/datasets+bug-reports:         https://github.com/diffusionkinetics/open/issues category:            Statistics, Machine Learning, Data Mining, Data Tested-With:         GHC == 7.10.2, GHC == 7.10.3, GHC == 8.0.1 extra-source-files:@@ -47,7 +47,7 @@  source-repository head   type:     git-  location: https://github.com/filopodia/open+  location: https://github.com/diffusionkinetics/open  Library    ghc-options:       -Wall

@@ -2,28 +2,35 @@ {-| Netflix prize dataset -From the README :--The movie rating files contain over 100 million ratings from 480 thousand randomly-chosen, anonymous Netflix customers over 17 thousand movie titles.  The data were collected between October, 1998 and December, 2005 and reflect the distribution of all ratings received during this period.  The ratings are on a scale from 1 to 5 (integral) stars. To protect customer privacy, each customer id has been replaced with a randomly-assigned id.  The date of each rating and the title and year of release for each movie id are also provided.+From the README : The movie rating files contain over 100 million ratings from 480 thousand randomly-chosen, anonymous Netflix customers over 17 thousand movie titles.  The data were collected between October, 1998 and December, 2005 and reflect the distribution of all ratings received during this period.  The ratings are on a scale from 1 to 5 (integral) stars. To protect customer privacy, each customer id has been replaced with a randomly-assigned id.  The date of each rating and the title and year of release for each movie id are also provided. -The competition ended on September, 2009, and the dataset was subsequently removed from the public domain by the company. +The competition ended on September, 2009, and the dataset was subsequently removed from the public domain by the company (see <http://netflixprize.com/>).  -We include in this repository a tiny subset of the original dataset for development purposes.+We include in the repository a tiny subset of the original dataset for development purposes. Since we use `file-embed` to load the data, the directories are hardcoded (see the Datasets section below); users may either symlink or copy the full dataset in the given directories. -For further information, see <http://netflixprize.com/>. -} -module Numeric.Datasets.Netflix where+module Numeric.Datasets.Netflix (+  -- * Dataset parsing and shaping+  parseTrainingSet, parseTestSet, parseMovies,+  -- * Types+  RD(..),+  UserId, MovieId,+  Train(..), Test(..), Movie(..), +  RatingDate(..),+  -- * Datasets+  trainingSet, testSet, movies+  ) where  import Prelude hiding (takeWhile)  import Numeric.Datasets -- import Data.Csv import Data.FileEmbed-import Data.ByteString hiding (map, takeWhile)+import Data.ByteString hiding (map, head, takeWhile) import Data.Time (Day, fromGregorian) -import Control.Applicative+-- import Control.Applicative import Data.Monoid (mconcat) import Data.Traversable (traverse) import qualified Data.Attoparsec.Internal.Types as PT (Parser) @@ -32,14 +39,19 @@   --- * Dataset files. The directories are scanned recursively and their contents are presented as (FilePath, ByteString) pairs+-- * Dataset files +-- The directories are scanned recursively and their contents are presented as (FilePath, ByteString) pairs++-- | The training set (a set of text files) is assumed to be in the directory `datafiles/netflix/training/` relative to the repository root trainingSet :: [(FilePath, ByteString)] trainingSet = $(embedDir "datafiles/netflix/training/") +-- | The test set (one text file) is assumed to be in the directory `datafiles/netflix/test/` relative to the repository root testSet :: [(FilePath, ByteString)] testSet = $(embedDir "datafiles/netflix/test/") +-- | The movies dataset (one text file) is assumed to be in the directory `datafiles/netflix/movies/` relative to the repository root movies :: [(FilePath, ByteString)] movies = $(embedDir "datafiles/netflix/movies/") @@ -47,25 +59,28 @@  -- * Data types +-- | A date-tagged movie rating data RatingDate = RatingDate {userId :: UserId,                               ratingDate :: Day} deriving (Eq, Show) --- | Training set+-- | User ID (anonymized) newtype UserId = UserId {unUserId :: Int} deriving Eq instance Show UserId where show = show . unUserId +-- | Training set item data Train = Train {trainRating :: RatingDate,                     rating :: Int } deriving (Eq, Show) --- | Movies file+-- | Movie ID newtype MovieId = MovieId {unMovieId :: Int} deriving Eq instance Show MovieId where show = show . unMovieId +-- | Movie dataset item data Movie = Movie { movieId :: MovieId,                      releaseYear :: Day,                      movieTitle :: ByteString } deriving (Eq, Show) --- | "Qualifying" file (test set)+-- | Test set item newtype Test = Test { testRating :: RatingDate } deriving (Eq, Show)  @@ -74,67 +89,110 @@ -- * Additional types and helper functions  -- Every file in the training set corresponds to a distinct column. The whole dataset can therefore be seen as a (very sparse) users vs. movies matrix-data TrainCol = TrainC { tcMovieId :: MovieId,-                         tcTrainSet :: [Train]} deriving (Eq, Show) --- A type for date-tagged ratings+data Col a = Col {cMovieId :: MovieId,+                  cSet :: [a]} deriving (Eq, Show)++newtype TrainCol = TrainC { unTrC :: Col Train } deriving (Eq, Show)++mkTrainCol :: MovieId -> [Train] -> TrainCol+mkTrainCol mid cs = TrainC (Col mid cs)++++newtype TestCol = TestC { unTeC :: Col Test } deriving (Eq, Show)++mkTestCol :: MovieId -> [Test] -> TestCol+mkTestCol mid cs = TestC (Col mid cs)++++-- | A type for date-tagged movie ratings data RD a = RD { rdRating :: a,                  rdDate :: Day} deriving (Eq, Show)  -- Convert a column of training data into (row, col, (rating, date)) coordinate format suitable for populating a sparse matrix-toCoordsCol :: Num a => TrainCol -> [(UserId, MovieId, RD a)]-toCoordsCol tc = map (f mid) tss where-  tss = tcTrainSet tc-  mid = tcMovieId tc+toCoordsTrainCol :: Num a => TrainCol -> [(UserId, MovieId, RD a)]+toCoordsTrainCol tc = map (f mid) tss where+  tss = cSet $ unTrC tc+  mid = cMovieId $ unTrC tc   f m ts = (uid, m, RD r d) where     r = fromIntegral $ rating ts     d = ratingDate $ trainRating ts     uid = userId $ trainRating ts +toCoordsTestCol :: TestCol -> [(UserId, MovieId, Day)]+toCoordsTestCol tc = map (f mid) tss where+  tss = cSet $ unTeC tc+  mid = cMovieId $ unTeC tc+  f m ts = (uid, m, d) where+    d = ratingDate $ testRating ts+    uid = userId $ testRating ts --- Parse the whole training set, convert to coordinate format and concatenate into a single list.++-- | Parse the whole training set, convert to coordinate format and concatenate into a single list. parseTrainingSet :: Num a => Either String [(UserId, MovieId, RD a)] parseTrainingSet = mconcat <$> parseTrainingSet' --- Parse the whole training set and convert to coordinate format+-- | Parse the whole training set and convert to coordinate format (each dataset file is parsed into a distinct inner list) parseTrainingSet' :: Num a => Either String [[(UserId, MovieId, RD a)]] parseTrainingSet' = do   d <- traverse (parseOnly trainingSetParser . snd) trainingSet-  pure $ map toCoordsCol d+  pure $ map toCoordsTrainCol d+   +-- | Parse the whole test set, convert to coordinate format and concatenate into a single list.+parseTestSet :: Either String [(UserId, MovieId, Day)]+parseTestSet = mconcat <$> parseTestSet' --- * Netflix dataset parsers+-- | Parse the whole test set and convert to coordinate format+parseTestSet' :: Either String [[(UserId, MovieId, Day)]]+parseTestSet' = do+  d <- traverse (parseOnly testSetParser . snd) testSet+  return $ map toCoordsTestCol $ mconcat d -{-The first line of each training set file contains the movie id followed by a-colon.  Each subsequent line in the file corresponds to a rating from a customer-and its date in the following format:+-- | Parse the whole movies file, convert to coordinate format and concatenate into a single list.+parseMovies :: Either String [Movie]+parseMovies = do+  d <- traverse (parseOnly moviesParser . snd) movies+  return $ mconcat d -CustomerID,Rating,Date -- MovieIDs range from 1 to 17770 sequentially.-- CustomerIDs range from 1 to 2649429, with gaps. There are 480189 users.-- Ratings are on a five star (integral) scale from 1 to 5.-- Dates have the format YYYY-MM-DD.-}+-- * Netflix dataset parsers++-- | The first line of each training set file contains the movie id followed by a+-- colon.  Each subsequent line in the file corresponds to a rating from a customer+-- and its date in the following format:+--+-- CustomerID,Rating,Date+--+-- - MovieIDs range from 1 to 17770 sequentially.+-- - CustomerIDs range from 1 to 2649429, with gaps. There are 480189 users.+-- - Ratings are on a five star (integral) scale from 1 to 5.+-- - Dates have the format YYYY-MM-DD. trainingSetParser :: PT.Parser ByteString TrainCol trainingSetParser = do   (mid, tr) <- stanza trainRow-  return $ TrainC mid tr+  return $ mkTrainCol mid tr -{-The test set ("qualifying") file consists of lines indicating a movie id, followed by a colon, and then customer ids and rating dates, one per line for that movie id.-The movie and customer ids are contained in the training set.  Of course the-ratings are withheld. There are no empty lines in the file.-}-testSetParser :: PT.Parser ByteString [(MovieId, [Test])]-testSetParser = many1 (stanza testRow)+-- | The test set ("qualifying") file consists of lines indicating a movie id, followed by a colon, and then customer ids and rating dates, one per line for that movie id.+-- The movie and customer ids are contained in the training set.  Of course the+-- ratings are withheld. There are no empty lines in the file.+testSetParser :: PT.Parser ByteString [TestCol]+testSetParser = do+  ll <- many1 (stanza testRow)+  return $ map (uncurry mkTestCol) ll -{-Movie information is in the following format: -MovieID,YearOfRelease,Title--- MovieID do not correspond to actual Netflix movie ids or IMDB movie ids.-- YearOfRelease can range from 1890 to 2005 and may correspond to the release of-  corresponding DVD, not necessarily its theaterical release.-- Title is the Netflix movie title and may not correspond to -  titles used on other sites.  Titles are in English.-}+-- | Movie information is in the following format:+--+-- MovieID,YearOfRelease,Title+--+-- - MovieID do not correspond to actual Netflix movie ids or IMDB movie ids.+-- - YearOfRelease can range from 1890 to 2005 and may correspond to the release of+--   corresponding DVD, not necessarily its theaterical release.+-- - Title is the Netflix movie title and may not correspond to +--   titles used on other sites.  Titles are in English. moviesParser :: PT.Parser ByteString [Movie] moviesParser = parseRows moviesRow 

@@ -14,234 +14,22 @@  module Numeric.Datasets.OldFaithful where -data OldFaithful = OF-  { recordingDay :: Int -- ^ Recording day. 1-8: 1978, 16-23: 1979-  , waiting :: Double -- ^ waiting time until next eruption+import Data.Csv+import Control.Applicative+import Numeric.Datasets++++data OldFaithful = OldFaithful+  { waiting :: Double -- ^ waiting time until next eruption   , duration :: Double -- ^ duration of eruption in minutes   } deriving Show --- The first comp-oldFaithful :: [OldFaithful]-oldFaithful =-  [ OF  1 78 4.4-  , OF  1 74 3.9-  , OF  1 68 4.0-  , OF  1 76 4.0-  , OF  1 80 3.5-  , OF  1 84 4.1-  , OF  1 50 2.3-  , OF  1 93 4.7-  , OF  1 55 1.7-  , OF  1 76 4.9-  , OF  1 58 1.7-  , OF  1 74 4.6-  , OF  1 75 3.4-  , OF  2 80 4.3-  , OF  2 56 1.7-  , OF  2 80 3.9-  , OF  2 69 3.7-  , OF  2 57 3.1-  , OF  2 90 4.0-  , OF  2 42 1.8-  , OF  2 91 4.1-  , OF  2 51 1.8-  , OF  2 79 3.2-  , OF  2 53 1.9-  , OF  2 82 4.6-  , OF  2 51 2.0-  , OF  3 76 4.5-  , OF  3 82 3.9-  , OF  3 84 4.3-  , OF  3 53 2.3-  , OF  3 86 3.8-  , OF  3 51 1.9-  , OF  3 85 4.6-  , OF  3 45 1.8-  , OF  3 88 4.7-  , OF  3 51 1.8-  , OF  3 80 4.6-  , OF  3 49 1.9-  , OF  3 82 3.5-  , OF  4 75 4.0-  , OF  4 73 3.7-  , OF  4 67 3.7-  , OF  4 68 4.3-  , OF  4 86 3.6-  , OF  4 72 3.8-  , OF  4 75 3.8-  , OF  4 75 3.8-  , OF  4 66 2.5-  , OF  4 84 4.5-  , OF  4 70 4.1-  , OF  4 79 3.7-  , OF  4 60 3.8-  , OF  4 86 3.4-  , OF  5 71 4.0-  , OF  5 67 2.3-  , OF  5 81 4.4-  , OF  5 76 4.1-  , OF  5 83 4.3-  , OF  5 76 3.3-  , OF  5 55 2.0-  , OF  5 73 4.3-  , OF  5 56 2.9-  , OF  5 83 4.6-  , OF  5 57 1.9-  , OF  5 71 3.6-  , OF  5 72 3.7-  , OF  5 77 3.7-  , OF  6 55 1.8-  , OF  6 75 4.6-  , OF  6 73 3.5-  , OF  6 70 4.0-  , OF  6 83 3.7-  , OF  6 50 1.7-  , OF  6 95 4.6-  , OF  6 51 1.7-  , OF  6 82 4.0-  , OF  6 54 1.8-  , OF  6 83 4.4-  , OF  6 51 1.9-  , OF  6 80 4.6-  , OF  6 78 2.9-  , OF  7 81 3.5-  , OF  7 53 2.0-  , OF  7 89 4.3-  , OF  7 44 1.8-  , OF  7 78 4.1-  , OF  7 61 1.8-  , OF  7 73 4.7-  , OF  7 75 4.2-  , OF  7 73 3.9-  , OF  7 76 4.3-  , OF  7 55 1.8-  , OF  7 86 4.5-  , OF  7 48 2.0-  , OF  8 77 4.2-  , OF  8 73 4.4-  , OF  8 70 4.1-  , OF  8 88 4.1-  , OF  8 75 4.0-  , OF  8 83 4.1-  , OF  8 61 2.7-  , OF  8 78 4.6-  , OF  8 61 1.9-  , OF  8 81 4.5-  , OF  8 51 2.0-  , OF  8 80 4.8-  , OF  8 79 4.1-  , OF 16 82 4.1-  , OF 16 80 4.2-  , OF 16 76 4.5-  , OF 16 56 1.9-  , OF 16 82 4.7-  , OF 16 47 2.0-  , OF 16 76 4.7-  , OF 16 61 2.5-  , OF 16 75 4.3-  , OF 16 72 4.4-  , OF 16 74 4.4-  , OF 16 69 4.3-  , OF 16 78 4.6-  , OF 16 52 2.1-  , OF 17 91 4.8-  , OF 17 66 4.1-  , OF 17 71 4.0-  , OF 17 75 4.0-  , OF 17 81 4.4-  , OF 17 77 4.1-  , OF 17 74 4.3-  , OF 17 70 4.0-  , OF 17 83 3.9-  , OF 17 53 3.2-  , OF 17 82 4.5-  , OF 17 62 2.2-  , OF 17 73 4.7-  , OF 17 84 4.6-  , OF 18 58 2.2-  , OF 18 82 4.8-  , OF 18 77 4.3-  , OF 18 75 3.8-  , OF 18 77 4.0-  , OF 18 77 4.1-  , OF 18 53 1.8-  , OF 18 75 4.4-  , OF 18 78 4.0-  , OF 18 51 2.2-  , OF 18 81 5.1-  , OF 18 52 1.9-  , OF 18 76 5.0-  , OF 18 73 4.4-  , OF 19 84 4.5-  , OF 19 72 3.8-  , OF 19 89 4.3-  , OF 19 75 4.4-  , OF 19 57 2.2-  , OF 19 81 4.8-  , OF 19 49 1.9-  , OF 19 87 4.7-  , OF 19 43 1.8-  , OF 19 94 4.8-  , OF 19 45 2.0-  , OF 19 81 4.4-  , OF 19 59 2.5-  , OF 19 82 4.3-  , OF 20 80 4.4-  , OF 20 54 1.9-  , OF 20 75 4.7-  , OF 20 73 4.3-  , OF 20 57 2.2-  , OF 20 80 4.7-  , OF 20 51 2.3-  , OF 20 77 4.6-  , OF 20 66 3.3-  , OF 20 77 4.2-  , OF 20 60 2.9-  , OF 20 86 4.6-  , OF 20 62 3.3-  , OF 20 75 4.2-  , OF 20 67 2.6-  , OF 20 69 4.6-  , OF 21 84 3.7-  , OF 21 58 1.8-  , OF 21 90 4.7-  , OF 21 82 4.5-  , OF 21 71 4.5-  , OF 21 80 4.8-  , OF 21 51 2.0-  , OF 21 80 4.8-  , OF 21 62 1.9-  , OF 21 84 4.7-  , OF 21 51 2.0-  , OF 21 81 5.1-  , OF 21 83 4.3-  , OF 21 84 4.8-  , OF 22 72 3.0-  , OF 22 54 2.1-  , OF 22 75 4.6-  , OF 22 74 4.0-  , OF 22 51 2.2-  , OF 22 91 5.1-  , OF 22 60 2.9-  , OF 22 80 4.3-  , OF 22 54 2.1-  , OF 22 80 4.7-  , OF 22 70 4.5-  , OF 22 60 1.7-  , OF 22 86 4.2-  , OF 22 78 4.3-  , OF 23 51 1.7-  , OF 23 83 4.4-  , OF 23 76 4.2-  , OF 23 51 2.2-  , OF 23 90 4.7-  , OF 23 71 4.0-  , OF 23 49 1.8-  , OF 23 88 4.7-  , OF 23 52 1.8-  , OF 23 79 4.5-  , OF 23 61 2.1-  , OF 23 81 4.2-  , OF 23 48 2.1-  , OF 23 84 5.2-  , OF 23 63 2.0 ]+instance FromRecord OldFaithful where+  parseRecord v = OldFaithful <$> v .! 2 <*> v.! 1++oldFaithful :: Dataset OldFaithful+oldFaithful +  = let src = URL "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/master/csv/datasets/faithful.csv"+    in Dataset src Nothing Nothing $ CSVRecord HasHeader defaultDecodeOptions+