diff --git a/changelog b/changelog --- a/changelog +++ b/changelog @@ -2,4 +2,9 @@ 0.0.0.1 * First version of the package. - +0.0.0.2 + * Added 'man/man1/fishfood.1' & 'fishfood.spec' to the distribution. +0.0.1.0 + * Migrated all but the I/O, from "Main" to a new module "FishFood.Profiler". + * Added module "FishFood.Test", containing "Test.QuickCheck"-implementation. + * Added Command-line options; '--deriveProbabilityMassFunction', '--binSizeRatio', '--runQuickChecks'. diff --git a/copyright b/copyright --- a/copyright +++ b/copyright @@ -1,8 +1,8 @@ Author: - Dr. Alistair Ward . + Dr. Alistair Ward . Copyright: - Copyright (C) 2010-2011 Dr. Alistair Ward. All Rights Reserved. + Copyright (C) 2013 Dr. Alistair Ward. All Rights Reserved. Home-page: http://functionalley.eu diff --git a/fishfood.cabal b/fishfood.cabal --- a/fishfood.cabal +++ b/fishfood.cabal @@ -1,13 +1,13 @@ --Package-properties Name: fishfood -Version: 0.0.0.1 +Version: 0.0.1.0 Cabal-Version: >= 1.6 Copyright: (C) 2013 Dr. Alistair Ward License: GPL License-file: LICENSE Author: Dr. Alistair Ward Stability: Unstable interface, incomplete features. -Synopsis: Finds the file-size frequency-distribution for the referenced files. +Synopsis: Calculates file-size frequency-distribution. Build-Type: Simple Description: Counts the number of files in a set of bins, each of which holds only those files which fall within a specific size-interval. Category: Utils @@ -16,7 +16,7 @@ Maintainer: fishfood functionalley eu Bug-reports: fishfood functionalley eu Data-files: -Extra-Source-Files: changelog copyright makefile +Extra-Source-Files: changelog copyright makefile man/man1/fishfood.1 fishfood.spec flag llvm Description: Whether the 'llvm' compiler-backend has been installed and is required for code-generation. @@ -28,13 +28,21 @@ Main-Is: Main.hs Other-modules: FishFood.Data.CommandOptions, - FishFood.Data.File + FishFood.Data.File, + FishFood.Data.Verbosity + FishFood.Profiler + FishFood.Test.Data.CommandOptions + FishFood.Test.Data.Verbosity + FishFood.Test.Profiler + FishFood.Test.QuickChecks Build-depends: base == 4.*, Cabal >= 1.6 && < 2, containers, factory >= 0.2.0.5, + mtl, + QuickCheck, toolshed >= 0.13, unix diff --git a/fishfood.spec b/fishfood.spec new file mode 100644 --- /dev/null +++ b/fishfood.spec @@ -0,0 +1,65 @@ +# Copyright (C) 2013 Dr. Alistair Ward +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +%define package %name-%version +%define tarBall %package.tar.gz +%define _bindir %prefix/bin +%define _sharedir %prefix/share +%define _docdir %_sharedir/doc/%package +%define _mandir %_sharedir/man + +Summary: Calculates file-size frequency-distribution. +Name: fishfood +Version: 0.0.1.0 +Release: 1 +License: GPLv3 +#From '/usr/share/doc/packages/rpm/GROUPS'. +Group: Applications/File +Source0: http://functionalley.eu/Downloads/sdist/%tarBall +URL: http://functionalley.eu +Prefix: /usr +BuildRequires: ghc + +%description +Counts the number of files in a set of bins, each of which holds only those files which fall within a specific size-interval. + +%prep +#NB: we've changed directory to %_builddir +(cd $OLDPWD && runhaskell Setup sdist) && tar -zxf $OLDPWD/dist/%tarBall #Make a source-distribution & unpack it into the build-directory. +cd '%package/' && runhaskell Setup configure --user --prefix=%prefix #Configure to use the user's personal package-database & to generate an appropriate "Paths" module. + +%build +cd '%package/' && runhaskell Setup build #Descend into the source-distribution and build according to the previously established configuration. + +%install +cd '%package/' #Descend into the build-directory. +runhaskell Setup copy --destdir=%buildroot #Install the package in the target-directory. +mkdir -p -- '%buildroot%_docdir' && mv changelog copyright LICENSE '%buildroot%_docdir/' +mkdir -p -- '%buildroot%_mandir' && mv 'man/man1' '%buildroot%_mandir/' + +%clean +make --directory='%package' clobber +rm -rf -- '%_builddir/%package/' '%buildroot' + +%files +%attr(0755, root, root) %_bindir/%name +%attr(0644, root, root) %doc %_docdir/changelog +%attr(0644, root, root) %doc %_docdir/copyright +%attr(0644, root, root) %doc %_docdir/LICENSE +%attr(0644, root, root) %doc %_mandir/man1/%name.1.gz + +%changelog +* Tue Jul 16 2013 Alistair Ward 0.0.0.2-1 +First cut. diff --git a/makefile b/makefile --- a/makefile +++ b/makefile @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -.PHONY: all build check clean clobber configure copy haddock help hlint install prof sdist +.PHONY: all build check clean clobber configure copy haddock help hlint install prof rpm sdist PACKAGE_NAME=fishfood @@ -49,6 +49,10 @@ check: sdist cabal upload --check --verbose=3 dist/*.tar.gz; + +rpm: $(PACKAGE_NAME).spec + rpmbuild -bb --nodeps -- $^ + mv -- `find ~/rpmbuild -type f -name '$(PACKAGE_NAME)*.rpm' -mtime 0 -print` ./ clean: runhaskell Setup $@ diff --git a/man/man1/fishfood.1 b/man/man1/fishfood.1 new file mode 100644 --- /dev/null +++ b/man/man1/fishfood.1 @@ -0,0 +1,166 @@ +.TH fishfood 1 +.SH NAME +\fBfishfood\fR - Calculates file-size frequency-distribution. +.SH SYNOPSIS +\fBfishfood\fR [\fIOPTIONS\fR] [\fIFile-path\fR ...] +.SH DESCRIPTION +.PP +Counts the number of the specified files which fall within each of a sequence of discrete size-intervals. +.SH OPTIONS +.TP +\fB-s\fR \fIInt\fR, \fB--binSizeIncrement=\fR\fIInt\fR +Defines the constant size-increase in the arithmetic sequence of bins into which the byte-sizes of files are categorised; defaulting to one standard-deviation. +.TP +\fB-r\fR \fIFloat\fR, \fB--binSizeRatio=\fR \fIFloat\fR +Defines the constant size-ratio in the geometric sequence of bins into which the byte-sizes of files are categorised; an alternative to "\fBbinSizeIncrement\fR". +.TP +\fB-p\fR[\fIBool\fR], \fB--deriveProbabilityMassFunction\fR[\fB=\fR\fIBool\fR] +Whether to derive the \fIProbability Mass Function\fR rather than the \fIFrequency-distribution\fR. +.br +The default value, in the absence of this option, is "\fBFalse\fR", +but in the absence of only the boolean argument, "\fBTrue\fR" will be inferred. +.TP +\fB-d\fR \fIInt\fR, \fB--nDecimalDigits=\fR\fIInt\fR +The precision to which fractional auxiliary data is displayed. +.TP +\fB--verbosity=\fR\fBSilent\fR|\fBNormal\fR|\fBVerbose\fR|\fBDeafening\fR +Produces additional output where appropriate; i.e. file-names, file-size statistics, & column-headers. +.SS "Generic Program-information" +.TP +\fB-v\fR, \fB--version\fR +Outputs version-information & then exits. +.TP +\fB-?\fR, \fB--help\fR +Displays help & then exits. +.SS "File-paths" +.TP +If \fIFile-path\fR is a single hyphen-minus (\fB-\fR), then the list of file-paths will be read from standard-input. Only plain files are acceptable; no directories, symlinks, sockets, ... +.SH EXIT-STATUS +\fB0\fR on success, & >\fB0\fR if an error occurs. +.SH EXAMPLES +.SS Example 1 +To find the frequency-distribution in the size of any Matroska video-files in a file-system: +.IP +\fBfishfood --verbosity=Verbose $(find / -name '*.mkv' 2>/dev/null)\fR #CAVEAT: for efficiency, one may want to be more precise with the path supplied to "\fBfind\fR". +.nf +Files=86, mean=394459250.942, standard-deviation=304129537.729 + Bin-size Frequency +========== ========= + 0 44 + 304129538 29 + 608259076 6 + 912388614 4 +1216518152 2 +1520647690 1 +.fi +.PP +The left-hand column defines an arithmetic sequence of bins, whilst the right-hand column defines the number of files from those specified which fall into each. +The choice of the increment between each bin has defaulted to one standard-deviation. +.br +From this data one can conclude that there are 44 files whose size lies in the semi-closed interval [0, 1) standard-deviations, +decaying monotonically to only one file whose size lies in the semi-closed interval [5, 6) standard-deviations. +.SS Example 2 +One can alternatively specify the arithmetic increment between bin-sizes, +& also derive the probability that a file-size lies in any specific bin. +.IP +.B fishfood --verbosity=Verbose --binSizeIncrement=100000000 --deriveProbabilityMassFunction $(find / -name '*.mkv' 2>/dev/null) +.nf +Files=86, mean=394459250.942, standard-deviation=304129537.729 + Bin-size Probability +========== =========== + 100000000 0.209 + 200000000 0.302 + 300000000 0.233 + 400000000 0.035 + 500000000 0.070 + 600000000 0.023 + 800000000 0.035 + 900000000 0.035 +1000000000 0.012 +1100000000 0.012 +1200000000 0.012 +1400000000 0.012 +1600000000 0.012 +.fi +.PP +CAVEAT: the total probability may differ from "\fB1\fR", due to round-errors; see "\fBnDecimalDigits\fR". +.SS Example 3 +One can alternatively define a geometric sequence of file-size bins, +& also read the file-names from standard-input, to bypass any limit applied by the shell to the length of the command-line. +.IP +.B find /etc -type f -readable 2>/dev/null | fishfood --verbosity=Verbose --binSizeRatio=10 - +.nf +Files=1735, mean=13846.622, standard-deviation=74846.621 + Bin-size Frequency +========== ========= + 0 4 #Though "\fB0\fR" isn't a member of the requested geometric sequence, it's the integral value beneath all fractional values which are. + 1 2 + 10 100 + 100 563 + 1000 794 + 10000 188 + 100000 83 + 1000000 1 +.fi +.PP +From this data one can conclude that there are 4 files whose size lies in the semi-closed interval [0, 1) bytes, 3 files in [1, 10), 100 files in [10, 100), ... +.IP +.B find $HOME -name '*.png' -o -name '*.gif' -o -name '*.jp*g' | fishfood --verbosity=Verbose -r 2 -p - +.nf +Files=878, mean=78365.943, standard-deviation=297831.014 + Bin-size Probability +========== =========== + 32 0.023 + 64 0.017 + 128 0.008 + 256 0.015 + 512 0.034 + 1024 0.046 + 2048 0.047 + 4096 0.096 + 8192 0.155 + 16384 0.179 + 32768 0.155 + 65536 0.157 + 131072 0.032 + 262144 0.017 + 524288 0.003 + 1048576 0.010 + 2097152 0.007 +.fi +.PP +When specifying an arithmetic sequence of bin-sizes, the lack of resolution amongst smaller files makes the distribution appear like the decaying exponential of a \fIgeometric distribution\fR, +but by using a geometric sequence of bin-sizes, it can be seen more clearly to be a \fIlog-normal distribution\fR; +see "A Large-Scale Study of File-System Contents" by John R. Douceur and William J. Bolosky. +.SH AUTHOR +Written by Dr. Alistair Ward. +.SH BUGS +.SS "REPORTING BUGS" +Report bugs to \fBfishfood\fR \fIat\fR \fBfunctionalley\fR \fIdot\fR \fBeu\fR +.SH COPYRIGHT +Copyright \(co 2013 Dr. Alistair Ward +.PP +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +.PP +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +.PP +You should have received a copy of the GNU General Public License along with this program. If not, see \fB\fR. +.SH "SEE ALSO" +.IP \(bu +Home-page: \fBhttp://functionalley.eu\fR +.IP \(bu +.B http://en.wikipedia.org/wiki/Interval_(mathematics) +.IP \(bu +.B http://en.wikipedia.org/wiki/Standard_deviation +.IP \(bu +.B http://en.wikipedia.org/wiki/Frequency_distribution +.IP \(bu +.B http://en.wikipedia.org/wiki/Geometric_distribution +.IP \(bu +.B http://en.wikipedia.org/wiki/Log-normal_distribution +.IP \(bu +.B http://en.wikipedia.org/wiki/Probability_mass_function +.IP \(bu +Source-documentation is generated by \fBHaddock\fR, & is available in the distribution. +.IP \(bu +.B http://www.haskell.org/haddock/ diff --git a/src/FishFood/Data/CommandOptions.hs b/src/FishFood/Data/CommandOptions.hs --- a/src/FishFood/Data/CommandOptions.hs +++ b/src/FishFood/Data/CommandOptions.hs @@ -28,14 +28,22 @@ module FishFood.Data.CommandOptions( -- * Types +-- ** Type-synonyms + BinSizeDelta, -- ** Data-types CommandOptions( -- MkCommandOptions, - getBinSize, - getIncludeEmpty, + getBinSizeDelta, + getDeriveProbabilityMassFunction, getNDecimalDigits, getVerbosity - ) + ), +-- * Functions +-- ** Constructors + mkCommandOptions, +-- ** Mutators + setBinSizeIncrement, + setBinSizeRatio ) where import qualified Data.Maybe @@ -44,33 +52,37 @@ import qualified ToolShed.Defaultable import qualified ToolShed.SelfValidate +-- | Either an arithmetic size-increase for which there's a default, or a geometric size-ratio. +type BinSizeDelta ratio = Either (Maybe Data.File.FileSize) ratio + -- | Declares a record to contain command-line options. -data CommandOptions = MkCommandOptions { - getBinSize :: Maybe Data.File.FileSize, -- ^ The intervals into which file-sizes are categorised. - getIncludeEmpty :: Bool, -- ^ Whether to display empty bins. - getNDecimalDigits :: Int, -- ^ The precision to which fractional auxiliary data is displayed. - getVerbosity :: Distribution.Verbosity.Verbosity -- ^ The threshold for ancillary information-output. +data CommandOptions ratio = MkCommandOptions { + getBinSizeDelta :: BinSizeDelta ratio, -- ^ Either the arithmetic size-increase (defaulting to one standard-deviation), or the geometric size-ratio, of the sequence of bins into which files are categorized. + getDeriveProbabilityMassFunction :: Bool, -- ^ Whether to derive the "Probability mass function" rather than the "Frequency-distribution". + getNDecimalDigits :: Int, -- ^ The precision to which fractional data is displayed. + getVerbosity :: Distribution.Verbosity.Verbosity -- ^ The threshold for ancillary information-output. } deriving Show -instance ToolShed.Defaultable.Defaultable CommandOptions where +instance ToolShed.Defaultable.Defaultable (CommandOptions ratio) where defaultValue = MkCommandOptions { - getBinSize = Nothing, --Interpreted as one standard-deviation. - getIncludeEmpty = False, - getNDecimalDigits = 3, - getVerbosity = Distribution.Verbosity.normal + getBinSizeDelta = Left Nothing, --Interpreted as one standard-deviation. + getDeriveProbabilityMassFunction = False, + getNDecimalDigits = 3, + getVerbosity = Distribution.Verbosity.normal } -instance ToolShed.SelfValidate.SelfValidator CommandOptions where +instance (Num ratio, Ord ratio, Show ratio) => ToolShed.SelfValidate.SelfValidator (CommandOptions ratio) where getErrors commandOptions@MkCommandOptions { - getBinSize = binSize, - getNDecimalDigits = nDecimalDigits + getBinSizeDelta = binSizeDelta, + getDeriveProbabilityMassFunction = deriveProbabilityMassFunction, + getNDecimalDigits = nDecimalDigits } = map snd $ filter fst [ ( - Data.Maybe.maybe False (<= 0) binSize, - "the bin-size must exceed zero; " ++ show commandOptions ++ "." + either (Data.Maybe.maybe False (<= 0)) (<= 1) binSizeDelta, + "either the bin-size's arithmetic increase must exceed zero, or it's geometric ratio must exceed one; " ++ show commandOptions ++ "." ), ( - nDecimalDigits < 1, - "the number of decimal digits must exceed zero; " ++ show commandOptions ++ "." + deriveProbabilityMassFunction && nDecimalDigits < 1, + "the number of decimal digits must exceed zero to adequately represent probabilities; " ++ show commandOptions ++ "." ), let maxNDecimalDigits = floor $ fromIntegral ( @@ -84,3 +96,18 @@ ) ] +-- | Smart constructor. +mkCommandOptions :: (Num ratio, Ord ratio, Show ratio) => BinSizeDelta ratio -> Bool -> Int -> Distribution.Verbosity.Verbosity -> CommandOptions ratio +mkCommandOptions binSizeDelta deriveProbabilityMassFunction nDecimalDigits verbosity + | ToolShed.SelfValidate.isValid commandOptions = commandOptions + | otherwise = error $ "FishFood.Data.CommandOptions.mkCommandOptions:\t" ++ ToolShed.SelfValidate.getFirstError commandOptions + where + commandOptions = MkCommandOptions binSizeDelta deriveProbabilityMassFunction nDecimalDigits verbosity + +-- | Mutator. +setBinSizeIncrement :: Data.File.FileSize -> CommandOptions ratio -> CommandOptions ratio +setBinSizeIncrement fileSize commandOptions = commandOptions { getBinSizeDelta = Left $ Just fileSize } + +-- | Mutator. +setBinSizeRatio :: ratio -> CommandOptions ratio -> CommandOptions ratio +setBinSizeRatio ratio commandOptions = commandOptions { getBinSizeDelta = Right ratio } diff --git a/src/FishFood/Data/Verbosity.hs b/src/FishFood/Data/Verbosity.hs new file mode 100644 --- /dev/null +++ b/src/FishFood/Data/Verbosity.hs @@ -0,0 +1,37 @@ +{-# OPTIONS_GHC -fno-warn-orphans #-} +{- + Copyright (C) 2013 Dr. Alistair Ward + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{- | + [@AUTHOR@] Dr. Alistair Ward + + [@DESCRIPTION@] Instance-definitions for 'Distribution.Verbosity.Verbosity'. +-} + +module FishFood.Data.Verbosity( +-- * Constants + range +) where + +import qualified Distribution.Verbosity +import qualified ToolShed.Defaultable + +instance ToolShed.Defaultable.Defaultable Distribution.Verbosity.Verbosity where + defaultValue = Distribution.Verbosity.normal + +-- | The constant complete range of values. +range :: [Distribution.Verbosity.Verbosity] +range = [minBound .. maxBound] diff --git a/src/FishFood/Profiler.hs b/src/FishFood/Profiler.hs new file mode 100644 --- /dev/null +++ b/src/FishFood/Profiler.hs @@ -0,0 +1,148 @@ +{- + Copyright (C) 2013 Dr. Alistair Ward + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{- | + [@AUTHOR@] Dr. Alistair Ward + + [@DESCRIPTION@] Profiles lists of file-sizes. +-} + +module FishFood.Profiler( +-- * Types +-- ** Type-synonyms +-- Probability, +-- Result, +-- FileSizeDistribution, +-- * Functions + calculateFileSizeDistribution, + formatFileSizeDistribution, +-- ** Accessors + getFileSize, + getValue +) where + +import Control.Arrow((&&&),(***)) +import qualified Control.Monad.Writer +import qualified Data.List +import qualified Data.Map +import qualified Data.Maybe +import qualified FishFood.Data.CommandOptions as Data.CommandOptions +import qualified FishFood.Data.File as Data.File +import FishFood.Data.Verbosity() +import qualified Text.Printf +import qualified ToolShed.Defaultable + +-- | Define a type to represent the fractional closed unit-interval. +type Probability = Double + +-- | Defines either the number of files or the probability that a files has a specific size. +type Result = (Data.File.FileSize, Either Int {-file-count-} Probability) + +-- | Accessor. +getFileSize :: Result -> Data.File.FileSize +getFileSize = fst + +-- | Accessor. +getValue :: Result -> Either Int {-file-count-} Probability +getValue = snd + +-- | Defines either a /Probability Mass Function/ or /Frequency-distribution/. +type FileSizeDistribution = [Result] + +-- | Calculates either the /Probability Mass Function/ or /Frequency-distribution/ for the specified files. +calculateFileSizeDistribution :: (Floating ratio, RealFrac ratio) => Data.CommandOptions.CommandOptions ratio -> [Data.File.FileSize] -> Control.Monad.Writer.Writer [String] FileSizeDistribution +calculateFileSizeDistribution commandOptions fileSizes = let + binSizeDelta = Data.CommandOptions.getBinSizeDelta commandOptions + deriveProbabilityMassFunction = Data.CommandOptions.getDeriveProbabilityMassFunction commandOptions + nDecimalDigits = Data.CommandOptions.getNDecimalDigits commandOptions + + mean, standardDeviation :: Double + (nFiles, mean, standardDeviation) = Data.File.getFileSizeStatistics fileSizes + in do + Control.Monad.Writer.tell [Text.Printf.printf "Files=%d, mean=%.*f, standard-deviation=%.*f" nFiles nDecimalDigits mean nDecimalDigits standardDeviation] + + return {-to Writer-monad-} $ if standardDeviation == 0 + then return {-to List-monad-} . (,) (head fileSizes) $ if deriveProbabilityMassFunction + then Right 1 --i.e. certainty. + else Left nFiles --i.e. all. + else let + getDefaultedBinSizeIncrement :: Maybe Data.File.FileSize -> Data.File.FileSize + getDefaultedBinSizeIncrement = Data.Maybe.fromMaybe $ round standardDeviation `max` 1 {-minimum increment-} --CAVEAT: guard against subsequent division by zero or infinite iteration. + + calculatedBinSizes :: [Data.File.FileSize] + calculatedBinSizes = map ( + \fileSize -> either ( + div {-round down-} fileSize . getDefaultedBinSizeIncrement {-non-zero-} + ) ( + floor {-round down-} . (`logBase` fromIntegral fileSize) --CAVEAT: converts file-size 0, to bin-size -infinity. + ) binSizeDelta + ) fileSizes --Each bin spans the semi-closed integral interval [size, succ size), so round down fractional values to match the lower bin. + + initialFrequencyDistribution :: Data.Map.Map Data.File.FileSize Int + initialFrequencyDistribution = Data.Map.fromAscList . ( + `zip` repeat 0 --The initial file-count. + ) . takeWhile ( + <= maximum calculatedBinSizes + ) . dropWhile ( + < minimum calculatedBinSizes + ) $ either ( + \maybeBinSizeIncrement -> iterate (+ getDefaultedBinSizeIncrement {-non-zero-} maybeBinSizeIncrement) 0 + ) ( + \binRatio -> map round {-file-sizes are integral-} $ iterate (* binRatio) 1 --The sequence could be started at fractional values in the open unit-interval, but the only value less than 1 which may be required is 0 (which isn't a sequence-member), which will be created later on demand. + ) binSizeDelta + + mapBinSizeToFileSize :: Data.Map.Map Data.File.FileSize value -> Data.Map.Map Data.File.FileSize value + mapBinSizeToFileSize = Data.Map.mapKeys $ \binSize -> either ( + (* binSize) . getDefaultedBinSizeIncrement + ) ( + ceiling {-round up-} . (^^ binSize) --Converts binSize -infinity, back to file-size 0. + ) binSizeDelta --Represent each bin by the minimum file-size it can accept. + in Data.Map.toList . ( + if deriveProbabilityMassFunction + then Data.Map.map Right . mapBinSizeToFileSize . Data.Map.map ((/ fromIntegral nFiles {-non-zero-}) . fromIntegral) + else Data.Map.map Left . mapBinSizeToFileSize + ) $ foldr ( + Data.Map.insertWith (+) `flip` 1 --Count the files allocated to each bin. + ) initialFrequencyDistribution calculatedBinSizes + +-- | Formats a file-size distribution. +formatFileSizeDistribution :: Data.CommandOptions.CommandOptions ratio -> FileSizeDistribution -> String +formatFileSizeDistribution commandOptions = Data.List.intercalate "\n" . map ( + \(fileSize, value) -> fileSize ++ " " ++ value + ) . ( + if Data.CommandOptions.getVerbosity commandOptions > ToolShed.Defaultable.defaultValue + then ( + [ + ( + ($ (fileSizeWidth, fileSizeHeader)) &&& ($ (valueWidth, valueHeader)) + ) . uncurry $ Text.Printf.printf "%*s", --Column-headers. + (`replicate` '=') *** (`replicate` '=') $ columnWidths --Separator-bar. + ] ++ + ) --Section. + else id + ) . map ( + Text.Printf.printf "%*d" fileSizeWidth *** either ( + Text.Printf.printf "%*d" valueWidth + ) ( + Text.Printf.printf "%.*f" $ Data.CommandOptions.getNDecimalDigits commandOptions + ) + ) where + fileSizeHeader, valueHeader :: String + headers@(fileSizeHeader, valueHeader) = (,) "Bin-size" $ if Data.CommandOptions.getDeriveProbabilityMassFunction commandOptions then "Probability" else "Frequency" + + fileSizeWidth, valueWidth :: Int + columnWidths@(fileSizeWidth, valueWidth) = (`max` 10) . length *** length $ headers --CAVEAT: the data-length may exceed the header-length, so define a minimum. + diff --git a/src/FishFood/Test/Data/CommandOptions.hs b/src/FishFood/Test/Data/CommandOptions.hs new file mode 100644 --- /dev/null +++ b/src/FishFood/Test/Data/CommandOptions.hs @@ -0,0 +1,61 @@ +{-# OPTIONS_GHC -fno-warn-orphans #-} +{- + Copyright (C) 2013 Dr. Alistair Ward + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{- | + [@AUTHOR@] Dr. Alistair Ward + + [@DESCRIPTION@] Implements 'Test.QuickCheck.Arbitrary' for 'Data.CommandOptions.CommandOptions'. +-} + +module FishFood.Test.Data.CommandOptions( +-- * Types +-- ** Type-synonyms + CommandOptions +) where + +import Control.Applicative((<$>), (<*>)) +import qualified Test.QuickCheck +import qualified FishFood.Data.CommandOptions as Data.CommandOptions +import FishFood.Test.Data.Verbosity() + +instance ( + Enum ratio, + Fractional ratio, + Ord ratio, + Show ratio, + Test.QuickCheck.Arbitrary ratio + ) => Test.QuickCheck.Arbitrary (Data.CommandOptions.CommandOptions ratio) where + arbitrary = do + deriveProbabilityMassFunction <- Test.QuickCheck.arbitrary + + Data.CommandOptions.mkCommandOptions <$> ( + either ( + Left . (succ . abs <$> {-into Maybe-monad-}) + ) ( + Right . (+ recip 1000000) . succ . abs + ) <$> Test.QuickCheck.arbitrary {-binSizeDelta-} + ) <*> return {-to Gen-monad-} deriveProbabilityMassFunction <*> ( + max ( + if deriveProbabilityMassFunction then 1 else 0 + ) . ( + `mod` 15 {-maximum permissible-} + ) <$> Test.QuickCheck.arbitrary {-nDecimalDigits-} + ) <*> Test.QuickCheck.arbitrary {-verbosity-} + +-- | Defines a concrete type for testing. +type CommandOptions = Data.CommandOptions.CommandOptions Double + diff --git a/src/FishFood/Test/Data/Verbosity.hs b/src/FishFood/Test/Data/Verbosity.hs new file mode 100644 --- /dev/null +++ b/src/FishFood/Test/Data/Verbosity.hs @@ -0,0 +1,31 @@ +{-# OPTIONS_GHC -fno-warn-orphans #-} +{- + Copyright (C) 2013 Dr. Alistair Ward + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{- | + [@AUTHOR@] Dr. Alistair Ward + + [@DESCRIPTION@] Implements 'Test.QuickCheck.Arbitrary' for 'Distribution.Verbosity.Verbosity'. +-} + +module FishFood.Test.Data.Verbosity() where + +import qualified Distribution.Verbosity +import qualified FishFood.Data.Verbosity as Data.Verbosity +import qualified Test.QuickCheck + +instance Test.QuickCheck.Arbitrary Distribution.Verbosity.Verbosity where + arbitrary = Test.QuickCheck.elements Data.Verbosity.range diff --git a/src/FishFood/Test/Profiler.hs b/src/FishFood/Test/Profiler.hs new file mode 100644 --- /dev/null +++ b/src/FishFood/Test/Profiler.hs @@ -0,0 +1,58 @@ +{- + Copyright (C) 2013 Dr. Alistair Ward + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{- | + [@AUTHOR@] Dr. Alistair Ward + + [@DESCRIPTION@] Implements 'Test.QuickCheck.Arbitrary' & defines /QuickCheck/-properties for 'Profiler'. +-} + +module FishFood.Test.Profiler( +-- * Functions + quickChecks +) where + +import qualified Control.Monad.Writer +import qualified Data.Either +import qualified Data.Set +import qualified FishFood.Data.CommandOptions as Data.CommandOptions +import qualified FishFood.Data.File as Data.File +import qualified FishFood.Profiler as Profiler +import qualified FishFood.Test.Data.CommandOptions as Test.Data.CommandOptions +import qualified Test.QuickCheck +import Test.QuickCheck((==>)) +import qualified ToolShed.SelfValidate + +-- | Defines invariant properties. +quickChecks :: IO () +quickChecks = Test.QuickCheck.quickCheck `mapM_` [prop_calculateProbabilityMassFunction, prop_calculateFileSizeFrequencyDistribution, prop_attendance] where + prop_calculateProbabilityMassFunction, prop_calculateFileSizeFrequencyDistribution, prop_attendance :: Test.Data.CommandOptions.CommandOptions -> [Data.File.FileSize] -> Test.QuickCheck.Property + prop_calculateProbabilityMassFunction commandOptions fileSizes = not (null fileSizes) && ToolShed.SelfValidate.isValid commandOptions' ==> Test.QuickCheck.label "prop_calculateProbabilityMassFunction" . (<= recip 1000000) . (+ negate 1) . sum . Data.Either.rights {-probabilities-} . map Profiler.getValue . fst {-distribution-} . Control.Monad.Writer.runWriter . Profiler.calculateFileSizeDistribution commandOptions' $ map abs fileSizes where + commandOptions' = commandOptions { Data.CommandOptions.getDeriveProbabilityMassFunction = True } + + prop_calculateFileSizeFrequencyDistribution commandOptions fileSizes = not (null fileSizes) && ToolShed.SelfValidate.isValid commandOptions' ==> Test.QuickCheck.label "prop_calculateFileSizeFrequencyDistribution" . (== length fileSizes) . sum . Data.Either.lefts {-frequency-} . map Profiler.getValue . fst {-distribution-} . Control.Monad.Writer.runWriter . Profiler.calculateFileSizeDistribution commandOptions' $ map abs fileSizes where + commandOptions' = commandOptions { Data.CommandOptions.getDeriveProbabilityMassFunction = False } + + prop_attendance commandOptions fileSizes = not (null fileSizes') && ToolShed.SelfValidate.isValid commandOptions' ==> Test.QuickCheck.label "prop_attendance" . ( + == Data.Set.fromList fileSizes' + ) . Data.Set.fromList . map Profiler.getFileSize . filter ( + (/= 0) . either fromIntegral id . Profiler.getValue --Remove file-sizes which match zero actual files. + ) . fst {-distribution-} . Control.Monad.Writer.runWriter $ Profiler.calculateFileSizeDistribution commandOptions' fileSizes' where + fileSizes' = map abs fileSizes + + commandOptions' :: Test.Data.CommandOptions.CommandOptions + commandOptions' = Data.CommandOptions.setBinSizeIncrement 1 commandOptions + diff --git a/src/FishFood/Test/QuickChecks.hs b/src/FishFood/Test/QuickChecks.hs new file mode 100644 --- /dev/null +++ b/src/FishFood/Test/QuickChecks.hs @@ -0,0 +1,38 @@ +{- + Copyright (C) 2013 Dr. Alistair Ward + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{- | + [@AUTHOR@] Dr. Alistair Ward + + [@DESCRIPTION@] Calls the /quickChecks/-functions for modules supporting this feature. +-} + +module FishFood.Test.QuickChecks( +-- * Functions + run +) where + +import qualified Control.Arrow +import qualified FishFood.Test.Profiler as Test.Profiler + +-- | Runs checks on invariant properties. +run :: IO () +run = mapM_ ( + uncurry (>>) . Control.Arrow.first putStrLn + ) [ + ("Profiler", Test.Profiler.quickChecks) + ] + diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -22,23 +22,26 @@ * Contains the entry-point of the application. * Processes the command-line arguments. + + * Delegates the task to "Profiler". -} module Main(main) where import Control.Applicative((<$>),(<*>)) -import Control.Arrow((&&&)) import qualified Control.Monad +import qualified Control.Monad.Writer import qualified Data.List -import qualified Data.Map import qualified Data.Maybe import qualified Data.Version import qualified Distribution.Package import qualified Distribution.Text -import qualified Distribution.Verbosity import qualified Distribution.Version import qualified FishFood.Data.CommandOptions as Data.CommandOptions import qualified FishFood.Data.File as Data.File +import qualified FishFood.Data.Verbosity as Data.Verbosity +import qualified FishFood.Profiler as Profiler +import qualified FishFood.Test.QuickChecks as Test.QuickChecks import qualified Paths_fishfood as Paths --Either local stub, or package-instance autogenerated by 'Setup.hs build'. import qualified System.Console.GetOpt as G import qualified System.Environment @@ -49,8 +52,11 @@ import qualified ToolShed.Defaultable import qualified ToolShed.SelfValidate +-- | Define the concrete type of command-options. +type CommandOptions = Data.CommandOptions.CommandOptions Double + -- | Used to thread user-defined command-line options, though the list of functions which implement them. -type CommandLineAction = Data.CommandOptions.CommandOptions -> IO Data.CommandOptions.CommandOptions --Supplied as the type-argument to 'G.OptDescr'. +type CommandLineAction = CommandOptions -> IO CommandOptions --Supplied as the type-argument to 'G.OptDescr'. -- | On failure to parse the specified string, returns an explanatory error. read' :: Read a => String -> String -> a @@ -83,31 +89,34 @@ progName <- System.Environment.getProgName let - defaultCommandOptions :: Data.CommandOptions.CommandOptions + defaultCommandOptions :: CommandOptions defaultCommandOptions = ToolShed.Defaultable.defaultValue optDescrList :: [G.OptDescr CommandLineAction] optDescrList = [ --- String [String] (G.ArgDescr CommandLineAction) String - G.Option "?" ["help"] (G.NoArg $ const printUsage) "Display this help, & then exit.", - G.Option "" ["verbosity"] ( - setVerbosity `G.ReqArg` show [minBound :: Distribution.Verbosity.Verbosity .. maxBound] - ) ("Define the log-level; default '" ++ show (Data.CommandOptions.getVerbosity defaultCommandOptions) ++ "'." +-- String [String] (G.ArgDescr CommandLineAction) String + G.Option "?" ["help"] (G.NoArg $ const printUsage) "Display this help, & then exit.", + G.Option "" ["verbosity"] ( + setVerbosity `G.ReqArg` show Data.Verbosity.range + ) ("Define the log-level; default '" ++ show (Data.CommandOptions.getVerbosity defaultCommandOptions) ++ "'." ), - G.Option "v" ["version"] (G.NoArg $ const printVersion) "Print version-information, & then exit.", - G.Option "b" ["binSize"] (setBinSize `G.ReqArg` "") "The range in bytes, accepted by any one of the bins into which file-sizes are categorised; defaulting to one standard-deviation.", - G.Option "d" ["nDecimalDigits"] (setNDecimalDigits `G.ReqArg` "") ("The precision to which fractional auxiliary data is displayed; default " ++ show (Data.CommandOptions.getNDecimalDigits defaultCommandOptions) ++ "."), - G.Option "z" ["includeEmpty"] (setIncludeEmpty `G.OptArg` "") ("Whether to display empty bins; default '" ++ show (Data.CommandOptions.getIncludeEmpty defaultCommandOptions) ++ "'.") + G.Option "v" ["version"] (G.NoArg $ const printVersion) "Print version-information, & then exit.", + G.Option "s" ["binSizeIncrement"] (setBinSizeIncrement `G.ReqArg` "") "The constant size-increase in the arithmetic sequence of bins into which the byte-sizes of files are categorised; defaulting to one standard-deviation.", + G.Option "r" ["binSizeRatio"] (setBinSizeRatio `G.ReqArg` "") "The constant size-ratio in the geometric sequence of bins into which the byte-sizes of files are categorised; an alternative to 'binSizeIncrement'.", + G.Option "p" ["deriveProbabilityMassFunction"] (setDeriveProbabilityMassFunction `G.OptArg` "") ("Whether to derive the \"Probability mass function\" rather than the \"Frequency-distribution\"; default '" ++ show (Data.CommandOptions.getDeriveProbabilityMassFunction defaultCommandOptions) ++ "'."), + G.Option "d" ["nDecimalDigits"] (setNDecimalDigits `G.ReqArg` "") ("The precision to which fractional auxiliary data is displayed; default " ++ show (Data.CommandOptions.getNDecimalDigits defaultCommandOptions) ++ "."), + G.Option "q" ["runQuickChecks"] (G.NoArg runQuickChecks) "Run Quick-checks using arbitrary data, & then exit." ] where - setBinSize, setNDecimalDigits, setVerbosity :: String -> CommandLineAction - setBinSize arg commandOptions = return {-to IO-monad-} commandOptions { Data.CommandOptions.getBinSize = Just $ readCommandArg arg } + setBinSizeIncrement, setBinSizeRatio, setNDecimalDigits, setVerbosity :: String -> CommandLineAction + setBinSizeIncrement arg = return {-to IO-monad-} . Data.CommandOptions.setBinSizeIncrement (readCommandArg arg) + setBinSizeRatio arg = return {-to IO-monad-} . Data.CommandOptions.setBinSizeRatio (readCommandArg arg) setNDecimalDigits arg commandOptions = return {-to IO-monad-} commandOptions { Data.CommandOptions.getNDecimalDigits = readBoundedIntegral arg } setVerbosity arg commandOptions = return {-to IO-monad-} commandOptions { Data.CommandOptions.getVerbosity = readCommandArg arg } - setIncludeEmpty :: Maybe String -> CommandLineAction - setIncludeEmpty arg commandOptions = return {-to IO-monad-} commandOptions { Data.CommandOptions.getIncludeEmpty = Data.Maybe.maybe True readCommandArg arg } + setDeriveProbabilityMassFunction :: Maybe String -> CommandLineAction + setDeriveProbabilityMassFunction arg commandOptions = return {-to IO-monad-} commandOptions { Data.CommandOptions.getDeriveProbabilityMassFunction = Data.Maybe.maybe True readCommandArg arg } - printVersion, printUsage :: IO Data.CommandOptions.CommandOptions + printVersion, printUsage :: IO CommandOptions printVersion = Text.Printf.printf "%s\n\n%s %s.\n%s.\n%s.\n%s %s.\n" packageName "Copyright (C) 2013" author "This program comes with ABSOLUTELY NO WARRANTY" "This is free software, & you are welcome to redistribute it under certain conditions" "Written by" author >> System.Exit.exitWith System.Exit.ExitSuccess where packageIdentifier :: Distribution.Package.PackageIdentifier packageIdentifier = Distribution.Package.PackageIdentifier { @@ -121,6 +130,9 @@ printUsage = System.IO.hPutStrLn System.IO.stderr ("Usage:\t" ++ G.usageInfo progName optDescrList ++ " [ ...]") >> System.Exit.exitWith System.Exit.ExitSuccess + runQuickChecks :: CommandOptions -> IO CommandOptions + runQuickChecks _ = Test.QuickChecks.run >> System.Exit.exitWith System.Exit.ExitSuccess + args <- System.Environment.getArgs case G.getOpt G.RequireOrder optDescrList args of @@ -151,43 +163,12 @@ else return filePaths else {-real fileNames-} return {-to IO-monad-} nonOptions - let (nDecimalDigits, verbosity) = Data.CommandOptions.getNDecimalDigits &&& Data.CommandOptions.getVerbosity $ commandOptions --Deconstruct. - - Control.Monad.when (verbosity == maxBound) $ System.IO.hPrint System.IO.stderr filePaths --CAVEAT: potentially very long. - - fileSizes <- mapM Data.File.findSize $ Data.List.nub filePaths --De-duplicate the file-paths, & find the corresponding file-sizes. - - let - mean, standardDeviation :: Double - (nFiles, mean, standardDeviation) = Data.File.getFileSizeStatistics fileSizes - - columnHeaders :: [String] - columnHeaders = ["Bin", "Frequency"] - - columnWidths :: [Int] - columnWidths = map length columnHeaders - - formatLine :: Data.File.FileSize -> Int -> IO () - formatLine bin = Text.Printf.printf "%*d\t%*d\n" (head columnWidths) bin (last columnWidths) - - Control.Monad.when (verbosity >= Distribution.Verbosity.verbose) $ do - Control.Monad.void $ Text.Printf.hPrintf System.IO.stderr "Files=%d, mean=%.*f, standard-deviation=%.*f\n" nFiles nDecimalDigits mean nDecimalDigits standardDeviation --Print statistics. - - let tabulate = Data.List.intercalate "\t" - - Text.Printf.hPrintf System.IO.stderr "%s\n%s\n" (tabulate columnHeaders) (tabulate $ map (`replicate` '=') columnWidths) --Print column-headers. + Control.Monad.when (Data.CommandOptions.getVerbosity commandOptions == maxBound) $ System.IO.hPrint System.IO.stderr filePaths --CAVEAT: potentially very long. - if standardDeviation == 0 - then formatLine 0 nFiles - else let - bins = map (`div` Data.Maybe.fromMaybe (round standardDeviation) (Data.CommandOptions.getBinSize commandOptions)) fileSizes + (fileSizeDistribution, statistics) <- Control.Monad.Writer.runWriter . Profiler.calculateFileSizeDistribution commandOptions <$> mapM Data.File.findSize (Data.List.nub filePaths) - frequencyDistribution - | Data.CommandOptions.getIncludeEmpty commandOptions = Data.Map.fromList $ [0 .. maximum bins] `zip` repeat 0 - | otherwise = Data.Map.empty - in mapM_ (uncurry formatLine) . Data.Map.toList $ foldr ( - uncurry (Data.Map.insertWith (+)) - ) frequencyDistribution $ bins `zip` repeat 1 + Control.Monad.when (Data.CommandOptions.getVerbosity commandOptions > ToolShed.Defaultable.defaultValue) $ mapM_ (System.IO.hPutStrLn System.IO.stderr) statistics + System.IO.putStrLn $ Profiler.formatFileSizeDistribution commandOptions fileSizeDistribution (_, _, errors) -> System.IO.Error.ioError . System.IO.Error.userError $ concatMap init {-chop-} errors