diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright Alexander Ignatyev (c) 2016 +Copyright Alexander Ignatyev (c) 2016-2018 All rights reserved. diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -39,6 +39,22 @@ ### Usage +#### OS X/macOS prerequisites setup + +* Using [Homebrew](https://brew.sh/): + +``` +brew install pkg-config gsl +``` + +or + +* Using [MacPorts](https://www.macports.org/): + +``` +sudo port install pkgconfig gsl +``` + #### Build the project stack build diff --git a/mltool.cabal b/mltool.cabal --- a/mltool.cabal +++ b/mltool.cabal @@ -1,5 +1,5 @@ name: mltool -version: 0.1.0.2 +version: 0.2.0.0 synopsis: Machine Learning Toolbox description: Haskell Machine Learning Toolkit @@ -12,7 +12,7 @@ license-file: LICENSE author: Alexander Ignatyev maintainer: ignatyev.alexander@gmail.com -copyright: (c) 2016-2017 Alexander Ignatyev +copyright: (c) 2016-2018 Alexander Ignatyev category: math build-type: Simple extra-source-files: README.md @@ -55,9 +55,9 @@ other-modules: MachineLearning.Classification.Internal build-depends: base >= 4.7 && < 5 , vector >= 0.11 - , hmatrix >= 0.17.0.1 + , hmatrix >= 0.18.0.0 , hmatrix-gsl >= 0.17 - , hmatrix-gsl-stats >= 0.4.1.3 + , hmatrix-morpheus >= 0.1.1.0 , ascii-progress >= 0.3.3.0 , deepseq , random >= 1.1 @@ -92,8 +92,8 @@ build-depends: base , mltool , vector >= 0.11 - , hmatrix >= 0.17.0.1 - , hmatrix-gsl-stats >= 0.4.1.3 + , hmatrix >= 0.18.0.0 + , hmatrix-morpheus >= 0.1.1.0 , random >= 1.1 , MonadRandom >= 0.4.2.3 , test-framework >= 0.8.1.1 diff --git a/src/MachineLearning.hs b/src/MachineLearning.hs --- a/src/MachineLearning.hs +++ b/src/MachineLearning.hs @@ -24,7 +24,7 @@ import MachineLearning.Types (Vector, Matrix) import qualified Numeric.LinearAlgebra as LA import Numeric.LinearAlgebra((|||), (??)) -import qualified Numeric.GSL.Statistics as Stat +import qualified Numeric.Morpheus.Statistics as Stat import Control.Monad (replicateM, mfilter, MonadPlus) import Data.List (sort, group, foldl') @@ -44,11 +44,9 @@ -- | Caclulates mean and stddev values of every feature. -- Takes feature matrix X, returns pair of vectors of means and stddevs. meanStddev x = - let cols = LA.toColumns x - means = map Stat.mean cols - stddevs = zipWith (\m col -> Stat.stddev_m m col) means cols - stddevs' = map (\s -> if s < 2 then 1 else s) stddevs - in (LA.row means, LA.row stddevs') + let means = Stat.columnMean x + stddevs = Stat.columnStddev_m means x + in (LA.asRow means, LA.asRow stddevs) featureNormalization (means, stddevs) x = (x - means) / stddevs @@ -64,7 +62,7 @@ makeTerm = foldl' (\c (index, power) -> c * (vv V.! index) ^ power) 1 terms :: Int -> [Vector] terms d = foldl' (\l x -> (makeTerm x) : l) [] $ polynomialTerms d [ncols-1, ncols-2 .. 0] - + polynomialTerms :: Ord a => Int -> [a] -> [[(a, Int)]] polynomialTerms degree terms = diff --git a/src/MachineLearning/MultiSvmClassifier.hs b/src/MachineLearning/MultiSvmClassifier.hs --- a/src/MachineLearning/MultiSvmClassifier.hs +++ b/src/MachineLearning/MultiSvmClassifier.hs @@ -1,7 +1,7 @@ {-| Module: MachineLearning.MultiSvmClassifier Description: Multiclass Support Vector Machines Classifier. -Copyright: (c) Alexander Ignatyev, 2017. +Copyright: (c) Alexander Ignatyev, 2017-2018. License: BSD-3 Stability: experimental Portability: POSIX @@ -18,6 +18,8 @@ where + +import Prelude hiding ((<>)) import qualified Numeric.LinearAlgebra as LA import Numeric.LinearAlgebra((<>), (<.>), (|||)) import qualified Data.Vector.Storable as V diff --git a/src/MachineLearning/NeuralNetwork/Layer.hs b/src/MachineLearning/NeuralNetwork/Layer.hs --- a/src/MachineLearning/NeuralNetwork/Layer.hs +++ b/src/MachineLearning/NeuralNetwork/Layer.hs @@ -2,7 +2,7 @@ {-| Module: MachineLearning.NeuralNetwork.Layer Description: Neural Network's Layer -Copyright: (c) Alexander Ignatyev, 2017 +Copyright: (c) Alexander Ignatyev, 2017-2018. License: BSD-3 Stability: experimental Portability: POSIX @@ -22,6 +22,7 @@ where +import Prelude hiding ((<>)) import MachineLearning.Types (R, Matrix) import MachineLearning.Utils (sumByColumns) import qualified Numeric.LinearAlgebra as LA diff --git a/src/MachineLearning/Optimization.hs b/src/MachineLearning/Optimization.hs --- a/src/MachineLearning/Optimization.hs +++ b/src/MachineLearning/Optimization.hs @@ -75,10 +75,13 @@ where theta_m = LA.asColumn theta eps_v = V.replicate (V.length theta) eps eps_m = LA.diag eps_v - theta_m1 = theta_m + eps_m - theta_m2 = theta_m - eps_m + -- +/- eps_m in case of zero theta + theta_m1 = theta_m * (1 + eps_m) + eps_m + theta_m2 = theta_m * (1 - eps_m) - eps_m cost1 = LA.vector $ map (cost model reg x y) $ LA.toColumns theta_m1 cost2 = LA.vector $ map (cost model reg x y) $ LA.toColumns theta_m2 - grad1 = (cost1 - cost2) / (LA.scalar $ 2*eps) + eps_v1 = LA.takeDiag $ theta_m1 - theta_m + eps_v2 = LA.takeDiag $ theta_m - theta_m2 + grad1 = (cost1 - cost2) / (eps_v1 + eps_v2) grad2 = gradient model reg x y theta diff --git a/src/MachineLearning/PCA.hs b/src/MachineLearning/PCA.hs --- a/src/MachineLearning/PCA.hs +++ b/src/MachineLearning/PCA.hs @@ -1,7 +1,7 @@ {-| Module: MachineLearning.PCA Description: Principal Component Analysis. -Copyright: (c) Alexander Ignatyev, 2017 +Copyright: (c) Alexander Ignatyev, 2017-2018. License: BSD-3 Stability: experimental Portability: POSIX @@ -18,6 +18,7 @@ where +import Prelude hiding ((<>)) import Data.Maybe (fromMaybe) import qualified Data.Vector.Storable as V import qualified Numeric.LinearAlgebra as LA diff --git a/src/MachineLearning/Regression.hs b/src/MachineLearning/Regression.hs --- a/src/MachineLearning/Regression.hs +++ b/src/MachineLearning/Regression.hs @@ -1,7 +1,7 @@ {-| Module: MachineLearning.Regression Description: Regression -Copyright: (c) Alexander Ignatyev, 2016-2017 +Copyright: (c) Alexander Ignatyev, 2016-2018. License: BSD-3 Stability: experimental Portability: POSIX @@ -20,6 +20,7 @@ where +import Prelude hiding ((<>)) import MachineLearning.Types (Vector, Matrix) import MachineLearning.Optimization as Optimization import MachineLearning.Model as Model diff --git a/src/MachineLearning/SoftmaxClassifier.hs b/src/MachineLearning/SoftmaxClassifier.hs --- a/src/MachineLearning/SoftmaxClassifier.hs +++ b/src/MachineLearning/SoftmaxClassifier.hs @@ -1,7 +1,7 @@ {-| Module: MachineLearning.SoftmaxClassifier Description: Softmax Classifier. -Copyright: (c) Alexander Ignatyev, 2017. +Copyright: (c) Alexander Ignatyev, 2017-2018. License: BSD-3 Stability: experimental Portability: POSIX @@ -18,6 +18,7 @@ where +import Prelude hiding ((<>)) import qualified Numeric.LinearAlgebra as LA import Numeric.LinearAlgebra((<>), (<.>), (|||)) import qualified Data.Vector.Storable as V diff --git a/test/MachineLearning/LeastSquaresModelTest.hs b/test/MachineLearning/LeastSquaresModelTest.hs --- a/test/MachineLearning/LeastSquaresModelTest.hs +++ b/test/MachineLearning/LeastSquaresModelTest.hs @@ -39,11 +39,11 @@ , testCase "gradient, lambda = 1000" $ assertVector "" 1e-5 gradient_l1000 (gradient LeastSquares (L2 1000) x1 y initialTheta) ] , testGroup "gradient checking" [ - testCase "non-zero theta, no reg" $ assertBool "" $ (checkGradient LeastSquares RegNone x1 y initialTheta 1e-4) < 10 - , testCase "non-zero theta, non-zero lambda" $ assertBool "" $ (checkGradient LeastSquares (L2 2) x1 y initialTheta 1e-4) < 10 - , testCase "zero theta, non-zero lambda" $ assertBool "" $ (checkGradient LeastSquares (L2 2) x1 y zeroTheta 1e-4) < 1 - , testCase "non-zero theta, zero lambda" $ assertBool "" $ (checkGradient LeastSquares (L2 0) x1 y initialTheta 1e-4) < 5 - , testCase "zero theta, zero lambda" $ assertBool "" $ (checkGradient LeastSquares (L2 0) x1 y zeroTheta 1e-4) < 1 + testCase "non-zero theta, no reg" $ assertApproxEqual "" 1 0 (checkGradient LeastSquares RegNone x1 y initialTheta 1e-4) + , testCase "non-zero theta, non-zero lambda" $ assertApproxEqual "" 1 0 (checkGradient LeastSquares (L2 2) x1 y initialTheta 1e-4) + , testCase "zero theta, non-zero lambda" $ assertApproxEqual "" 1 0 (checkGradient LeastSquares (L2 2) x1 y zeroTheta 1e-4) + , testCase "non-zero theta, zero lambda" $ assertApproxEqual "" 1 0 (checkGradient LeastSquares (L2 0) x1 y initialTheta 1e-4) + , testCase "zero theta, zero lambda" $ assertApproxEqual "" 1 0(checkGradient LeastSquares (L2 0) x1 y zeroTheta 1e-4) ] ] diff --git a/test/MachineLearning/LogisticModelTest.hs b/test/MachineLearning/LogisticModelTest.hs --- a/test/MachineLearning/LogisticModelTest.hs +++ b/test/MachineLearning/LogisticModelTest.hs @@ -32,7 +32,7 @@ checkGradientTest lambda theta = do let diffs = take 5 $ map (\e -> checkGradient Logistic lambda x1 y theta e) [1e-3, 1.1e-3 ..] - diff = minimum $ filter (not . isNaN) diffs + diff = minimum diffs assertApproxEqual (show theta) gradientCheckingEps 0 diff diff --git a/test/MachineLearning/MultiSvmClassifierTest.hs b/test/MachineLearning/MultiSvmClassifierTest.hs --- a/test/MachineLearning/MultiSvmClassifierTest.hs +++ b/test/MachineLearning/MultiSvmClassifierTest.hs @@ -59,7 +59,7 @@ checkGradientTest lambda theta eps = do let diffs = take 5 $ map (\e -> checkGradient model lambda x1 y theta e) [1e-3, 1.1e-3 ..] - diff = minimum $ filter (not . isNaN) diffs + diff = minimum diffs assertApproxEqual "" eps 0 diff diff --git a/test/MachineLearning/NeuralNetwork/WeightInitializationTest.hs b/test/MachineLearning/NeuralNetwork/WeightInitializationTest.hs --- a/test/MachineLearning/NeuralNetwork/WeightInitializationTest.hs +++ b/test/MachineLearning/NeuralNetwork/WeightInitializationTest.hs @@ -13,7 +13,7 @@ import Control.Monad (replicateM) import qualified Data.Vector.Storable as V import qualified Numeric.LinearAlgebra as LA -import qualified Numeric.GSL.Statistics as Stat +import qualified Numeric.Morpheus.Statistics as Stat import qualified Control.Monad.Random as RndM import MachineLearning.NeuralNetwork.WeightInitialization diff --git a/test/MachineLearning/Optimization/GradientDescentTest.hs b/test/MachineLearning/Optimization/GradientDescentTest.hs --- a/test/MachineLearning/Optimization/GradientDescentTest.hs +++ b/test/MachineLearning/Optimization/GradientDescentTest.hs @@ -27,7 +27,8 @@ xNorm = ML.featureNormalization muSigma x x1 = ML.addBiasDimension xNorm initialTheta = LA.konst 0 (LA.cols x1) -lsExpectedTheta = LA.vector [340412.660, 110630.879, -8737.743] +-- Normal Equation's Result: 340412.660,110631.050,-6649.474 +lsExpectedTheta = LA.vector [340412.660, 110630.886, -6649.310] eps = 1e-3 diff --git a/test/MachineLearning/Optimization/MinibatchGradientDescentTest.hs b/test/MachineLearning/Optimization/MinibatchGradientDescentTest.hs --- a/test/MachineLearning/Optimization/MinibatchGradientDescentTest.hs +++ b/test/MachineLearning/Optimization/MinibatchGradientDescentTest.hs @@ -29,7 +29,7 @@ x1 = ML.addBiasDimension xNorm initialTheta :: Vector initialTheta = LA.konst 0 (LA.cols x1) -lsExpectedTheta = LA.vector [325009.354,113890.981,6876.935] +lsExpectedTheta = LA.vector [325010.120,113889.649,5234.404] eps = 1e-3 diff --git a/test/MachineLearning/SoftmaxClassifierTest.hs b/test/MachineLearning/SoftmaxClassifierTest.hs --- a/test/MachineLearning/SoftmaxClassifierTest.hs +++ b/test/MachineLearning/SoftmaxClassifierTest.hs @@ -44,7 +44,7 @@ yExpected = LA.vector [1, 1, 0, 0, 1, 0] -checkSoftmaxGradient theta eps lambda = minimum . take 5 . filter (not . isNaN) $ map check [eps, eps+0.001 ..] +checkSoftmaxGradient theta eps lambda = minimum . take 5 $ map check [eps, eps+0.001 ..] where check e = checkGradient model lambda x1 y theta e