sgd 0.5.0.0 → 0.6.0.0

raw patch · 8 files changed

+152/−11 lines, 8 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• sgd.cabal +3/−2
• src/Numeric/SGD.hs +5/−3
• src/Numeric/SGD/AdaDelta.hs +2/−1
• src/Numeric/SGD/Adam.hs +103/−0
• src/Numeric/SGD/DataSet.hs +2/−1
• src/Numeric/SGD/Momentum.hs +2/−1
• src/Numeric/SGD/ParamSet.hs +30/−1
• src/Numeric/SGD/Type.hs +5/−2

@@ -1,5 +1,5 @@ name:               sgd-version:            0.5.0.0+version:            0.6.0.0 synopsis:           Stochastic gradient descent description:     Stochastic gradient descent library.@@ -42,8 +42,9 @@       , Numeric.SGD.Type       , Numeric.SGD.DataSet       , Numeric.SGD.ParamSet-      , Numeric.SGD.AdaDelta       , Numeric.SGD.Momentum+      , Numeric.SGD.AdaDelta+      , Numeric.SGD.Adam       , Numeric.SGD.Sparse       , Numeric.SGD.Sparse.Momentum       , Numeric.SGD.Sparse.LogSigned

@@ -51,6 +51,7 @@   -- * SGD variants     Mom.momentum   , Ada.adaDelta+  , Adam.adam    -- * Pure SGD   , run@@ -85,8 +86,9 @@ import qualified Pipes.Prelude as P import           Pipes ((>->)) -import qualified Numeric.SGD.AdaDelta as Ada import qualified Numeric.SGD.Momentum as Mom+import qualified Numeric.SGD.AdaDelta as Ada+import qualified Numeric.SGD.Adam as Adam import           Numeric.SGD.Type import           Numeric.SGD.ParamSet import           Numeric.SGD.DataSet@@ -131,7 +133,7 @@     -- no guarantee of seeing each training sample in every epoch.   , reportEvery :: Double     -- ^ How often the value of the objective function should be reported (with-    -- `1` meaning once per pass over the training data)+    -- @1@ meaning once per pass over the training data)   } deriving (Show, Eq, Ord, Generic)  instance Default Config where@@ -227,7 +229,7 @@ result pDef = fmap (maybe pDef id) . P.last  --- | Apply the given function every `k` param sets flowing downstream.+-- | Apply the given function every @k@ param sets flowing downstream. every :: (Monad m) => Int -> (p -> m ()) -> P.Pipe p p m x every k f = do   go (1 `mod` k)

@@ -2,7 +2,8 @@ {-# LANGUAGE DeriveGeneric #-}  --- | AdaDelta algorithm as described in the following paper:+-- | Provides the `adaDelta` function which implements the AdaDelta algorithm+-- as described in the following paper: -- --     * https://arxiv.org/pdf/1212.5701.pdf 

@@ -0,0 +1,103 @@+{-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE DeriveGeneric #-}+++-- | Provides the `adam` function which implements the Adam algorithm based on+-- the paper:+--+--     * https://arxiv.org/pdf/1412.6980+++module Numeric.SGD.Adam+  ( Config(..)+  , adam+  ) where+++import           GHC.Generics (Generic)++import           Prelude hiding (div)+-- import           Control.Monad (when)++import           Data.Default++import qualified Pipes as P++import           Numeric.SGD.Type+import           Numeric.SGD.ParamSet+++-- | AdaDelta configuration+data Config = Config+  { alpha :: Double+    -- ^ Step size+  , beta1 :: Double+    -- ^ 1st exponential moment decay+  , beta2 :: Double+    -- ^ 1st exponential moment decay+  , eps   :: Double+    -- ^ Epsilon+  } deriving (Show, Eq, Ord, Generic)++instance Default Config where+  def = Config+    { alpha = 0.001+    , beta1 = 0.9+    , beta2 = 0.999+    , eps = 1.0e-8+    }+++-- | Perform gradient descent using the Adam algorithm.  +-- See "Numeric.SGD.Adam" for more information.+adam+  :: (Monad m, ParamSet p)+  => Config+    -- ^ Adam configuration+  -> (e -> p -> p)+    -- ^ Gradient on a training element+  -> SGD m e p+adam Config{..} gradient net0 =++  let zr = zero net0 +   in go (1 :: Integer) zr zr net0++  where++    go t m v net = do+      x <- P.await+      let g = gradient x net+          m' = pmap (*beta1) m `add` pmap (*(1-beta1)) g+          v' = pmap (*beta2) v `add` pmap (*(1-beta2)) (g `mul` g)+          -- bias-corrected moment estimates +          mb = pmap (/(1-beta1^t)) m'+          vb = pmap (/(1-beta2^t)) v'+          newNet = net `sub`+            ( pmap (*alpha) mb `div`+              (pmap (+eps) (pmap sqrt vb))+            )+      P.yield newNet+      go (t+1) m' v' newNet+++-------------------------------+-- Utils+-------------------------------+++-- -- | Scaling+-- scale :: ParamSet p => Double -> p -> p+-- scale x = pmap (*x)+-- {-# INLINE scale #-}+-- +-- +-- -- | Root square+-- squareRoot :: ParamSet p => p -> p+-- squareRoot = pmap sqrt+-- {-# INLINE squareRoot #-}+-- +-- +-- -- | Square+-- square :: ParamSet p => p -> p+-- square x = x `mul` x+-- {-# INLINE square #-}

@@ -1,7 +1,8 @@ {-# LANGUAGE RecordWildCards #-}  --- | Dataset abstraction.+-- | Provides the `DataSet` type which abstracts over the actual (IO-based)+-- representation of the training dataset.   module Numeric.SGD.DataSet

@@ -2,7 +2,8 @@ {-# LANGUAGE DeriveGeneric #-}  --- | Stochastic gradient descent with momentum, following:+-- | Provides the `momentum` function which implements stochastic gradient+-- descent with momentum, following: -- --     * http://ruder.io/optimizing-gradient-descent/index.html#momentum 

@@ -6,9 +6,25 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE DeriveGeneric #-} +-- {-# OPTIONS_GHC -O -ddump-rule-firings #-} ++-- | Provides the class `ParamSet` which is used to represent the set of+-- parameters of a particular model.  The goal of SGD is then to find the+-- parameter values which minimize a given objective function.++ module Numeric.SGD.ParamSet-  ( ParamSet(..)+  ( +  -- * Class+    ParamSet(..)+  -- * Generics+  , GPMap+  , GAdd+  , GSub+  , GDiv+  , GMul+  , GNorm2   ) where  @@ -84,6 +100,7 @@   pmap = genericPMap   {-# INLINE pmap #-} +   default add :: (Generic a, GAdd (Rep a)) => a -> a -> a   add = genericAdd   {-# INLINE add #-}@@ -103,6 +120,18 @@   default norm_2 :: (Generic a, GNorm2 (Rep a)) => a -> Double   norm_2 = genericNorm2   {-# INLINE norm_2 #-}+++{-# RULES+"ParamSet pmap/pmap" forall f g p. pmap f (pmap g p) = pmap (f . g) p+  #-}+++-- {-# RULES+-- "ParamSet pmap/add/pmap" forall f g p h q. +--   pmap f (add (pmap g p) (pmap h q))+--   = add (pmap (f . g) p) (pmap (f . h) q)+--   #-}   -- -- | 'add' using GHC Generics; works if all fields are instances of

@@ -1,3 +1,6 @@+-- | Provides the basic `SGD` pipe type.++ module Numeric.SGD.Type   ( SGD   ) where@@ -7,6 +10,6 @@   -- | SGD is a pipe which, given the initial parameter values, consumes training--- elements of type `e` and outputs the subsequently calculated parameter sets--- of type `p`.+-- elements of type @e@ and outputs the subsequently calculated parameter sets+-- of type @p@. type SGD m e p = p -> P.Pipe e p m ()