Layer.hs

{-# OPTIONS_HADDOCK show-extensions #-}

{-# LANGUAGE DataKinds #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE ScopedTypeVariables #-}

{-|
Module      : Numeric.Neural.Layer
Description : layer components
Copyright   : (c) Lars Brünjes, 2016
License     : MIT
Maintainer  : brunjlar@gmail.com
Stability   : experimental
Portability : portable

This modules defines special "layer" components and convenience functions for the creation of such layers.
-}

module Numeric.Neural.Layer
    ( Layer
    , linearLayer
    , layer
    , tanhLayer
    , logisticLayer
    , reLULayer
    , softmax
    ) where

import Control.Arrow
import Control.Category
import Data.Proxy
import GHC.TypeLits
import GHC.TypeLits.Witnesses
import Numeric.Neural.Model
import Prelude                 hiding (id, (.))
import Data.Utils.Analytic
import Data.Utils.Matrix
import Data.Utils.Random
import Data.Utils.Vector

-- | A @'Layer' i o@ is a component that maps a vector of length @i@ to a vector of length @j@.
--
type Layer i o = Component (Vector i Analytic) (Vector o Analytic)

linearLayer' :: ParamFun (Matrix o (i + 1)) (Vector i Analytic) (Vector o Analytic)
linearLayer' = ParamFun $ \xs ws -> ws <%%> cons 1 xs

-- | Creates a /linear/ 'Layer', i.e. a layer that multiplies the input with a weight matrix and adds a bias to get the output.
--   
--   Random initialization follows the recommendation from chapter 3 of the online book 
--   <http://neuralnetworksanddeeplearning.com/ Neural Networks and Deep Learning> by Michael Nielsen.
linearLayer :: forall i o. (KnownNat i, KnownNat o) => Layer i o
linearLayer = withNatOp (%+) p (Proxy :: Proxy 1) Component
    { weights = pure 0
    , compute = linearLayer'
    , initR   = sequenceA $ mgenerate r
    }

  where

    p = Proxy :: Proxy i

    s = 1 / sqrt (fromIntegral $ natVal p)

    r (_, 0) = boxMuller
    r (_, _) = boxMuller' 0 s

-- | Creates a 'Layer' as a combination of a linear layer and a non-linear activation function.
--
layer :: (KnownNat i, KnownNat o) => (Analytic -> Analytic) -> Layer i o
layer f = arr (fmap f) . linearLayer

-- | This is simply 'layer', specialized to 'tanh'-activation. Output values are all in the interval [0,1].
--
tanhLayer :: (KnownNat i, KnownNat o) => Layer i o
tanhLayer = layer tanh

-- | This is simply 'layer', specialized to the logistic function as activation. Output values are all in the interval [-1,1].
--
logisticLayer :: (KnownNat i, KnownNat o) => Layer i o
logisticLayer = layer $ \x -> 1 / (1 + exp (- x))

-- | This is simply 'layer', specialized to the /rectified linear unit/ activation function. 
--   Output values are all non-negative.
--
reLULayer :: (KnownNat i, KnownNat o) => Layer i o
reLULayer = layer $ \x -> max 0 x

-- | The 'softmax' function normalizes a vector, so that all entries are in [0,1] with sum 1. 
--   This means the output entries can be interpreted as probabilities.
--
softmax :: (Floating a, Functor f, Foldable f) => f a -> f a
softmax xs = let xs' = exp <$> xs
                 s   = sum xs'
             in  (/ s) <$> xs'