Dense.hs

{- | Provides dense layer implementation.

'Dense' datatype represents densely-connected neural network layer and
it performs following operation: @x `matMul` w + b@, where @w@ is weights and @b@ is bias (if present) of a layer.
-}


-- 'TypeFamilies' are needed to instantiate 'DType'.
{-# LANGUAGE TypeFamilies #-}


module Synapse.NN.Layers.Dense
    ( -- * 'Dense' datatype

      Dense (Dense, denseWeights, denseBias, denseConstraints, denseRegularizers)
    , layerDenseWith
    , layerDense
    ) where



import Synapse.Tensors (DType, SingletonOps(singleton), MatOps(addMatRow, matMul))

import Synapse.Tensors.Vec (Vec)

import Synapse.Tensors.Mat (Mat)
import qualified Synapse.Tensors.Mat as M

import Synapse.Autograd (Symbolic, SymbolIdentifier(SymbolIdentifier), symbol, SymbolMat)

import Synapse.NN.Layers.Layer (AbstractLayer(..), LayerConfiguration)
import Synapse.NN.Layers.Initializers (Initializer(Initializer), zeroes, ones)
import Synapse.NN.Layers.Constraints (Constraint(Constraint))
import Synapse.NN.Layers.Regularizers (Regularizer(Regularizer))


{- | 'Dense' datatype represents densely-connected neural network layer.

'Dense' performs following operation: @x `matMul` w + b@, where @w@ is weights and @b@ is bias (if present) of a layer.
-}
data Dense a = Dense
    { denseWeights      :: Mat a                           -- ^ Matrix that represents weights of dense layer.
    , denseBias         :: Vec a                           -- ^ Vector that represents bias of dense layer.

    , denseConstraints  :: (Constraint a, Constraint a)    -- ^ Constraints on weights and bias of dense layer.
    , denseRegularizers :: (Regularizer a, Regularizer a)  -- ^ Regularizers on weights and bias of dense layer.
    }

-- | Creates symbol for weights.
weightsSymbol :: SymbolIdentifier -> Mat a -> SymbolMat a
weightsSymbol prefix = symbol (prefix <> SymbolIdentifier "1")


-- | Creates symbol for bias.
biasSymbol :: SymbolIdentifier -> Vec a -> SymbolMat a
biasSymbol prefix = symbol (prefix <> SymbolIdentifier "2") . M.rowVec

type instance DType (Dense a) = a

instance AbstractLayer Dense where
    inputSize = Just . M.nRows . denseWeights
    outputSize = Just . M.nCols . denseWeights

    nParameters _ = 2
    getParameters prefix (Dense weights bias _ _) = [weightsSymbol prefix weights, biasSymbol prefix bias]
    updateParameters (Dense _ _ constraints@(Constraint weightsConstraintFn, Constraint biasConstraintFn) regularizers) [weights', biasMat'] =
        Dense (weightsConstraintFn weights') (M.indexRow (biasConstraintFn biasMat') 0) constraints regularizers
    updateParameters _ _ = error "Parameters update failed - wrong amount of parameters was given"

    symbolicForward prefix input (Dense weights bias _ (Regularizer weightsRegularizerFn, Regularizer biasRegularizerFn)) =
        let symbolWeights = weightsSymbol prefix weights
            symbolBias = biasSymbol prefix bias
        in
        ( (input `matMul` symbolWeights) `addMatRow` symbolBias
        , weightsRegularizerFn symbolWeights + biasRegularizerFn symbolBias
        )

-- | Creates configuration of dense layer.
layerDenseWith
    :: Symbolic a
    => (Initializer a, Constraint a, Regularizer a)  -- ^ Weights initializer, constraint and regularizer.
    -> (Initializer a, Constraint a, Regularizer a)  -- ^ Bias initializer, constraint and regularizer.
    -> Int                                           -- ^ Amount of neurons.
    -> LayerConfiguration (Dense a)
layerDenseWith (Initializer weightsInitializer, weightsConstraints, weightsRegularizer)
               (Initializer biasInitializer, biasConstraints, biasRegularizer)
               neurons input =
    Dense (weightsInitializer (input, neurons)) (M.indexRow (biasInitializer (1, neurons)) 0) 
          (weightsConstraints, biasConstraints) (weightsRegularizer, biasRegularizer)

-- | Creates default configuration of dense layer - no constraints and weight are initialized with ones, bias is initialized with zeroes.
layerDense :: Symbolic a => Int -> LayerConfiguration (Dense a)
layerDense = layerDenseWith (Initializer ones, Constraint id, Regularizer (const $ singleton 0))
                            (Initializer zeroes, Constraint id, Regularizer (const $ singleton 0))