normalize 0.3.1.0 → 0.3.1.1
raw patch · 4 files changed
+49/−43 lines, 4 filesdep +vector-algorithmsdep −sparse-linear-algebra
Dependencies added: vector-algorithms
Dependencies removed: sparse-linear-algebra
Files
- app/Main.hs +1/−1
- normalize.cabal +2/−2
- src/Normalize.hs +14/−17
- src/Utility.hs +32/−23
app/Main.hs view
@@ -45,7 +45,7 @@ , bySampleRemoveSynonyms :: Bool <?> "When normalizing by sample, if the divisor appears multiple times we assume those are synonyms. Here, we would remove the synonym with the smaller intensity. If not set, errors out and provides the synonym name." , method :: Maybe String- <?> "([StandardScore] | UpperQuartile | None) The method for standardization of the samples."+ <?> "([StandardScore] | UpperQuartile | QuantileMedian | QuantileAverage | None) The method for standardization of the samples. The Quantile* methods expect the same number of entities for each sample. " , filterEntitiesMissing :: Maybe Int <?> "([0] | INT) Whether to remove entities that appear less than this many times after normalizing." , filterEntitiesValue :: Maybe Double
normalize.cabal view
@@ -1,5 +1,5 @@ name: normalize-version: 0.3.1.0+version: 0.3.1.1 synopsis: Normalize data using a variety of methods. description: Normalize data using a variety of methods. For use with csv files. homepage: http://github.com/GregorySchwartz/normalize#readme@@ -24,10 +24,10 @@ , cassava , containers , lens- , sparse-linear-algebra , statistics , text , vector+ , vector-algorithms ghc-options: -O2 default-language: Haskell2010
src/Normalize.hs view
@@ -19,6 +19,7 @@ -- Standard import Data.Ord import Data.List+import Data.Maybe (fromMaybe) import qualified Data.Map.Strict as Map import qualified Data.Sequence as Seq import qualified Data.Foldable as F@@ -27,7 +28,6 @@ -- Cabal import Control.Lens import Statistics.Quantile-import qualified Data.Sparse.Common as S import qualified Data.Text as T import qualified Data.Vector as V import qualified Statistics.Sample as Stat@@ -53,16 +53,10 @@ -> Map.Map Sample (V.Vector Entity) normalize StandardScore = Map.map standardScore normalize UpperQuartile = Map.map upperQuartileNormalize+normalize method@QuantileMedian = quantileNormalize method+normalize method@QuantileAverage = quantileNormalize method normalize None = id-normalize _ = error "Method not supported by normalize." --- | Normalize all samples by a specific method using a sparse matrix.-normalizeSparse :: Method -> S.SpMatrix Double -> S.SpMatrix Double-normalizeSparse method@QuantileMedian = quantileNormalize method-normalizeSparse method@QuantileAverage = quantileNormalize method-normalizeSparse None = id-normalizeSparse _ = error "Method not supported by normalizeSparse."- -- | Normalize a sample (1) by another sample (2) by division. The -- NormSampleString contains the string that differentiates (1) from (2). -- NormSampleString must be within (2) and must make, upon its removal from (2),@@ -156,15 +150,18 @@ zeroFiltered = V.filter ((> 0) . _value) xs uqVal = continuousBy (ContParam 1 1) 3 4 . fmap _value --- | Quantile normalization for sparse matrices, ignoring zeros.-quantileNormalize :: Method -> S.SpMatrix Double -> S.SpMatrix Double+-- | Quantile normalization. Important: assumes the same number of entities in+-- each sample.+quantileNormalize :: Method+ -> Map.Map Sample (V.Vector Entity)+ -> Map.Map Sample (V.Vector Entity) quantileNormalize method mat =- fmap (\x -> S.lookupDenseSV (x - 1) summaryVec) rankMat+ Map.map (fmap (over value (fromMaybe 0 . (V.!?) summaryVec . round))) rankMat where- summaryFunc QuantileMedian = medianSparseVector- summaryFunc QuantileAverage = avgSparseVector+ summaryFunc QuantileMedian = medianVector+ summaryFunc QuantileAverage = avgVector summaryFunc _ = error "Unsupported method for quantile normalization." summaryVec =- S.sparsifySV . S.vr . fmap (summaryFunc method) . S.toRowsL $ sortMat- sortMat = S.fromColsL . fmap sortSparseVector . S.toColsL $ mat- rankMat = S.fromColsL . fmap rankSparseVector . S.toColsL $ mat+ V.fromList . fmap (summaryFunc method) . transposeSamples $ sortMat+ sortMat = Map.map sortVector mat+ rankMat = Map.map rankVector mat
src/Utility.hs view
@@ -7,10 +7,11 @@ {-# LANGUAGE BangPatterns #-} module Utility- ( sortSparseVector- , rankSparseVector- , medianSparseVector- , avgSparseVector+ ( sortVector+ , rankVector+ , medianVector+ , avgVector+ , transposeSamples ) where -- Standard@@ -23,33 +24,41 @@ import Statistics.Sample (mean) import qualified Data.Map.Strict as Map import qualified Data.Set as Set-import qualified Data.Sparse.Common as S-import qualified Data.Vector.Unboxed as V+import qualified Data.Vector as V+import qualified Data.Vector.Algorithms.Merge as V -- Local+import Types --- | Find the average of a sparse vector, ignoring zeros.-avgSparseVector :: S.SpVector Double -> Double-avgSparseVector xs = mean . V.fromList . fmap snd . S.toListSV $ xs+-- | Find the average of a vector.+avgVector :: V.Vector Double -> Double+avgVector = mean --- | Find the median of a sparse vector, ignoring zeros.-medianSparseVector :: S.SpVector Double -> Double-medianSparseVector xs =- continuousBy s 2 4 . V.fromList . fmap snd . S.toListSV $ xs+-- | Find the median of a vector.+medianVector :: V.Vector Double -> Double+medianVector = continuousBy s 2 4 --- | Sort a sparse vector, ignoring zeros.-sortSparseVector :: S.SpVector Double -> S.SpVector Double-sortSparseVector xs =- S.fromListDenseSV (S.svDim xs) . sort . fmap snd . S.toListSV $ xs+-- | Sort a vector of elements.+sortVector :: V.Vector Entity -> V.Vector Entity+sortVector = V.modify (V.sortBy (compare `on` view value)) --- | Get the rank transformed vector of a sparse vector, ignoring zeros.-rankSparseVector :: S.SpVector Double -> S.SpVector Int-rankSparseVector xs = fmap (\k -> Map.findWithDefault 0 k rankMap) xs+-- | Get the rank transformed vector of elements.+rankVector :: V.Vector Entity -> V.Vector Entity+rankVector xs = V.map (over value (flip (Map.findWithDefault 0) rankMap)) xs where rankMap = Map.fromList- . flip zip [1,2..]+ . flip zip [0,1..] . Set.toList . Set.fromList- . fmap snd- . S.toListSV+ . fmap (view value)+ . V.toList $ xs++-- | Transpose sample map. Important: assumes same number of entities in each+-- sample.+transposeSamples :: Map.Map Sample (V.Vector Entity)+ -> [V.Vector Double]+transposeSamples = fmap V.fromList+ . transpose+ . fmap (V.toList . fmap (view value))+ . Map.elems