diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -98,7 +98,7 @@ , minDistance :: Maybe Double "([Nothing] | DOUBLE) Stopping criteria to stop at the node immediate after a node with DOUBLE distance. So a node N with L and R children will stop with this criteria the distance at N to L and R is < DOUBLE. Includes L and R in the final result." , smartCutoff :: Maybe Double "([Nothing] | DOUBLE) Whether to set the cutoffs for --min-size, --max-proportion, and --min-distance based off of the distributions (median + (DOUBLE * MAD)) of all nodes. To use smart cutoffs, use this argument and then set one of the three arguments to an arbitrary number, whichever cutoff type you want to use. --min-size distribution is log2 transformed." , dendrogramOutput :: Maybe String "([dendrogram.svg] | FILE) The filename for the dendrogram. Supported formats are PNG, PS, PDF, and SVG." - , matrixOutput :: Maybe String "([Nothing] | FOLDER) Output the filtered and normalized (not including TfIdfNorm) matrix in this folder." + , matrixOutput :: Maybe String "([Nothing] | FOLDER | FILE.csv) Output the filtered and normalized (not including TfIdfNorm) matrix in this folder in matrix market format or, if a csv file is specified, a dense csv format." , drawLeaf :: Maybe String "([DrawText] | DrawItem DrawItemType) How to draw leaves in the dendrogram. DrawText is the number of items in that leaf. DrawItem is the collection of items represented by circles, consisting of: DrawItem DrawLabel, where each item is colored by its label, DrawItem (DrawContinuous FEATURE), where each item is colored by the expression of FEATURE (corresponding to a feature name in the input matrix), DrawItem (DrawThresholdContinuous [(FEATURE, DOUBLE)]), where each item is colored by the binary high / low expression of FEATURE based on DOUBLE and multiple FEATUREs can be used to combinatorically label items (FEATURE1 high / FEATURE2 low, etc.), DrawItem DrawSumContinuous, where each item is colored by the sum of the post-normalized columns (use --normalization NoneNorm for UMI counts, default), and DrawItem DrawDiversity, where each node is colored by the diversity based on the labels of each item and the color is normalized separately for the leaves and the inner nodes. The default is DrawText, unless --labels-file is provided, in which DrawItem DrawLabel is the default. If the label or feature cannot be found, the default color will be black (check your spelling!)." , drawCollection :: Maybe String "([PieChart] | PieRing | PieNone | CollectionGraph MAXWEIGHT THRESHOLD [NODE]) How to draw item leaves in the dendrogram. PieRing draws a pie chart ring around the items. PieChart only draws a pie chart instead of items. PieNone only draws items, no pie rings or charts. (CollectionGraph MAXWEIGHT THRESHOLD [NODE]) draws the nodes and edges within leaves that are descendents of NODE (empty list [] indicates draw all leaf networks) based on the input matrix, normalizes edges based on the MAXWEIGHT, and removes edges for display less than THRESHOLD (after normalization, so for CollectionGraph 2 0.5 [26], draw the leaf graphs for all leaves under node 26, then a edge of 0.7 would be removed because (0.7 / 2) < 0.5)." , drawMark :: Maybe String "([MarkNone] | MarkModularity) How to draw annotations around each inner node in the tree. MarkNone draws nothing and MarkModularity draws a black circle representing the modularity at that node, darker black means higher modularity for that next split." @@ -333,7 +333,7 @@ . unHelpful . dendrogramOutput $ opts - matrixOutput' = fmap (MatrixFile . (unOutputDirectory output' FP.)) + matrixOutput' = fmap (getMatrixOutputType . (unOutputDirectory output' FP.)) . unHelpful . matrixOutput $ opts diff --git a/src/TooManyCells/File/Types.hs b/src/TooManyCells/File/Types.hs --- a/src/TooManyCells/File/Types.hs +++ b/src/TooManyCells/File/Types.hs @@ -37,9 +37,6 @@ newtype DendrogramFile = DendrogramFile { unDendrogramFile :: FilePath } newtype CellFile = CellFile { unCellFile :: FilePath } newtype GeneFile = GeneFile { unGeneFile :: FilePath } -newtype MatrixFile = MatrixFile - { unMatrixFile :: FilePath - } deriving (Read,Show) newtype ProjectionFile = ProjectionFile { unProjectionFile :: FilePath } newtype CellWhitelistFile = CellWhitelistFile { unCellWhitelistFile :: FilePath @@ -50,3 +47,4 @@ newtype OutputDirectory = OutputDirectory { unOutputDirectory :: FilePath } -- Advanced +data MatrixFileFolder = MatrixFile FilePath | MatrixFolder FilePath diff --git a/src/TooManyCells/Matrix/Load.hs b/src/TooManyCells/Matrix/Load.hs --- a/src/TooManyCells/Matrix/Load.hs +++ b/src/TooManyCells/Matrix/Load.hs @@ -55,14 +55,14 @@ loadCellrangerData :: GeneFile -> CellFile - -> MatrixFile + -> MatrixFileFolder -> IO SingleCells -loadCellrangerData gf cf mf = do +loadCellrangerData _ _ (MatrixFile mf) = error (mf <> " must be a folder for matrix market format.") +loadCellrangerData gf cf (MatrixFolder mf) = do let csvOptsTabs = CSV.defaultDecodeOptions { CSV.decDelimiter = fromIntegral (ord '\t') } m <- fmap (MatObsRow . HS.transposeSM . matToSpMat) -- We want observations as rows . readMatrix - . unMatrixFile $ mf -- m <- fmap (MatObsRow . HS.transposeSM) . loadMatrixMarket $ mf -- We want observations as rows g <- fmap (\ x -> either error (fmap (Gene . fst)) ( CSV.decodeWith csvOptsTabs CSV.NoHeader x @@ -89,9 +89,10 @@ -- | Load an H Matrix in CSV format (rows were features) with row names and -- column names. loadHMatrixData :: Delimiter - -> MatrixFile + -> MatrixFileFolder -> IO SingleCells -loadHMatrixData (Delimiter delim) mf = do +loadHMatrixData _ (MatrixFolder mf) = error $ mf <> " must be a csv for dense format." +loadHMatrixData (Delimiter delim) (MatrixFile mf) = do let csvOpts = CSV.defaultDecodeOptions { CSV.decDelimiter = fromIntegral (ord delim) } all <- fmap (\ x -> either error id ( CSV.decodeWith csvOpts CSV.NoHeader x @@ -99,7 +100,6 @@ ) ) . B.readFile - . unMatrixFile $ mf let c = fmap Cell . V.drop 1 . V.head $ all @@ -122,16 +122,16 @@ -- | Load a sparse matrix in CSV format (rows were features) with row names and -- column names. loadSparseMatrixData :: Delimiter - -> MatrixFile + -> MatrixFileFolder -> IO SingleCells -loadSparseMatrixData (Delimiter delim) mf = do +loadSparseMatrixData _ (MatrixFolder mf) = error $ mf <> " must be a csv for dense format." +loadSparseMatrixData (Delimiter delim) (MatrixFile mf) = do let csvOpts = CSV.defaultDecodeOptions { CSV.decDelimiter = fromIntegral (ord delim) } strictRead path = (evaluate . force) =<< B.readFile path all <- fmap (\x -> either error id $ (CSV.decodeWith csvOpts CSV.NoHeader x :: Either String (V.Vector [T.Text]))) . strictRead - . unMatrixFile $ mf let c = V.fromList . fmap Cell . drop 1 . V.head $ all @@ -152,9 +152,10 @@ -- | Load a sparse matrix streaming in CSV format (rows were features) with row -- names and column names. loadSparseMatrixDataStream :: Delimiter - -> MatrixFile + -> MatrixFileFolder -> IO SingleCells -loadSparseMatrixDataStream (Delimiter delim) mf = do +loadSparseMatrixDataStream _ (MatrixFolder mf) = error $ mf <> " must be a csv for dense format." +loadSparseMatrixDataStream (Delimiter delim) (MatrixFile mf) = do let csvOpts = S.defaultDecodeOptions { S.decDelimiter = fromIntegral (ord delim) } cS = fmap (S.first (fmap Cell . drop 1 . fromMaybe (error "\nNo header."))) @@ -166,7 +167,7 @@ res <- flip with return $ do - contents <- SW.withBinaryFileContents . unMatrixFile $ mf + contents <- SW.withBinaryFileContents mf (c S.:> g S.:> m S.:> _) <- fmap (either (error . show) id) diff --git a/src/TooManyCells/Matrix/Utility.hs b/src/TooManyCells/Matrix/Utility.hs --- a/src/TooManyCells/Matrix/Utility.hs +++ b/src/TooManyCells/Matrix/Utility.hs @@ -19,11 +19,16 @@ , loadMatrixMarket , extractSc , writeMatrixLike + , isCsvFile + , getMatrixOutputType ) where -- Remote import BirchBeer.Types +import Control.Monad.Managed (runManaged) import Control.Monad.State (MonadState (..), State (..), evalState, execState, modify) +import Data.Bool (bool) +import Data.Char (toUpper) import Data.Function (on) import Data.List (maximumBy) import Data.Maybe (fromMaybe) @@ -32,19 +37,27 @@ import Language.R as R import Language.R.QQ (r) import System.FilePath (()) +import TextShow (showt) import qualified Control.Lens as L +import qualified Data.ByteString.Char8 as B +import qualified Data.ByteString.Streaming.Char8 as BS import qualified Data.Clustering.Hierarchical as HC import qualified Data.Graph.Inductive as G import qualified Data.Map.Strict as Map import qualified Data.Sequence as Seq import qualified Data.Sparse.Common as S import qualified Data.Text as T +import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL import qualified Data.Text.Lazy.Read as TL import qualified Data.Vector as V import qualified Numeric.LinearAlgebra as H +import qualified Streaming as Stream +import qualified Streaming.Cassava as Stream +import qualified Streaming.Prelude as Stream +import qualified Streaming.With.Lifted as SW import qualified System.Directory as FP -- Local @@ -122,7 +135,7 @@ $ mat -- | Load a matrix market format. -loadMatrixMarket :: MatrixFile -> IO (S.SpMatrix Double) +loadMatrixMarket :: MatrixFileFolder -> IO (S.SpMatrix Double) loadMatrixMarket (MatrixFile file) = do let toDouble [r, c, v] = ( either error fst . TL.decimal $ r @@ -149,8 +162,8 @@ extractSc = fromMaybe (error "Need to provide matrix in --matrix-path for this functionality.") -- | Write a matrix to a file. -writeMatrixLike :: MatrixLike (a) => MatrixFile -> a -> IO () -writeMatrixLike (MatrixFile folder) mat = do +writeSparseMatrixLike :: MatrixLike (a) => MatrixFileFolder -> a -> IO () +writeSparseMatrixLike (MatrixFolder folder) mat = do -- Where to place output files. FP.createDirectoryIfMissing True folder @@ -167,3 +180,41 @@ $ mat return () + +-- | Print a dense matrix to a streaming string. +printDenseMatrixLike :: (MatrixLike a, Monad m) + => a + -> BS.ByteString m () +printDenseMatrixLike mat = Stream.encode (Just $ Stream.header header) + . Stream.zipWith (:) rowN + . Stream.each + . fmap (fmap showt . S.toDenseListSV) + . S.toRowsL + . S.transposeSM -- To have cells as columns + . getMatrix + $ mat + where + header = (B.empty :) + . fmap T.encodeUtf8 + . V.toList + . getRowNames -- To have rows as columns + $ mat + rowN = Stream.each . V.toList . getColNames $ mat -- To have columns as rows + +-- | Write a matrix to a dense file. +writeDenseMatrixLike :: (MatrixLike a) => MatrixFileFolder -> a -> IO () +writeDenseMatrixLike (MatrixFile file) = + runManaged . SW.writeBinaryFile file . printDenseMatrixLike + +-- | Write a MatrixLike to a file (dense) or folder (sparse). +writeMatrixLike :: (MatrixLike a) => MatrixFileFolder -> a -> IO () +writeMatrixLike o@(MatrixFolder _) = writeSparseMatrixLike o +writeMatrixLike o@(MatrixFile _) = writeDenseMatrixLike o + +-- | Check if a name ends with .csv +isCsvFile :: FilePath -> Bool +isCsvFile = (== ".CSV") . fmap toUpper . reverse . take 4 . reverse + +-- | Get matrix output format from input name. +getMatrixOutputType :: FilePath -> MatrixFileFolder +getMatrixOutputType x = bool (MatrixFolder x) (MatrixFile x) . isCsvFile $ x diff --git a/too-many-cells.cabal b/too-many-cells.cabal --- a/too-many-cells.cabal +++ b/too-many-cells.cabal @@ -1,6 +1,6 @@ cabal-version: >=1.10 name: too-many-cells -version: 0.1.1.0 +version: 0.1.2.0 license: GPL-3 license-file: LICENSE copyright: 2019 Gregory W. Schwartz