diff --git a/ChangeLog b/ChangeLog --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +Version 1.4.1 2022-06-03 by luispedro + * Fix bug with split mapping + * Fix packaging for hackage + Version 1.4.0 2022-05-30 by luispedro * Add --trace argument to all modes * Update --create-reference-pack mode to newer format (previous version worked, but diff --git a/Html/index.html b/Html/index.html new file mode 100644 --- /dev/null +++ b/Html/index.html @@ -0,0 +1,86 @@ + + + + + + + + + Ngless + + + + + + +
+
+

NGLess Log

+
+
+
+

Result of running script file {{output.processed.name}} at {{output.processed.time}}.

+
+
+

Script

+
+

QC Stats {{line[1]}}

+
+
+
+
+ +

NGLess Output

+ +

Show timestampHide timestamp. + +

+

+ Line {{oline.lno}} [{{oline.time}}]: {{oline.message}}
+
+
+
+ + + + + + + + + + + + + + + + diff --git a/Html/ngless.css b/Html/ngless.css new file mode 100644 --- /dev/null +++ b/Html/ngless.css @@ -0,0 +1,233 @@ +.textTitle { + padding: 80px 15px; + text-align: center; +} + +.frameFastQ { + height: 900px; + width: 100%; + display: block; +} + +.timetext { + float:right ; +} + +.titletext { + float: left; +} + +div.panel-title { + padding-bottom: 30px; + padding-top: 10px; +} + +.panel-default>.panel-heading { + color: #fff; + background-color: #E8A10C; + border-color: #ddd; +} + + +/* Sidenav for Docs +-------------------------------------------------- */ + +.sidenav { + width: 228px; + margin: 30px 0 0; + padding: 0; + background-color: #fff; + -webkit-border-radius: 6px; + -moz-border-radius: 6px; + border-radius: 6px; + -webkit-box-shadow: 0 1px 4px rgba(0,0,0,.065); + -moz-box-shadow: 0 1px 4px rgba(0,0,0,.065); + box-shadow: 0 1px 4px rgba(0,0,0,.065); +} +.sidenav > li > a { + display: block; + width: 190px \9; + margin: 0 0 -1px; + padding: 8px 14px; + border: 1px solid #e5e5e5; +} +.sidenav > li:first-child > a { + -webkit-border-radius: 6px 6px 0 0; + -moz-border-radius: 6px 6px 0 0; + border-radius: 6px 6px 0 0; +} +.sidenav > li:last-child > a { + -webkit-border-radius: 0 0 6px 6px; + -moz-border-radius: 0 0 6px 6px; + border-radius: 0 0 6px 6px; +} +.sidenav > .active > a { + position: relative; + z-index: 2; + padding: 9px 15px; + border: 0; + text-shadow: 0 1px 0 rgba(0,0,0,.15); + -webkit-box-shadow: inset 1px 0 0 rgba(0,0,0,.1), inset -1px 0 0 rgba(0,0,0,.1); + -moz-box-shadow: inset 1px 0 0 rgba(0,0,0,.1), inset -1px 0 0 rgba(0,0,0,.1); + box-shadow: inset 1px 0 0 rgba(0,0,0,.1), inset -1px 0 0 rgba(0,0,0,.1); +} + +.bs-docs-masthead, .bs-docs-header { + position: relative; + padding: 30px 15px; + color: #cdbfe3; + text-align: center; + text-shadow: 0 1px 0 rgba(0, 0, 0, .1); + background-color: #53B4FF; + background-image: -webkit-linear-gradient(top, #236DFF 0, #53B4FF 100%); + background-image: linear-gradient(to bottom, #236DFF 0, #53B4FF 100%); + background-repeat: repeat-x; +} + + +.bs-docs-header h1 { + color: #fff +} + +@media (min-width:768px){ + .bs-docs-header { + padding-top: 10px; + padding-bottom: 10px; + font-size: 16px; + text-align: left + } + + .bs-docs-header h1 { + font-size: 60px; + line-height: 1 + } +} + +@media (min-width:992px){.bs-docs-header h1, .bs-docs-header p { + margin-right: 380px + } +} + + +.navbar { + position:relative; + min-height:40px; + margin:0px; + /* Add padding: 0 */ + border:0px solid transparent; +} + +#nav.affix { + position: fixed; + top: 70px; + width: 100% +} + +.headTemplate { + color: #53B4FF; +} + + +html,body { + height: 100%; + /* The html and body elements cannot have any padding or margin. */ +} + +/* Wrapper for page content to push down footer */ +#wrap { + min-height: 100%; + height: auto !important; + height: 100%; + /* Negative indent footer by it's height */ + margin: 0 auto -60px; +} + +/* Set the fixed height of the footer here */ +#push, +#footer { + height: 60px; +} + +#footer { + background-color: #f5f5f5; + padding-top: 20px; +} + + + +.section-body { + position: relative; + padding: 30px 15px; + color: #cdbfe3; + text-align: center; + text-shadow: 0 1px 0 rgba(0, 0, 0, .1); + background-color: #fff; + background-image: -webkit-linear-gradient(top, #fff 0, #fff 100%); + background-image: linear-gradient(to bottom, #fff 0, #fff 100%); + background-repeat: repeat-x; +} + + +.section-body h1 { + color: #000000 +} + +@media (min-width:768px){ + .section-body { + padding-top: 60px; + padding-bottom: 60px; + font-size: 14px; + text-align: center + } + + .section-body h1 { + font-size: 20px; + line-height: 1 + } +} + + +.sec-collapse{ + text-shadow: 0 1px 0 rgba(0, 0, 0, .1); + background-color: #5CDBE5; + background-image: -webkit-linear-gradient(bottom, #fff 0, #5CDBE5 100%); + background-image: linear-gradient(to top, #fff 0, #5CDBE5 100%); + background-repeat: repeat-x; +} + +@media (min-width:768px){ + .sec-collapse { + padding-top: 20px; + padding-bottom: 60px; + } + + .sec-collapse h1 { + font-size: 20px; + line-height: 1 + } +} + +#leftcolumn { + background-color: grey; + font-size: 12px; + text-align: left; + color: white; + width: 20%; + float:left; +} + +#rightCont { + width: 70%; + float:right; + margin-left: 10px; + margin-right: 50px; +} + +#leftitem { + margin-left: 6px; + color: white; +} + +#headList { + text-align: center; +} diff --git a/Html/nglessMapStats.html b/Html/nglessMapStats.html new file mode 100644 --- /dev/null +++ b/Html/nglessMapStats.html @@ -0,0 +1,66 @@ + + + + + + + + + + + + + Mapping Statistics + + + + +

Mapping Statistics

+
+
+ + + + + + + + + + + + + + + + + + + + + +
Line NumberReferenceInput fileTotal readsTotal alignedTotal unique
{{m.lno}}{{m.reference}}{{m.inputFile}}{{m.totalReads}}{{m.totalAligned}}{{m.totalUnique}}
+
+ + +
+ + diff --git a/Html/nglessQC.html b/Html/nglessQC.html new file mode 100644 --- /dev/null +++ b/Html/nglessQC.html @@ -0,0 +1,130 @@ + + + + + + + + + + + + + + FastQ Quality Statistics + + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
{{fq.fileName}}
%GC: {{fq.gcContent*100. | number : 1}}%
Encoding: {{fq.encoding}}
Number of Sequences: {{fq.numSeqs}}
Sequence length: {{fq.seqLength[0]}}–{{fq.seqLength[1]}}
Plot below
+
+ +
+
+ +
+
+ + + +
+
+ + + + diff --git a/Html/perBaseQualScores.css b/Html/perBaseQualScores.css new file mode 100644 --- /dev/null +++ b/Html/perBaseQualScores.css @@ -0,0 +1,40 @@ + +#table { + position: relative; + top: 30px; +} + +body { + overflow-y:scroll; + margin: 0; + padding: 0; +} + +svg { + overflow: hidden; +} + +div { + border: 0; + margin: 0; +} + + +#offsetDiv { + margin-left: 100px; + margin-top: 100px; + margin-right: 100px; +} + + +.nv-legendWrap { + margin-bottom: 300px; +} + + +#qualityPlot { + margin: 0; + height: 500px; + min-width: 800px; +} + diff --git a/NGLess.cabal b/NGLess.cabal --- a/NGLess.cabal +++ b/NGLess.cabal @@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: NGLess -version: 1.4.0 +version: 1.4.1 synopsis: NGLess implements ngless, a DSL for processing sequencing data description: NGLess implements a domain-specific language for processing next generation data, particularly metagenomics. category: Domain Specific Language @@ -17,7 +17,14 @@ license-file: COPYING build-type: Simple extra-source-files: + README.md ChangeLog + Html/index.html + Html/ngless.css + Html/nglessMapStats.html + Html/nglessQC.html + Html/perBaseQualScores.css + NGLess/Interpretation/Count/RefSeqInfoVector.h source-repository head type: git diff --git a/NGLess/Data/Fasta.hs b/NGLess/Data/Fasta.hs --- a/NGLess/Data/Fasta.hs +++ b/NGLess/Data/Fasta.hs @@ -49,7 +49,7 @@ getdata !n header toks = C.await >>= \case Nothing -> C.yield $ FastaSeq header (B.concat $ reverse toks) Just (ByteLine next) - | B.null next -> throwDataError ("Unexpected empty string at line " ++ show (n+1) ++ " (expected header line).") + | B.null next -> getdata (n+1) header toks | B.head next == greaterThanSign -> do C.yield $ FastaSeq header (B.concat $ reverse toks) getdata (n+1) (B.drop 1 next) [] diff --git a/NGLess/Interpretation/Count/RefSeqInfoVector.h b/NGLess/Interpretation/Count/RefSeqInfoVector.h new file mode 100644 --- /dev/null +++ b/NGLess/Interpretation/Count/RefSeqInfoVector.h @@ -0,0 +1,71 @@ +#include +#include +#include +#include + +const unsigned block_size = 8192 - 64; // 64 is for padding +struct PoolC { + PoolC():free_(0) { } + ~PoolC() { + for (std::vector::iterator it = data_.begin(), past = data_.end(); it != past; ++it) { + operator delete(*it); + } + } + const char* strdup(const char* input) { + const unsigned int len = std::strlen(input) + 1; + if (len > block_size) { + char* n = static_cast(operator new(len)); + std::strcpy(n, input); + data_.push_back(n); + free_ = 0; + return n; + } + if (len > free_) { + data_.push_back(operator new(block_size)); + free_ = block_size; + } + char* r = static_cast(data_.back()); + r += block_size - free_; + std::strcpy(r, input); + free_ -= len; + return r; + } + private: + + std::vector data_; + size_t free_; +}; + + +struct RefSeqInfo { + RefSeqInfo(const char* s, double v) + :str(s) + ,val(v) { } + bool operator <(const RefSeqInfo& rhs) const { return strcmp(str, rhs.str) < 0; } + const char* str; + double val; +}; + + +struct RefSeqInfoVector { + RefSeqInfoVector() { } + ~RefSeqInfoVector() { } + void insert(std::string f, double v) { + data_.push_back(RefSeqInfo(pool_.strdup(f.c_str()), v)); + } + RefSeqInfo& at(int ix) { return data_.at(ix); } + int find(const char* f) { + std::vector::const_iterator it = std::lower_bound(data_.begin(), data_.end(), RefSeqInfo(f, 0)); + if (it == data_.end() || strcmp(it->str, f)) { return -1; } + return it - data_.begin(); + } + void sort() { std::sort(data_.begin(), data_.end()); } + unsigned size() const { return data_.size(); } + std::vector data_; + PoolC pool_; +}; + +extern "C" { + void rsiv_free(void* p) { delete static_cast(p); } +} + diff --git a/NGLess/Interpretation/Map.hs b/NGLess/Interpretation/Map.hs --- a/NGLess/Interpretation/Map.hs +++ b/NGLess/Interpretation/Map.hs @@ -1,4 +1,4 @@ -{- Copyright 2013-2019 NGLess Authors +{- Copyright 2013-2022 NGLess Authors - License: MIT -} {-# LANGUAGE FlexibleContexts #-} @@ -24,7 +24,6 @@ import qualified Data.Conduit.Algorithms.Async as CAlg import Data.Conduit ((.|)) import Data.Conduit.Algorithms.Utils (awaitJust) -import Data.Conduit.Algorithms.Async (conduitPossiblyCompressedFile) import Control.Monad.Extra (unlessM) import Data.List (sort) @@ -142,6 +141,7 @@ forM_ blocks (ensureIndexExists 0 mapper) return blocks +ensureSplitsExist :: Int -> FilePath -> NGLessIO [FilePath] ensureSplitsExist blockSize fafile = do fafile' <- getIndexOutput False fafile let ofafile = FilePath.takeDirectory fafile' FilePath.takeBaseName fafile <.> "splits_" ++ show blockSize ++ "m" @@ -189,12 +189,12 @@ , maxAge = 36 * 3000 , whenExistsStrategy = IfLockedRetry { nrLockRetries = 120, timeBetweenRetries = 60 } , mtimeUpdate = True - } $ C.runConduit $ - conduitPossiblyCompressedFile ifile - .| faConduit - .| splitWriter + } $ + CAlg.withPossiblyCompressedFile ifile $ \ic -> + C.runConduit (ic .| faConduit .| splitWriter) where maxBPS = 1000 * 1000 * megaBPS + splitWriter :: C.ConduitT FastaSeq C.Void NGLessIO [FilePath] splitWriter = splitWriter' [] (0 :: Int) splitWriter' fs n = do let f = ofileBase ++ "." ++ show n ++ ".fna" @@ -314,13 +314,13 @@ mergeSamFiles inputs = do lift $ outputListLno' TraceOutput ["Merging SAM files: ", show inputs] forM_ inputs $ \f -> - CB.sourceFile f + CAlg.conduitPossiblyCompressedFile f .| linesC .| readSamHeaders -- This is sub-optimal as we reparse the file. -- There are also obvious opportunities to make this code take advantage of parallelism C.sequenceSources - [CB.sourceFile f + [CAlg.conduitPossiblyCompressedFile f .| linesVC 4096 .| readSamGroupsC' 1 True .| CC.concat -- TODO: Remove this and adapt `mergeSAMGroups` to work directly on vectors diff --git a/NGLess/Version.hs b/NGLess/Version.hs --- a/NGLess/Version.hs +++ b/NGLess/Version.hs @@ -17,10 +17,10 @@ versionStr = showVersion version versionStrLong :: String -versionStrLong = "1.4.0" +versionStrLong = "1.4.1" dateStr :: String -dateStr = "May 30 2022" +dateStr = "June 3 2022" embeddedStr :: String #ifdef NO_EMBED_SAMTOOLS_BWA diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,143 @@ +# NGLess: NGS Processing with Less Work + +![NGLess logo](NGLess-logo-128x64.png) Ngless is a domain-specific language for +NGS (next-generation sequencing data) processing. + +[![Build Status](https://travis-ci.com/ngless-toolkit/ngless.svg?branch=master)](https://travis-ci.com/ngless-toolkit/ngless) +[![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/hyperium/hyper/master/LICENSE) +[![Install with Bioconda](https://anaconda.org/bioconda/ngless/badges/installer/conda.svg)](https://anaconda.org/bioconda/ngless) +[![Install with Bioconda](https://anaconda.org/bioconda/ngless/badges/downloads.svg)](https://anaconda.org/bioconda/ngless) +[![Citation for NGLess](https://img.shields.io/badge/CITATION-DOI%3A10.1186%252Fs40168--019--0684--8-brightgreen.svg)](https://doi.org/10.1186/s40168-019-0684-8) +[![Join the chat at https://gitter.im/ngless-toolkit](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ngless-toolkit) + + +For questions and discussions, please use the [ngless mailing +list](https://groups.google.com/forum/#!forum/ngless). + +If you are using NGLess, please cite: + +> _NG-meta-profiler: fast processing of metagenomes using NGLess, a +> domain-specific language_ by Luis Pedro Coelho, Renato Alves, Paulo Monteiro, +> Jaime Huerta-Cepas, Ana Teresa Freitas, Peer Bork, Microbiome (2019) +> [https://doi.org/10.1186/s40168-019-0684-8](https://doi.org/10.1186/s40168-019-0684-8) + +![NGLess cartoon](docs/NGLess-cartoon.svg) + +## Example + + ngless "1.4" + input = fastq(['ctrl1.fq','ctrl2.fq','stim1.fq','stim2.fq']) + input = preprocess(input) using |read|: + read = read[5:] + read = substrim(read, min_quality=26) + if len(read) < 31: + discard + + mapped = map(input, + reference='hg19') + write(count(mapped, features=['gene']), + ofile='gene_counts.csv', + format={csv}) + +## Installing + +See the [install documentation](https://ngless.embl.de/install.html) for more +information. + +### Bioconda + +The recommended way to install NGLess is through +[bioconda](https://bioconda.github.io): + + conda install -c bioconda ngless + +### Docker + +Alternatively, a docker container with NGLess is available at +[docker hub](https://hub.docker.com/r/nglesstoolkit/ngless): + + docker run -v $PWD:/workdir -w /workdir -it nglesstoolkit/ngless:1.4.0 ngless --version + +Adapt the mount flags (``-v``) as needed. + +### Linux + +You can download a [statically linked version of NGless +1.4.0](https://github.com/ngless-toolkit/ngless/releases/download/v1.4.0/NGLess-v1.4.0-Linux-static-full) + +This should work across a wide range of Linux versions (please +[report](https://github.com/ngless-toolkit/ngless/issues) any issues you encounter): + + curl -L -O https://github.com/ngless-toolkit/ngless/releases/download/v1.4.0/NGLess-v1.4.0-Linux-static-full + chmod +x NGLess-v1.4.0-Linux-static-full + ./NGLess-v1.4.0-Linux-static-full + +This downloaded file bundles bwa, samtools and megahit (also statically linked). + +### From Source + +Installing/compiling from source is also possible. Clone +[https://github.com/ngless-toolkit/ngless](https://github.com/ngless-toolkit/ngless) + +#### Dependencies + +The simplest way to get an environment with all the dependencies is to use conda: + + conda create -n ngless + conda activate ngless + conda config --add channels conda-forge + conda install stack cairo bzip2 gmp zlib perl wget xz pkg-config make + +You should have `gcc` installed (or another C-compiler). + +The following sequence of commands should download and build the software + + git clone https://github.com/ngless-toolkit/ngless + cd ngless + stack setup + make + +To install, you can use the following command (replace `` with +the directory where you wish to install, default is `/usr/local`): + + make make + +## Running Sample Test Scripts on Local Machine + +For developers who have successfully compiled and installed NGless, running the +test scripts in the `tests` folder would be the next line of action to have the +output of sample test cases. + + cd tests + +Once in the `test` directory, select any of the test folders to run NGless. + +For example, here we would run the `regression-fqgz` test: + + cd regression-fqgz + ngless ungzip.ngl + +After running this script open the newly generated folder `ungzip.ngl.output_ngless` and view the template in the **index.html** file. + +For developers who have done this much more datasets for testing purposes can be referenced and used by reading these documentation links: +**[Human Gut Metagenomics Functional & Taxonomic Profiling](https://ngless.embl.de/tutorial-gut-metagenomics.html#)** +**[Ocean Metagenomics Functional Profiling](https://ngless.embl.de/tutorial-ocean-metagenomics.html)** +**[Ocean Metagenomics Assembly and Gene Prediction](https://ngless.embl.de/tutorial-assembly-gp.html)** + + +## More information + +- [Full documentation](https://ngless.embl.de/) +- [Frequently Asked Questions (FAQ)](https://ngless.embl.de/faq.html) +- [ngless mailing list](https://groups.google.com/forum/#!forum/ngless) +- [What's new log](https://ngless.embl.de/whatsnew.html) +- [NGless 1.4.0 Release Documentation](https://ngless.embl.de/whatsnew.html#version-1-4-0) + +## Authors + +- [Luis Pedro Coelho](https://luispedro.org) (email: [luispedro@big-data-biology.org](mailto:luispedro@big-data-biology.org)) (on twitter: [@luispedrocoelho](https://twitter.com/luispedrocoelho)) +- Paulo Monteiro +- Renato Alves +- [Ana Teresa Freitas](https://web.tecnico.ulisboa.pt/ana.freitas/) +- Peer Bork +