diff --git a/image_development/Dockerfile_nextits_2 b/image_development/Dockerfile_nextits_2 index 9caddcc2..bd25ae4e 100644 --- a/image_development/Dockerfile_nextits_2 +++ b/image_development/Dockerfile_nextits_2 @@ -1,10 +1,31 @@ +ARG NEXTITS_VERSION=1.1.0 +ARG NEXTFLOW_VERSION=25.10.4 -# Stage 1 (Nextflow - minimal stage) -FROM nextflow/nextflow:25.10.2 AS nextflow +FROM vmikk/nextits:${NEXTITS_VERSION} AS main -## Build stage 3 - Main -FROM vmikk/nextits:1.1.0 AS main +## Re-declare build args inside this stage so RUN can use them +ARG NEXTITS_VERSION +ARG NEXTFLOW_VERSION -# Copy Nextflow binary from the Nextflow stage -COPY --from=nextflow /usr/local/bin/nextflow /usr/local/bin/nextflow +ENV NEXTITS_DIR=/opt/pipelines/NextITS +ENV NXF_HOME=/opt/software/conda/bin +# ENV PATH="${NXF_HOME}:${PATH}" + +USER root + +RUN set -eux; \ + mkdir -p "${NXF_HOME}" "${NEXTITS_DIR}"; \ + ## Install Nextflow into $NXF_HOME + NXF_VER="${NEXTFLOW_VERSION}" curl -s https://get.nextflow.io | bash ; \ + mv nextflow "${NXF_HOME}/nextflow"; \ + chmod 0755 "${NXF_HOME}/nextflow"; \ + "${NXF_HOME}"/nextflow plugin install nf-schema@2.4.1; \ + rm -rf ./.nextflow; \ + ## Pull NextITS into $NEXTITS_DIR + git clone --depth 1 --branch "${NEXTITS_VERSION}" https://github.com/vmikk/NextITS "${NEXTITS_DIR}" ; \ + rm -rf "${NEXTITS_DIR}/.git" ; \ + ## Ensure the runtime user can access everything + chown -R nextits:nextits "${NXF_HOME}" "${NEXTITS_DIR}" + +USER nextits diff --git a/src/pipecraft-core/service_scripts/NextITS/.gitattributes b/src/pipecraft-core/service_scripts/NextITS/.gitattributes deleted file mode 100644 index b21921b9..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/.gitattributes +++ /dev/null @@ -1,4 +0,0 @@ -*.config linguist-language=nextflow -*.nf.test linguist-language=nextflow -modules/** linguist-generated -subworkflows/** linguist-generated diff --git a/src/pipecraft-core/service_scripts/NextITS/.gitignore b/src/pipecraft-core/service_scripts/NextITS/.gitignore deleted file mode 100644 index 5124c9ac..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -.nextflow* -work/ -data/ -results/ -.DS_Store -testing/ -testing* -*.pyc diff --git a/src/pipecraft-core/service_scripts/NextITS/CHANGELOG.md b/src/pipecraft-core/service_scripts/NextITS/CHANGELOG.md deleted file mode 100644 index 0b92d7a7..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/CHANGELOG.md +++ /dev/null @@ -1,115 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -This project tries to adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -For version numbering, we use the following convention: `MAJOR.MINOR.PATCH`. -Each element increases numerically (e.g., `1.9.0` -> `1.10.0` -> `1.11.0`). - - -## [1.1.0] - 2026-01-22 - -- Fixed handling of unknown barcode combinations (in `dual asymmetric` mode); thanks to Alice Retter for reporting -- Refactored and optimized the tag-jump removal step -- Fixed a bug with duplicated sequences in the tag-jump removal step; thanks to Valentin Étienne for reporting -- Implemented a chunking option for splitting the dataset into smaller parts prior to clustering in Step-2 (pre-clustering, clustering, and denoising moved to a separate sub-workflow), using MMseqs2 -- Added possibility to disable reference-based and/or de novo chimera removal steps and tag-jump removal -- New parameters added: - - `lima_remove_unknown` (default, `false`; if `true`, unknown barcode combinations are removed from demultiplexed data) - - `chunking_n` (number of chunks to split the dataset into prior to clustering) - - `chunking_id` (minimum sequence identity used for splitting the dataset into chunks) - - `chimera_methods` (specifies which chimera removal methods to use - "ref" for reference-based, "denovo" for de novo, or "ref,denovo" for both; could be also "none" or `null` to disable chimera removal) - - `tj` (specifies whether to run tag-jump removal - "true" or "false") -- Added DADA2 denoising (`--preclustering dada2`; also works with `--clustering none`) -- Implemented automated documentation for analysis procedures (generates `README_Step1_Methods.txt` and `README_Step2_Methods.txt` in the `pipeline_info` directory) -- Refactored parameter validation (using `nf-schema` plugin) -- Refactored the runtime parameter summary and help message -- Added test profiles (`test`, `test1`, `test2`) -- Improved run summary for Step-1 -- Default parameters changed: - - ITSx now checks only a single strand (option `ITSx_complement` set to `F`). This should be safe for most cases, as amplicons were re-oriented using primers during the pipeline run. However, we recommend checking the results carefully (e.g., columns `ITSx_Extracted_Reads` and `ITSx_Yield_Percent` in the run summary) - - Prior to tag-jump removal, sequences are now dereplicated at 100% identity (option `tj_id` set to `1`). It is possible to pre-cluster sequences at a lower similarity threshold (e.g., `--tj_id 0.99`) but this will take much longer. This change should also be safe for most cases, as amplicons undergo homopolymer-correction -- Fixed a minor bug in extraction of sample IDs at the ref-based chimera rescue step; thanks to Valentin Étienne for reporting - -## [1.0.0] - 2025-03-24 - -- Added support of asymmetric barcoding scheme for demultiplexing of PacBio data -- Added support of BAM files (CCS) as input -- Added support for SWARM *d*=1 pre-clustering -- Changed the selection of representative sequences (sequence with the highest quality score is taken as the representative; using [`phredsort`](https://github.com/vmikk/phredsort)) -- Refactored sequence quality estimation -- Improved processing speed (using DuckDB and Parquet format) -- Improved tag valiadion for demultiplexing -- Improved compression speed for output files (runs in parallel using `pigz`) -- Update of the database for reference-based chimera detection (using the [EUKARYOME database](https://eukaryome.org/)) -- New parameters added: - - `step` (specifies which pipeline step to run - "Step1" or "Step2") - - `storagemode` (Adjusts how files are directed to the results folder) - - `gzip_compression` (Controls GZIP compression level in output files) - - categorical `lima_barcodetype` replaces boolean `lima_dualbarcode` - - `lima_minendscore` (For asymmetric and dual barcoding scheme) - - `lima_minrefspan` (Controls barcode coverage) - - `lima_minscoringregions` (Controls the number of reqired barcodes for demultiplexing using dual barcodes) -- Added auxilarry output files: - - [Step-1] All rRNA parts extracted by ITSx (pooled within sequencing run - useful for extracting these regions for representative sequences) - - [Step-1] File with quality scores for full-length sequences (after QC and trimming) - - [Step-2] File with joined sequence memebership (dereplication, pre-clustering, and clustering) -- Primer trimming prior ITSx is now default (sequence quality is also estimated on trimmed sequence) -- Fixed VSEARCH clustering on denoised reads -- Resolved an issue where no *de novo* chimeras were detected -- Reconfigured parameter specification -- Introduced a parameter schema and enhanced parameter validation -- Container updates to included the latest versions of dependencies -- New dependencies - specialized tools written in Go to speed up the processing: - - [`phredsort` (https://github.com/vmikk/phredsort)](https://github.com/vmikk/phredsort) (Sorts sequences by quality score) - - [`seqhasher` (https://github.com/vmikk/seqhasher)](https://github.com/vmikk/seqhasher) (Hashes sequences) - - [`ucs` (https://github.com/vmikk/ucs)](https://github.com/vmikk/ucs) (Parses UC files and converts them to parquet format) - - -## [0.5.0] - 2023-08-08 - -- New `seqstats` sub-workflow (only dereplication, primer validation, and basic run stats) -- Add SWARM clustering ([Mahé et al., 2022 DOI:10.1093/bioinformatics/btab493](https://academic.oup.com/bioinformatics/article/38/1/267/6318385)) -- Add post-clustering curation with LULU ([Frøslev et al., 2017 DOI:doi.org/10.1038](https://www.nature.com/articles/s41467-017-01312-x)) -- Add barcode validation step -- Add SSU and LSU region-based output sequences -- Add support for UNOISE-only output (without clustering) -- Add `merge_replicates` parameter (Step-2) for merging or keeping separate sample replicates -- Update Step-1 run summary (add homopolymer stats) -- Deprecate taxonomy annotation workflow at Step-1 -- Fixed different extensions in demultiplexed input -- Experimental: UNITE-style dereplication (allows query sequences to vary in length at 100% similarity) -- Experimental: support of alternative alignment penalty scores (ITS-specific feature) - - -## [0.4.0] - 2023-05-08 - -- Add Step-2 workflow for pooling, dereplicating, and clustering sequences from Step-1 - - Read clustering with VSEARCH ([Rognes et al., 2016 DOI:10.7717/peerj.2584](https://peerj.com/articles/2584/)) - - Error-correction with UNOISE2 ([Edgar, 2016 DOI:10.1101/081257](https://www.biorxiv.org/content/10.1101/081257v1)) -- Add run summary for Step-1 (read counts at different pipeline stages) -- Separate config for HPC clusters -- Add Docker container - - -## [0.3.0] - 2023-03-02 - -- Add support for pre-demultiplexed data as input -- Add option for semi-full-length ITS (especially useful when forward primer is located at the very end of SSU and the HMM site can not be recognized by ITSx) -- Add removal of long homopolymer artefacts at QC stage -- Correct handling of a case with no valid sequences at primer checking step (thank to Taavi Riit for reporting the bug) -- Bug fixed in `assemble_its` (thanks to Kadri Põldmaa for discovering the error) -- Addition of ITSx detailed results (with information on the HMM profile used for ITS extraction) -- Fixed sample names for the rescued chimeric sequences -- Minor fixes related with the Singularity container, output directory, help message, and single-end QC - -## [0.2.0] - 2022-09-30 - -- Add Ilumina-based workflow (see `--seqplatform` flag) -- Publish Singularity image to Singularity library -- Minor bugfixes in `primer_check` (multiprimer artefacts), `pool_seqs` (sequence headers), and `prep_asvtab` (aggregation of non-unique joined Illumina sequences) processes -- New logo design (thanks to Olesya Dulya) - -## [0.0.1] - 2022-07-07 - -- Initial release diff --git a/src/pipecraft-core/service_scripts/NextITS/CITATION.cff b/src/pipecraft-core/service_scripts/NextITS/CITATION.cff deleted file mode 100644 index 67541b19..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/CITATION.cff +++ /dev/null @@ -1,19 +0,0 @@ -cff-version: 1.2.0 -title: "NextITS: a pipeline for metabarcoding eukaryotes with full-length ITS sequenced with PacBio" -type: software -authors: -- family-names: "Mikryukov" - given-names: "Vladimir" - orcid: "https://orcid.org/0000-0003-2786-2690" -- family-names: "Anslan" - given-names: "Sten" - orcid: "https://orcid.org/0000-0002-2299-454X" -- family-names: "Tedersoo" - given-names: "Leho" - orcid: "https://orcid.org/0000-0002-1635-1249" -version: 1.1.0 -doi: 10.5281/zenodo.15074881 -date-released: 2025-03-24 -url: "https://github.com/vmikk/NextITS" -license: Apache-2.0 -repository-code: "https://github.com/vmikk/NextITS" diff --git a/src/pipecraft-core/service_scripts/NextITS/CITATIONS.md b/src/pipecraft-core/service_scripts/NextITS/CITATIONS.md deleted file mode 100644 index 56acadd6..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/CITATIONS.md +++ /dev/null @@ -1,50 +0,0 @@ -# Citations - -## **NextITS** - -> Mikryukov V., Anslan S., Tedersoo L. NextITS: a pipeline for metabarcoding fungi and other eukaryotes with full-length ITS sequenced with PacBio. [https://github.com/vmikk/NextITS](https://github.com/vmikk/NextITS), doi: 10.5281/zenodo.15074881 - -## [**Nextflow**](https://pubmed.ncbi.nlm.nih.gov/28398311/) - -> Di Tommaso P, Chatzou M, Floden EW, Barja PP, Palumbo E, Notredame C. Nextflow enables reproducible computational workflows. Nat Biotechnol. 2017 Apr 11;35(4):316-319. doi: 10.1038/nbt.3820. PubMed PMID: 28398311. - -## Pipeline tools - -### Core tools - -- [ITSx](https://besjournals.onlinelibrary.wiley.com/doi/10.1111/2041-210X.12073) - - > Bengtsson-Palme, J., Ryberg, M., Hartmann, M., Branco, S., Wang, Z., Godhe, A., De Wit, P., Sánchez-García, M., Ebersberger, I., de Sousa, F., Amend, A., Jumpponen, A., Unterseher, M., Kristiansson, E., Abarenkov, K., Bertrand, Y.J.K., Sanli, K., Eriksson, K.M., Vik, U., Veldre, V. and Nilsson, R.H.. Improved software detection and extraction of ITS1 and ITS2 from ribosomal ITS sequences of fungi and other eukaryotes for analysis of environmental sequencing data. Methods Ecol Evol 2013, 4: 914-919. doi: 10.1111/2041-210X.12073. - -- [VSEARCH](https://peerj.com/articles/2584/) - - > Rognes T, Flouri T, Nichols B, Quince C, Mahé F. VSEARCH: a versatile open source tool for metagenomics. PeerJ. 2016 4:e2584. doi: 10.7717/peerj.2584 - -- VSEARCH option usearch_global implements the [USEARCH](https://doi.org/10.1093/bioinformatics/btq461) algorithm - - > Edgar RC. Search and clustering orders of magnitude faster than BLAST. Bioinformatics. 2010 26(19) 2460-2461 - -- [Cutadapt](https://journal.embnet.org/index.php/embnetjournal/article/view/200/479) - - > Marcel, M. Cutadapt removes adapter sequences from high-throughput sequencing reads. EMBnet. journal 17.1 (2011): pp-10. doi: 10.14806/ej.17.1.200. - -### Database for reference-based chimera removal - -- [EUKARYOME database](https://eukaryome.org/) - - > Tedersoo L, Hosseyni Moghaddam MS, Mikryukov V, Hakimzadeh A, Bahram M, Nilsson RH, Yatsiuk I, Geisen S, Schwelm A, Piwosz K, Prous M, Sildever S, Chmolowska D, Rueckert S, Skaloud P, Laas P, Tines M, Jung J-H, Choi JH, Alkahtani S, Anslan S (2024). EUKARYOME: The rRNA gene reference database for identification of all eukaryotes. Database, 2024, baae043. doi: 10.1093/database/baae043 - - -### Taxonomic classification and databases - -- [UNITE - eukaryotic nuclear ribosomal ITS region](https://unite.ut.ee/) - -Abarenkov K, Nilsson RH, Larsson K-H, Taylor AFS, May TW, Frøslev TG, Pawlowska J, Lindahl B, Põldmaa K, Truong C, Vu D, Hosoya T, Niskanen T, Piirmann T, Ivanov F, Zirk A, Peterson M, Cheeke TE, Ishigami Y, Jansson AT, Stjernegaard Jeppesen T, Kristiansson E, Mikryukov V, Miller JT, Oono R, Ossandon FJ, Paupério J, Saar I, Schigel D, Suija A, Tedersoo L, Kõljalg U (2023). The UNITE database for molecular identification and taxonomic communication of fungi and other eukaryotes: Sequences, taxa and classifications reconsidered. Nucleic Acids Research, gkad1039. doi: 10.1093/nar/gkad1039 - - -## Software packaging/containerisation tools - -- [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241) - -- [Singularity](https://pubmed.ncbi.nlm.nih.gov/28494014/) - > Kurtzer GM, Sochat V, Bauer MW. Singularity: Scientific containers for mobility of compute. PLoS One. 2017 May 11;12(5):e0177459. doi: 10.1371/journal.pone.0177459. eCollection 2017. PubMed PMID: 28494014; PubMed Central PMCID: PMC5426675. diff --git a/src/pipecraft-core/service_scripts/NextITS/LICENSE b/src/pipecraft-core/service_scripts/NextITS/LICENSE deleted file mode 100644 index 521e13f5..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [2021-2025] [Vladimir Mikruykov] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/pipecraft-core/service_scripts/NextITS/README.md b/src/pipecraft-core/service_scripts/NextITS/README.md deleted file mode 100644 index 3f504e26..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# NextITS - -![GitHub (latest release)](https://img.shields.io/github/v/release/vmikk/NextITS?label=GitHub%20release&color=23aa62) -[![Nextflow](https://img.shields.io/badge/version-%E2%89%A525.04.2-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/) -[![GitHub license](https://img.shields.io/github/license/vmikk/NextITS)](https://github.com/vmikk/NextITS/blob/main/LICENSE) - -[![Runs with Singularity](https://img.shields.io/badge/Runs%20with-Singularity-blue?style=flat&logo=singularity)](https://cloud.sylabs.io/library/vmiks/nextits/nextits) -[![Runs with Docker](https://img.shields.io/badge/Runs%20with-Docker-blue?style=flat&logo=docker)](https://hub.docker.com/r/vmikk/nextits/tags) -[![Runs with Podman](https://img.shields.io/badge/Runs%20with-Podman-blue?style=flat&logo=podman)](https://podman.io/) - -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.15074881.svg)](https://doi.org/10.5281/zenodo.15074881) - -NextITS is an automated pipeline for metabarcoding fungi and other eukaryotes with full-length ITS sequenced with PacBio. -Amplicons obtained with Illumina are also supported. - -## Introduction - -The most widely used genetic markers for metabarcoding fungal communities are highly variable rRNA ITS1 and ITS2 sub-regions of the internal transcribed spacer. High-throughput metabarcoding has greatly improved our understanding of fungal community ecology. Here, we present NextITS, an automated pipeline for analyzing full-length ITS sequences (ITS1-5.8S-ITS2) from the Pacific Biosciences (PacBio) third-generation sequencing platform. Although the PacBio HiFi reads are highly accurate, the primary type of sequencing error is insertions or deletions in homopolymeric sites, which are also naturally common in fungal ITS. In the pipeline, we implemented correction of homopolymer errors, detection of tag-switching artefacts, and recovery of sequences false-positively annotated as chimeric. The pipeline is built using Nextflow workflow manager, with all the software dependencies packaged into Docker and Singularity containers. - -## User Documentation - -User documentation: https://Next-ITS.github.io/ - -## Quick Start - -``` -nextflow run vmikk/NextITS -r main \ - -profile singularity \ - -resume \ - --input "pacbio_ccs.fastq.gz" \ - --barcodes "sample_barcodes.fasta" \ - --primer_forward "GTACACACCGCCCGTCG" \ - --primer_reverse "CCTSCSCTTANTDATATGC" \ - --its_region "full" \ - --outdir "Results" -``` - -## Citation - -Mikryukov V., Anslan S., Tedersoo L. NextITS: a pipeline for metabarcoding fungi and other eukaryotes with full-length ITS sequenced with PacBio. [https://github.com/vmikk/NextITS](https://github.com/vmikk/NextITS). DOI:10.5281/zenodo.15074881 - diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/ITSx_to_DuckDB.sh b/src/pipecraft-core/service_scripts/NextITS/bin/ITSx_to_DuckDB.sh deleted file mode 100755 index 84b69ae5..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/ITSx_to_DuckDB.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash - -## Import sequences into DuckDB-compatible tables - -## Input = FASTA formatted sequences (header = "hash;size=...") -## Output = Table in DuckDB-native format or Parquet - -# Define usage function -usage() { - echo "Usage: $0 [-i input_file] [-o output_file] [-f format]" - echo " -i : Input FASTA file (required)" - echo " -o : Output file (optional, defaults to input filename with .db/.parquet extension)" - echo " -f : Output format (optional): 'duckdb' or 'parquet' (default, 'parquet')" - exit 1 -} - -# Parse command line arguments -input_file="" -output_file="" -format="parquet" # format="duckdb" - -while getopts "i:o:f:h" opt; do - case $opt in - i) input_file="$OPTARG" ;; - o) output_file="$OPTARG" ;; - f) format="$OPTARG" ;; - h) usage ;; - ?) usage ;; - esac -done - -# Validate required parameters -if [ -z "$input_file" ]; then - echo "Error: Input file is required" - usage -fi - -# Validate output format -if [ "$format" != "duckdb" ] && [ "$format" != "parquet" ]; then - echo "Error: Format must be either 'duckdb' or 'parquet'" - usage -fi - -## Extract rRNA region name from filename -if [[ $input_file =~ ([^.]+)\.fasta\.gz$ ]]; then - rRNA_part="${BASH_REMATCH[1]}" -else - echo "Error in extracting rRNA region name from filename" - rRNA_part="X" -fi - -## Check if rRNA region name is valid -VALID_PARTS=("full" "SSU" "ITS1" "5_8S" "ITS2" "LSU") -if [[ ! " ${VALID_PARTS[@]} " =~ " ${rRNA_part} " ]]; then - echo "..Error: Invalid rRNA region name. Supported names are: ${VALID_PARTS[*]}" - rRNA_part="X" -fi - -## 'full' is a reserved keyword in DuckDB, rename to ITS -if [ "$rRNA_part" == "full" ]; then - rRNA_part="ITS" -fi - -## DuckDB table name cannot start with a number -if [[ "$rRNA_part" == "5_8S" ]]; then - rRNA_part="S58" -fi - -## Extract sample name from filename -sample_name="${input_file/.fasta.gz/}" - -# Set output file if not specified -if [ -z "${output_file}" ]; then - if [ "${format}" == "duckdb" ]; then - output_file="${sample_name}.db" - else - output_file="${sample_name}.parquet" - fi -fi - -## Check if input file exists -if [ ! -f "${input_file}" ]; then - echo "..Error: File ${input_file} not found" - exit 1 -fi - -echo "..Importing ${input_file} into ${output_file} (format: ${format})" - -if [ "$format" == "duckdb" ]; then - seqkit fx2tab "${input_file}" \ - | sed 's/;size=/\t/' \ - | duckdb "${output_file}" \ - " - DROP TABLE IF EXISTS ${rRNA_part}; - CREATE TABLE ${rRNA_part} ( - SeqID VARCHAR PRIMARY KEY, - Abundance INTEGER, - Sequence VARCHAR - ); - - INSERT INTO ${rRNA_part} - SELECT * FROM read_csv( - '/dev/stdin', - header = false, delim = '\t', - columns = { - 'SeqID': 'VARCHAR', - 'Abundance': 'INTEGER', - 'Sequence': 'VARCHAR' - } - );" -else - seqkit fx2tab "${input_file}" \ - | sed 's/;size=/\t/' \ - | duckdb -c " - COPY ( - SELECT * FROM read_csv( - '/dev/stdin', - header = false, delim = '\t', - columns = { - 'SeqID': 'VARCHAR', - 'Abundance': 'INTEGER', - 'Sequence': 'VARCHAR' - } - ) - ) TO '${output_file}' (FORMAT PARQUET, COMPRESSION 'ZSTD', COMPRESSION_LEVEL 12);" -fi - -echo "..Data imported to ${output_file}" - - -#### Check the data -# duckdb "$db_file" -# -# -- Show all tables -# SHOW TABLES; -# SELECT * FROM information_schema.tables; -# -# -- Show all column names and their types -# DESCRIBE ITS1; -# -# -- Show first 10 rows -# SELECT * FROM ITS1 LIMIT 10; -# -# -- Get count of rows -# SELECT COUNT(*) FROM ITS1; -# \ No newline at end of file diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/aggregate_sequences.R b/src/pipecraft-core/service_scripts/NextITS/bin/aggregate_sequences.R deleted file mode 100755 index 8f71d9f2..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/aggregate_sequences.R +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to aggregate sequences from multiple runs into a single file (for dereplication and subsequent clustering) -## Also, performs removal of de novo chimeras with high scores (with option to recover sequences that occurred in multiple runs) - -## Do-novo chimera recovery: -# if a sequence identified as putative chimera was observed in the other samples, -# where there is no evidence that it is chimeric, it will be recovered - - -## Function to load packages -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(".. ", paste(pkg, packageVersion(pkg), "\n")) -} - -cat("Loading packages:\n") - -load_pckg("optparse") -load_pckg("data.table") -load_pckg("Biostrings") -load_pckg("plyr") -load_pckg("arrow") -# load_pckg("dplyr") - - -cat("Parsing input options and arguments...\n") - -option_list <- list( - make_option("--seqtabs", action="store", default=NA, type='character', help = "Direcotry containing long tables with quality-filtered sequences (Parquet format)"), - make_option("--maxchim", action="store", default=0.6, type='numeric', help = "Maximum de novo chimera score to remove"), - make_option("--recoverdenovo", action="store", default=TRUE, type='logical', help="Recover de-novo chimeras (logical)"), - make_option("--output", action="store", default="Seqs", type='character', help = "Output prefix"), - make_option("--threads", action="store", default=4, type='integer', help = "Number of CPU threads to use") -) - -opt <- parse_args(OptionParser(option_list=option_list)) - -## Function to convert text "NA"s to NA -to_na <- function(x){ - if(x %in% c("NA", "null", "Null")){ x <- NA } - return(x) -} - -## Replaces "null"s from Nextflow with NA -opt <- lapply(X = opt, FUN = to_na) - - -## Validation of the required arguments -if (is.na(opt$seqtabs)) { - stop("Input directory with quality-filtered sequences is not specified\n") -} - -## Assign variables -SEQTABS <- opt$seqtabs -MAXCHIM <- opt$maxchim -RECOV_DENOVO <- opt$recoverdenovo -OUTPUT <- opt$output -CPUTHREADS <- as.numeric( opt$threads ) - -## Log assigned variables -cat(paste("Path to sequence tables: ", SEQTABS, "\n", sep="")) -cat(paste("Max de novo chimera score: ", MAXCHIM, "\n", sep="")) -cat(paste("De novo chimera recovery: ", RECOV_DENOVO, "\n", sep="")) -cat(paste("Output prefix: ", OUTPUT, "\n", sep="")) -cat(paste("CPU threads: ", CPUTHREADS, "\n", sep="")) - -cat("\n") - -## Set CPU thread number -cat("Setting number of CPU threads to: ", CPUTHREADS, "\n") -setDTthreads(threads = CPUTHREADS) # for data.table -set_cpu_count(CPUTHREADS) # for arrow - - -###################################### -###################################### Process the data -###################################### - -## Load sequence tables -cat("\n..Looking for sequence tables\n") -TABS <- list.files(path = SEQTABS, pattern = ".parquet", full.names = TRUE, recursive = TRUE) -cat("... Tables found: ", length(TABS), "\n") - -cat("\n..Loading sequence tables\n") -TAB <- alply(.data = TABS, .margins = 1, .fun = function(x){ - res <- arrow::read_parquet(x) - setDT(res) - return(res) -}) -TAB <- rbindlist(TAB, use.names = TRUE, fill = TRUE) -cat("... Total number of records: ", nrow(TAB), "\n") -cat("... Total number unique sequences: ", length(unique(TAB$Sequence)), "\n") -cat("... Total number unique samples (fastq files): ", length(unique(TAB$SampleID)), "\n") - - -## Filter sequences by chimeric score (MAXCHIM) -if(!is.na(MAXCHIM)){ - - cat("\n..Filtering data by max de novo chimera score\n") - nrecs <- nrow(TAB) - nabun <- sum(TAB$Abundance, na.rm = TRUE) - - cat("... Max de novo chimera score observed: ", max(TAB$DeNovo_Chimera_Score, na.rm = TRUE), "\n") - - ## If no chimera recovery is required - if(RECOV_DENOVO == FALSE){ - - TAB <- TAB[ DeNovo_Chimera_Score < MAXCHIM | is.na(DeNovo_Chimera_Score) ] - - ## If we need to recover chimeras - } else { - - ## Find putative chimeras - CHIMERAS <- TAB[ DeNovo_Chimera_Score >= MAXCHIM, .(SeqID___SampleID, DeNovo_Chimera_Score, Sequence, Abundance) ] - NONCHIMERAS <- TAB[ ! SeqID___SampleID %in% CHIMERAS$SeqID___SampleID ] - - ## Recover false-positives - chim_seqs <- unique(CHIMERAS$Sequence) - nonchim_seqs <- unique(NONCHIMERAS$Sequence) - fp_chims <- chim_seqs %in% nonchim_seqs - if(any(fp_chims)){ - cat(".... Probably there are a few false-positive chimeras\n") - cat(".... Recovering ", sum(fp_chims), "sequences\n") - fp_seqs <- chim_seqs[ fp_chims ] - CHIMERAS <- CHIMERAS[ ! Sequence %in% fp_seqs ] - rm(fp_seqs) - } - - TAB <- TAB[ ! Sequence %in% CHIMERAS$Sequence ] - rm(CHIMERAS, NONCHIMERAS) - - } # end of chimera recovery - - ## Data summary after filtering - nrecs_delta <- nrecs - nrow(TAB) - nabun_delta <- nabun - sum(TAB$Abundance, na.rm = TRUE) - - cat("... Records removed: ", nrecs_delta, " (", round(nrecs_delta/nrecs * 100, 1), "%)\n") - cat("... Reads removed: ", nabun_delta, " (", round(nabun_delta/nabun * 100, 1), "%)\n") - - rm(nrecs_delta, nabun_delta) - -} # end of MAXCHIM filtering - - -cat("\n..Sorting table by abundance, quality score\n") -setorder(x = TAB, -Abundance, -PhredScore, SeqID) - -cat("..Preparing FASTA file\n") - -SQF <- DNAStringSet(x = TAB$Sequence) -names(SQF) <- paste0(TAB$SeqID, ";size=", TAB$Abundance) # , ";sample=", TAB$SampleID, ";" - -## Export FASTA -cat("..Exporting FASTA file with filtered sequences\n") - -writeXStringSet( - x = SQF, - filepath = paste0(OUTPUT, ".fa.gz"), - compress = TRUE, format = "fasta", width = 9999) - -## Export FASTA -cat("..Exporting filtered table\n") - -write_parquet( - x = TAB, - sink = paste0(OUTPUT, ".parquet"), - compression = "zstd", - compression_level = 10, - use_dictionary = TRUE) - -cat("All done.\n") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/bucketize_db.R b/src/pipecraft-core/service_scripts/NextITS/bin/bucketize_db.R deleted file mode 100755 index e5b0874c..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/bucketize_db.R +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env Rscript - -## Aim - evenly distribute sequence clusters across a specified number of buckets. -## The goal is to have the total length of sequences in each bucket as equal as possible. - -## Number of buckets can be automatically selected -## (e.g., to avoid the DADA2s' error message `long vectors not supported yet`, related with > 2^31 elements) - -## Usage examples: -# bucketize_db.R \ -# --db stat_clusters.txt \ -# --fasta Input.fa.gz \ -# --summary bucket_summary.txt \ -# --numbuckets 10 \ -# --threads 10 - - -## Check time -start_time <- Sys.time() - -cat("\nParsing input options and arguments...\n") - -suppressPackageStartupMessages(require(optparse)) - -## Parse arguments -option_list <- list( - make_option(c("-d", "--db"), action="store", default="DB_clu.tsv", type='character', help="Clustering database"), - make_option(c("-f", "--fasta"), action="store", default="Input.fa.gz", type='character', help="Input sequences in FASTA format"), - make_option(c("-s", "--summary"), action="store", default="bucket_summary.txt", type='character', help="Output file summary information"), - make_option(c("-n", "--numbuckets"), action="store", default=NA, type='integer', help="Number of output buckets (NA, for automatic selection)"), - make_option(c("-t", "--threads"), action="store", default=4, type='integer', help="Number of CPU threads to use") -) -opt <- parse_args(OptionParser(option_list=option_list)) - -# Validation of the required arguments -if(is.na(opt$fasta)){ - stop("Input file with sequences is not specified\n") -} -if(is.na(opt$db)){ - stop("Clustering results are not specified\n") -} -if(!is.na(opt$numbuckets) & opt$numbuckets <= 1){ - stop("Number of buckets should be > 1\n") -} - - -## Assign variables -DATABASE <- opt$db -FASTA <- opt$fasta -SUMMARY <- opt$summary -NBUCKETS <- opt$numbuckets -THREADS <- opt$threads - -## Log assigned variables -cat("\nParameters specified:\n") -cat(paste("Clustering database: " , DATABASE, "\n", sep = "")) -cat(paste("Input sequences (FASTA): " , FASTA, "\n", sep = "")) -cat(paste("Output with bucket summary: ", SUMMARY, "\n", sep = "")) -if(is.na(NBUCKETS)){ - cat(paste("Number of buckets: ", "auto", "\n", sep = "")) -} else { - cat(paste("Number of buckets: ", NBUCKETS, "\n", sep = "")) -} - -cat(paste("CPU threads: ", THREADS, "\n", sep = "")) -cat("\n") - - -############################################## Load packages - -cat("Loading R packages...\n") - -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("data.table") -load_pckg("Biostrings") -load_pckg("plyr") - -if(THREADS < 1){ THREADS <- 1 } -if(THREADS > 1){ - cat("Preparing multi-threaded setup\n") - - load_pckg("doFuture") - registerDoFuture() - plan(multicore, workers = THREADS) - options(future.globals.maxSize = 6e10) # 60GB - - setDTthreads(threads = THREADS) # for data.table - - parall <- TRUE - -} else { - parall <- FALSE - setDTthreads(threads = 1) -} - -cat("\n") - - -############################################## Workflow - -## Load seq stats -cat("..Loading input sequences\n") -seqs <- readDNAStringSet(filepath = FASTA) - -## Load clustering file -cat("..Loading clustering file\n") -DB <- fread(file = DATABASE, - sep = "\t", header = FALSE, - col.names = c("Cluster", "Member")) - -## Estimate sequence length -cat("..Estimating total length of the sequences\n") -seqt <- data.table(Member = names(seqs), Len = width(seqs)) -DB <- merge(x = DB, y = seqt, by = "Member", all.x = TRUE) -rm(seqt) - -## Estimate number of sequences per cluster and the total length of sequences -cat("..Estimating cluster sizes\n") -datt <- DB[ , .(num_seqs = .N, sum_len = sum(Len, na.rm = TRUE)), by = "Cluster" ] - -## Sort clusters by the number of sequenes in descending order -cat("..Sorting clusters\n") -setorder(datt, -sum_len, -num_seqs) - - -cat("..Bucketizing\n") - -if(is.na(NBUCKETS)){ - cat("...Number of buckets is not specified, using automatic selection\n") - - ## For DADA2, a matrix with quality values is required `as(Biostrings::quality(fq), "matrix")` - ## It should not exceed 2^31 (2147483648) elemens, - ## Meaning that `num_seq * len_seq` must be < 2^31 - - ## Calculate approximate estimate for the maximum number of sequences per bucket - maxseqs <- 2^31 / max(DB$Len) # quantile(x = DB$Len, probs = 0.99) - - ## Number of buckets - NBUCKETS <- ceiling(nrow(DB) / maxseqs) - - cat("...The sugested number of buckets is ", NBUCKETS, "\n") -} - - -## Initializing buckets and bucket sizes -buckets <- vector("list", length = NBUCKETS) -bucket_size_numseqs <- numeric(NBUCKETS) -bucket_size_lenseqs <- numeric(NBUCKETS) - -## Distributing files into buckets -## By starting with the largest files and placing each one in the currently smallest bucket, -## we try to prevent any single bucket from becoming significantly larger than the others -for (i in 1:nrow(datt)) { - - ## Find the bucket with the minimum total sequence length - min_bucket_index <- which.min(bucket_size_lenseqs) - - ## Add the cluster ID to the chosen bucket - buckets[[ min_bucket_index ]] <- c( - buckets[[ min_bucket_index ]], - datt[i, ]$Cluster - ) - - # Updating the total sequence length of the chosen bucket - bucket_size_lenseqs[ min_bucket_index ] <- bucket_size_lenseqs[min_bucket_index] + datt[i, ]$sum_len - bucket_size_numseqs[ min_bucket_index ] <- bucket_size_numseqs[min_bucket_index] + datt[i, ]$num_seqs - -} - -cat("..Bucket summary:\n\n") - -## Prepare bucket summary -smr <- data.table( - BucketID = 1:length(buckets), - Num_clusters = laply(.data = buckets, .fun = function(x){ length(x) }), - sum_len = bucket_size_lenseqs, - num_seqs = bucket_size_numseqs) - -print(smr) - -## Add percentages -smr[ , NumClust_Percent := round(Num_clusters / sum(Num_clusters) * 100, 2) ] -smr[ , TotLen_Percent := round(sum_len / sum(sum_len) * 100, 2) ] -smr[ , TotSeqs_Percent := round(num_seqs / sum(num_seqs) * 100, 2) ] - - -cat("\n\n..Exporting FASTA file for each bucket\n") - -## Exporting function -export_bucket <- function(clustnum = 1){ - - cat("...Bucket ", clustnum, "\n") - - ## IDs of cluster representatives - clustids <- buckets[[ clustnum ]] - - ## Find sequence IDs to export - ids <- data.table(SeqID = DB[ Cluster %in% clustids ]$Member) - - ## Sort sequences by size - ids[ , Size := tstrsplit(SeqID, split = ";", keep = 2) ] - ids[ , Size := as.numeric( sub(pattern = "size=", replacement = "", x = Size) ) ] - setorder(ids, -Size, SeqID) - - ## Cluster ID with leading zero - cl <- sprintf(paste0("%0", nchar(NBUCKETS), "d"), clustnum) - - ## Extract and export - writeXStringSet( - x = seqs[ ids$SeqID ], - filepath = paste0("bucket_", cl, ".fa.gz"), - compress = TRUE, - format = "fasta", - width = 9999) - -} - -a_ply( - .data = seq_along(buckets), - .margins = 1, - .fun = export_bucket, - .parallel = parall) - - -## Bucket summary -cat("..Exporting bucket summary\n") -fwrite(x = smr, file = SUMMARY, sep = "\t", col.names = TRUE) - - -cat("\nAll done.\n") - - -##################### Session info - -## Check time -end_time <- Sys.time() - -tmm <- as.numeric(difftime(end_time, start_time, units = "min")) -cat("\nElapsed time: ", tmm, " minutes\n") - -cat("\n") -cat("Session info:\n") -sessionInfo() -cat("\n") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/chimera_rescue.R b/src/pipecraft-core/service_scripts/NextITS/bin/chimera_rescue.R deleted file mode 100755 index b53f3a3f..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/chimera_rescue.R +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to rescue sequences that were annotated as chimeric, -## but have high occurrence within sequenceing run (occurrence > 2) - -# Input is given as positional arguments: -# 1. List of all chimeric sequences (`All_chimeras.txt.gz`) -# 2. Min sequence occurrence to be preserved (e.g., 2) -# 3. Output file name (`Rescued_Chimeric_sequences.fa.gz`) - -suppressMessages(library(data.table)); setDTthreads(threads = 1) -suppressMessages(library(Biostrings)) - -args <- commandArgs(trailingOnly = TRUE) - -MINOCC <- as.numeric( args[2] ) - -## Load sequences -cat("..Loading chimeric sequences\n") -CH <- try( - fread(file = args[1], - sep = "\t", header = F, - col.names = c("SeqID", "Seq")) - ) - -if("try-error" %in% class(CH)){ - cat("\nCould not read the file with chimeric sequences\n") - cat("Most likely, the file file is empty (no chimeras)\n") - q(save = "no", status = 0, runLast = FALSE) -} - -cat("..Total number of chimeric records: ", nrow(CH), "\n") - -if(nrow(CH) > 0){ - - ## Extract sample name and sequencing run ID - CH[, SampleID := tstrsplit(x = SeqID, split = ";", keep = 2) ] - CH[, SampleID := gsub(pattern = "sample=", replacement = "", x = SampleID) ] - - ## Estimate sequence frequency - cat("..Estimating chimera occurrence\n") - CF <- CH[, .(Occurrence = .N), by = "Seq"] - - ## Exclude sequences with low occurrence (most probably chimeric) - ## Sequences with higher occurrence should be "real" sequences - CF <- CF[ Occurrence > MINOCC ] - - cat("..Total number of unique chimeric sequences: ", length(unique(CH$Seq)), ".\n") - - ## Export sequences - if(nrow(CF) > 0){ - cat("..There are", nrow(CF), "unique sequence to rescue.\n") - NCH <- CH[ Seq %in% CF$Seq ] - SQS <- DNAStringSet(x = NCH$Seq) - names(SQS) <- NCH$SeqID - - cat("..Exporting rescued sequences\n") - writeXStringSet(x = SQS, filepath = args[3], - compress=TRUE, format="fasta", width=9999) - } else { - cat("..No sequences were rescued.\n") - } - -} \ No newline at end of file diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/convert_IUPAC.sh b/src/pipecraft-core/service_scripts/NextITS/bin/convert_IUPAC.sh deleted file mode 100755 index 4a250b74..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/convert_IUPAC.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -## Function to convert IUPAC codes in primers -# Based on PipeCraft2 scripts -# https://github.com/SuvalineVana/pipecraft/blob/main/src/pipecraft-core/service_scripts/submodules/framework.functions.sh -# Git commit 5650545 (Jun 9, 2022) -# Author - Sten Anslan - -echo "$1" | \ -if grep -q -E "R|Y|S|W|K|M|B|D|H|V|N|I" ; then - - ## Define IUPAC codes - R=$"[AG]" - Y=$"[CT]" - S=$"[GC]" - W=$"[AT]" - K=$"[GT]" - M=$"[AC]" - B=$"[CGT]" - D=$"[AGT]" - H=$"[ACT]" - V=$"[ACG]" - N=$"[ATGC]" - I=$"[ATGC]" - - ## Replace IUPAC codes - primer=$(echo "$1" | \ - sed -e "s/R/$R/g; s/Y/$Y/g; \ - s/S/$S/g; s/W/$W/g; s/K/$K/g; \ - s/M/$M/g; s/B/$B/g; s/D/$D/g; \ - s/H/$H/g; s/V/$V/g; s/N/$N/g; \ - s/I/$I/g") - - ## Return convered primer - echo "$primer" -else - ## Return original primer when no IUPAC codes were detected - echo "$1" -fi - -## Example: -# ./convert_IUPAC.sh "CGACCWGCGGARGGATCATTA" # CGACC[AT]GCGGA[AG]GGATCATTA diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/count_homopolymer_stats.sh b/src/pipecraft-core/service_scripts/NextITS/bin/count_homopolymer_stats.sh deleted file mode 100755 index c2c4af75..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/count_homopolymer_stats.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# $1 = input file -# $2 = text to add to the resulting file - -zcat "$1" \ - | awk \ - -F '\t' -v OFS='\t' \ - -v fnm="$2" \ - '$1 ~ /H/ { print fnm , $9 , $10 }' \ - | sed 's/_uch.uc//' - diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/count_number_of_reads.sh b/src/pipecraft-core/service_scripts/NextITS/bin/count_number_of_reads.sh deleted file mode 100755 index ae811632..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/count_number_of_reads.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -## Count number of reads in the dereplicated file -## Size annotations should be in USEARCH-style (e.g., size=100) - -# $1 = input file -# $2 = text to add to the resulting file - -seqkit seq --name "$1" \ - | grep -Po ';size=[0-9]+' \ - | sed 's/;size=//g' \ - | awk -F '\t' -v OFS='\t' -v fnm="$2" '{sum+=$1} END {print fnm , sum}' diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/dada2_no_quals.R b/src/pipecraft-core/service_scripts/NextITS/bin/dada2_no_quals.R deleted file mode 100755 index 930c4363..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/dada2_no_quals.R +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env Rscript - -## Perform sequence denoising with DADA2 - -### Notes: -## - `USE_QUALS = FALSE` will be deprecated -## https://github.com/benjjneb/dada2/issues/816#issuecomment-521836313 -## therefore, use `noqualErrfun` -## - - -## TO DO: -## - Benchmark params (especially BAND_SIZE, DETECT_SINGLETONS, and OMEGA_A) - - -## Usage example: -# dada2_no_quals.R \ -# --input input.fq.gz \ -# --nbases 1e6 \ -# --bandsize 16 \ -# --detectsingletons TRUE \ -# --omegaA 1e-20 \ -# --omegaC 1e-40 \ -# --omegaP 1e-4 \ -# --maxconsist 10 \ -# --match 4 \ -# --mismatch -5 \ -# --gappenalty -8 \ -# --threads 8 - -## Outputs: -# - DADA2_ErrorRates_noqualErrfun.RData -# - DADA2_denoised.fa.gz -# - DADA2_denoised.uc.gz -# - DADA2_UC.qs -# - DADA2_denoising_summary.txt - - -############################################## Parse input parameters - -## Check time -start_time <- Sys.time() - - -cat("\nParsing input options and arguments...\n") - -suppressPackageStartupMessages(require(optparse)) - -## Parse arguments -option_list <- list( - make_option(c("-i", "--input"), action="store", default=NA, type='character', help=""), - make_option(c("-n", "--nbases"), action="store", default=1e6, type='double', help=""), - make_option(c("-b", "--bandsize"), action="store", default=16, type='double', help=""), - make_option(c("-s", "--detectsingletons"), action="store", default=TRUE, type='logical', help=""), - make_option(c("-A", "--omegaA"), action="store", default=1e-20, type='double', help=""), - make_option(c("-C", "--omegaC"), action="store", default=1e-40, type='double', help=""), - make_option(c("-P", "--omegaP"), action="store", default=1e-4, type='double', help=""), - make_option(c("-x", "--maxconsist"), action="store", default=10, type='integer', help=""), - make_option("--match", action="store", default=4, type='double', help=""), - make_option("--mismatch", action="store", default=-5, type='double', help=""), - make_option("--gappenalty", action="store", default=-8, type='double', help=""), - make_option("--hpgap", action="store", default=NULL, type='double', help=""), - make_option(c("-t", "--threads"), action="store", default=4L, type='integer', help="Number of CPU threads for arrow, default 4") -) -opt <- parse_args(OptionParser(option_list=option_list)) - -# Validation of the required argiments -if(is.na(opt$input)){ - cat("Input file is not specified: ....\n", file=stderr()) - stop() -} - - -## Function to convert text "NA"s to NA -# to_na <- function(x){ -# if(x %in% c("NA", "null", "Null")){ x <- NA } -# return(x) -# } - -## Assign variables -INPUT <- opt$input -NBASES <- opt$nbases -BAND_SIZE <- opt$bandsize -DETECT_SINGLETONS <- opt$detectsingletons -OMEGA_A <- opt$omegaA -OMEGA_C <- opt$omegaC -OMEGA_P <- opt$omegaP -MAX_CONSIST <- opt$maxconsist -MATCH <- opt$match -MISMATCH <- opt$mismatch -GAP_PENALTY <- opt$gappenalty -HOMOPOLYMER_GAP_PENALTY <- opt$hpgap # PacBio CCS does not make homopolymer errors at a higher rate than normal indels -> NULL -CPUTHREADS <- opt$threads - - -## Log assigned variables -cat("\nParameters specified:\n") -cat(paste("Input file: " , INPUT, "\n", sep="")) -cat(paste("Number of bases to use for error rate learning: ", NBASES, "\n", sep = "")) -cat(paste("Band size for the Needleman-Wunsch alignment: ", BAND_SIZE, "\n", sep = "")) -cat(paste("Singleton detection: ", DETECT_SINGLETONS, "\n", sep = "")) -cat(paste("OMEGA_A: ", OMEGA_A, "\n", sep = "")) -cat(paste("OMEGA_C: ", OMEGA_C, "\n", sep = "")) -cat(paste("OMEGA_P: ", OMEGA_P, "\n", sep = "")) -cat(paste("Number of iterations of the self-consistency loop: ", MAX_CONSIST, "\n", sep = "")) -cat(paste("Alignment for matches: ", MATCH, "\n", sep = "")) -cat(paste("Alignment for mismatches: ", MISMATCH, "\n", sep = "")) -cat(paste("Gap penalty: ", GAP_PENALTY, "\n", sep = "")) -cat(paste("Homopolymer gap penalty: ", HOMOPOLYMER_GAP_PENALTY, "\n", sep = "")) -cat(paste("Number of CPU threads to use: ", CPUTHREADS, "\n", sep="")) - -cat("\n") - - - -############################################## Load packages - -cat("Loading R packages...\n") - -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("Biostrings") -load_pckg("ShortRead") -load_pckg("data.table") -load_pckg("dada2") - -cat("\n") - -## Set CPU thread number -cat("Setting number of CPU threads to: ", CPUTHREADS, "\n") -setDTthreads(threads = CPUTHREADS) # for data.table - -## Set seed -set.seed(111) - -## Set DADA options -cat("Setting DADA2 options\n") -setDadaOpt( - BAND_SIZE = BAND_SIZE, # dada2 default, 16 - DETECT_SINGLETONS = DETECT_SINGLETONS, # dada2 default, FALSE - OMEGA_A = OMEGA_A, # dada2 default, 1e-40 - OMEGA_C = OMEGA_C, # dada2 default, 1e-40 - OMEGA_P = OMEGA_P, # dada2 default, 1e-4 - MAX_CONSIST = MAX_CONSIST, # dada2 default, 10 - GAP_PENALTY = GAP_PENALTY, # dada2 default, -8 - MATCH = MATCH, # dada2 default, 4 - MISMATCH = MISMATCH, # dada2 default, -5 - HOMOPOLYMER_GAP_PENALTY = HOMOPOLYMER_GAP_PENALTY # PacBio CCS does not make homopolymer errors at a higher rate than normal indels - ) - -## Get DADA options -# getDadaOpt() - -############################################## Workflow - - -## Load FASTQ file -cat("\nLoading input data\n") -fq <- readFastq(dirPath = INPUT, qualityType = "FastqQuality") - -## Extract sequence headers -cat("Processing sequences\n") -sq <- as.data.table(fq@id) -setnames(x = sq, new = "SeqName") -sq[ , c("SeqID", "Abundance") := tstrsplit(x = SeqName, split = ";size=", keep = 1:2) ] -sq[ , Abundance := as.numeric(Abundance) ] -sq[ , Sequence := as.character(sread(fq))] - -## Extract sequence qualities -cat("Processing sequence quality scores\n") -seq_quals <- as(quality(fq), "matrix") -# dada2:::qtables2(fq) - -## Summary stats -num_seqs <- nrow(sq) -num_singl <- nrow(sq[ Abundance < 2 ]) -num_reads <- sum(sq$Abundance, na.rm = TRUE) -perc_nonsingleton <- round((num_seqs - num_singl) / num_seqs * 100, 2) - -cat("\n") -cat("Number of unique sequences detected: ", num_seqs, "\n") -cat("Number of singleton sequences: ", num_singl, "\n") -cat("Total abundance of sequences: ", num_reads, "\n") -cat("Percentage of non-singleton sequences: ", round(perc_nonsingleton, 2), "\n") - -## Test the rule of thumb, https://github.com/benjjneb/dada2/issues/1663#issuecomment-1359905397 -if(perc_nonsingleton < 10){ - cat("WARNING: <10% of reads are duplicates of other reads,\n") - cat(" meaning that DADA2 might not be the right algorithmic choice\n") -} - - -## Manually create a derep-class object -## See also https://github.com/benjjneb/dada2/blob/004ce26909268e1318a2f68e0ea26807412c7a2d/R/sequenceIO.R#L240-L242 -# https://github.com/benjjneb/dada2/blob/004ce26909268e1318a2f68e0ea26807412c7a2d/R/sequenceIO.R#L45 - -## Prepare derep-class object -cat("\nPreparing derep-class object\n") -uniques <- sq$Abundance -names(uniques) <- as.character(sread(fq)) # names = full amplicon sequence -rownames(seq_quals) <- names(uniques) - -derep <- list( - uniques = uniques, - quals = seq_quals, - map = NULL, - SeqID = sq$SeqID # add allso sequence IDs - ) - -derep <- as(derep, "derep") - -## Clean up -rm(uniques, seq_quals) - - - -## Estimate error rates for each type of transition while ignoring quality scores -cat("\nEstimating error rates\n") -errors <- try( - learnErrors( - fls = derep, - nbases = NBASES, - errorEstimationFunction = noqualErrfun, - qualityType = "FastqQuality", - verbose = 1, - multithread = CPUTHREADS - ) - ) - -## Retry if multithreading failed -if("try-error" %in% class(errors) & CPUTHREADS > 1){ - cat("..Multi-threaded error rate estimation failed\n") - cat("..Trying to resume with a single CPU thread\n") - - errors <- learnErrors( - fls = derep, - nbases = NBASES, - errorEstimationFunction = noqualErrfun, - qualityType = "FastqQuality", - verbose = 1, - multithread = 1 - ) -} -if("try-error" %in% class(errors) & CPUTHREADS == 1){ - stop("..Error rate estimation failed\n") -} - - -## Export results -cat("\nExporting error rates\n") -saveRDS(object = errors, - file = "DADA2_ErrorRates_noqualErrfun.RData", - compress = "xz") - - -## Plot observed and estimated error rates -# plotErrors(errors) - - -## Run sample inference with DADA2 -cat("\nRunning sample inference\n") -dadares <- dada( - derep = derep, - err = errors, - errorEstimationFunction = noqualErrfun, - selfConsist = FALSE, - verbose = 1, - multithread = CPUTHREADS) - -cat("\nExporting DADA2 object\n") -saveRDS(object = dadares, - file = "DADA2_InferedSeqs_noqualErrfun.RData", - compress = "xz") - - -## Prepare resulting data -cat("Preparing resulting table\n") -res <- data.table( - Sequence = dadares$sequence, - Abundance = dadares$denoised) - -## Add sequence IDs -res[ , SeqNumID := .I ] -res <- merge( - x = res, - y = sq[, .(SeqID, Sequence)], - by = "Sequence", all.x = TRUE) - -## Sort by abundance -setorder(res, -Abundance, SeqID, na.last = TRUE) - -## Export denoised sequences -cat("Exporting denoised sequences\n") -ASVS <- DNAStringSet(x = res$Sequence) -names(ASVS) <- paste0(res$SeqID, ";size=", res$Abundance) - -writeXStringSet( - x = ASVS, - filepath = "DADA2_denoised.fa.gz", - compress = TRUE, - format = "fasta", - width = 20000) - - - -## Create UC file -cat("Preparing pseudo-UC file\n") -UC <- data.table( - DerepSeqID = derep$SeqID, - SeqNumID = dadares$map, - Abundance = derep$uniques) - -UC <- merge( - x = UC, - y = res[ , .(SeqNumID, SeqID) ], - by = "SeqNumID", all.x = TRUE) - -setorder(UC, SeqNumID, na.last = TRUE) -setnames(x = UC, old = "SeqID", new = "ASV") - -## Export pre-UC file -cat("Exporting pre-UC file\n") -# saveRDS( -# object = UC, -# file = "DADA2_UC.RData", -# compress = "xz") -qs::qsave(x = UC, file = "DADA2_UC.qs", - preset = "custom", algorithm = "zstd", compress_level = 15L, nthreads = CPUTHREADS) - -## Summary stats -num_asvs <- nrow(res) -num_asvreads <- sum(res$Abundance, na.rm = T) -num_merged <- nrow(UC[ !is.na(SeqNumID) & DerepSeqID != ASV ]) # excluding representative seqs -num_dsc <- nrow(UC[ is.na(SeqNumID) ]) -num_dscreads <- sum(UC[ is.na(SeqNumID) ]$Abundance) -perc_dsc <- round(num_dsc / num_seqs * 100, 2) -perc_dscreads <- round(num_dscreads / num_reads * 100, 2) - -cat("\nRun summary:\n") -cat("Number of ASVs infered: ", num_asvs, "\n") -cat("Number of reads in ASV table: ", num_asvreads , "\n") -cat("Number of sequences merged into ASVs: ", num_merged, "\n") -cat("Number of discarded sequences (%): ", num_dsc, "(", perc_dsc, "% )\n") -cat("Number of reads of discarded sequences (%): ", num_dscreads, "(", perc_dscreads, "% )\n") - - -## Format pseudo-UC file -# 1 Record type S, H, C or N (see table below) -# 2 Cluster number (0-based) -# 3 Sequence length (S, N and H) or cluster size (C) -# 4 For H records, percent identity with target -# 5 For H records, the strand: + or - for nucleotides, . for proteins -# 6 Not used, parsers should ignore this field. Included for backwards compatibility -# 7 Not used, parsers should ignore this field. Included for backwards compatibility -# 8 Compressed alignment or the symbol '=' (equals sign). The = indicates that the query is 100% identical to the target sequence (field 10) -# 9 Label of query sequence (always present) -# 10 Label of target sequence (H records only) - -## Remove noisy sequences -UC <- UC[ ! is.na(SeqNumID), .(DerepSeqID, ASV) ] -UC[ , RecordType := fifelse(DerepSeqID == ASV, "C", "H", na = NA) ] -UC[ , `:=` (ClustNum = NA, SeqLen = NA, Ident = NA, Strand = "+", V6 = NA, V7 = NA, ALN = ".") ] - -setcolorder(x = UC, - neworder = c("RecordType", "ClustNum", "SeqLen", "Ident", "Strand", "V6", "V7", "ALN", "DerepSeqID", "ASV")) - -## Export UC file -cat("\nExporting pseudo-UC file\n") -fwrite(x = UC, - file = "DADA2_denoised.uc.gz", - quote = FALSE, sep = "\t", - col.names = FALSE, row.names = FALSE, - compress = "gzip") - - - -## Write summary -cat("Exporting run statistics\n") -smr <- rbind( - data.table(Param = "Number of unique sequences (prior denoising)", Value = num_seqs), - data.table(Param = "Number of singleton sequences (prior denoising)", Value = num_singl), - data.table(Param = "Total abundance of sequences (prior denoising)", Value = num_reads), - data.table(Param = "Percentage of non-singleton sequences (prior denoising)", Value = perc_nonsingleton), - - data.table(Param = "Number of ASVs infered", Value = num_asvs), - data.table(Param = "Number of reads in ASV table", Value = num_asvreads), - data.table(Param = "Number of sequences merged into ASVs (excluding representative seqs)", Value = num_merged), - data.table(Param = "Number of discarded sequences", Value = num_dsc), - data.table(Param = "Percentage of discarded sequences", Value = perc_dsc), - data.table(Param = "Number of reads of discarded sequences", Value = num_dscreads), - data.table(Param = "Percentage of reads of discarded sequences", Value = perc_dscreads) - ) - -fwrite(x = smr, - file = "DADA2_denoising_summary.txt", - quote = FALSE, sep = "\t") - - - -## Construct sequence table (rows = samples, cols = ASVs) -# makeSequenceTable(dadares, orderBy = "abundance") - - - -cat("\nAll done.\n") - - -##################### Session info - -## Check time -end_time <- Sys.time() - -tmm <- as.numeric(difftime(end_time, start_time, units = "min")) -cat("\nElapsed time: ", tmm, " minutes\n") - -cat("\n") -cat("Session info:\n") -sessionInfo() -cat("\n") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/disambiguate_primers.R b/src/pipecraft-core/service_scripts/NextITS/bin/disambiguate_primers.R deleted file mode 100755 index 0ba61102..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/disambiguate_primers.R +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env Rscript - -## The script to disambiguate sequences -## (expand ambiguous nucleotides into all combinations) -## Based on IUPAC codes - -# Input is given as positional arguments: -# 1. A text string (e.g., "ACTGNK") -# 2. output file name (e.g., "Primer_F.fasta") - -# Output: -# - FASTA with disambiguated sequences - -args <- commandArgs(trailingOnly = TRUE) - -cat("..Loading packages\n") -suppressMessages(library(DECIPHER)) -suppressMessages(library(Biostrings)) - -## Convert input string into DNAStringSet object -cat("..Preparing DNAStringSet\n") -dna <- DNAStringSet(args[1]) - -## Disambiguate -cat("..Disambiguating\n") -res <- Disambiguate(dna)[[1]] - -## Assign names -names(res) <- paste0("seq", 1:length(res), sep = "") - -## Export FASTA -cat("..Exporting FASTA\n") -writeXStringSet(x = res, - filepath = args[2], - compress=FALSE, format="fasta", width=9999) - -cat("..done\n") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/document_s1.R b/src/pipecraft-core/service_scripts/NextITS/bin/document_s1.R deleted file mode 100755 index 755bb888..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/document_s1.R +++ /dev/null @@ -1,337 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to document the Step-1 workflow of the NextITS pipeline. - -## Usage: -## Rscript document_s1.R [output_path] - -## Input: -## - software_versions.yml -## - pipeline_params.tsv - -## Output: -## - README_Step1_Methods.txt -## with two sections: methods and references - - -## Function to load packages -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("glue") -load_pckg("data.table") -load_pckg("yaml") - -## Parse arguments -args <- commandArgs(trailingOnly = TRUE) -if (length(args) < 2) { - cat("Usage: document_s1.R [output_path]\n") - stop() -} - -versions_path <- args[[1]] -params_path <- args[[2]] -output_path <- ifelse(length(args) >= 3, args[[3]], "README_Step1_Methods.txt") - - -## Validation -if(is.null(versions_path) || versions_path == ""){ - stop("Versions YAML not specified") -} -if(is.null(params_path) || params_path == ""){ - stop("Params table not specified") -} - -if(!file.exists(versions_path)){ - stop(glue("Versions YAML not found: {versions_path}")) -} -if(!file.exists(params_path)){ - stop(glue("Params table not found: {params_path}")) -} - - -################################## -################################## Data for debugging -################################## - -# versions_path <- "software_versions.yml" -# params_path <- "pipeline_params.tsv" -# output_path <- "README_Step1_Methods.txt" - - -################################## -################################## References -################################## - -## Citation registry -citation_db <- list( - nextits = "Mikryukov V, Anslan S, Tedersoo L (2025) NextITS - A pipeline for metabarcoding fungi and other eukaryotes with full-length ITS sequenced with PacBio. DOI:10.5281/zenodo.15074882", - nextflow = "Di Tommaso P, et al. (2017) Nextflow enables reproducible computational workflows. Nat Biotechnol 35, 316-319, DOI:10.1038/nbt.3820", - lima = "Pacific Biosciences (2025) LIMA - The PacBio barcode demultiplexer and primer remover. URL: https://lima.how/", - seqkit = "Shen W, Sipos B, Zhao L (2024) SeqKit2: A Swiss Army Knife for Sequence and Alignment Processing. iMeta e191. DOI:10.1002/imt2.191", - csvtk = "Shen W (2025) csvtk - a cross-platform, efficient and practical CSV/TSV toolkit. URL: https://github.com/shenwei356/csvtk", -# brename = "Shen W (2025) brename - batch renaming safely, URL: https://github.com/shenwei356/brename", - cutadapt = "Martin M (2011) Cutadapt removes adapter sequences. EMBnet.journal 17(1):10-12, DOI:10.14806/ej.17.1.200", - itsx = "Bengtsson-Palme J, et al (2013) Improved software detection and extraction of ITS1 and ITS2 from ribosomal ITS sequences of fungi and other eukaryotes for analysis of environmental sequencing data. Methods Ecol Evol 4:914-919, DOI:10.1111/2041-210X.12073", - vsearch = "Rognes T, Flouri T, Nichols B, Quince C, Mahé F (2016) VSEARCH: a versatile open source tool for metagenomics. PeerJ 4:e2584. DOI:10.7717/peerj.2584", - uchime2 = "Edgar RC (2016) UCHIME2: improved chimera prediction for amplicon sequencing. bioRxiv 074252. DOI:10.1101/074252", - uncross2 = "Edgar RC (2018) UNCROSS2: identification of cross-talk in 16S rRNA OTU tables. bioRxiv 400762. DOI:10.1101/400762", - chimscore = "Nilsson RH, et al. (2015) A Comprehensive, Automatically Updated Fungal ITS Sequence Dataset for Reference-Based Chimera Control in Environmental Sequencing Efforts. Microbes Environ. 30(2), 145-50. DOI:10.1264/jsme2.ME14121", - bedtools = "Quinlan AR, Hall IM (2010) BEDTools: a flexible suite of utilities for comparing genomic features. Bioinformatics 26:841-842. DOI:10.1093/bioinformatics/btq033", - duckdb = "Raasveldt M, Mühleisen H (2019) DuckDB: an Embeddable Analytical Database. SIGMOD '19: Proceedings of the 2019 International Conference on Management of Data, 1981-1984. DOI:10.1145/3299869.332021", - parallel = "Tange O (2011) GNU Parallel: The command-line power tool. Usenix Mag 36 (1), 42", - eukaryome = "Tedersoo L, et al. (2024). EUKARYOME: the rRNA gene reference database for identification of all eukaryotes. Database (Oxford) 12:baae043. DOI:10.1093/database/baae043", - R = "R Core Team (2025) R: A Language and Environment for Statistical Computing. R Foundation for Statistical Computing, Vienna, Austria. URL: https://www.R-project.org/", - arrow = "Richardson N, Cook I, Crane N, Dunnington D, François R, Keane J, Moldovan-Grünfeld D, Ooms J, Wujciak-Jens J, and Apache Arrow (2025) arrow: Integration to Apache Arrow. URL: https://github.com/apache/arrow/", - ggplot2 = "Wickham H (2016) ggplot2: Elegant Graphics for Data Analysis. Springer. DOI:10.1007/978-3-319-24277-4", - biostrings= "Pagès H, Aboyoun P, Gentleman R, DebRoy S (2025) Biostrings: Efficient manipulation of biological strings. DOI:10.18129/B9.bioc.Biostrings", - datatable = "Barrett T, Dowle M, Srinivasan A, Gorecki J, Chirico M, Hocking T, Schwendinger B, Krylov I (2025) data.table: Extension of data.frame. URL: " -) - - -################################## -################################## Helpers -################################## - -## Get version number -getv <- function(v, process, tool){ - # v = list (from YAML file) - # process = process name - # tool = tool name - - if(is.null(v[[process]]) || is.null(v[[process]][[tool]])){ return("") } - as.character( v[[process]][[tool]] ) -} -# E.g., getv(versions, "demux", "lima") - - -## Get parameter -getp <- function(p, pname, default = NA){ - # p = table with parameters (two columns: name and value) - # pname = parameter name - # default = default value if parameter is not found - - pp <- p[ name == pname ]$value - if(is.null(pp) || is.na(pp)){ return(default) } - return(pp) -} -# E.g., getp(params, "lima_minscore", 93) - - -## Remove NAs and empty strings (to curate the citations) -trim_na <- function(x){ - x[ !is.na(x) & nzchar(x) ] -} - -################################## -################################## Body builders -################################## - -emit_nextits <- function(v) { - nextits_v <- if(!is.null(v$NextITS$version)){ as.character(v$NextITS$version) } else { "" } - glue("Bioinformatic processing was performed using the \\ - NextITS pipeline v.{nextits_v} (Mikryukov et al., 2025).") -} - -emit_nextflow <- function(v) { - nextflow_v <- if(!is.null(v$Nextflow$version)){ as.character(v$Nextflow$version) } else { "" } - glue("Workflow management was performed using \\ - Nextflow v.{nextflow_v} (Di Tommaso et al., 2017).") -} - -emit_demux_pacbio <- function(p, v) { - ms <- getp(p, "lima_minscore", 93) - mb <- getp(p, "lima_barcodetype", "dual_symmetric") - vs <- getv(v, "demux", "lima") - switch(mb, - "single" = {barcode_type <- "single-end barcodes"}, - "dual_symmetric" = {barcode_type <- "symmetric dual-end barcodes"}, - "dual_asymmetric" = {barcode_type <- "asymmetric dual-end barcodes"}, - "dual" = {barcode_type <- "combination of symmetric and asymmetric dual-end barcodes"}) - - glue("Demultiplexed PacBio reads using LIMA v.{vs} (Pacific Biosciences) with min score {ms} and {barcode_type}.") -} - -emit_qc_pacbio <- function(p, v) { - glue("Quality control was performed using \\ - VSEARCH v.{getv(v,'qc_se','vsearch')} (Rognes et al., 2016) and \\ - seqkit v.{getv(v,'qc_se','seqkit')} (Shen et al., 2024). \\ - Reads with the number of ambiguous bases >= {getp(p,'qc_maxn',4)}, \\ - expected error rate >= {getp(p,'qc_maxeerate',0.01)}, \\ - or homopolymer stretches longer than {getp(p,'qc_maxhomopolymerlen',25)} nt were removed.") -} - -# emit_demux_illumina <- function(p, v) { -# c( -# glue("- Illumina PE QC and merging; demultiplexed merged reads with cutadapt v.{getv(v,'primer_check','cutadapt')} using barcode window {getp(p,'barcode_window',30)}, max errors {getp(p,'barcode_errors',1)}, min overlap {getp(p,'barcode_overlap',11)}."), -# glue("- Non-merged reads optionally retained (join padding '{getp(p,'illumina_joinpadgap','NNNNNNNNNN')}').") -# ) -# } - -emit_primer_check <- function(p, v) { - glue("Primers were trimmed using \\ - cutadapt v.{getv(v, 'primer_check', 'cutadapt')} (Martin, 2011) \\ - with <= {getp(p, 'primer_mismatches', 2)} mismatches. \\ - Reads without both primers were discarded.") -} - -emit_itsx <- function(p, v) { - switch(getp(p,'its_region','full'), - "full" = {its_region <- "full-length ITS"}, - "SSU" = {its_region <- "SSU"}, - "ITS1" = {its_region <- "ITS1"}, - # "5_8S" = {its_region <- "5.8S"}, # not-yet-implemented - "ITS2" = {its_region <- "ITS2"}, - "LSU" = {its_region <- "LSU"}, - "ITS1_5.8S_ITS2" = {its_region <- "near-full-length ITS"}) - - - glue("Extraction of rRNA regions ({its_region}) was performed using \\ - ITSx v.{getv(v,'itsx','ITSx')} (Bengtsson-Palme et al., 2013).") -} - -emit_assemble_its <- function(p, v) { - glue("To assemble near-full-length ITS sequences, we ... (TODO)") -} - -emit_hp_and_chimeras <- function(p, v, did_hp) { - res <- character() - if(isTRUE(did_hp)){ - res <- c(res, glue( - "Homopolymer correction of sequences was performed using an algorithm implemented in NextITS \\ - with support of VSEARCH v.{getv(v,'homopolymer','vsearch')} and seqkit v.{getv(v,'homopolymer','seqkit')}.") ) - } else { - res <- c(res, "Homopolymer correction of sequences was not performed.") - } - res <- c(res, - glue( - "Two-step chimera detection was done using VSEARCH v.{getv(v,'chimera_denovo','vsearch')}: - - de novo using UCHIME2 algorithm (Edgar, 2016) with max score {getp(p,'max_ChimeraScore',0.6)} (Nilsson et al., 2015), - - then reference-based against the EUKARYOME database (Tedersoo et al., 2024).") - ) - res <- paste0(res, collapse = "\n") - return(res) -} - -emit_tj <- function(p, v) { - glue("Tag-jump detection and removal was performed using \\ - UNCROSS2 algorithm (Edgar, 2018) with the parameter f = {getp(p,'tj_f',0.01)}.") -} - -emit_seqtab <- function(p, v) { - glue("Sequence counts table was generated using \\ - R v.{getv(v,'prep_seqtab','R')} (R Core Team, 2025), \\ - data.table v.{getv(v,'prep_seqtab','data.table')} (Barrett et al., 2025), \\ - and Apache Arrow v.{getv(v,'prep_seqtab','arrow')} (Richardson et al., 2025) \\ - packages.") -} - - - -################################## -################################## Workflow-dependent method descriptions -################################## - -## Function to assembly the workflow description and references -build_docs <- function(versions, params){ - body <- character() - tools_used <- character() - - body <- c(body, emit_nextits(versions)) - tools_used <- c(tools_used, "nextits") - - body <- c(body, emit_nextflow(versions)) - tools_used <- c(tools_used, "nextflow") - - demuxed <- tolower(as.character(getp(params, "demultiplexed", FALSE))) %in% c("true", "t", "1") - platform <- getp(params, "seqplatform", "PacBio") - - if(!demuxed){ - if(platform %in% "PacBio"){ - - body <- c(body, emit_demux_pacbio(params, versions)) - tools_used <- c(tools_used, c("lima")) - - body <- c(body, emit_qc_pacbio(params, versions)) - tools_used <- c(tools_used, c("vsearch", "seqkit")) - - } else { - body <- c(body, emit_demux_illumina(params, versions)) - tools_used <- c(tools_used, c("cutadapt")) - } - } else { - if(platform %in% "PacBio"){ - body <- c(body, emit_qc_pacbio(params, versions)) - tools_used <- c(tools_used, c("vsearch", "seqkit")) - } else { - ## TODO - } - } - - ## Primer trimming - body <- c(body, emit_primer_check(params, versions)) - tools_used <- c(tools_used, c("cutadapt")) - - ## ITS extraction - its_region <- getp(params, "its_region", "full") - if(its_region %in% c("full", "ITS1", "ITS2", "SSU", "LSU")){ - body <- c(body, emit_itsx(params, versions)) - tools_used <- c(tools_used, c("itsx", "vsearch", "duckdb", "seqkit", "cutadapt")) - } else if (its_region %in% "ITS1_5.8S_ITS2") { - body <- c(body, emit_itsx(params, versions)) # , emit_assemble_its(params, versions)) - tools_used <- c(tools_used, c("itsx", "vsearch", "duckdb", "seqkit", "cutadapt")) - } - - did_hp <- tolower(as.character(getp(params, "hp", TRUE))) %in% c("true", "t", "1") - body <- c(body, emit_hp_and_chimeras(params, versions, did_hp)) - tools_used <- c(tools_used, c("vsearch", "uchime2", "eukaryome")) - - body <- c(body, emit_tj(params, versions)) - tools_used <- c(tools_used, c("uncross2")) - - body <- c(body, emit_seqtab(params, versions)) - tools_used <- c(tools_used, c("arrow", "datatable", "R")) - - tools_used <- unique(tools_used) - citations <- trim_na( unlist(citation_db[tools_used]) ) - citations <- sort(unique(citations)) - - res <- list( - body = body, - citations = citations) - - return(res) -} - - -################################## -################################## Assemble body and citations -################################## - -## Load inputs -cat("Loading versions YAML...\n") -versions <- yaml::read_yaml(versions_path) - -cat("Loading params table...\n") -params <- data.table::fread(params_path, sep = "\t", header = TRUE, na.strings = c("", "NA")) -setnames(params, new = c("name", "value")) - -## Build body and citations -res <- build_docs(versions, params) - -## Write output -con <- file(output_path, open = "wt") - -writeLines("Methods:", con) -writeLines(res$body, con) - -writeLines("", con) - -writeLines("References:", con) -writeLines(paste0("- ", res$citations), con) - -close(con) - -cat("All done.\n") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/document_s2.R b/src/pipecraft-core/service_scripts/NextITS/bin/document_s2.R deleted file mode 100755 index 1469ba39..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/document_s2.R +++ /dev/null @@ -1,339 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to document the Step-2 workflow of the NextITS pipeline. - -## Usage: -## Rscript document_s2.R [output_path] - - - -## Function to load packages -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("glue") -load_pckg("data.table") -load_pckg("yaml") - -## Parse arguments -args <- commandArgs(trailingOnly = TRUE) -if (length(args) < 2) { - cat("Usage: document_s2.R [output_path]\n") - stop() -} - -versions_path <- args[[1]] -params_path <- args[[2]] -output_path <- ifelse(length(args) >= 3, args[[3]], "README_Step2_Methods.txt") - - -## Validation -if(is.null(versions_path) || versions_path == ""){ - stop("Versions YAML not specified") -} -if(is.null(params_path) || params_path == ""){ - stop("Params table not specified") -} - -if(!file.exists(versions_path)){ - stop(glue("Versions YAML not found: {versions_path}")) -} -if(!file.exists(params_path)){ - stop(glue("Params table not found: {params_path}")) -} - - - -################################## -################################## References -################################## - -## Citation registry -citation_db <- list( - nextits = "Mikryukov V, Anslan S, Tedersoo L (2025) NextITS - A pipeline for metabarcoding fungi and other eukaryotes with full-length ITS sequenced with PacBio. DOI:10.5281/zenodo.15074882", - nextflow = "Di Tommaso P, et al. (2017) Nextflow enables reproducible computational workflows. Nat Biotechnol 35, 316-319, DOI:10.1038/nbt.3820", - vsearch = "Rognes T, Flouri T, Nichols B, Quince C, Mahé F (2016) VSEARCH: a versatile open source tool for metagenomics. PeerJ 4:e2584. DOI:10.7717/peerj.2584", - dada2 = "Callahan BJ, et al. (2016) DADA2: High-resolution sample inference from Illumina amplicon data. Nat Methods 13:581-583. DOI:10.1038/nmeth.3869", - unoise = "Edgar RC (2016) UNOISE2: improved error-correction for Illumina 16S and ITS amplicon sequencing. bioRxiv 081257. DOI:10.1101/081257", - swarm = "Mahé F, Czech L, Stamatakis A, Quince C, de Vargas C, Dunthorn M, Rognes T. (2021) Swarm v3: towards tera-scale amplicon clustering. Bioinformatics 38(1), 267-269. DOI:10.1093/bioinformatics/btab493", - lulu = "Frøslev TG, et al. (2017) Algorithm for post-clustering curation of DNA amplicon data yields reliable biodiversity estimates. Nat Commun 8:1188. DOI:10.1038/s41467-017-01312-x", - mumu = "Mahé F (2025) MUMU: C++ implementation of LULU, a R package for post-clustering curation of metabarcoding data. URL: https://github.com/frederic-mahe/mumu", - ucs = "Mikryukov V (2025) ucs - USEARCH cluster file parser. URL: https://github.com/vmikk/ucs", - duckdb = "Raasveldt M, Mühleisen H (2019) DuckDB: an Embeddable Analytical Database. SIGMOD '19: Proceedings of the 2019 International Conference on Management of Data, 1981-1984. DOI:10.1145/3299869.332021", - R = "R Core Team (2025) R: A Language and Environment for Statistical Computing. R Foundation for Statistical Computing, Vienna, Austria. URL: https://www.R-project.org/", - arrow = "Richardson N, Cook I, Crane N, Dunnington D, François R, Keane J, Moldovan-Grünfeld D, Ooms J, Wujciak-Jens J, and Apache Arrow (2025) arrow: Integration to Apache Arrow. URL: https://github.com/apache/arrow/", - biostrings= "Pagès H, Aboyoun P, Gentleman R, DebRoy S (2025) Biostrings: Efficient manipulation of biological strings. DOI:10.18129/B9.bioc.Biostrings", - datatable = "Barrett T, Dowle M, Srinivasan A, Gorecki J, Chirico M, Hocking T, Schwendinger B, Krylov I (2025) data.table: Extension of data.frame. URL: " -) - - -################################## -################################## Helpers -################################## - -## Get version number -getv <- function(v, process, tool){ - # v = list (from YAML file) - # process = process name - # tool = tool name - - if(is.null(v[[process]]) || is.null(v[[process]][[tool]])){ return("") } - as.character( v[[process]][[tool]] ) -} -# E.g., getv(versions, "dereplication", "vsearch") - - -## Get parameter -getp <- function(p, pname, default = NA){ - # p = table with parameters (two columns: name and value) - # pname = parameter name - # default = default value if parameter is not found - - pp <- p[ name == pname ]$value - if(is.null(pp) || is.na(pp)){ return(default) } - return(pp) -} -# E.g., getp(params, "otu_id", 0.98) - - -## Remove NAs and empty strings (to curate the citations) -trim_na <- function(x){ - x[ !is.na(x) & nzchar(x) ] -} - -################################## -################################## Body builders -################################## - -emit_nextits <- function(v) { - nextits_v <- if(!is.null(v$NextITS$version)){ as.character(v$NextITS$version) } else { "" } - glue("Bioinformatic processing was performed using the \\ - NextITS pipeline v.{nextits_v} (Mikryukov et al., 2025).") -} - -emit_nextflow <- function(v) { - nextflow_v <- if(!is.null(v$Nextflow$version)){ as.character(v$Nextflow$version) } else { "" } - glue("Workflow management was performed using \\ - Nextflow v.{nextflow_v} (Di Tommaso et al., 2017).") -} - -emit_aggregation <- function(p, v) { - glue("Sequences from all sequencing runs were aggregated and \\ - de novo chimeric sequences with chimera score >= {getp(p,'max_ChimeraScore',0.6)} were removed.") -} - -emit_dereplication <- function(p, v) { - minlen <- getp(p, "ampliconlen_min", NA) - maxlen <- getp(p, "ampliconlen_max", NA) - - length_filter <- "" - if(!is.na(minlen) && !is.na(maxlen)){ - length_filter <- glue(" Sequences shorter than {minlen} nt or longer than {maxlen} nt were excluded.") - } else if(!is.na(minlen)){ - length_filter <- glue(" Sequences shorter than {minlen} nt were excluded.") - } else if(!is.na(maxlen)){ - length_filter <- glue(" Sequences longer than {maxlen} nt were excluded.") - } - - glue("Global sequence dereplication was performed using \\ - VSEARCH v.{getv(v,'dereplication','vsearch')} (Rognes et al., 2016).\\ - {length_filter}") -} - -emit_preclustering <- function(p, v) { - preclustering_method <- getp(p, "preclustering", "none") - - res <- switch(preclustering_method, - - "none" = "", # No pre-clustering or denoising was performed - - "homopolymer" = glue( - "Global homopolymer correction was performed using an algorithm implemented in NextITS \\ - with support of VSEARCH v.{getv(v,'homopolymer','vsearch')} (Rognes et al., 2016)."), - - "unoise" = glue( - "Sequence denoising was performed using the UNOISE3 algorithm (Edgar, 2016) \\ - implemented in VSEARCH v.{getv(v,'unoise','vsearch')} (Rognes et al., 2016) \\ - with alpha parameter {getp(p,'unoise_alpha',6.0)} and minimum size {getp(p,'unoise_minsize',1)}."), - - "dada2" = glue( - "Sequence denoising was performed using \\ - DADA2 v.{getv(v,'dada2','dada2')} (Callahan et al., 2016)"), - # using {getp(p,'dada2_pooling','global')} pooling strategy." - - "swarm_d1" = glue( - "Pre-clustering was performed using \\ - SWARM v.{getv(v,'precluster_swarm','swarm')} (Mahé et al., 2021) \\ - with d=1 and fastidious option enabled.") - ) - - return(res) -} - -emit_clustering <- function(p, v) { - clustering_method <- getp(p, "clustering", "vsearch") - preclustering_method <- getp(p, "preclustering", "none") - - ## Handle special case where SWARM pre-clustering = SWARM clustering with same d - # if(preclustering_method == "swarm_d1" && clustering_method == "swarm" && getp(p, "swarm_d", 1) == 1){ - # return("No additional clustering was performed (SWARM pre-clustering with d=1 was used as final clustering).") - # } - - cls <- switch(clustering_method, - - "vsearch" = glue( - "VSEARCH v.{getv(v,'cluster_vsearch','vsearch')} (Rognes et al., 2016) \\ - with { as.numeric(getp(p,'otu_id',0.98))*100}% similarity threshold."), - - "swarm" = { - fastidious_text <- if(getp(p, "swarm_fastidious", TRUE) && getp(p, "swarm_d", 1) == 1) { - " with fastidious option enabled" - } else { - "" - } - glue("SWARM v.{getv(v,'cluster_swarm','swarm')} (Mahé et al., 2021) \\ - with parameter d={getp(p,'swarm_d',1)}{fastidious_text}.") - }) - - res <- glue("OTU clustering was performed using {cls}") - return(res) -} - -emit_uc_merging <- function(p, v) { - glue("UC files from dereplication, pre-clustering, and clustering steps were merged using \\ - ucs v.{getv(v,'merge_uc','ucs')} (Mikryukov, 2025) and \\ - DuckDB v.{getv(v,'merge_uc','duckdb')} (Raasveldt & Mühleisen, 2019) \\ - to track sequence membership through all processing steps.") -} - -emit_lulu <- function(p, v) { - glue("Post-clustering curation was performed using \\ - LULU algorithm (Frøslev et al., 2017) \\ - as implemented in MUMU v.{getv(v,'lulu','mumu')} (Mahé, 2025) \\ - with {getp(p,'lulu_match',95.0)}% minimum similarity, \\ - {getp(p,'lulu_ratio',1.0)} minimum abundance ratio, \\ - and {getp(p,'lulu_relcooc',0.95)} minimum relative co-occurrence. \\ - Pairwise sequence similarities were calculated using \\ - VSEARCH v.{getv(v,'lulu','vsearch')} (Rognes et al., 2016).") -} - - -################################## -################################## Workflow-dependent method descriptions -################################## - -## Function to assembly the workflow description and references -build_docs <- function(versions, params){ - body <- character() - tools_used <- character() - - ## Pipeline version - body <- c(body, emit_nextits(versions)) - tools_used <- c(tools_used, "nextits") - - ## Nextflow version - body <- c(body, emit_nextflow(versions)) - tools_used <- c(tools_used, "nextflow") - - ## Sequence aggregation - body <- c(body, emit_aggregation(params, versions)) - - ## Sequence dereplication and amplicon length filtering - body <- c(body, emit_dereplication(params, versions)) - tools_used <- c(tools_used, "vsearch") - - ## Conditional: pre-clustering/denoising - preclustering_method <- getp(params, "preclustering", "none") - if(preclustering_method != "none" && !is.na(preclustering_method)){ - body <- c(body, emit_preclustering(params, versions)) - - switch(preclustering_method, - "homopolymer" = { - tools_used <- c(tools_used, "vsearch") - }, - "unoise" = { - tools_used <- c(tools_used, c("vsearch", "unoise")) - }, - "dada2" = { - tools_used <- c(tools_used, "dada2") - }, - "swarm_d1" = { - tools_used <- c(tools_used, "swarm") - } - ) - } - - ## Conditional: clustering - clustering_method <- getp(params, "clustering", "vsearch") - if(clustering_method != "none" && !is.na(clustering_method)){ - ## Skip clustering description if it's redundant with pre-clustering - if(!(preclustering_method == "swarm_d1" && clustering_method == "swarm" && getp(params, "swarm_d", 1) == 1)){ - body <- c(body, emit_clustering(params, versions)) - - switch(clustering_method, - "vsearch" = { - tools_used <- c(tools_used, "vsearch") - }, - "swarm" = { - tools_used <- c(tools_used, "swarm") - } - ) - } - } - - ## UC file merging - body <- c(body, emit_uc_merging(params, versions)) - tools_used <- c(tools_used, c("ucs", "duckdb")) - - ## Conditional: LULU post-clustering curation - lulu_enabled <- tolower(as.character(getp(params, "lulu", "true"))) %in% c("true", "t", "1") - if(lulu_enabled){ - body <- c(body, emit_lulu(params, versions)) - tools_used <- c(tools_used, c("mumu", "lulu", "vsearch")) - } - - ## Generate citations - tools_used <- unique(tools_used) - citations <- trim_na( unlist(citation_db[tools_used]) ) - citations <- sort(unique(citations)) - - res <- list( - body = body, - citations = citations) - - return(res) -} - - -################################## -################################## Assemble body and citations -################################## - -## Load inputs -cat("Loading versions YAML...\n") -versions <- yaml::read_yaml(versions_path) - -cat("Loading params table...\n") -params <- data.table::fread(params_path, sep = "\t", header = TRUE, na.strings = c("", "NA")) -setnames(params, new = c("name", "value")) - -## Build body and citations -res <- build_docs(versions, params) - -## Write output -con <- file(output_path, open = "wt") - -writeLines("Methods:", con) -writeLines(res$body, con) - -writeLines("", con) - -writeLines("References:", con) -writeLines(paste0("- ", res$citations), con) - -close(con) - -cat("All done.\n") - diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/hash_sequences.sh b/src/pipecraft-core/service_scripts/NextITS/bin/hash_sequences.sh deleted file mode 100755 index d36f6ba0..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/hash_sequences.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -awk \ - '{ print $0 "\t" encodeData( $2 ) } - function encodeData( fld, cmd, output ) { - cmd = "printf \047" fld "\047 | sha1sum" - if ( (cmd | getline output) > 0 ) { - sub(/ .*/,"",output) - } - else { - print "failed to hash " fld | "cat>&2" - output = fld - } - close( cmd ) - return output - }' \ - "$1" diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/homopolymer_collapse_len.sh b/src/pipecraft-core/service_scripts/NextITS/bin/homopolymer_collapse_len.sh deleted file mode 100755 index c7575431..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/homopolymer_collapse_len.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -## Script to compress homopolymer stretches (e.g., prior to k-mer counting) -## All homopolymer stretches will be collapsed to max H length (default, 1) - -## Input: -# $1 = Parameter H (max homopolymer length) -# $2 = input FASTA file - -## Usage example: -# -# cat > input.fasta <<'EOT' -# >s1 -# AACCCTTTTGGGGG -# >s2 -# ACCTTTGGGGGGGG -# >s3 -# ACTGACTGACTGAC -# EOT -# -# homopolymer_compression.sh 2 input.fasta - - - -awk -v H="$1" '\ - -BEGIN { - if (H < 1) H = 1; -} - -# If the line is a header, print it as is -/^>/ { - print; - next; -} - -# Process sequence lines -{ - sequence = $0; - collapsedSeq = ""; - count = 1; - - for (i = 2; i <= length(sequence); i++) { - if (substr(sequence, i, 1) == substr(sequence, i - 1, 1)) { - count++; - } else { - collapsedSeq = collapsedSeq substr(sequence, i - count, (count > H) ? H : count); - count = 1; - } - } - - # Handle the last homopolymer stretch - collapsedSeq = collapsedSeq substr(sequence, length(sequence) - count + 1, (count > H) ? H : count); - - print collapsedSeq; -} -' "$2" - diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/homopolymer_compression.sh b/src/pipecraft-core/service_scripts/NextITS/bin/homopolymer_compression.sh deleted file mode 100755 index 03e66d5e..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/homopolymer_compression.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -## Script to collapse homopolymer stretches to a single letter - -bioawk \ - '{ gsub(/[A]+/,"A");gsub(/[C]+/,"C");gsub(/[T]+/,"T");gsub(/[G]+/,"G");gsub(/[N]+/,"N") }1' \ - "$1" diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/max_ee.R b/src/pipecraft-core/service_scripts/NextITS/bin/max_ee.R deleted file mode 100755 index 3031d40d..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/max_ee.R +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to combine Phred scores and MaxEE estimates - -# Input is given as positional arguments: -# 1. Phred score table (`tmp_hash_table.txt`) -# 2. MaxEE table (`tmp_ee.txt`) -# 3. Output file name (`${sampID}_hash_table.txt`) - - -args <- commandArgs(trailingOnly = TRUE) - -## Debug: -# args <- c( -# "tmp_hash_table.txt", -# "tmp_ee.txt", -# "res_hash_table.txt" -# ) - -suppressMessages(library(data.table)) - - -## Load table with Phred scores -cat("..Loading Phred scores\n") -T1 <- fread( - file = args[1], - sep = "\t", header = FALSE, - col.names = c("SeqID", "SeqHash", "Len", "PhredScore"), - colClasses = c("character", "character", "numeric", "numeric")) - -if(any(is.na(T1$Len))){ - cat("WARNING: non-numeric data detected. Maybe there are some empty sequences\n") -} - -## Load table with Phred scores -cat("..Loading MaxEE estimates\n") -T2 <- fread( - file = args[2], - sep = "\t", header = FALSE, - col.names = c("SeqID", "MaxEE")) - -## Merge tables -cat("..Merging tables\n") -TAB <- merge(x = T1, y = T2, by = "SeqID", all.x = TRUE) - -## Estimate the MEEP score (Koparde et al., DOI:10.1504/IJCBDD.2017.10006006) -## Maximum number of probable incorrect base calls per every 100 bases in the read -cat("..Estimating MEEP score\n") -TAB[ , MEEP := 100 * MaxEE / Len ] - -## Export results -cat("..Exporting results\n") -fwrite(x = TAB, - file = args[3], - sep = "\t", - compress = "none") - diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/merge_hash_tables.sh b/src/pipecraft-core/service_scripts/NextITS/bin/merge_hash_tables.sh deleted file mode 100755 index ec5313f4..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/merge_hash_tables.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash - -## Usage: -# merge_hash_tables.sh \ -# -i '/path/to/input/directory' \ -# -o '/path/to/output.parquet' \ -# -t 4 - -## Input data: -# - Tab-delimited tables with columns: -# SampleID - Hash - PacBioID - AvgPhredScore - MaxEE - MEEP - Sequence - Quality - Length - -## Notes -# - memory constraints might reduce the number of threads used -# - when saving to parquet, the ROW_GROUP_SIZE param might be adjusted to reduce memory usage (but the effect is not very significant): -# default ROW_GROUP_SIZE = 122,880 (with DuckDB's vector size = 2,048 -> 60 row groups) -# here, a half of the default value is used (ROW_GROUP_SIZE = 61,440 -> 30 row groups) - -## Function to display usage information -usage() { - echo "Usage: $0 -i INPUTDIR -o OUTPUT [-t THREADS] [-m MEMORY] [-x TEMP_DIR] [-z COMPRESSION]" - echo " -i INPUTDIR : Input directory with text files" - echo " -o OUTPUT : Output Parquet file path" - echo " -t THREADS : Number of CPU threads to use (optional)" - echo " -m MEMORY : Memory limit (e.g., '100GB') (optional)" - echo " -z COMPRESSION : ZSTD compression level (0-22) (optional, default: 12)" - exit 1 -} - -## Initialize variables -INPUT="" -OUTPUT="" -THREADS="" -MEMORY="" -COMPRESSION="12" - -## Parse command-line options -while getopts "i:o:t:m:z:" opt; do - case $opt in - i) INPUT="$OPTARG" ;; - o) OUTPUT="$OPTARG" ;; - t) THREADS="$OPTARG" ;; - m) MEMORY="$OPTARG" ;; - z) COMPRESSION="$OPTARG" ;; - *) usage ;; - esac -done - - -## Validate input parameters -if [[ -z "$INPUT" || -z "$OUTPUT" ]]; then - echo -e "Error: Missing required parameters!\n" - usage -fi - -## Threads should be a positive integer -if [[ -n "$THREADS" && "$THREADS" -le 0 ]]; then - echo -e "Error: Threads must be a positive integer!\n" - usage -fi - -## Validate compression level -if ! [[ "$COMPRESSION" =~ ^[0-9]+$ ]] || [ "$COMPRESSION" -lt 0 ] || [ "$COMPRESSION" -gt 22 ]; then - echo -e "Error: Compression level must be an integer between 0 and 22!\n" - usage -fi - -## View user-supplied parameters -echo -e "\nInput parameters:" -echo "Input directory: $INPUT" -echo "Output file: $OUTPUT" -if [[ -n "$THREADS" ]]; then - echo "Threads: $THREADS" -fi -if [[ -n "$MEMORY" ]]; then - echo "Memory: $MEMORY" -fi -echo "Parquet compression level (ZSTD): $COMPRESSION" - - -SQL_COMMAND="" - -## Add configuration settings (if provided) -if [[ -n "$THREADS" ]]; then - SQL_COMMAND+=" -SET threads TO ${THREADS}; -" -fi - -if [[ -n "$MEMORY" ]]; then - SQL_COMMAND+=" -SET memory_limit = '${MEMORY}'; -" -fi - -SQL_COMMAND+=" -COPY ( - SELECT - column0 as SampleID, - column1 as Hash, - column2 as PacBioID, - column3 as AvgPhredScore, - column4 as MaxEE, - column5 as MEEP, - column6 as Sequence, - column7 as Quality, - column8 as Length - FROM read_csv('${INPUT}/*.txt.gz', - header = false, - delim = '\t', - quote = '', - columns = { - 'column0': 'VARCHAR', - 'column1': 'VARCHAR', - 'column2': 'VARCHAR', - 'column3': 'DOUBLE', - 'column4': 'DOUBLE', - 'column5': 'DOUBLE', - 'column6': 'VARCHAR', - 'column7': 'VARCHAR', - 'column8': 'INTEGER' - } - ) -) TO '${OUTPUT}' (FORMAT PARQUET, ROW_GROUP_SIZE 61_440, COMPRESSION 'ZSTD', COMPRESSION_LEVEL ${COMPRESSION}); -" - -## Execute the SQL command -echo -e "\nExecuting DuckDB command" - -duckdb -c "${SQL_COMMAND}" - diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/merge_sequnce_qualities.sh b/src/pipecraft-core/service_scripts/NextITS/bin/merge_sequnce_qualities.sh deleted file mode 100755 index c30735ce..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/merge_sequnce_qualities.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# $1 = input file -# $2 = text to add to the resulting file - -zcat "$1" \ - | awk \ - -F '\t' -v OFS='\t' \ - -v fnm="$2" \ - 'NR>1 { print fnm , $2 , $3 , $4 , $5 , $6 }' \ - | sed 's/_hash_table.txt//' \ - | sed '1i SampleID\tSeqID\tSeqLen\tPhredScore\tMaxEE\tMEEP' diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/merge_tj_memberships.sh b/src/pipecraft-core/service_scripts/NextITS/bin/merge_tj_memberships.sh deleted file mode 100755 index 683b834c..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/merge_tj_memberships.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash - -## Usage: -# merge_tj_memberships.sh \ -# -d 'Dereplicated.parquet' \ -# -c 'Clustered.parquet' \ -# -o 'TJPreclust.uc.parquet' \ -# -t 4 - -## Input data: -# - Parsed UC file from dereplication (`Dereplicated.parquet`) -# - Parsed UC file from clustering (`Clustered.parquet`) - -## Function to display usage information -usage() { - echo "Usage: $0 -d DEREP -c CLUST -o OUTPUT [-t THREADS]" - echo " -d DEREP : Parquet file from dereplication" - echo " -c CLUST : Parquet file from clustering" - echo " -o OUTPUT : Output Parquet file path" - echo " -t THREADS : Number of CPU threads to use (optional)" - exit 1 -} - -## Initialize variables -DEREP="" -CLUST="" -OUTPUT="TJPreclust.uc.parquet" # default output file name -THREADS="" - -## Parse command-line options -while getopts "d:c:o:t:" opt; do - case $opt in - d) DEREP="$OPTARG" ;; - c) CLUST="$OPTARG" ;; - o) OUTPUT="$OPTARG" ;; - t) THREADS="$OPTARG" ;; - *) usage ;; - esac -done - - -## Validate input parameters -if [[ -z "$DEREP" || -z "$CLUST" ]]; then - echo -e "Error: Missing required parameters!\n" - usage -fi - -## Threads should be a positive integer -if [[ -n "$THREADS" && "$THREADS" -le 0 ]]; then - echo -e "Error: Threads must be a positive integer!\n" - usage -fi - -## View user-supplied parameters -echo -e "\nInput parameters:" -echo "Parquet file from dereplication: $DEREP" -echo "Parquet file from clustering: $CLUST" -echo "Output file: $OUTPUT" -if [[ -n "$THREADS" ]]; then - echo "Threads: $THREADS" -fi - -SQL_COMMAND="" - -## Add configuration settings (if provided) -if [[ -n "$THREADS" ]]; then - SQL_COMMAND+=" -SET threads TO ${THREADS}; -" -fi - - -SQL_COMMAND+=" -COPY ( - SELECT - d.query AS SeqID, - c.target AS OTU - FROM read_parquet('${DEREP}') AS d - LEFT JOIN read_parquet('${CLUST}') AS c - ON d.target = c.query -) TO '${OUTPUT}' (FORMAT PARQUET, COMPRESSION 'ZSTD', COMPRESSION_LEVEL 8); -" - -## Execute the SQL command -echo -e "\nExecuting DuckDB command" - -duckdb -c "${SQL_COMMAND}" diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/merge_uc_files.R b/src/pipecraft-core/service_scripts/NextITS/bin/merge_uc_files.R deleted file mode 100755 index 94be7771..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/merge_uc_files.R +++ /dev/null @@ -1,264 +0,0 @@ -#!/usr/bin/env Rscript - -## Merge UC files from different steps (dereplication, pre-clustering, clustering) into a single file - -cat("Joining parquet files\n\n") - -## Check time -start_time <- Sys.time() - -## Function to load packages -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(".. ", paste(pkg, packageVersion(pkg), "\n")) -} - -cat("Loading packages:\n") -load_pckg("DBI") -load_pckg("duckdb") -# load_pckg("qs") -# load_pckg("data.table") -# load_pckg("arrow") -# load_pckg("dplyr") - -cat("\nParsing input options and arguments...\n") - -suppressPackageStartupMessages(require(optparse)) - -## Parse arguments -option_list <- list( - make_option("--ucderep", action="store", default=NA, type='character', help="UC file from global dereplication"), - make_option("--ucpreclust", action="store", default=NA, type='character', help="UC file from pre-clustering (optional)"), - make_option("--ucclust", action="store", default=NA, type='character', help="UC file from clustering"), - make_option("--output", action="store", default="UC_Pooled.parquet", type='character', help="Output file name"), - make_option(c("-t", "--threads"), action="store", default=4L, type='integer', help="Number of CPU threads for arrow, default 4") -) -opt <- parse_args(OptionParser(option_list=option_list)) - -## Function to convert text "NA"s to NA -to_na <- function(x){ - if(x %in% c("NA", "null", "Null")){ x <- NA } - return(x) -} - -## Replaces "null"s from Nextflow with NA -opt <- lapply(X = opt, FUN = to_na) - - -## Validation of the required argiments -if(is.na(opt$ucderep)){ - cat("Input file is not specified: UC file from global dereplication.\n", file=stderr()) - stop() -} -if(is.na(opt$ucclust)){ - cat("Input file is not specified: UC file from clustering.\n", file=stderr()) - stop() -} - - -## Assign variables -UCDEREP <- opt$ucderep -UCPRECLUST <- opt$ucpreclust -UCCLUST <- opt$ucclust -OUTPUT <- opt$output -CPUTHREADS <- as.numeric( opt$threads ) - -## Log assigned variables -cat("\nParameters specified:\n") -cat(paste(" UC file from global dereplication: ", UCDEREP, "\n", sep="")) -cat(paste(" UC file from pre-clustering or denoising: ", UCPRECLUST, "\n", sep="")) -cat(paste(" UC file from clustering: ", UCCLUST, "\n", sep="")) -cat(paste(" Output file name: ", OUTPUT, "\n", sep="")) -cat(paste(" Number of CPU threads to use: ", CPUTHREADS, "\n", sep="")) - -cat("\n") - -## Data for debugging -# UCDEREP <- "UC_derep.parquet" -# UCPRECLUST <- "UC_preclust.parquet" # "NoPrecluster" -# UCCLUST <- "UC_clust.parquet" -# OUTPUT <- "UC_Pooled.parquet" -# CPUTHREADS <- 4 - -if(UCPRECLUST == "NoPrecluster") { - UCPRECLUST <- NA -} - - -###################################### -###################################### Load and process the data [duckdb] -###################################### - -## Initialize DuckDB connection -cat("..Initializing DuckDB connection\n") -con <- DBI::dbConnect(duckdb::duckdb()) -# duckdb::duckdb(dbdir = tempfile(pattern = "nextits.duckdb.") - -## Register parquet files as tables -cat("..Registering parquet files as tables\n") -cat("...Dereplication UC\n") -dbExecute(con, sprintf("CREATE VIEW derep_seqs AS SELECT * FROM parquet_scan('%s')", UCDEREP)) - -if(!is.na(UCPRECLUST)) { - cat("...Pre-clustering UC\n") - dbExecute(con, sprintf("CREATE VIEW preclust_seqs AS SELECT * FROM parquet_scan('%s')", UCPRECLUST)) -} - -cat("...Clustering UC\n") -dbExecute(con, sprintf("CREATE VIEW clust_seqs AS SELECT * FROM parquet_scan('%s')", UCCLUST)) - -## Set number of threads -cat("..Setting number of threads for DuckDB\n") -dbExecute(con, sprintf("SET threads TO %d;", CPUTHREADS)) - - -## Process and merge the data -if(is.na(UCPRECLUST)) { - - ## Two-file merge (no pre-clustering) - cat("..Merging UC files [no pre-clustering or denoising]\n") - dbExecute(con, sprintf(" - COPY ( - WITH derep AS ( - SELECT DISTINCT - query as SeqID, - target as DerepID - FROM derep_seqs - QUALIFY ROW_NUMBER() OVER (PARTITION BY query ORDER BY target) = 1 - ), - clust AS ( - SELECT DISTINCT - query as DerepID, - target as OTU - FROM clust_seqs - QUALIFY ROW_NUMBER() OVER (PARTITION BY query ORDER BY target) = 1 - ) - SELECT - d.SeqID, - d.DerepID, - c.OTU - FROM derep d - LEFT JOIN clust c ON d.DerepID = c.DerepID - ) TO '%s' - (FORMAT 'parquet', COMPRESSION 'ZSTD', COMPRESSION_LEVEL 8)", - OUTPUT)) - -} else { - - ## Three-file merge (with pre-clustering) - cat("..Merging UC files [with pre-clustering or denoising]\n") - - dbExecute(con, sprintf(" - COPY ( - WITH derep AS ( - SELECT DISTINCT - query as SeqID, - target as DerepID - FROM derep_seqs - QUALIFY ROW_NUMBER() OVER (PARTITION BY query ORDER BY target) = 1 - ), - preclust AS ( - SELECT DISTINCT - query as DerepID, - target as PreclusterID - FROM preclust_seqs - QUALIFY ROW_NUMBER() OVER (PARTITION BY query ORDER BY target) = 1 - ), - clust AS ( - SELECT DISTINCT - query as PreclusterID, - target as OTU - FROM clust_seqs - QUALIFY ROW_NUMBER() OVER (PARTITION BY query ORDER BY target) = 1 - ) - SELECT - d.SeqID, - d.DerepID, - p.PreclusterID, - c.OTU - FROM derep d - LEFT JOIN preclust p ON d.DerepID = p.DerepID - LEFT JOIN clust c ON p.PreclusterID = c.PreclusterID - ) TO '%s' - (FORMAT 'parquet', COMPRESSION 'ZSTD', COMPRESSION_LEVEL 8)", - OUTPUT)) - -} - -## Clean up -cat("..Disconnecting from DuckDB\n") -dbDisconnect(con, shutdown = TRUE) - - -cat("..Done!\n") - - - -###################################### -###################################### Load and process the data [arrow + dplyr + data.table] -###################################### - -# ## Set number of threads for data.table -# cat("..Setting number of threads\n") -# setDTthreads(threads = CPUTHREADS) # for data.table -# set_cpu_count(CPUTHREADS) # for arrow -# -# ## Globally dereplicated sequences (remove multi-target matches) -# cat("..Loading globally dereplicated sequences\n") -# UCA <- open_dataset(UCDEREP) %>% -# rename(SeqID = query, DerepID = target) %>% -# to_duckdb() %>% -# distinct(SeqID, .keep_all = TRUE) %>% -# collect() %>% -# setDT() -# -# ## Pre-clustered sequences (remove multi-target matches) -# if(!is.na(UCPRECLUST)){ -# cat("..Loading pre-clustered sequences\n") -# UCP <- open_dataset(UCPRECLUST) %>% -# rename(DerepID = query, PreclusterID = target) %>% -# to_duckdb() %>% -# distinct(DerepID, .keep_all = TRUE) %>% -# collect() %>% -# setDT() -# } -# -# ## Clustered sequences (remove multi-target matches) -# cat("..Loading clustering UC file\n") -# if(!is.na(UCPRECLUST)){ -# UCO <- open_dataset(UCCLUST) %>% -# rename(PreclusterID = query, OTU = target) %>% -# to_duckdb() %>% -# distinct(PreclusterID, .keep_all = TRUE) %>% -# collect() %>% -# setDT() -# } else { -# UCO <- open_dataset(UCCLUST) %>% -# rename(DerepID = query, OTU = target) %>% -# to_duckdb() %>% -# distinct(DerepID, .keep_all = TRUE) %>% -# collect() %>% -# setDT() -# } -# -# ## Merge UC files -# if(is.na(UCPRECLUST)){ -# -# ## No pre-clustering or denoising -# cat("..Merging UC files [no pre-clustering or denoising ]\n") -# RES <- merge(x = UCA, y = UCO, by = "DerepID", all.x = TRUE) -# -# } else { -# -# ## Merge UC files with pre-clustering or denoising -# cat("..Merging UC files [with pre-clustering or denoising ]\n") -# -# cat("... Adding pre-cluster or denoised IDs\n") -# RES <- merge(x = UCA, y = UCP, by = "DerepID", all.x = TRUE) -# -# cat("... Adding clustering IDs\n") -# RES <- merge(x = RES, y = UCO, by = "PreclusterID", all.x = TRUE) -# -# } -# - diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/pool_seqs_clean_header.sh b/src/pipecraft-core/service_scripts/NextITS/bin/pool_seqs_clean_header.sh deleted file mode 100755 index 773efa4b..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/pool_seqs_clean_header.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# $1 = input file (full path) -# $2 = sample ID (e.g., basename of the input file) - -zcat "${1}" \ - | sed -r '/^>/ s/;sample=[^;]*/;/g ; s/;;/;/g' \ - | sed "s/>.*/&;sample=${2}; / ; s/_NoChimera.fa//g ; s/_RescuedChimera.fa//g ; s/_JoinedPE//g ; s/Rescued_Chimeric_sequences.part_//g" \ - | sed -r '/^>/ s/;;/;/g' - - diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/quick_stats.R b/src/pipecraft-core/service_scripts/NextITS/bin/quick_stats.R deleted file mode 100755 index a132c364..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/quick_stats.R +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env Rscript - -## Summarise number of reads (demultiplexed and primer-checked) - -# quick_stats.R \ -# --raw Counts_1.RawData.txt \ -# --qc Counts_2.QC.txt \ -# --demuxed Counts_3.Demux.txt \ -# --primer Counts_4.PrimerCheck.txt \ -# --primerartef Counts_4.PrimerArtefacts.txt \ -# --threads 4 - - - -############################################## Parse input parameters - -## Check time -start_time <- Sys.time() - - -cat("Parsing input options and arguments...\n") - -suppressPackageStartupMessages(require(optparse)) - -## Parse arguments -option_list <- list( - - make_option("--raw", action="store", default=NA, type='character', help="Raw read counts"), - make_option("--qc", action="store", default=NA, type='character', help="Counts of reads passed QC"), - make_option("--demuxed", action="store", default=NA, type='character', help="Counts of demultiplexed reads"), - make_option("--primer", action="store", default=NA, type='character', help="Counts of reads with both primers detected"), - make_option("--primerartef",action="store", default=NA, type='character', help="Counts of primer artefacts"), - make_option(c("-t", "--threads"), action="store", default=4L, type='integer', help="Number of CPU threads for arrow, default 4") -) -opt <- parse_args(OptionParser(option_list=option_list)) - - -## Validation of the required argiments -# if(is.na(opt$raw)){ -# cat("Input file is not specified: ....\n", file=stderr()) -# stop() -# } - - - -## Function to convert text "NA"s to NA -to_na <- function(x){ - if(x %in% c("NA", "null", "Null")){ x <- NA } - return(x) -} - -## Assign variables -RAW <- opt$raw -QC <- opt$qc -DEMUXED <- opt$demuxed -PRIMER <- opt$primer -PRIMERARTEF <- opt$primerartef -CPUTHREADS <- as.numeric( opt$threads ) - -## Log assigned variables -cat(paste("Counts - RawData: " , RAW, "\n", sep="")) -cat(paste("Counts - QC: " , QC, "\n", sep="")) -cat(paste("Counts - Demux: " , DEMUXED, "\n", sep="")) -cat(paste("Counts - PrimerCheck: " , PRIMER, "\n", sep="")) -cat(paste("Counts - Primer Artefacts: " , PRIMERARTEF, "\n", sep="")) -cat(paste("Number of CPU threads to use: ", CPUTHREADS, "\n", sep="")) - -cat("\n") - - -############################################## data for debuging - -# RAW <- "Counts_1.RawData.txt" -# QC <- "Counts_2.QC.txt" -# DEMUXED <- "Counts_3.Demux.txt" -# PRIMER <- "Counts_4.PrimerCheck.txt" -# PRIMERARTEF <- "Counts_4.PrimerArtefacts.txt" -# CPUTHREADS <- 6 - - -############################################## Load packages and data - -cat("Loading R packages...\n") - -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("data.table") -load_pckg("plyr") -load_pckg("metagMisc") -load_pckg("openxlsx") - -cat("\n") - -## Set CPU thread number -cat("Setting number of CPU threads to: ", CPUTHREADS, "\n") -setDTthreads(threads = CPUTHREADS) # for data.table - - -###################################### -###################################### Load the data -###################################### - -cat("\nLoading input data\n") - - -#### Per-dataset stats - -## Load ASV table -cat("..Loading raw counts\n") -RAW <- fread(RAW) - -cat("..Loading QC counts\n") -QC <- fread(QC) - -#### Per-sample stats - -SEQKITCOUNTS <- list() - -cat("..Loading demux counts\n") -SEQKITCOUNTS$DEMUXED <- fread(DEMUXED) - -cat("..Loading primer-checked data counts\n") -SEQKITCOUNTS$PRIMER <- fread(PRIMER) -SEQKITCOUNTS$PRIMERARTEF <- fread(PRIMERARTEF) - - -## Remove NULL-files -null_seqk <- laply(.data = SEQKITCOUNTS, .fun = nrow) - -if(any(null_seqk == 0)){ - cat("Some files with counts are missing:\n") - to_rm <- which(null_seqk == 0) - cat(".. ", paste(names(SEQKITCOUNTS)[ to_rm ], collapse = ", "), "\n") - SEQKITCOUNTS[ to_rm ] <- NULL - rm(to_rm) -} - - -## Process seqkit counts -seqkit_process <- function(x){ - if(nrow(x) > 0){ - - ## Remove reudndant columns - x <- x[ , .(file, num_seqs) ] - - ## Remove file extensions - x[ , file := sub(pattern = ".fastq.gz", replacement = "", x = file) ] - x[ , file := sub(pattern = ".fq.gz", replacement = "", x = file) ] - x[ , file := sub(pattern = ".fa.gz", replacement = "", x = file) ] - x[ , file := sub(pattern = "_PrimerChecked$", replacement = "", x = file) ] - x[ , file := sub(pattern = "_PrimerArtefacts$",replacement = "", x = file) ] - - } - return(x) -} - - -cat("Processing data\n") -SEQKITCOUNTS <- llply(.data = SEQKITCOUNTS, .fun = seqkit_process) - -## Rename columns -if(!is.null(SEQKITCOUNTS$DEMUXED)){ -setnames(x = SEQKITCOUNTS$DEMUXED, old = "num_seqs", new = "Demultiplexed_Reads", skip_absent = TRUE) -} -if(!is.null(SEQKITCOUNTS$PRIMER)){ -setnames(x = SEQKITCOUNTS$PRIMER, old = "num_seqs", new = "PrimerChecked_Reads", skip_absent = TRUE) -} -if(!is.null(SEQKITCOUNTS$PRIMERARTEF)){ -setnames(x = SEQKITCOUNTS$PRIMERARTEF, old = "num_seqs", new = "PrimerArtefacts_Reads", skip_absent = TRUE) -} - -## Merge seqkit and custom counts into a single list -cat("Pooling per-sample counts\n") -COUNTS <- SEQKITCOUNTS - -## Pool per-file estimates -merge_dt <- function(x,y){ merge(x, y, by = "file", all = TRUE) } -PER_SAMPLE_COUNTS_merged <- Reduce(f = merge_dt, x = COUNTS) - -## Estimate percentage of multiprimer artefacts -PER_SAMPLE_COUNTS_merged[ , - PrimerArtefacts_Percent := round( - PrimerArtefacts_Reads / (PrimerChecked_Reads + PrimerArtefacts_Reads) * 100, - 2) - ] - - -### ... update -# .. replace NAs with zero -# .. reorder columns -# .. estimate percentages -# .. add tag-jump summary -# .. add final counts from the Seq table -# .. add positive / negative counts (based on default sample names) - -## Prepare per-run stats -PER_RUN_COUNTS_merged <- data.table( - Total_Number_Of_Reads = RAW$num_seqs, - Reads_Passed_QC = QC$num_seqs, - Reads_Demultiplexed = sum(PER_SAMPLE_COUNTS_merged$Demultiplexed_Reads, na.rm = TRUE), - Reads_PrimerChecked = sum(PER_SAMPLE_COUNTS_merged$PrimerChecked_Reads, na.rm = TRUE) - ) - -## Estimate percentages -cat("..Estimating per-run percentages\n") -PER_RUN_COUNTS_merged[ , Percentage_QC := - round(Reads_Passed_QC / Total_Number_Of_Reads * 100, 1) ] - -PER_RUN_COUNTS_merged[ , Percentage_Demultiplexed := - round(Reads_Demultiplexed / Total_Number_Of_Reads * 100, 1) ] - -PER_RUN_COUNTS_merged[ , Percentage_Passed := - round(Reads_PrimerChecked / Total_Number_Of_Reads * 100, 1) ] - - -## Export summary stats -write.xlsx(list( - "per_sample" = PER_SAMPLE_COUNTS_merged, - "per_run" = PER_RUN_COUNTS_merged - ), - file = "Run_summary.xlsx", colNames = TRUE) - - -cat("\nAll done.\n") - - -##################### Session info - -## Check time -end_time <- Sys.time() - -tmm <- as.numeric(difftime(end_time, start_time, units = "min")) -cat("\nElapsed time: ", tmm, " minutes\n") - -cat("\n") -cat("Session info:\n") -sessionInfo() -cat("\n") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/rc.sh b/src/pipecraft-core/service_scripts/NextITS/bin/rc.sh deleted file mode 100755 index 77f6ee69..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/rc.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -## Function to reverse-complement DNA sequences (with the support of IUPAC codes) - -echo "$1" \ - | tr \ - "[ATGCUatgcuNnYyRrSsWwKkMmBbDdHhVv]" \ - "[TACGAtacgaNnRrYySsWwMmKkVvHhDdBb]" \ - | rev diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/read_count_summary.R b/src/pipecraft-core/service_scripts/NextITS/bin/read_count_summary.R deleted file mode 100755 index ea399aa1..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/read_count_summary.R +++ /dev/null @@ -1,500 +0,0 @@ -#!/usr/bin/env Rscript - -## Summarise number of reads per process - -# read_count_summary.R \ -# --raw Counts_1.RawData.txt \ -# --qc Counts_2.QC.txt \ -# --demuxed Counts_3.Demux.txt \ -# --primer Counts_4.PrimerCheck.txt \ -# --primerartef Counts_4.PrimerArtefacts.txt \ -# --itsx Counts_5.ITSx_or_PrimTrim.txt \ -# --homopolymer Counts_5.Homopolymers.txt \ -# --chimrefn Counts_6.ChimRef_reads.txt \ -# --chimrefu Counts_6.ChimRef_uniqs.txt \ -# --chimdenovo Counts_7.ChimDenov.txt \ -# --chimrecovn Counts_8.ChimRecov_reads.txt \ -# --chimrecovu Counts_8.ChimRecov_uniqs.txt \ -# --tj TagJump_scores.qs \ -# --seqtab Seqs.parquet \ -# --maxchim 0.6 \ -# --threads 4 - - -############################################## Parse input parameters - -## Check time -start_time <- Sys.time() - - -cat("\nParsing input options and arguments...\n") - -suppressPackageStartupMessages(require(optparse)) - -## Parse arguments -option_list <- list( - - make_option("--raw", action="store", default=NA, type='character', help="Raw read counts"), - make_option("--qc", action="store", default=NA, type='character', help="Counts of reads passed QC"), - make_option("--demuxed", action="store", default=NA, type='character', help="Counts of demultiplexed reads"), - make_option("--primer", action="store", default=NA, type='character', help="Counts of reads with both primers detected"), - make_option("--primerartef",action="store", default=NA, type='character', help="Counts of primer artefacts"), - make_option("--itsx", action="store", default=NA, type='character', help="Read counts after ITSx or primer removal"), - make_option("--homopolymer",action="store", default=NA, type='character', help="Homopolymer correction results"), - make_option("--chimrefn", action="store", default=NA, type='character', help="Number of reads for reference-based chimeras"), - make_option("--chimrefu", action="store", default=NA, type='character', help="Number of unique sequences detected as reference-based chimeras"), - make_option("--chimdenovo", action="store", default=NA, type='character', help="Number of de novo chimeras"), - make_option("--chimrecovn", action="store", default=NA, type='character', help="Number of resued reads for de novo chimeras (false positives)"), - make_option("--chimrecovu", action="store", default=NA, type='character', help="Number of resued unique sequences detected as de novo chimeras (false positives)"), - make_option("--tj", action="store", default=NA, type='character', help="Tag jump removal data (serialized in qs format)"), - make_option("--seqtab", action="store", default=NA, type='character', help="Final seq table (Parquet format)"), - make_option("--maxchim", action="store", default=0.6, type='numeric', help = "Maximum de novo chimera score to remove"), - make_option(c("-t", "--threads"), action="store", default=4L, type='integer', help="Number of CPU threads for arrow, default 4") -) -opt <- parse_args(OptionParser(option_list=option_list)) - - -## Validation of the required argiments -# if(is.na(opt$raw)){ -# cat("Input file is not specified: ....\n", file=stderr()) -# stop() -# } - - - -## Function to convert text "NA"s to NA -to_na <- function(x){ - if(x %in% c("NA", "null", "Null")){ x <- NA } - return(x) -} - -## Assign variables -RAW <- opt$raw -QC <- opt$qc -DEMUXED <- opt$demuxed -PRIMER <- opt$primer -PRIMERARTEF <- opt$primerartef -ITSX <- opt$itsx -HOMOPOLY <- opt$homopolymer -CHIMREFN <- opt$chimrefn -CHIMREFU <- opt$chimrefu -CHIMDENOVO <- opt$chimdenovo -CHIMRECOVN <- opt$chimrecovn -CHIMRECOVU <- opt$chimrecovu -TJ <- opt$tj -SEQTAB <- opt$seqtab -MAXCHIM <- opt$maxchim -CPUTHREADS <- as.numeric( opt$threads ) - -## Log assigned variables -cat(paste("Counts - RawData: " , RAW, "\n", sep="")) -cat(paste("Counts - QC: " , QC, "\n", sep="")) -cat(paste("Counts - Demux: " , DEMUXED, "\n", sep="")) -cat(paste("Counts - PrimerCheck: " , PRIMER, "\n", sep="")) -cat(paste("Counts - Primer Artefacts: " , PRIMERARTEF, "\n", sep="")) -cat(paste("Counts - ITSx or Primer Trim: " , ITSX, "\n", sep="")) -cat(paste("Counts - Homopolymer correction results: " , HOMOPOLY, "\n", sep="")) -cat(paste("Counts - Chimera Ref-based, reads: " , CHIMREFN, "\n", sep="")) -cat(paste("Counts - Chimera Ref-based, unique sequences: " , CHIMREFU, "\n", sep="")) -cat(paste("Counts - Chimera de novo: " , CHIMDENOVO, "\n", sep="")) -cat(paste("Counts - Chimera Ref-based recoverd, reads: " , CHIMRECOVN, "\n", sep="")) -cat(paste("Counts - Chimera Ref-based recoverd, unique sequences: " , CHIMRECOVU, "\n", sep="")) -cat(paste("Tag-jump data: " , TJ, "\n", sep="")) -cat(paste("Final sequence table: " , SEQTAB, "\n", sep="")) -cat(paste("Maximum de novo chimera score: ", MAXCHIM, "\n", sep="")) -cat(paste("Number of CPU threads to use: ", CPUTHREADS, "\n", sep="")) - -cat("\n") - - -############################################## data for debuging - -# RAW <- "Counts_1.RawData.txt" -# QC <- "Counts_2.QC.txt" -# DEMUXED <- "Counts_3.Demux.txt" -# PRIMER <- "Counts_4.PrimerCheck.txt" -# PRIMERARTEF <- "Counts_4.PrimerArtefacts.txt" -# ITSX <- "Counts_5.ITSx_or_PrimTrim.txt" -# HOMOPOLY <- "Counts_5.Homopolymers.txt" -# CHIMREFN <- "Counts_6.ChimRef_reads.txt" -# CHIMREFU <- "Counts_6.ChimRef_uniqs.txt" -# CHIMDENOVO <- "Counts_7.ChimDenov.txt" -# CHIMRECOVN <- "Counts_8.ChimRecov_reads.txt" -# CHIMRECOVU <- "Counts_8.ChimRecov_uniqs.txt" -# TJ <- "TagJump_scores.qs" -# SEQTAB <- "Seqs.parquet" -# MAXCHIM <- 0.6 -# CPUTHREADS <- 6 - - -############################################## Load packages and data - -cat("Loading R packages...\n") - -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("data.table") -load_pckg("plyr") -load_pckg("arrow") -# load_pckg("dplyr") -load_pckg("metagMisc") -load_pckg("openxlsx") - -cat("\n") - -## Set CPU thread number -cat("Setting number of CPU threads to: ", CPUTHREADS, "\n") -setDTthreads(threads = CPUTHREADS) # for data.table - - -###################################### -###################################### Load the data -###################################### - -cat("\nLoading input data\n") - - -#### Per-dataset stats - -## Load ASV table -cat("..Loading raw counts\n") -RAW <- fread(RAW) - -cat("..Loading QC counts\n") -QC <- fread(QC) - -#### Per-sample stats - -SEQKITCOUNTS <- list() -CUSTOMCOUNTS <- list() - -cat("..Loading demux counts\n") -SEQKITCOUNTS$DEMUXED <- fread(DEMUXED) - -cat("..Loading primer-checked data counts\n") -SEQKITCOUNTS$PRIMER <- fread(PRIMER) -SEQKITCOUNTS$PRIMERARTEF <- fread(PRIMERARTEF) - -cat("..Loading ITSx or primer trim counts\n") -CUSTOMCOUNTS$ITSX <- fread(ITSX) - -cat("..Loading homopolymer correction results\n") -HOMOPOLY_data <- fread(HOMOPOLY) - -cat("..Loading ref-based chimera counts\n") -CUSTOMCOUNTS$CHIMREFN <- fread(CHIMREFN) -SEQKITCOUNTS$CHIMREFU <- fread(CHIMREFU) - -cat("..Loading de novo chimera counts\n") -CHIMDENOVO <- fread(CHIMDENOVO) # incorporate to the main table - -cat("..Loading rescued ref-based chimera counts\n") -CUSTOMCOUNTS$CHIMRECOVN <- fread(CHIMRECOVN) -SEQKITCOUNTS$CHIMRECOVU <- fread(CHIMRECOVU) - -if(!is.na(TJ) && TJ != "no_tj" && file.exists(TJ)){ - cat("..Loading tag-jump filtration data\n") - TJ <- qs::qread(TJ) - tjdata <- TRUE -} else { - cat("..No tag-jump filtration data found\n") - tjdata <- FALSE -} - -cat("..Loading sequence table\n") -SEQTAB <- arrow::open_dataset(SEQTAB) - - -## Remove NULL-files -null_cust <- laply(.data = CUSTOMCOUNTS, .fun = nrow) -null_seqk <- laply(.data = SEQKITCOUNTS, .fun = nrow) - -if(any(null_cust == 0)){ - cat("Some files with counts are missing:\n") - to_rm <- which(null_cust == 0) - cat(".. ", paste(names(CUSTOMCOUNTS)[ to_rm ], collapse = ", "), "\n") - CUSTOMCOUNTS[ to_rm ] <- NULL - rm(to_rm) -} - -if(any(null_seqk == 0)){ - cat("Some files with counts are missing:\n") - to_rm <- which(null_seqk == 0) - cat(".. ", paste(names(SEQKITCOUNTS)[ to_rm ], collapse = ", "), "\n") - SEQKITCOUNTS[ to_rm ] <- NULL - rm(to_rm) -} - - -## Process seqkit counts -seqkit_process <- function(x){ - if(nrow(x) > 0){ - - ## Remove reudndant columns - x <- x[ , .(file, num_seqs) ] - - ## Remove file extensions - x[ , file := sub(pattern = ".fastq.gz$", replacement = "", x = file) ] - x[ , file := sub(pattern = ".fq.gz$", replacement = "", x = file) ] - x[ , file := sub(pattern = ".fa.gz$", replacement = "", x = file) ] - x[ , file := sub(pattern = ".full.fasta$", replacement = "", x = file) ] - x[ , file := sub(pattern = ".ITS1.fasta.gz$", replacement = "", x = file) ] - x[ , file := sub(pattern = ".ITS2.fasta.gz$", replacement = "", x = file) ] - x[ , file := sub(pattern = "_PrimerChecked$", replacement = "", x = file) ] - x[ , file := sub(pattern = "_PrimerArtefacts$", replacement = "", x = file) ] - x[ , file := sub(pattern = "_Chimera$", replacement = "", x = file) ] - x[ , file := sub(pattern = "_RescuedChimera$", replacement = "", x = file) ] - x[ , file := sub(pattern = "^Rescued_Chimeric_sequences.part_", replacement = "", x = file) ] - - } - return(x) -} - -## Process custom counts -custom_process <- function(x){ - if(nrow(x) > 0){ - - ## There should be just two columns - `SampleID` & `NumReads` - - ## Rename "SampleID" into "file" - setnames(x = x, old = "SampleID", new = "file") - - ## Remove file extensions - x[ , file := sub(pattern = ".full.fasta$", replacement = "", x = file) ] - x[ , file := sub(pattern = "_ITS1_58S_ITS2.fasta$", replacement = "", x = file) ] - x[ , file := sub(pattern = "_Chimera.fa$", replacement = "", x = file) ] - x[ , file := sub(pattern = "_RescuedChimera.fa$", replacement = "", x = file) ] - x[ , file := sub(pattern = "^Rescued_Chimeric_sequences.part_", replacement = "", x = file) ] - - } - return(x) -} - - -cat("Processing data\n") -SEQKITCOUNTS <- llply(.data = SEQKITCOUNTS, .fun = seqkit_process) -CUSTOMCOUNTS <- llply(.data = CUSTOMCOUNTS, .fun = custom_process) - -cat("Estimating homopolymer stats\n") -if(nrow(HOMOPOLY_data) > 0){ - HOMOPOLY_counts <- HOMOPOLY_data[ , .( - N_UniqSequences_AfterITSx_or_PrimerTrimming = .N, - N_UniqSequences_AfterHomopolymerCorrection = length(unique(Target)) - ), - by = "SampleID" ] -} else { - cat("..No homopolymer correction data found\n") -} - -# HOMOPOLY_counts[, Num_HomopolymerCorrectedSequences := -# N_UniqSequences_AfterITSx_or_PrimerTrimming - N_UniqSequences_AfterHomopolymerCorrection ] - - -## Rename columns -if(!is.null(SEQKITCOUNTS$DEMUXED)){ -setnames(x = SEQKITCOUNTS$DEMUXED, old = "num_seqs", new = "Demultiplexed_Reads", skip_absent = TRUE) -} -if(!is.null(SEQKITCOUNTS$PRIMER)){ -setnames(x = SEQKITCOUNTS$PRIMER, old = "num_seqs", new = "PrimerChecked_Reads", skip_absent = TRUE) -} -if(!is.null(SEQKITCOUNTS$PRIMERARTEF)){ -setnames(x = SEQKITCOUNTS$PRIMERARTEF, old = "num_seqs", new = "PrimerArtefacts_Reads", skip_absent = TRUE) -} -if(!is.null(SEQKITCOUNTS$CHIMREFU)){ -setnames(x = SEQKITCOUNTS$CHIMREFU, old = "num_seqs", new = "ReferenceBasedChimera_NumUniqSequences", skip_absent = TRUE) -} -if(!is.null(SEQKITCOUNTS$CHIMRECOVU)){ -setnames(x = SEQKITCOUNTS$CHIMRECOVU, old = "num_seqs", new = "Recovered_ReferenceBasedChimea_NumUniqSequences", skip_absent = TRUE) -} - -if(!is.null(CUSTOMCOUNTS$ITSX)){ -setnames(x = CUSTOMCOUNTS$ITSX, old = "NumReads", new = "ITSx_Extracted_Reads", skip_absent = TRUE) -} -if(!is.null(CUSTOMCOUNTS$CHIMREFN)){ -setnames(x = CUSTOMCOUNTS$CHIMREFN, old = "NumReads", new = "ReferenceBasedChimera_Reads", skip_absent = TRUE) -} -if(!is.null(CUSTOMCOUNTS$CHIMRECOVN)){ -setnames(x = CUSTOMCOUNTS$CHIMRECOVN, old = "NumReads", new = "Recovered_ReferenceBasedChimea_Reads", skip_absent = TRUE) -} - -## Merge seqkit and custom counts into a single list -cat("Pooling per-sample counts\n") -COUNTS <- c(SEQKITCOUNTS, CUSTOMCOUNTS) - -## Pool per-file estimates -merge_dt <- function(x,y){ merge(x, y, by = "file", all = TRUE) } -PER_SAMPLE_COUNTS_merged <- Reduce(f = merge_dt, x = COUNTS) - -## If there are no primer artefacts -if(is.null(SEQKITCOUNTS$PRIMERARTEF)){ - PER_SAMPLE_COUNTS_merged[ , PrimerArtefacts_Reads := 0 ] -} - -## Estimate percentage of primer artefacts -cat("Estimating percentage of primer artefacts\n") -PER_SAMPLE_COUNTS_merged[ , - PrimerArtefacts_Percent := round( - PrimerArtefacts_Reads / (PrimerChecked_Reads + PrimerArtefacts_Reads) * 100, - 2) - ] - -## Estimate tag-jump stats -if(tjdata == TRUE){ - cat("Estimating tag-jump removal yields\n") - TJ_stats <- TJ[ TagJump == TRUE, .( - TagJump_Events = .N, - TagJump_Reads = sum(Abundance, na.rm = TRUE)), - by = "SampleID" ] - - if(nrow(TJ_stats) > 0){ - PER_SAMPLE_COUNTS_merged <- merge( - x = PER_SAMPLE_COUNTS_merged, - y = TJ_stats, - by.x = "file", by.y = "SampleID", all.x = TRUE) - } else { - PER_SAMPLE_COUNTS_merged[ , TagJump_Events := 0 ] - PER_SAMPLE_COUNTS_merged[ , TagJump_Reads := 0 ] - } -} - -## Add homopolymer stats -if(nrow(HOMOPOLY_data) > 0){ - cat("Adding homopolymer stats\n") - PER_SAMPLE_COUNTS_merged <- merge( - x = PER_SAMPLE_COUNTS_merged, - y = HOMOPOLY_counts, - by.x = "file", by.y = "SampleID", all.x = TRUE) -} - -## Add de novo chimera stats -cat("Adding de novo chimera stats\n") -denovo_stats <- SEQTAB |> - dplyr::filter(DeNovo_Chimera_Score >= MAXCHIM) |> - dplyr::group_by(SampleID) |> - dplyr::summarize( - DeNovoChimeras_NumReads = sum(Abundance, na.rm = TRUE), - DeNovoChimeras_NumUniqSeqs = n()) |> - dplyr::collect() |> - setDT() - -PER_SAMPLE_COUNTS_merged <- merge( - x = PER_SAMPLE_COUNTS_merged, - y = denovo_stats, - by.x = "file", by.y = "SampleID", all.x = TRUE) - - - -## Number of reads and unique sequences in the sequence table (per sample) -## Excluding putative de novo chimeras (with score >= MAXCHIM) -cat("Adding sequence table stats\n") -seqtab_stats <- SEQTAB |> - dplyr::filter(DeNovo_Chimera_Score < MAXCHIM | is.na(DeNovo_Chimera_Score) ) |> - dplyr::group_by(SampleID) |> - dplyr::summarize( - SeqTable_NumReads = sum(Abundance, na.rm = TRUE), - SeqTable_NumUniqSeqs = n()) |> - dplyr::collect() |> - setDT() - -PER_SAMPLE_COUNTS_merged <- merge( - x = PER_SAMPLE_COUNTS_merged, - y = seqtab_stats, - by.x = "file", by.y = "SampleID", all.x = TRUE) - - -## Replace NAs with zeros -cat("Replacing NAs with zero\n") -for (j in seq_len(ncol(PER_SAMPLE_COUNTS_merged))){ - set(PER_SAMPLE_COUNTS_merged, which(is.na(PER_SAMPLE_COUNTS_merged[[j]])), j, 0) -} -rm(j) - -## Estimate percentage of reads retained (starting from demultiplexed reads) -cat("Estimating percentage of reads retained\n") -PER_SAMPLE_COUNTS_merged[ , Percentage_Reads_Retained := round( SeqTable_NumReads / Demultiplexed_Reads * 100, 2) ] - -## Estimate percentage of reads retained after ITSx -if("ITSx_Extracted_Reads" %in% colnames(PER_SAMPLE_COUNTS_merged)){ - PER_SAMPLE_COUNTS_merged[ , ITSx_Yield_Percent := round( ITSx_Extracted_Reads / PrimerChecked_Reads * 100, 2) ] -} - -### TODO: -# .. estimate percentages -# .. add per-run positive / negative counts (based on default sample names) - - -## Reorder columns -setcolorder(PER_SAMPLE_COUNTS_merged, - skip_absent = TRUE, - neworder = c( - "file", "Demultiplexed_Reads", - "PrimerChecked_Reads", "PrimerArtefacts_Reads", "PrimerArtefacts_Percent", - "ReferenceBasedChimera_Reads", "ReferenceBasedChimera_NumUniqSequences", - "Recovered_ReferenceBasedChimea_Reads", "Recovered_ReferenceBasedChimea_NumUniqSequences", - "DeNovoChimeras_NumReads", "DeNovoChimeras_NumUniqSeqs", - "ITSx_Extracted_Reads", "ITSx_Yield_Percent", - "N_UniqSequences_AfterITSx_or_PrimerTrimming", - "N_UniqSequences_AfterHomopolymerCorrection", - # "Num_HomopolymerCorrectedSequences", - "TagJump_Reads", "TagJump_Events", - "SeqTable_NumReads", "SeqTable_NumUniqSeqs", - "Percentage_Reads_Retained")) - - - - -## Prepare per-run stats -cat("Preparing per-run stats\n") -PER_RUN_COUNTS_merged <- data.table( - Total_Number_Of_Reads = sum(RAW$num_seqs, na.rm = TRUE), - Reads_Demultiplexed = sum(PER_SAMPLE_COUNTS_merged$Demultiplexed_Reads, na.rm = TRUE), - Reads_Passed_QC = sum(QC$num_seqs, na.rm = TRUE), - Reads_PrimerChecked = sum(PER_SAMPLE_COUNTS_merged$PrimerChecked_Reads, na.rm = TRUE) - ) - -if("ITSx_Extracted_Reads" %in% colnames(PER_SAMPLE_COUNTS_merged)){ - PER_RUN_COUNTS_merged[ , Reads_ITSx_Extracted := sum(PER_SAMPLE_COUNTS_merged$ITSx_Extracted_Reads, na.rm = TRUE) ] -} - -## Estimate percentage of reads passed primer checking -cat("..Estimating per-run percentages\n") -PER_RUN_COUNTS_merged[ , Percentage_Demultiplexed := - round(Reads_Demultiplexed / Total_Number_Of_Reads * 100, 1) ] - -PER_RUN_COUNTS_merged[ , Percentage_PrimerChecked := - round(Reads_PrimerChecked / Total_Number_Of_Reads * 100, 1) ] - -## Final per-run num reads -PER_RUN_COUNTS_merged[ , SeqTable_NumReads := sum(PER_SAMPLE_COUNTS_merged$SeqTable_NumReads, na.rm = TRUE) ] -PER_RUN_COUNTS_merged[ , Percentage_Reads_Retained := round( SeqTable_NumReads / Total_Number_Of_Reads * 100, 2) ] - -NumUniqSeqs <- SEQTAB |> dplyr::select(SeqID) |> dplyr::summarize(N = n()) |> dplyr::collect() -PER_RUN_COUNTS_merged$SeqTable_NumUniqueSequences <- NumUniqSeqs$N - -## Export summary stats -cat("Exporting results\n") -write.xlsx(list( - "per_sample" = PER_SAMPLE_COUNTS_merged, - "per_run" = PER_RUN_COUNTS_merged - ), - file = "Run_summary.xlsx", colNames = TRUE) - - -cat("\nAll done.\n") - - -##################### Session info - -## Check time -end_time <- Sys.time() - -tmm <- as.numeric(difftime(end_time, start_time, units = "min")) -cat("\nElapsed time: ", tmm, " minutes\n") - -cat("\n") -cat("Session info:\n") -sessionInfo() -cat("\n") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/seq_table_assembly.R b/src/pipecraft-core/service_scripts/NextITS/bin/seq_table_assembly.R deleted file mode 100755 index 161fc594..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/seq_table_assembly.R +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to perform tag-jump removal - -## To do: -# - add HMM profile ID if ITSx was used - -# Inputs: -# 1. tag-jump-filtered Seq table (`Seq_tab_TagJumpFiltered.txt.gz`) -# 2. Sequences in fasta (`Seq_not_filtered.fa.gz`) -# 3. de novo chimera scores (`DeNovo_Chimera.txt`) -# 4. sequence qualities (`SeqQualities.parquet`) - -# Outputs: -# - FASTA with filtered Seqs `Seqs.fa.gz` -# - Seq table in long format `Seqs.txt.gz` (with additional sequence info) -# - Data in Parquet format `Seqs.parquet` - - -## Function to load packages -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(".. ", paste(pkg, packageVersion(pkg), "\n")) -} - -cat("Loading packages:\n") - -load_pckg("optparse") -load_pckg("data.table") -load_pckg("Biostrings") -load_pckg("plyr") -load_pckg("arrow") -# load_pckg("dplyr") -# load_pckg("openxlsx") - - -cat("\nParsing input options and arguments...\n") - -option_list <- list( - make_option("--seqtab", action="store", default=NA, type='character', help = "Sequence table (tab-delimited, long format)"), - make_option("--fasta", action="store", default=NA, type='character', help = "Sequences in FASTA format"), - make_option("--chimera", action="store", default=NA, type='character', help = "De novo chimera scores"), - make_option("--quality", action="store", default=NA, type='character', help = "Sequence qualities (Parquet format)"), - make_option("--threads", action="store", default=4, type='integer', help = "Number of CPU threads to use") -) - -opt <- parse_args(OptionParser(option_list=option_list)) - -## Function to convert text "NA"s to NA -to_na <- function(x){ - if(x %in% c("NA", "null", "Null")){ x <- NA } - return(x) -} - -## Replaces "null"s from Nextflow with NA -opt <- lapply(X = opt, FUN = to_na) - - -## Validation of the required arguments -required_args <- c("seqtab", "fasta", "quality") -missing_args <- required_args[ sapply(required_args, function(x) is.na(opt[[x]])) ] -if (length(missing_args) > 0) { - stop("Missing required arguments: ", paste(missing_args, collapse=", ")) -} - -## Assign variables -SEQTAB <- opt$seqtab -FASTA <- opt$fasta -CHIMERA <- opt$chimera -QUALITY <- opt$quality -CPUTHREADS <- as.numeric( opt$threads ) - -## Log assigned variables -cat(paste("Input sequence table: ", SEQTAB, "\n", sep="")) -cat(paste("Sequences in FASTA format: ", FASTA, "\n", sep="")) -cat(paste("De novo chimera scores: ", CHIMERA, "\n", sep="")) -cat(paste("Sequence qualities: ", QUALITY, "\n", sep="")) -cat(paste("Number of CPU threads to use: ", CPUTHREADS, "\n", sep="")) - -cat("\n") - - -## Debug: -# SEQTAB <- "Seq_tab_TagJumpFiltered.txt.gz" -# FASTA <- "Seq_not_filtered.fa.gz" -# CHIMERA <- "DeNovo_Chimera.txt" -# QUALITY <- "SeqQualities.parquet" -# CPUTHREADS <- 4 - - -## Set CPU thread number -cat("Setting number of CPU threads to: ", CPUTHREADS, "\n") -setDTthreads(threads = CPUTHREADS) # for data.table -set_cpu_count(CPUTHREADS) # for arrow - -###################################### -###################################### Load the data -###################################### - -## Load sequnece table -cat("\n\n..Loading sequence table\n") - -TAB <- fread( - file = SEQTAB, - sep = "\t", header = TRUE) - -## Load sequences in fasta format -cat("..Loading sequences in FASTA format\n") -SQS <- readDNAStringSet(filepath = FASTA) - -## Load de novo chimera scores -cat("..Loading de novo chimera scores\n") - -CHI <- try( - fread( - file = CHIMERA, - header = FALSE, sep = "\t", - col.names = c("SeqID", "DeNovo_Chimera_Score", "SampleID")) - ) - -if("try-error" %in% class(CHI)){ - cat("\nCould not read the file with de novo chimeric scores\n") - cat("Most likely, the file file is empty (no de novo chimeras)\n") - - ## Initialize empty data table - CHI <- data.table(SeqID = character(), DeNovo_Chimera_Score = numeric(), SampleID = character()) -} - - -## Load sequence quality scores -cat("..Loading sequence quality scores\n") -QLT <- arrow::open_dataset(QUALITY) |> - dplyr::select(Hash, Length, AvgPhredScore, MaxEE, MEEP) |> - dplyr::collect() |> - dplyr::filter(Hash %in% unique(TAB$SeqID)) |> - setDT() - -setnames(QLT, - old = c("Hash", "Length", "AvgPhredScore"), - new = c("SeqID", "SeqLen", "PhredScore")) - -## Quality data: -# old header: c("SampleID", "SeqID", "SeqLen", "PhredScore", "MaxEE", "MEEP") -# new header: c("SampleID", "Hash", "PacBioID", "PhredScore", "MaxEE", "MEEP", "Sequence", "Quality", "Length") - - -## Create SeqID___SampleID column -TAB[, SeqID___SampleID := paste0(SeqID, "___", SampleID) ] -# QLT[, SeqID___SampleID := paste0(SeqID, "___", SampleID) ] - - -###################################### -###################################### Add quality scores, for non-singleton use max score -###################################### - -cat("\n\n..Adding quality scores\n") - -cat("...Prepareing quality scores\n") -setorder(QLT, SeqID, -PhredScore) -QLT <- QLT[QLT[, .I[which.max(PhredScore)], by=SeqID]$V1] - -cat("...Adding data to the main table\n") -if(any(! TAB$SeqID %in% QLT$SeqID)){ - cat("WARNING: Some sequences are not present in the quality table\n") -} - -TAB <- merge(x = TAB, y = QLT, by = "SeqID", all.x = TRUE) - -# with(TAB, plot(Abundance, PhredScore)) - - -###################################### -###################################### Add chimera info -###################################### - -cat("..Adding info about de novo chimeric sequences\n") - -if(nrow(CHI) > 0){ - - TAB <- merge(x = TAB, y = CHI, - by = c("SeqID", "SampleID"), all.x = TRUE) - - ## Convert variables to numeric scores - TAB[ , DeNovo_Chimera_Score := as.numeric(DeNovo_Chimera_Score) ] - - ## Classify sequences into putative chimeras - TAB[ !is.na(DeNovo_Chimera_Score), DeNovo_Chimera := TRUE ] - TAB[ is.na(DeNovo_Chimera_Score), DeNovo_Chimera := FALSE ] - - cat("... ", sum( TAB$DeNovo_Chimera), " putative de novo chimeras found\n") - cat("... ", sum(!TAB$DeNovo_Chimera), " non-chimeric sequences\n") - -} else { - - ## No de novo chimeras - - TAB[ , DeNovo_Chimera_Score := as.numeric(NA) ] - TAB[ , DeNovo_Chimera := FALSE ] - - cat("... ", 0, " putative de novo chimeras found\n") - cat("... ", nrow(TAB), " non-chimeric sequences\n") - -} - - -###################################### -###################################### Add sequences -###################################### - -cat("\n\n..Processing sequences\n") - -SQTAB <- data.table( - SeqHeader = names(SQS), - Sequence = as.character(SQS)) - -## Split the header (`feb76b9;size=1;sample=ABCD;` ) -SQTAB[ , c("SeqID", "SampleID") := tstrsplit(x = SeqHeader, split = ";", keep = c(1,2)) ] -SQTAB[ , SeqHeader := NULL ] -SQTAB[ , SampleID := gsub(pattern = "sample=", replacement = "", x = SampleID) ] - - -SQTAB[ , SeqID___SampleID := paste0(SeqID, "___", SampleID) ] -SQTAB[ , c("SeqID", "SampleID") := NULL ] - -cat("..Adding sequences to the main table\n") - -TAB <- merge(x = TAB, y = SQTAB, - by = c("SeqID___SampleID"), all.x = TRUE) - - -cat("..Sorting table by abundance, quality score, and SampleID\n") - -setorder(x = TAB, -Abundance, -PhredScore, SampleID) - - -cat("..Preparing FASTA file with filtered sequences\n") - -SQF <- DNAStringSet(x = TAB$Sequence) -names(SQF) <- paste0(TAB$SeqID, ";size=", TAB$Abundance, ";sample=", TAB$SampleID, ";") - -## Export FASTA -cat("..Exporting FASTA file with filtered sequences\n") - -writeXStringSet(x = SQF, - filepath = "Seqs.fa.gz", - compress = TRUE, format = "fasta", width = 9999) - - - - -###################################### -###################################### Export results -###################################### - -# cat("..Reshaping sequence table into wide format\n") -# -# TABW <- dcast(data = TAB, -# formula = SeqID ~ SampleID, -# value.var = "Abundance", -# fill = 0) - - -cat("..Exporting result\n") - -setcolorder( - x = TAB, - neworder = c( - "SeqID___SampleID", "SampleID", "SeqID", - "Abundance", "SeqLen", "PhredScore", "MaxEE", "MEEP", - "DeNovo_Chimera", "DeNovo_Chimera_Score", - "Sequence")) - -# cat("...Exporting RData\n") -# saveRDS(object = TAB, file = "Seqs.RData", compress = "xz") - -cat("...Exporting Parquet\n") - -write_parquet( - x = TAB, - sink = "Seqs.parquet", - compression = "zstd", - compression_level = 10, - use_dictionary = TRUE) - - -## Long table -cat("...Exporting long table\n") - -TAB[ , SeqID___SampleID := NULL ] - -fwrite(x = TAB, file = "Seqs.txt.gz", sep = "\t", compress = "gzip") - -## Wide table -# cat("...Exporting wide table\n") - -# fwrite(x = TABW, file = "Seq_tab.txt.gz", sep = "\t", compress = "gzip") - - -cat("All done.") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/substitute_compressed_seqs.R b/src/pipecraft-core/service_scripts/NextITS/bin/substitute_compressed_seqs.R deleted file mode 100755 index 0f66d715..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/substitute_compressed_seqs.R +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to replace homopolymer-comressed sequences with non-compressed seqs -## + update size annotation - -# Input is given as positional arguments: -# 1. Uncompressed sequences (`inp_tab.txt`) -# 2. Homopolymer-compressed sequences (`clust_tab.txt`) -# 3. Name of the output FASTA file (`res.fa`) - -suppressMessages(library(data.table)); setDTthreads(threads = 1) -suppressMessages(library(Biostrings)) - -args <- commandArgs(trailingOnly = TRUE) - -## Load data - Uncompressed (inp_tab.txt) -cat("..Loading original sequences\n") -d1 <- fread(file = args[1], - header=FALSE, sep = "\t", quote = F, col.names = c("SeqID", "Seq_OK"), selec = 1:2) - -## Load data - Compressed (clust_tab.txt) -cat("..Loading compressed sequences\n") -d2 <- fread(file = args[2], - header=FALSE, sep = "\t", quote = F, col.names = c("SeqID", "Seq_Compr"), selec = 1:2) - -cat("...Number of raw sequences: ", nrow(d1), "\n") -cat("...Number of compressed sequences: ", nrow(d2), "\n") - -cat("..Processing data\n") - -## Remove multiple separators -d1[, SeqID := gsub(pattern = ";;", replacement = ";", x = SeqID)] -d2[, SeqID := gsub(pattern = ";;", replacement = ";", x = SeqID)] - -## Split seq ID -d1[, c("Hash", "Size") := tstrsplit(SeqID, ";", keep=1:2)] -d2[, c("Hash", "Size") := tstrsplit(SeqID, ";", keep=1:2)] - -## Drop seq ID -d1[, SeqID := NULL ] -d2[, SeqID := NULL ] - -## Replace seqs -res <- merge( - x = d2[, .(Hash, Size)], - y = d1[, .(Seq_OK, Hash)], - by = "Hash", all.x = TRUE) - -res[, SeqID := do.call(paste, c(.SD, sep = ";")), .SDcols = c("Hash", "Size")] - -## Verify the number of reads - should be the same -# sum(as.numeric(gsub(pattern = "size=", replacement = "", x = d1$Size))) -# sum(as.numeric(gsub(pattern = "size=", replacement = "", x = d2$Size))) -# sum(as.numeric(gsub(pattern = "size=", replacement = "", x = res$Size))) - -cat("...Total number of reads: ", - sum(as.numeric(gsub(pattern = "size=", replacement = "", x = res$Size))), - "\n") - -## Prepare sequences -cat("..Exporting results\n") -sqs <- DNAStringSet(x = res$Seq_OK) -names(sqs) <- res$SeqID - -## Export FASTA -writeXStringSet(x = sqs, filepath = args[3], - compress=FALSE, format="fasta", width=9999) diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/summarize_clustered_data.R b/src/pipecraft-core/service_scripts/NextITS/bin/summarize_clustered_data.R deleted file mode 100755 index c6e0b6fc..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/summarize_clustered_data.R +++ /dev/null @@ -1,400 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to pool remove low-quality singletons and summarize sequence abundance at OTU level (per sample) - -# Input: -# 1. Sequence tables in long format with de novo chimeras removed (`Seqs.parquet`) -# 2. UC file (`UC_Pooled.parquet`) -# 3. FASTA file with OTU sequences (`Clustered.fa.gz`) -# 4. Max MEEP score - -# Outputs: -# - OTU table in long format (`OTU_table_long.txt.gz` & `OTU_table_long.RData`) -# - OTU table in wide format (`OTU_table_wide.txt.gz` & `OTU_table_wide.RData`) -# - FASTA file with sequences (`OTUs.fa.gz`) - -## Usage: -# ./summarize_clustered_data.R \ -# --seqtab "Seqs.parquet" \ -# --uc "UC_Pooled.parquet" \ -# --otus "Clustered.fa.gz" \ -# --maxmeep 0.6 \ -# --recoversinglet TRUE \ -# --mergesamples TRUE \ -# --threads 4 - - -## Quality threshold: -# MEEP score of 0.6 corresponds approximately to the average Phred score of 22.2 - -## Singleton recovery: -# If enabled, then singleton OTUs with MEEP score <= 0.6 & will be preserved -# Otherwise, singleton OTUs will be removed - - -############################################## Parse input parameters - -## Check time -start_time <- Sys.time() - - -cat("Parsing input options and arguments:\n") - -suppressPackageStartupMessages(require(optparse)) - -## Parse arguments -option_list <- list( - make_option("--seqtab", action="store", default=NA, type='character', help="Sequence tables in long format with de novo chimeras removed (Parquet format)"), - make_option("--uc", action="store", default=NA, type='character', help="UC file (Parquet format)"), - make_option("--otus", action="store", default=NA, type='character', help="FASTA file with OTU sequences"), - make_option("--maxmeep", action="store", default=0.5, type='double', help="Max MEEP score"), - make_option("--recoversinglet", action="store", default=TRUE, type='logical', help="Recover singletons"), - make_option(c("-m", "--mergesamples"), action="store", default=FALSE, type='logical', help="Merge sample replicates (default, false)"), - make_option(c("-t", "--threads"), action="store", default=4L, type='integer', help="Number of CPU threads for arrow, default 4") - # make_option(c("-s", "--scriptdir"), action="store", default=getwd(), type='character', help="Directory containing source scripts") -) -opt <- parse_args(OptionParser(option_list=option_list)) - -## Function to convert text "NA"s to NA -to_na <- function(x){ - if(x %in% c("NA", "null", "Null")){ x <- NA } - return(x) -} - -## Replaces "null"s from Nextflow with NA -opt <- lapply(X = opt, FUN = to_na) - - -## Validation of the required argiments -if(is.na(opt$seqtab)){ - cat("Input file is not specified: sequence tables in Parquet format.\n", file=stderr()) - stop() -} -if(is.na(opt$uc)){ - cat("Input file is not specified: UC file is required.\n", file=stderr()) - stop() -} -if(is.na(opt$otus)){ - cat("Input file is not specified: FASTA file with OTU sequences.\n", file=stderr()) - stop() -} -if(opt$recoversinglet == TRUE && is.na(opt$maxmeep)){ - cat("For singleton recovery, the max MEEP score must be specified.\n", file=stderr()) - stop() -} - -## Assign variables -SEQTAB <- opt$seqtab -UCF <- opt$uc -MAXMEEP <- as.numeric( opt$maxmeep ) -RECOV_SINGLET <- as.logical(opt$recoversinglet) -MERGE_SAMPLES <- as.logical(opt$mergesamples) -OTUS <- opt$otus - -CPUTHREADS <- as.numeric( opt$threads ) -# SCRIPTDIR <- opt$scriptdir - -## Log assigned variables -cat(paste("Sequence tables (Parquet format): ", SEQTAB, "\n", sep="")) -cat(paste("UC file (Parquet format): ", UCF, "\n", sep="")) -cat(paste("Max MEEP score: ", MAXMEEP, "\n", sep="")) -cat(paste("Low-quality singleton recovery: ", RECOV_SINGLET, "\n", sep="")) -cat(paste("Merge sample replicates: ", MERGE_SAMPLES, "\n", sep="")) -cat(paste("OTU sequences: ", OTUS, "\n", sep="")) -cat(paste("Number of CPU threads to use: ", CPUTHREADS, "\n", sep="")) -# cat(paste("Directory containing source scripts: ", SCRIPTDIR, "\n", sep="")) - -cat("\n") - - - -############################################## Data for debugging - -# SEQTAB <- "Seqs.parquet" -# UCF <- "UC_Pooled.parquet" -# MAXMEEP <- 0.5 -# RECOV_SINGLET <- TRUE -# MERGE_SAMPLES <- TRUE -# OTUS <- "Clustered.fa.gz" -# CPUTHREADS <- 4 - - -############################################## Load packages and data - -cat("Loading R packages:\n") - -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("data.table") -load_pckg("plyr") -load_pckg("metagMisc") -load_pckg("Biostrings") -load_pckg("arrow") - - -cat("\n") - - -# cat("Loading additional R funcitons...\n") -# source(file.path(SCRIPTDIR, "R_functions.R")) -# cat("\n") - - -## Set CPU thread number -cat("Setting number of CPU threads to: ", CPUTHREADS, "\n") -setDTthreads(threads = CPUTHREADS) # for data.table -set_cpu_count(CPUTHREADS) # for arrow - -###################################### -###################################### Load the data -###################################### - -## Load sequence tables -cat("\n..Loading sequence tables\n") -TAB <- arrow::read_parquet(SEQTAB) -setDT(TAB) -cat("... Total number of records: ", nrow(TAB), "\n") -cat("... Total number unique sequences: ", length(unique(TAB$Sequence)), "\n") -cat("... Total number unique samples (files): ", length(unique(TAB$SampleID)), "\n") - -## Load UC file for globally dereplicated sequences -cat("..Loading pooled UC file\n") -UC <- open_dataset(UCF) |> dplyr::collect() |> setDT() - -## Add OTU IDs to seq table -cat("... Adding OTU IDs to sequence table\n") -cat(".... Number of records in sequence table before merging: ", nrow(TAB), "\n") -TAB <- merge(x = TAB, y = UC, by = "SeqID", all.x = TRUE) -cat(".... Number of records in sequence table after merging: ", nrow(TAB), "\n") - -## Remove NA OTUs -- probably excluded seqs -if(any(is.na(TAB$OTU))){ - cat("WARNING: not all sequences were assigned to OTUs\n") - cat("..Removing missing/excluded sequences\n") - cat(".. ", sum(is.na(TAB$OTU)), " sequences with total abundance ", - sum(TAB[ is.na(OTU) ]$Abundance, na.rm = TRUE), " reads will be excluded\n") - TAB <- TAB[ ! is.na(OTU) ] -} - - -## Find singleton OTUs -cat("\n..Finding singleton OTUs\n") -SINGLETONS <- TAB[ , .(Abundance = sum(Abundance, na.rm = TRUE)), by = .(OTU) ][ Abundance < 2 ] -cat("... Number of singleton OTUs: ", nrow(SINGLETONS), "\n") - -## If singleton recovery is reqired -if(RECOV_SINGLET == TRUE && nrow(SINGLETONS) > 0){ - - ## Add quality scores - SINGLETONS <- merge(x = SINGLETONS, y = TAB[ , .(SeqID, MEEP)], by.x = "OTU", by.y = "SeqID", all.x = TRUE) - - ## Filter by MEEP score - SINGLETONS <- SINGLETONS[ MEEP > MAXMEEP ] - cat("... Number of singleton OTUs after filtering by MEEP score: ", nrow(SINGLETONS), "\n") - -} - -if(nrow(SINGLETONS) > 0){ - cat("..Removing singleton OTUs\n") - TAB <- TAB[ ! OTU %in% SINGLETONS$OTU ] - cat("... Number of records in sequence table after removing singleton OTUs: ", nrow(TAB), "\n") -} - - -## Summarize abundance by sample and OTU -cat("\n..Summarizing OTU abundance\n") -if(MERGE_SAMPLES == TRUE){ - - cat("\n... Merging sample replicates (e.g., re-sequenced samples)\n") - - ## Extract sample names - cat(".... Extracting sample names\n") - TAB[ , SampleName := tstrsplit(x = SampleID, split = "__", keep = 2) ] - - cat(".... Summarizing abundance by sample and OTU\n") - RES <- TAB[ , - .( Abundance = sum(Abundance, na.rm = TRUE) ), - by = c("OTU", "SampleName") ] - - setnames(x = RES, old = "SampleName", new = "SampleID") - -} else { - - cat("... Summarizing abundance by sample and OTU\n") - RES <- TAB[ , - .( Abundance = sum(Abundance, na.rm = TRUE) ), - by = c("OTU", "SampleID") ] - -} - -#### Reshape to wide table -cat("\nReshaping table into wide format\n") - -## Check if we can reshape the table in a single pass -n_otu <- length(unique(RES$OTU)) -n_smp <- length(unique(RES$SampleID)) -n_cll <- as.numeric(n_otu) * as.numeric(n_smp) -cat("...In total, there are ", n_otu, " OTUs and ", n_smp, " samples\n") -cat("...The total number of cells in the wide table will be ", n_cll, "\n") - -## Reshape data in one pass -if(n_cll < 50000000){ - REW <- dcast(data = RES, - formula = OTU ~ SampleID, - fun.aggregate = sum, fill = 0, value.var = "Abundance") -} else { -## Split data into chunks, reshape, and merge back - - cat("..The input table is too large to reshape in a single pass, reshaping by chunks\n") - - ## Function to split vector into N chunks - chunk <- function(x, n){ - if(n > 1) { res <- split(x, cut(seq_along(x), n, labels = FALSE)) } - if(n == 1){ res <- list(); res[[1]] <- x } - return(res) - } - - ## Choose the number of chunks - n_chunks <- data.table::fcase( - n_cll < 9e7, 2L, - n_cll >= 9e7 & n_cll < 5e8, 5L, - n_cll >= 5e8 & n_cll < 5e9, 6L, - n_cll >= 5e9 & n_cll < 5e10, 7L, - n_cll >= 5e10, 8L) - - cat("...The number of chunks to process: , ", n_chunks, "\n") - - ch <- chunk(x = sort(unique(RES$SampleID)), n = n_chunks) - - ## Chunk-and-reshape loop - REWL <- plyr::llply( - .data = ch, - .fun = function(x){ - - ## Reshape to wide - res <- dcast( - data = RES[ SampleID %in% x , ], - formula = OTU ~ SampleID, - fill = 0, fun.aggregate = sum, value.var = "Abundance") - - ## Create key on a data.table (should improve merging speed) - setkey(res, OTU) - - return(res) - }, - .progress = "text") - - cat("...Chunk reshaping finished\n") - cat("..Merging data into a single wide table\n") - - ## Merge chunks into a single wide table - merge_dt <- function(x,y){ data.table::merge.data.table(x, y, by = "OTU", all = TRUE) } - REW <- Reduce(f = merge_dt, x = REWL) - cat("...Merging finished\n") - - ## Clean up - cat("...Cleaning up\n") - rm(REWL); gc() - - ## Replace NAs with zeros - cat("...Filling missing values with zeros\n") - for (j in seq_len(ncol(REW))){ - set(REW, which(is.na(REW[[j]])), j, 0) - } - -} ## end of reshaping - -cat("...Reshaping to the wide format done!\n") - - -## Reorder OTU rows -cat("\n..Reordering OTU rows by total abundance\n") -otu_tots <- rowSums(REW[, -1], na.rm = TRUE) -REW <- REW[ order(otu_tots, decreasing = T), ] - -## Add attributes if samples were merged -setattr(x = RES, name = "Samples_merged", value = MERGE_SAMPLES) -setattr(x = REW, name = "Samples_merged", value = MERGE_SAMPLES) - - -cat("\nExporting results\n") - -## Export data -saveRDS.gz <- function(object, file, threads = parallel::detectCores()) { - con <- pipe(paste0("pigz -p",threads," > ",file),"wb") - saveRDS(object, file = con) - close(con) -} - -cat("..Exporting long table [R]\n") -saveRDS.gz(object = RES, - file = "OTU_table_long.RData", - threads = CPUTHREADS) - -cat("..Exporting wide table [R]\n") -saveRDS.gz(object = REW, - file = "OTU_table_wide.RData", - threads = CPUTHREADS) - -cat("..Exporting long table [tab-delimited]\n") -fwrite(x = RES, file = "OTU_table_long.txt.gz", sep = "\t", compress = "gzip") - -cat("..Exporting wide table [tab-delimited]\n") -fwrite(x = REW, file = "OTU_table_wide.txt.gz", sep = "\t", compress = "gzip") - - -cat("\nExporting OTU sequences to FASTA\n") - -cat("..Preparing sequences\n") - -## Take sequences from the data (NB! there are a several different sequence per OTU) -# SQS <- unique(RES[, .(OTU) ]) -# tmp_OTUs <- unique(TAB[ OTU %in% SQS$OTU & SeqID == OTU , .(OTU, Sequence) ]) -# SQS <- merge(x = SQS, y = tmp_OTUs, by = "OTU", all.x = TRUE) -# rm(tmp_OTUs) -# -# cat("...Preparing XStringSet object\n") -# SQF <- DNAStringSet(x = SQS$Sequence) -# names(SQF) <- SQS$OTU - -## Take sequnces from the OTU file -cat("... Loading FASTA file\n") -SQS <- readDNAStringSet(filepath = OTUS, format="fasta") -cat("... Extracting sequence IDs\n") -names(SQS) <- tstrsplit(x = names(SQS), split = ";", keep = 1)[[1]] - -if(any(duplicated(names(SQS)))){ - cat("WARNING: duplicated OTU names detected!\n") -} - -cat("... Subsetting OTUs\n") -SQF <- SQS[ names(SQS) %in% unique(REW$OTU) ] - -cat("....Total number of OTUs in input sequences: ", length(SQS), "\n") -cat("....Number of OTUs to export: ", length(SQF), "\n") -cat("....Number of OTUs in the OTU table: ", nrow(REW), "\n") - -cat("... Writing FASTA file\n") -writeXStringSet(x = SQF, - filepath = "OTUs.fa.gz", - compress = TRUE, format = "fasta", width = 9999) - - -cat("\nAll done.\n") - - -##################### Session info - -## Check time -end_time <- Sys.time() - -tmm <- as.numeric(difftime(end_time, start_time, units = "min")) -cat("\nElapsed time: ", tmm, " minutes\n") - -cat("\n") -cat("Session info:\n") -sessionInfo() -cat("\n") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/summarize_dereplicated_data.R b/src/pipecraft-core/service_scripts/NextITS/bin/summarize_dereplicated_data.R deleted file mode 100755 index 5b1bb285..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/summarize_dereplicated_data.R +++ /dev/null @@ -1,390 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to pool dereplicated sequences (non-clustered or denoised), remove low-quality data, and summarize sequence abundance per sample - -# Input: -# 1. Sequence tables in long format with de novo chimeras removed (`Seqs.parquet`) -# 2. UC file from dereplication (`UC_Pooled.parquet`) -# 3. FASTA file with sequences (`Dereplicated.fa.gz`) -# 4. Max MEEP score - -# Outputs: -# - OTU table in long format (`OTU_table_long.txt.gz` & `OTU_table_long.RData`) -# - OTU table in wide format (`OTU_table_wide.txt.gz` & `OTU_table_wide.RData`) -# - FASTA file with sequences (`OTUs.fa.gz`) - -## Usage: -# ./summarize_dereplicated_data.R \ -# --seqtab "Seqs.parquet" \ -# --uc "UC_Pooled.parquet" \ -# --seqs "Dereplicated.fa.gz" \ -# --maxmeep 0.6 \ -# --recoversinglet TRUE \ -# --mergesamples TRUE \ -# --threads 4 - - -## Quality threshold: -# MEEP score of 0.6 corresponds approximately to the average Phred score of 22.2 - -## Singleton recovery: -# If enabled, then singleton sequences with MEEP score <= 0.6 & will be preserved -# Otherwise, low-quality singleton sequences will be removed - - -############################################## Parse input parameters - -## Check time -start_time <- Sys.time() - - -cat("Parsing input options and arguments:\n") - -suppressPackageStartupMessages(require(optparse)) - -## Parse arguments -option_list <- list( - make_option("--seqtab", action="store", default=NA, type='character', help="Sequence tables in long format with de novo chimeras removed (Parquet format)"), - make_option("--uc", action="store", default=NA, type='character', help="UC file (Parquet format)"), - make_option("--seqs", action="store", default=NA, type='character', help="FASTA file with sequences"), - make_option("--maxmeep", action="store", default=0.5, type='double', help="Max MEEP score"), - make_option("--recoversinglet", action="store", default=TRUE, type='logical', help="Recover singletons"), - make_option(c("-m", "--mergesamples"), action="store", default=FALSE, type='logical', help="Merge sample replicates (default, false)"), - make_option(c("-t", "--threads"), action="store", default=4L, type='integer', help="Number of CPU threads for arrow, default 4") - # make_option(c("-s", "--scriptdir"), action="store", default=getwd(), type='character', help="Directory containing source scripts") -) -opt <- parse_args(OptionParser(option_list=option_list)) - -## Function to convert text "NA"s to NA -to_na <- function(x){ - if(x %in% c("NA", "null", "Null")){ x <- NA } - return(x) -} - -## Replaces "null"s from Nextflow with NA -opt <- lapply(X = opt, FUN = to_na) - - -## Validation of the required argiments -if(is.na(opt$seqtab)){ - cat("Input file is not specified: sequence tables in Parquet format.\n", file=stderr()) - stop() -} -if(is.na(opt$uc)){ - cat("Input file is not specified: UC file is required.\n", file=stderr()) - stop() -} -if(is.na(opt$seqs)){ - cat("Input file is not specified: FASTA file with sequences.\n", file=stderr()) - stop() -} -if(opt$recoversinglet == TRUE && is.na(opt$maxmeep)){ - cat("For singleton recovery, the max MEEP score must be specified.\n", file=stderr()) - stop() -} - -## Assign variables -SEQTAB <- opt$seqtab -UCF <- opt$uc -MAXMEEP <- as.numeric( opt$maxmeep ) -RECOV_SINGLET <- as.logical(opt$recoversinglet) -MERGE_SAMPLES <- as.logical(opt$mergesamples) -SEQS <- opt$seqs - -CPUTHREADS <- as.numeric( opt$threads ) -# SCRIPTDIR <- opt$scriptdir - -## Log assigned variables -cat(paste("Sequence tables (Parquet format): ", SEQTAB, "\n", sep="")) -cat(paste("UC file (Parquet format): ", UCF, "\n", sep="")) -cat(paste("Max MEEP score: ", MAXMEEP, "\n", sep="")) -cat(paste("Low-quality singleton recovery: ", RECOV_SINGLET, "\n", sep="")) -cat(paste("Merge sample replicates: ", MERGE_SAMPLES, "\n", sep="")) -cat(paste("Sequences: ", SEQS, "\n", sep="")) -cat(paste("Number of CPU threads to use: ", CPUTHREADS, "\n", sep="")) -# cat(paste("Directory containing source scripts: ", SCRIPTDIR, "\n", sep="")) - -cat("\n") - - - -############################################## Data for debugging - -# SEQTAB <- "Seqs.parquet" -# UCF <- "UC_Pooled.parquet" -# MAXMEEP <- 0.5 -# RECOV_SINGLET <- TRUE -# MERGE_SAMPLES <- TRUE -# SEQS <- "Dereplicated.fa.gz" -# CPUTHREADS <- 4 - - -############################################## Load packages and data - -cat("Loading R packages:\n") - -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("data.table") -load_pckg("plyr") -load_pckg("metagMisc") -load_pckg("Biostrings") -load_pckg("arrow") - - -cat("\n") - - -# cat("Loading additional R funcitons...\n") -# source(file.path(SCRIPTDIR, "R_functions.R")) -# cat("\n") - - -## Set CPU thread number -cat("Setting number of CPU threads to: ", CPUTHREADS, "\n") -setDTthreads(threads = CPUTHREADS) # for data.table -set_cpu_count(CPUTHREADS) # for arrow - -###################################### -###################################### Load the data -###################################### - -## Load sequence tables -cat("\n..Loading sequence tables\n") -TAB <- arrow::read_parquet(SEQTAB) |> setDT() -cat("... Total number of records: ", nrow(TAB), "\n") -cat("... Total number unique sequences: ", length(unique(TAB$Sequence)), "\n") -cat("... Total number unique samples (files): ", length(unique(TAB$SampleID)), "\n") - -## Load UC file for globally dereplicated sequences -cat("..Loading pooled UC file\n") -UC <- open_dataset(UCF) |> dplyr::collect() |> setDT() -setnames(UC, new = c("SeqID", "DerepID")) - -## Add dereplicated IDs to seq table -cat("... Adding dereplicated IDs to sequence table\n") -cat(".... Number of records in sequence table before merging: ", nrow(TAB), "\n") -TAB <- merge(x = TAB, y = UC, by = "SeqID", all.x = TRUE) -cat(".... Number of records in sequence table after merging: ", nrow(TAB), "\n") - -## Remove NA IDs -- probably excluded seqs -if(any(is.na(TAB$DerepID))){ - cat("WARNING: not all sequences are present in the dereplicated data (could be due to length-filteing)\n") - cat("..Removing missing/excluded sequences\n") - cat(".. ", sum(is.na(TAB$DerepID)), " sequences with total abundance ", - sum(TAB[ is.na(DerepID) ]$Abundance, na.rm = TRUE), " reads will be excluded\n") - TAB <- TAB[ ! is.na(DerepID) ] -} - - -## Find singleton sequences -cat("\n..Finding singleton sequences\n") -SINGLETONS <- TAB[ , .(Abundance = sum(Abundance, na.rm = TRUE)), by = .(DerepID) ][ Abundance < 2 ] -cat("... Number of singleton sequences: ", nrow(SINGLETONS), "\n") - -## If singleton recovery is reqired -if(RECOV_SINGLET == TRUE && nrow(SINGLETONS) > 0){ - - ## Add quality scores - SINGLETONS <- merge(x = SINGLETONS, y = TAB[ , .(SeqID, MEEP)], by.x = "DerepID", by.y = "SeqID", all.x = TRUE) - - ## Filter by MEEP score - SINGLETONS <- SINGLETONS[ MEEP > MAXMEEP ] - cat("... Number of singleton sequences after filtering by MEEP score: ", nrow(SINGLETONS), "\n") - -} - -if(nrow(SINGLETONS) > 0){ - cat("..Removing singleton sequences\n") - TAB <- TAB[ ! DerepID %in% SINGLETONS$DerepID ] - cat("... Number of records in sequence table after removing singleton sequences: ", nrow(TAB), "\n") -} - - -## Summarize abundance by sample and dereplicated ID -cat("\n..Summarizing sequence abundance\n") -if(MERGE_SAMPLES == TRUE){ - - cat("\n... Merging sample replicates (e.g., re-sequenced samples)\n") - - ## Extract sample names - cat(".... Extracting sample names\n") - TAB[ , SampleName := tstrsplit(x = SampleID, split = "__", keep = 2) ] - - cat(".... Summarizing abundance by sample and OTU\n") - RES <- TAB[ , - .( Abundance = sum(Abundance, na.rm = TRUE) ), - by = c("DerepID", "SampleName") ] - - setnames(x = RES, old = "SampleName", new = "SampleID") - -} else { - - cat("... Summarizing abundance by sample and dereplicated ID\n") - RES <- TAB[ , - .( Abundance = sum(Abundance, na.rm = TRUE) ), - by = c("DerepID", "SampleID") ] - -} - -#### Reshape to wide table -cat("\nReshaping table into wide format\n") - -## Check if we can reshape the table in a single pass -n_seq <- length(unique(RES$DerepID)) -n_smp <- length(unique(RES$SampleID)) -n_cll <- as.numeric(n_seq) * as.numeric(n_smp) -cat("...In total, there are ", n_seq, " sequences and ", n_smp, " samples\n") -cat("...The total number of cells in the wide table will be ", n_cll, "\n") - -## Reshape data in one pass -if(n_cll < 50000000){ - REW <- dcast(data = RES, - formula = DerepID ~ SampleID, - fun.aggregate = sum, fill = 0, value.var = "Abundance") -} else { -## Split data into chunks, reshape, and merge back - - cat("..The input table is too large to reshape in a single pass, reshaping by chunks\n") - - ## Function to split vector into N chunks - chunk <- function(x, n){ - if(n > 1) { res <- split(x, cut(seq_along(x), n, labels = FALSE)) } - if(n == 1){ res <- list(); res[[1]] <- x } - return(res) - } - - ## Choose the number of chunks - n_chunks <- data.table::fcase( - n_cll < 9e7, 2L, - n_cll >= 9e7 & n_cll < 5e8, 5L, - n_cll >= 5e8 & n_cll < 5e9, 6L, - n_cll >= 5e9 & n_cll < 5e10, 7L, - n_cll >= 5e10, 8L) - - cat("...The number of chunks to process: , ", n_chunks, "\n") - - ch <- chunk(x = sort(unique(RES$SampleID)), n = n_chunks) - - ## Chunk-and-reshape loop - REWL <- plyr::llply( - .data = ch, - .fun = function(x){ - - ## Reshape to wide - res <- dcast( - data = RES[ SampleID %in% x , ], - formula = DerepID ~ SampleID, - fill = 0, fun.aggregate = sum, value.var = "Abundance") - - ## Create key on a data.table (should improve merging speed) - setkey(res, DerepID) - - return(res) - }, - .progress = "text") - - cat("...Chunk reshaping finished\n") - cat("..Merging data into a single wide table\n") - - ## Merge chunks into a single wide table - merge_dt <- function(x,y){ data.table::merge.data.table(x, y, by = "DerepID", all = TRUE) } - REW <- Reduce(f = merge_dt, x = REWL) - cat("...Merging finished\n") - - ## Clean up - cat("...Cleaning up\n") - rm(REWL); gc() - - ## Replace NAs with zeros - cat("...Filling missing values with zeros\n") - for (j in seq_len(ncol(REW))){ - set(REW, which(is.na(REW[[j]])), j, 0) - } - -} ## end of reshaping - -cat("...Reshaping to the wide format done!\n") - - -## Reorder OTU rows -cat("\n..Reordering OTU rows by total abundance\n") -otu_tots <- rowSums(REW[, -1], na.rm = TRUE) -REW <- REW[ order(otu_tots, decreasing = T), ] - -## Add attributes if samples were merged -setattr(x = RES, name = "Samples_merged", value = MERGE_SAMPLES) -setattr(x = REW, name = "Samples_merged", value = MERGE_SAMPLES) - - -cat("\nExporting results\n") - -## Export data -saveRDS.gz <- function(object, file, threads = parallel::detectCores()) { - con <- pipe(paste0("pigz -p",threads," > ",file),"wb") - saveRDS(object, file = con) - close(con) -} - -cat("..Exporting long table [R]\n") -saveRDS.gz(object = RES, - file = "OTU_table_long.RData", - threads = CPUTHREADS) - -cat("..Exporting wide table [R]\n") -saveRDS.gz(object = REW, - file = "OTU_table_wide.RData", - threads = CPUTHREADS) - -cat("..Exporting long table [tab-delimited]\n") -fwrite(x = RES, file = "OTU_table_long.txt.gz", sep = "\t", compress = "gzip") - -cat("..Exporting wide table [tab-delimited]\n") -fwrite(x = REW, file = "OTU_table_wide.txt.gz", sep = "\t", compress = "gzip") - - -cat("\nExporting sequences to FASTA\n") - -cat("..Preparing sequences\n") - -## Take sequnces from the FASTA file -cat("... Loading FASTA file\n") -SQS <- readDNAStringSet(filepath = SEQS, format="fasta") -cat("... Extracting sequence IDs\n") -names(SQS) <- tstrsplit(x = names(SQS), split = ";", keep = 1)[[1]] - -if(any(duplicated(names(SQS)))){ - cat("WARNING: duplicated OTU names detected!\n") -} - -cat("... Subsetting OTUs\n") -SQF <- SQS[ names(SQS) %in% unique(REW$DerepID) ] - -cat("....Total number of sequences in input FASTA: ", length(SQS), "\n") -cat("....Number of sequences to export: ", length(SQF), "\n") -cat("....Number of sequences in the abundance table: ", nrow(REW), "\n") - -cat("... Writing FASTA file\n") -writeXStringSet(x = SQF, - filepath = "OTUs.fa.gz", - compress = TRUE, format = "fasta", width = 9999) - - -cat("\nAll done.\n") - - -##################### Session info - -## Check time -end_time <- Sys.time() - -tmm <- as.numeric(difftime(end_time, start_time, units = "min")) -cat("\nElapsed time: ", tmm, " minutes\n") - -cat("\n") -cat("Session info:\n") -sessionInfo() -cat("\n") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/tag_jump_removal.R b/src/pipecraft-core/service_scripts/NextITS/bin/tag_jump_removal.R deleted file mode 100755 index c6fb3e56..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/tag_jump_removal.R +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to perform tag-jump removal - -# Input is given as positional arguments: -# 1. OTU table (`OTU_tab_not_filtered.txt.gz`) -# 2. f-parameter of UNCROSS (e.g., 0.01) -# 3. p-parameter (e.g., 1.0) - -# Outputs: -# - Tag-jumpfiltered OTU table (`OTU_tab_TagJumpFiltered.txt.gz`) -# - Table with tag-jumps (`TagJump_OTUs.RData`) -# - Plot (`TagJump_plot.pdf`) - -args <- commandArgs(trailingOnly = TRUE) - -suppressMessages(library(data.table)) -suppressMessages(library(ggplot2)) -# library(openxlsx) - -theme_set(theme_classic(base_size = 14)) - -## Load OTU table -cat("..Loading OTU table\n") -OTUTABW <- fread( - file = args[1], - sep = "\t", header = TRUE) - -colnames(OTUTABW)[1] <- "OTU" - -cat("...Number of OTUs: ", nrow(OTUTABW), "\n") -cat("...Number of samples: ", ncol(OTUTABW) - 1, "\n") - -## Convert to long format -cat("..Converting OTU table to long format\n") -OTUTAB <- melt(data = OTUTABW, id.vars = "OTU", - variable.name = "SampleID", value.name = "Abundance") - -## Remove zero-OTUs -OTUTAB <- OTUTAB[ Abundance > 0 ] -cat("...Number of non-zero records: ", nrow(OTUTAB), "\n") - - -## Estimate total abundance of sequence per plate -cat("..Estimating total OTU abundance\n") -OTUTAB[ , Total := sum(Abundance, na.rm = TRUE), by = "OTU" ] - -## UNCROSS score (with original parameter - take a root from the exp in denominator, to make curves more steep) -uncross_score <- function(x, N, n, f = 0.01, tmin = 0.1, p = 1){ - # x = OTU abundance in a sample - # N = total OTU abundance - # n = number of samples - # f = expected cross-talk rate, e.g. 0.01 - # tmin = min score to be considered as cross-talk - # p = power to rise the exponent (default, 1; use 1/2 or 1/3 to make cureves more stepp) - - z <- f * N / n # Expected treshold - sc <- 2 / (1 + exp(x/z)^p) # t-score - res <- data.table(Score = sc, TagJump = sc >= tmin) - return(res) -} - -## Esimate UNCROSS score -cat("..Estimating UNCROSS score\n") -OTUTAB <- cbind( - OTUTAB, - uncross_score( - x = OTUTAB$Abundance, - N = OTUTAB$Total, - n = length(unique(OTUTAB$SampleID)), - f = as.numeric(args[2]), - p = as.numeric(args[3]) - ) - ) - -## Truncate singletons with total OTU abundance > 99 reads -# OTUTAB[ Abundance == 1 & Total > 99 , TagJump := TRUE ] -# OTUTAB[ Abundance == 2 & Total > 999 , TagJump := TRUE ] - -cat("...Number of tag-jumps: ", sum(OTUTAB$TagJump, na.rm = TRUE), "\n") - - -## Plot -cat("..Making a plot\n") -PP <- ggplot(data = OTUTAB, aes(x = Total, y = Abundance, color = TagJump)) + - geom_point() + scale_x_log10() + scale_y_log10() + - scale_color_manual(values = c("#0C7C59", "#D64933")) + - labs(x = "Total abundance of OTU, reads", y = "Abundance of OTU in a sample, reads") - -cat("..Exporting a plot\n") -pdf(file = "TagJump_plot.pdf", width = 12, height = 9.5, useDingbats = FALSE) - PP -dev.off() - - -## TJ stats -cat("..Calculating tag-jump summary\n") -TJ <- data.table( - Total_reads = sum(OTUTAB$Abundance), - Number_of_TagJump_Events = sum(OTUTAB$TagJump), - TagJump_reads = sum(OTUTAB[ TagJump == TRUE ]$Abundance, na.rm = T) - ) - -TJ$ReadPercent_removed <- with(TJ, (TagJump_reads / Total_reads * 100)) - -fwrite(x = TJ, file = "TagJump_stats.txt", sep = "\t") - - -## Exporting tag-jump data -cat("..Exporting tag-jump data\n") -JMPS <- OTUTAB[ TagJump == TRUE, .(OTU, SampleID) ] - -saveRDS(object = JMPS, - file = "TagJump_OTUs.RData", - compress = "xz") - - -## Prepare OTU tables, remove tag-jumps -cat("..Removing tag-jumps\n") - -OTUTAB <- OTUTAB[ TagJump == FALSE ] - -## Convert to wide format -RES <- dcast(data = OTUTAB, - formula = OTU ~ SampleID, - value.var = "Abundance", fill = 0) - -## Sort rows (by total abundance) -clz <- colnames(RES)[-1] -otu_sums <- rowSums(RES[, ..clz], na.rm = TRUE) -RES <- RES[ order(otu_sums, decreasing = TRUE) ] - - -cat("..Exporting tag-jump filtered table\n") - -fwrite(x = RES, - file = "OTU_tab_TagJumpFiltered.txt.gz", - sep = "\t", compress = "gzip") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/tag_jump_removal_longtab.R b/src/pipecraft-core/service_scripts/NextITS/bin/tag_jump_removal_longtab.R deleted file mode 100755 index c7bd52ee..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/tag_jump_removal_longtab.R +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to perform tag-jump removal - -# Input arguments: -# 1. Sequence table in long format, no header (`Seq_tab_not_filtered.txt.gz`), -# with columns: `SeqID`, `Abundance`, `SampleID` -# 2. Dereplicated or pre-clustered membership table (`TJPreclust.uc.parquet`) -# 2. f-parameter of UNCROSS (e.g., 0.01) -# 3. p-parameter (e.g., 1.0) - -# Outputs: -# - Tag-jump-filtered sequence table (`Seq_tab_TagJumpFiltered.txt.gz`) -# - Table with tag-jump scores (`TagJump_scores.qs`) -# - Plot (`TagJump_plot.pdf`) - -cat("\nParsing input options and arguments...\n") - -suppressPackageStartupMessages(require(optparse)) - -## Parse arguments -option_list <- list( - make_option(c("-s", "--seqtab"), action="store", default="seqtab.txt.gz", type='character', help="Sequence table in long format"), - make_option(c("-c", "--precls"), action="store", default="precls.txt.gz", type='character', help="Table with pre-clustered sequence membership"), - make_option(c("-f", "--uncross_f"), action="store", default=0.01, type='numeric', help="f-parameter of UNCROSS"), - make_option(c("-p", "--uncross_p"), action="store", default=1, type='numeric', help="Additional p-parameter for UNCROSS") -) -opt <- parse_args(OptionParser(option_list=option_list)) - -## Function to convert text "NA"s to NA -to_na <- function(x){ - if(x %in% c("NA", "null", "Null")){ x <- NA } - return(x) -} - -## Replaces "null"s from Nextflow with NA -opt <- lapply(X = opt, FUN = to_na) - -## Validation of the required arguments -if(is.na(opt$seqtab)){ - stop("Input file with sequence table is not specified\n") -} -if(is.na(opt$precls)){ - stop("Input file with pre-clustered membership table is not specified\n") -} - -## Set default params if not specified -if(is.na(opt$uncross_f) | is.null(opt$uncross_f) | is.nan(opt$uncross_f) | !is.numeric(opt$uncross_f)){ - cat("f-parameter is not specified, using default value: 0.01\n") - opt$uncross_f <- 0.01 -} -if(is.na(opt$uncross_p) | is.null(opt$uncross_p) | is.nan(opt$uncross_p) | !is.numeric(opt$uncross_p)){ - cat("p-parameter is not specified, using default value: 1\n") - opt$uncross_p <- 1 -} - -## Assign variables -SEQTAB <- opt$seqtab -PRECLS <- opt$precls -F <- opt$uncross_f -P <- opt$uncross_p - -## Log assigned variables -cat("\nParameters specified:\n") -cat(paste("Sequence table: " , SEQTAB, "\n", sep = "")) -cat(paste("Pre-clustered membership table: ", PRECLS, "\n", sep = "")) -cat(paste("f-parameter of UNCROSS: ", F, "\n", sep = "")) -cat(paste("p-parameter of UNCROSS: ", P, "\n", sep = "")) - -cat("\n") - - -############################################## Data for debugging - -# SEQTAB <- "Seq_tab_not_filtered.txt.gz" -# PRECLS <- "TJPreclust.uc.parquet" -# F <- 0.01 -# P <- 1 - - -############################################## Load packages - -cat("Loading R packages...\n") - -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("data.table") -load_pckg("arrow") -load_pckg("ggplot2") -# load_pckg("qs") - -theme_set(theme_classic(base_size = 14)) - -cat("\n") - -############################################## Workflow - -## Load sequence table -cat("..Loading sequence table\n") -SEQTAB <- fread(file = SEQTAB, - sep = "\t", header = TRUE) # "SeqID", "SampleID", "Abundance" - -## Load sequence membership table -cat("..Loading sequence membership table\n") -PRECLS <- read_parquet(file = PRECLS) -setDT(PRECLS) -setnames(PRECLS, new = c("SeqID", "OTU")) - -## Remove ambiguous mappings (should not happen, but just in case) -if(any(duplicated(PRECLS$SeqID))){ - cat("WARNING: ambiguous mapping of sequences in membership table detected - excluding duplicates\n") - PRECLS <- unique(PRECLS, by = "SeqID") -} - -## Add cluster membership to the sequence table -SEQTAB <- merge(x = SEQTAB, y = PRECLS, by = "SeqID", all.x = TRUE) - -if(any(is.na(SEQTAB$OTU))){ - cat("WARNING: Sequences without cluster membership detected\n") - cat("WARNING: Excluding these records from the analysis\n") - SEQTAB <- SEQTAB[ !is.na(OTU) ] -} - -cat("...Number of unique sequences: ", length(unique(SEQTAB$SeqID)), "\n") -cat("...Number of clusters: ", length(unique(SEQTAB$OTU)), "\n") -cat("...Number of samples: ", length(unique(SEQTAB$SampleID)), "\n") - -## Summarize by sequence clusters -cat("..Summarizing by sequence clusters\n") -OTUTAB <- SEQTAB[ , .(Abundance = sum(Abundance, na.rm = TRUE)), by = c("OTU", "SampleID") ] - -## Estimate total abundance of sequence per plate -cat("..Estimating total OTU abundance\n") -OTUTAB[ , Total := sum(Abundance, na.rm = TRUE), by = "OTU" ] - -## UNCROSS score (with original parameter - take a root from the exp in denominator, to make curves more steep) -uncross_score <- function(x, N, n, f = 0.01, tmin = 0.1, p = 1){ - # x = OTU abundance in a sample - # N = total OTU abundance - # n = number of samples - # f = expected cross-talk rate, e.g. 0.01 - # tmin = min score to be considered as cross-talk - # p = power to rise the exponent (default, 1; use 1/2 or 1/3 to make cureves more stepp) - - z <- f * N / n # Expected treshold - sc <- 2 / (1 + exp(x/z)^p) # t-score - res <- data.table(Score = sc, TagJump = sc >= tmin) - return(res) -} - -## Esimate UNCROSS score -cat("..Estimating UNCROSS score\n") -OTUTAB <- cbind( - OTUTAB, - uncross_score( - x = OTUTAB$Abundance, - N = OTUTAB$Total, - n = length(unique(OTUTAB$SampleID)), - f = as.numeric(F), - p = as.numeric(P) - ) - ) - -## Truncate singletons with total OTU abundance > 99 reads -# OTUTAB[ Abundance == 1 & Total > 99 , TagJump := TRUE ] -# OTUTAB[ Abundance == 2 & Total > 999 , TagJump := TRUE ] - -cat("...Number of tag-jumps: ", sum(OTUTAB$TagJump, na.rm = TRUE), "\n") - -## Export tag-jump scores -setcolorder(OTUTAB, - c("OTU", "SampleID", "TagJump", "Score", "Abundance", "Total")) - -setorder(OTUTAB, OTU, -Abundance, SampleID) - -cat("..Exporting tag-jump scores\n") -qs::qsave(OTUTAB, - "TagJump_scores.qs", - preset = "custom", algorithm = "zstd", compress_level = 5L, nthreads = 1L) - - -## Plot -cat("..Making a plot\n") -PP <- ggplot(data = OTUTAB, aes(x = Total, y = Abundance, color = TagJump)) + - geom_point() + scale_x_log10() + scale_y_log10() + - scale_color_manual(values = c("#0C7C59", "#D64933")) + - labs(x = "Total abundance of OTU, reads", y = "Abundance of OTU in a sample, reads") - -cat("..Exporting a plot\n") -pdf(file = "TagJump_plot.pdf", width = 12, height = 9.5, useDingbats = FALSE) - PP -dev.off() - - - -## Exporting tag-jump data -# cat("..Exporting tag-jump data\n") -# JMPS <- OTUTAB[ TagJump == TRUE, .(OTU, SampleID) ] -# -# saveRDS(object = JMPS, -# file = "TagJump_OTUs.RData", -# compress = "xz") - - -## Prepare filtered sequence table, remove tag-jumps -cat("..Removing tag-jumps\n") - -## Add tag-jump info to the sequence table -n1 <- nrow(SEQTAB) - -RES <- merge( - x = SEQTAB, - y = OTUTAB[ , .(OTU, SampleID, TagJump) ], - by = c("OTU", "SampleID"), all.x = TRUE) - -n2 <- nrow(RES) -if(n1 != n2){ - cat("WARNING: merging went wrong likely\n") - cat("WARNING: There might be duplicated sequences\n") -} - - -## TJ stats -cat("..Calculating tag-jump summary\n") -TJ <- data.table( - Total_reads = sum(RES$Abundance), - Number_of_TagJump_Events = sum(RES$TagJump), - TagJump_reads = sum(RES[ TagJump == TRUE ]$Abundance, na.rm = T) - ) - -TJ$ReadPercent_removed <- with(TJ, (TagJump_reads / Total_reads * 100)) - -fwrite(x = TJ, file = "TagJump_stats.txt", sep = "\t") - - -## Keep only non-tag-jump reads -RES <- RES[ TagJump == FALSE , .(SampleID, SeqID, Abundance) ] -setorder(RES, SampleID, -Abundance) - -cat("..Exporting tag-jump filtered table\n") - -fwrite(x = RES, - file = "Seq_tab_TagJumpFiltered.txt.gz", - sep = "\t", compress = "gzip") diff --git a/src/pipecraft-core/service_scripts/NextITS/bin/validate_tags.R b/src/pipecraft-core/service_scripts/NextITS/bin/validate_tags.R deleted file mode 100755 index 6cb4d4ee..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/bin/validate_tags.R +++ /dev/null @@ -1,488 +0,0 @@ -#!/usr/bin/env Rscript - -## Script to validate tags (barcodes) used during sample multiplexing -## - Tags should be unique -## - Tag names should be unique -## - Tag names must be alphanumeric (ASCII-only) and must not contain whitespace, dot, comma, semicolon, or dash -## - Sequencing run ID could be present in tag names (before double underscore) -## - Checks the presence of positive and negative controls -## - Estimates number of unqiue tags and their length -## - For dual assymetric tags, -## unique barcodes are converted into a "long" format, -## a biosample tables (`biosamples_asym.csv` and `biosamples_sym.csv`), -## file naming scheme (`file_renaming.tsv`), -## and `unknown_combinations.tsv` are exported as well - -## Usage: -# validate_tags.R \ -# --tags tags.fasta \ -# --output tags_validated.fasta - - - -cat("Parsing input options and arguments...\n") - -suppressPackageStartupMessages(require(optparse)) - -## Parse arguments -option_list <- list( - make_option("--tags", action="store", default=NA, type='character', help="FASTA file with tags"), - make_option("--output", action="store", default=NA, type='character', help="FASTA file with validated tags") -) -opt <- parse_args(OptionParser(option_list=option_list)) - - -## Validation of the required argiments -if(is.na(opt$tags)){ - cat("Input file is not specified!\n", file=stderr()) - stop() -} -if(is.na(opt$output)){ - cat("Output file is not specified!\n", file=stderr()) - stop() -} - -## Assign variables -TAGS <- opt$tags -OUTP <- opt$output - -## Log assigned variables -cat(paste("FASTA file with tags: ", TAGS, "\n", sep="")) -cat(paste("Output file with validated tags: ", OUTP, "\n", sep="")) - -cat("\n") - - - -############## - -cat("Loading R packages...\n") - -load_pckg <- function(pkg = "data.table"){ - suppressPackageStartupMessages( library(package = pkg, character.only = TRUE) ) - cat(paste(pkg, packageVersion(pkg), "\n")) -} - -load_pckg("data.table") -load_pckg("Biostrings") - -cat("\n") - - -## Load FASTA sequences -cat("..Loading sequence tables\n") -TAGS <- try( readDNAStringSet(filepath = TAGS, format="fasta") ) - -if("try-error" %in% class(TAGS)){ - cat("Error in reading FASTA file!\n", file=stderr()) - stop(TAGS) -} - -cat("Number of records in the file: ", length(TAGS), "\n") - - -######################## -######################## Validate sample names -######################## - -cat("\n\n===== Validating sample names =====\n\n") - -cat("Positive control: ", - ifelse(test = any(grepl(pattern = "PosC", x = names(TAGS))), - yes = "Present", no = "Absent"), "\n") - -cat("Negative control: ", - ifelse(test = any(grepl(pattern = "NegC", x = names(TAGS))), - yes = "Present", no = "Absent"), "\n") - -## Validate names -cat("\nValidating tag names\n") -newnames <- names(TAGS) - -cat("..Replacing leading and trailing spaces and tabs\n") -newnames <- trimws(x = newnames, which = "both") - -cat("..Replacing duplicated spaces, dashes, dots, commas, or semicolons\n") -newnames <- gsub(pattern = "\\s+", replacement = " ", x = newnames) -newnames <- gsub(pattern = "\\-+", replacement = "-", x = newnames) -newnames <- gsub(pattern = "\\.+", replacement = ".", x = newnames) -newnames <- gsub(pattern = ",+", replacement = ",", x = newnames) -newnames <- gsub(pattern = ";+", replacement = ";", x = newnames) - -cat("..Replacing disallowed symbols\n") -newnames <- iconv(newnames, from = "UTF-8", to = "ASCII//TRANSLIT") -newnames <- gsub(pattern = "[^[:alnum:]]", replacement = "_", x = newnames) - -cat("..Replacing the second occurrence of double underscore\n") -## need something like `sed 's/__/_/2g'` -newnames <- sub(pattern = "__", replacement = "TEMPTEMPTEMPTEMP", x = newnames) -newnames <- gsub(pattern = "_+", replacement = "_", x = newnames) -newnames <- sub(pattern = "TEMPTEMPTEMPTEMP", replacement = "__", x = newnames) - -## test: c("A__B", "B__C__D", "E__F__G__H", "A___3", "B___4___E") - - -## Find out which names were changed -renamed <- data.table( - OriginalName = names(TAGS), - NewName = newnames) - -renamed[ , Renamed := OriginalName != NewName ] -renamed <- renamed[ Renamed == TRUE ] - -if(nrow(renamed) > 0){ - cat("..The following tag names were corrected:\n") - print( - renamed[, .(OriginalName, NewName)], - nrows = nrow(renamed), trunc.cols = FALSE) - - cat("...Exporting renamed tag names\n") - fwrite(x = renamed[ , .(OriginalName, NewName)], - file = "tag_names_renamed.tsv", quote = FALSE, sep = "\t", col.names = FALSE) -} - -names(TAGS) <- newnames - - -## Check tag name uniqness -nuniq <- length(unique(names(TAGS))) -cat("\nAll tag names unique: ", - ifelse(test = (length(TAGS) == nuniq), - yes = "TRUE", no = "FALSE"), "\n") - -if(length(TAGS) != nuniq){ - cat("..Not all tag names are unique!\n") - cat("..Resolving tag name uniqness by adding sequential numbers to non-unique names\n") - - dups <- unique(names(TAGS)[ which(duplicated(names(TAGS))) ]) - cat("..Number of duplicates: ", length(dups), "\n") - cat("..Duplicated names: ", paste(dups, collapse = ", "), "\n") - - dtt <- data.table(ID = 1:length(TAGS), TagName = names(TAGS)) - dtt[ , Duplicated := TagName %in% dups ] - dtt[ , NewName := TagName ] - dtt[ - Duplicated == TRUE, - NewName := paste0(TagName, "_", 1:.N), - by = "TagName" ] - setorder(x = dtt, ID) - - names(TAGS) <- dtt$NewName - - rm(dtt) -} - - -## Check run name -TESTRUN <- grepl(pattern = "__", x = names(TAGS)) - -cat("\nTag names contain sequencing run ID: ", - ifelse(test = any(TESTRUN), - yes = "TRUE", no = "FALSE"), "\n") - -if(any(TESTRUN)){ - - ## Check that all samples contain RunID - cat("\nAll samples contain sequencing run ID: ", - ifelse(test = sum(TESTRUN) == length(names(TAGS)), - yes = "TRUE", no = "FALSE"), "\n") - - ## Check run name uniqness - dtt <- data.table(TagName = names(TAGS)) - dtt[ , c("RunID", "SampleID") := tstrsplit(x = TagName, split = "__", keep = 1:2) ] - - cat("Number of run IDs in tag names (ideally, should be = 1): ", length(unique(dtt$RunID)), "\n") - cat("Run IDs detected: ", paste(unique(dtt$RunID), collapse = ", "), "\n") - - rm(dtt) -} - - -######################## -######################## Validate sequences -######################## - -cat("\n\n===== Validating sequences =====\n\n") - - -DUAL <- grepl(pattern = "\\.\\.\\.", x = as.character(TAGS)) - -if(any(DUAL)){ - cat("Barcode type detected: Dual\n") - - if(any(!DUAL)){ - cat("WARNING: mixture of single and dual tags detected!\n") - print(names(TAGS)[ !DUAL ]) - stop("\nPlease fix the tag sequences (remove single tags or add double dots to dual tags)!\n") - } - -} else { - cat("Barcode type detected: Single (or dual symmetric)\n") -} - - -##### Single tag - -if(any(DUAL) == FALSE){ - - cat("Tag length: ", paste(unique(width(TAGS)), collapse = ", "), "\n") - - suniq <- length(unique(as.character(TAGS))) - cat("\nAll tag sequences unique: ", - ifelse(test = (length(TAGS) == suniq), - yes = "TRUE", no = "FALSE"), "\n") - - if(length(TAGS) != suniq){ - cat("..Not all tag sequences are unique!\n") - cat("..This should be resolved manually!\n") - - dup_name <- unique( names(TAGS)[ duplicated(as.character(TAGS)) ]) - dup_tags <- as.character(TAGS[ dup_name ]) - - cat("..Number of duplicated tags: ", length(dup_name), "\n") - - dupss <- TAGS[ TAGS %in% dup_tags ] - dups <- data.table( - TagNames = names(dupss), - Tags = as.character(dupss)) - - dup_smr <- dups[ , .( - TagNames = paste0("[ ", paste(TagNames, collapse = ", "), " ]") - ), - by = "Tags"] - - cat("..Duplicates: \n") - print(dup_smr, nrows = length(TAGS), trunc.cols = FALSE) - - stop("\nPlease fix the tag sequences!\n") - } - - ## Export FASTA - cat("Exporting validated tags in FASTA format\n") - - writeXStringSet( - x = TAGS, - filepath = OUTP, - compress = FALSE, - format = "fasta", - width = 9999) - -} # end of single tag - - - -##### Dual tags - -if(any(DUAL) == TRUE){ - - ## Convert to tabular format - dtt <- data.table( - SampleID = names(TAGS), - Tags = as.character(TAGS)) - - ## Split dual tags - dtt[ , c("Tag1", "Tag2") := tstrsplit(x = Tags, split = "\\.\\.\\.", keep = c(1,2)) ] - - ## Check if there are any missing tags - missing_tags <- dtt[ is.na(Tag1) | is.na(Tag2) ] - if(nrow(missing_tags) > 0){ - cat("WARNING: missing dual tags detected!\n") - print(missing_tags) - stop("\nPlease fix the tag sequences!\n") - } - - cat("..Forward tag length: ", paste(sort(unique(nchar(dtt$Tag1))), collapse = ", "), "\n") - cat("..Reverse tag length: ", paste(sort(unique(nchar(dtt$Tag2))), collapse = ", "), "\n") - - cat("\n") - cat("..Number of unique forward tags: ", length(unique(dtt$Tag1)), "\n") - cat("..Number of unique reverse tags: ", length(unique(dtt$Tag2)), "\n") - - ## Find unique barcodes - bu <- data.table(Sequence = unique(c(dtt$Tag1, dtt$Tag2))) - - ## Name unique barcodes - len <- nchar(nrow(bu)) - bu[ , ID := .I ] - bu[ , ID := sprintf(paste("%0", len, "d", sep = ""), ID) ] - bu[ , ID := paste0("bc", ID) ] - - ## Convert to FASTA - seqs <- DNAStringSet(x = bu$Sequence) - names(seqs) <- bu$ID - - - - ## Add bacrode IDs - dtt <- merge(x = dtt, y = bu, by.x = "Tag1", by.y = "Sequence", all.x = TRUE) - setnames(x = dtt, old = "ID", new = "ID1") - - dtt <- merge(x = dtt, y = bu, by.x = "Tag2", by.y = "Sequence", all.x = TRUE) - setnames(x = dtt, old = "ID", new = "ID2") - - dtt[ , Barcodes := paste0(ID1, "--", ID2)] - - - dtt[ , TagSymmetry := fifelse(Tag1 == Tag2, yes = "symmetric", no = "asymmetric", na = NA) ] - cat("Number of symmetric tag combinations: ", sum(dtt$TagSymmetry %in% "symmetric"), "\n") - cat("Number of asymmetric tag combinations: ", sum(dtt$TagSymmetry %in% "asymmetric"), "\n") - - ## Validate barcode combination uniqness - if(nrow(dtt) != length(unique(dtt$Barcodes))){ - cat("WARNING: non-unique barcode combination detected!\n") - - dups <- dtt$Barcodes[ duplicated(dtt$Barcodes) ] - print( dtt[ Barcodes %in% dups ] ) - - stop("\nPlease fix the tag sequences!\n") - } - - - ## Find unique barcode combinations (taking into account reverse complements) - dtt[, tag_pair_unordered := paste( - pmin(Tag1, Tag2), - pmax(Tag1, Tag2), - sep="|") ] - - dtt[, tag_pair_ordered := paste(Tag1, Tag2, sep="|") ] - - ## Swapped-pair ambiguity: X-Y exists AND Y-X exists (cannot disambiguate if you don't know direction) - swapped <- dtt[, .( - n_samples = .N, - n_ordered = uniqueN(tag_pair_ordered), - samples = paste(sort(SampleID), collapse=", "), - ordered_set = paste(sort(unique(tag_pair_ordered)), collapse=", ") - ), by = tag_pair_unordered][ n_samples > 1 ] - - if(nrow(swapped) > 0){ - cat("\nWARNING: swapped-pair tag ambiguity detected!\n") - cat("..Number of swapped-pair tag combinations: ", nrow(swapped), "\n") - cat("..Swapped-pair tag combinations: ", "\n") - print(swapped[ , .(samples, tag_pair_unordered) ]) - stop("\nIt is impossible to assign sample ID to sequences with swapped-pair tag combinations!\nPlease fix the tag sequences (e.g., combine primer sequence with the tag sequence!\n") - } - - - ## Prepare biosample table for LIMA - # https://lima.how/faq/biosample.html - cat("\nExporting biosample tables: 'biosamples_sym.csv' and 'biosamples_asym.csv'\n") - - res <- data.table( - Barcodes = dtt$Barcodes, - `Bio Sample` = dtt$SampleID, - TagSymmetry = dtt$TagSymmetry) - - setorder(x = res, `Bio Sample`) - - fwrite( - x = res[ TagSymmetry %in% "symmetric", .(Barcodes, `Bio Sample`) ] , - file = "biosamples_sym.csv", - quote = FALSE, sep = ",", col.names = TRUE) - - fwrite( - x = res[ TagSymmetry %in% "asymmetric", .(Barcodes, `Bio Sample`) ] , - file = "biosamples_asym.csv", - quote = FALSE, sep = ",", col.names = TRUE) - - - ## File for sample renaming - res[ , OldName := paste0("lima.", Barcodes, ".fq.gz") ] - res[ , NewName := paste0(`Bio Sample`, ".fq.gz") ] - - ## The order of tags can be different in the FASTQ file names - ## Ensure that we keep track of both options (x--y and y--x) - tmp <- copy(res) - tmp[ , c("ID1", "ID2") := tstrsplit(x = Barcodes, split = "--", keep = 1:2) ] - tmp[ , Barcodes := paste0(ID2, "--", ID1) ] - tmp[ , OldName := paste0("lima.", Barcodes, ".fq.gz") ] - tmp[ , ID1 := NULL ] - tmp[ , ID2 := NULL ] - - res <- rbind(res, tmp) - rm(tmp) - res <- unique(res, by = "OldName") - setorder(x = res, Barcodes) - - cat("Exporting file naming scheme: 'file_renaming.tsv'\n") - - fwrite(x = res[ , .(OldName, NewName)], - file = "file_renaming.tsv", quote = F, sep = "\t", col.names = FALSE) - - ## Export unique barcodes - cat("Exporting unique tags in FASTA format\n") - - writeXStringSet( - x = seqs, - filepath = OUTP, - compress = FALSE, - format = "fasta", - width = 9999) - - - ## Prepare unknown combinations - cat("Preparing unknown tag combinations\n") - - UNKN <- CJ( - Tag1 = unique(dtt$Tag1), - Tag2 = unique(dtt$Tag2)) - - UNKN <- merge(x = UNKN, y = bu, by.x = "Tag1", by.y = "Sequence", all.x = TRUE) - setnames(x = UNKN, old = "ID", new = "ID1") - - UNKN <- merge(x = UNKN, y = bu, by.x = "Tag2", by.y = "Sequence", all.x = TRUE) - setnames(x = UNKN, old = "ID", new = "ID2") - - ## Trying to parse RunID from the first sample - if(any(TESTRUN)){ - cat("WARNING: in assumption that there is a single sequencing run, RunID of the first sample will be used!\n") - RUNID <- tstrsplit(dtt$SampleID[1], split = "__", keep = 1)[[1]] - if(is.na(RUNID)){ - cat("WARNING: RunID is not found in the sample name\n") - RUNID <- "unknown" - } - } else { - RUNID <- "unknown" - } - - UNKN1 <- copy(UNKN) - UNKN1[ , IDS := paste0(ID1, "--", ID2) ] - UNKN1[ , OldName := paste0("lima.", IDS, ".fq.gz") ] - UNKN1[ , Barcodes := paste0(Tag1, "_", Tag2) ] - UNKN1[ , NewName := paste0(RUNID, "__", Barcodes, ".fq.gz") ] - - UNKN2 <- copy(UNKN) - UNKN2[ , IDS := paste0(ID2, "--", ID1) ] - UNKN2[ , OldName := paste0("lima.", IDS, ".fq.gz") ] - UNKN2[ , Barcodes := paste0(Tag2, "_", Tag1) ] - UNKN2[ , NewName := paste0(RUNID, "__", Barcodes, ".fq.gz") ] - - UNKN <- rbind(UNKN1, UNKN2) - rm(UNKN1, UNKN2) - UNKN <- unique(UNKN, by = "OldName") - setorder(x = UNKN, OldName) - - ## Remove known combinations - UNKN <- UNKN[ !IDS %in% res$Barcodes ] - - cat("Number of possible unknown combinations: ", nrow(UNKN), "\n") - - ## Export unknown combinations - cat("Exporting unknown combinations\n") - - fwrite(x = UNKN[ , .(OldName, NewName)], - file = "unknown_combinations.tsv", quote = F, sep = "\t", col.names = FALSE) - -} # end of dual tags - - -cat("\nValidation finished\n") - - -##################### Session info - -cat("\nAll done.\n") -cat("\n") -cat("Session info:\n") -sessionInfo() -cat("\n") - diff --git a/src/pipecraft-core/service_scripts/NextITS/conf/docker.config b/src/pipecraft-core/service_scripts/NextITS/conf/docker.config deleted file mode 100644 index 16c0eab8..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/conf/docker.config +++ /dev/null @@ -1,12 +0,0 @@ -// Container specifications for the Docker profile - -process { - - // Container from Docker Hub - container = 'vmikk/nextits:1.2.0' - - //// Container from Quay.io registry - // container = 'docker://quay.io/vmikk/nextits:1.1.0' - -} - diff --git a/src/pipecraft-core/service_scripts/NextITS/conf/hpc.config b/src/pipecraft-core/service_scripts/NextITS/conf/hpc.config deleted file mode 100644 index 07877a47..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/conf/hpc.config +++ /dev/null @@ -1,214 +0,0 @@ -/* - * ----------------------------------------------------------- - * Config file for execution on HPC - * ----------------------------------------------------------- - * Specification for a larger resource amount (e.g., CPU number) for each process - * - */ - -// HPC-specific params -params { - max_cpus = 60 - max_memory = "200.GB" - max_time = "240.h" -} - -// Process configuration -process { - - ////////// Step-1 processes - - // Converting BAM to FASTQ - withName: 'S1:bam2fastq' { - cpus = 12 - } - - // Primer disambiguation - withName: 'S1:disambiguate' { - cpus = 1 - } - - // Validate tags for demultiplexing - withName: 'S1:tag_validation' { - cpus = 1 - } - - // QC - PacBio single-end reads - // vsearch currently does not suppot multithreading for `--fastq_filter` - // see https://github.com/torognes/vsearch/issues/466 - withName: 'S1:qc_se' { - cpus = 1 - } - - // QC - Illumina paired-end reads - withName: 'S1:qc_pe' { - // max threads for fastp = 16 - cpus = 12 - } - - // Demultiplexing of PacBio reads (with LIMA) - withName: 'S1:demux' { - cpus = 12 - } - - // Demultiplexing merged Illumina reads - withName: 'S1:demux_illumina' { - cpus = 16 - } - - // Demultiplexing non-merged Illumina reads - withName: 'S1:demux_illumina_notmerged' { - cpus = 10 - } - - // Merging of Illumina PE reads - withName: 'S1:merge_pe' { - cpus = 8 - } - - // Demultiplexing of Illumina reads (with cutadapt) - withName: 'S1:demux_illumina' { - cpus = 16 - } - - // Check primers - withName: 'S1:primer_check' { - cpus = 1 - } - - // ITSx - withName: 'S1:itsx' { - cpus = 6 - } - - // Collect all ITS parts extracted by ITSx - withName: 'S1:itsx_collect' { - cpus = 1 - } - - // Sequence quality tables - withName: 'S1:seq_qual' { - cpus = 6 - memory = null - } - - // Homopolymer compression - withName: 'S1:homopolymer' { - cpus = 2 - } - - // Reference-based chimera removal - withName: 'S1:CHIMERA_REMOVAL:chimera_ref' { - cpus = 4 - } - - // Chimera rescue - withName: 'S1:CHIMERA_REMOVAL:chimera_rescue' { - cpus = 1 - } - - // De novo chimera search - withName: 'S1:CHIMERA_REMOVAL:chimera_denovo' { - cpus = 1 - } - - // Aggregate de novo chimeras - withName: 'S1:CHIMERA_REMOVAL:chimera_denovo_agg' { - cpus = 1 - } - - // Pool sequences (for tag-jump removal and final sequence table creation) - withName: 'S1:pool_seqs' { - cpus = 3 - } - - // Dereplication or pre-clustering prior to tag-jump removal - withName: 'S1:tj_preclust' { - cpus = 20 - } - - // Tag-jump removal - withName: 'S1:tj' { - cpus = 1 - } - - // Create sequence table - withName: 'S1:prep_seqtab' { - cpus = 8 - } - - // Read count summary - withName: 'S1:read_counts' { - cpus = 4 - } - - - ////////// Step-2 processes - - // Aggregate sequences, remove de novo chimeras - withName: 'S2:aggregate_sequences' { - cpus = 8 - } - - // Dereplication (only comression is multithreaded) - withName: 'S2:dereplication' { - cpus = 20 - } - - // 100% clustering with sequence length variation allowed (UNITE-style) - withName: 'S2:dereplication_unite' { - cpus = 20 - } - - // Pre-clustering - withName: 'S2:linclust' { - cpus = 8 - } - - // Bucketize sequences into clusters - withName: 'S2:bucketize' { - cpus = 6 - } - - // UNOISE - withName: 'S2:CLUSTERING:unoise' { - cpus = 20 - } - - // DADA2 - withName: 'S2:CLUSTERING:dada2' { - cpus = 20 - } - - // VSEARCH clustering - withName: 'S2:CLUSTERING:cluster_vsearch' { - cpus = 20 - } - - // SWARM clustering - withName: 'S2:CLUSTERING:cluster_swarm' { - cpus = 20 - } - - // Bucketizing workflow - merge chunks into a single file - withName: 'S2:merge_buckets' { - cpus = 4 - } - - // Merge UC files - withName: 'S2:merge_uc' { - cpus = 4 - } - - // Summarize sequence abundance by OTU - withName: 'S2:summarize' { - cpus = 12 - } - - // Post-clustering curation with LULU - withName: 'S2:lulu' { - cpus = 20 - } - -} - diff --git a/src/pipecraft-core/service_scripts/NextITS/conf/hpc_utslurm.config b/src/pipecraft-core/service_scripts/NextITS/conf/hpc_utslurm.config deleted file mode 100644 index 676654fc..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/conf/hpc_utslurm.config +++ /dev/null @@ -1,340 +0,0 @@ -/* - * ----------------------------------------------------------- - * Config file for execution on HPC - * ----------------------------------------------------------- - * Specification for a larger resource amount (e.g., CPU number) for each process - * - */ - -// HPC-specific params -executor { - name = 'slurm' - queueSize = 200 - submitRateLimit = '5 sec' - pollInterval = '1sec' - retry.delay = '1sec' -} - -env { - OPENBLAS_NUM_THREADS=1 - OMP_NUM_THREADS=1 -} - -// Process configuration -process { - - // Cluster-specific options (e.g., partition name, billing account, etc.) - clusterOptions = '-p amd' - - // Error strategy - errorStrategy = { task.exitStatus in [1,104,125,130,134,135,137,139,140,143,255] ? 'retry' : 'finish' } - maxRetries = 3 - maxErrors = '-1' - - // Default resources - cpus = { check_max( 1 * task.attempt, 'cpus' ) } - memory = { check_max( 2.GB * task.attempt, 'memory' ) } - time = { check_max( 3.h * task.attempt, 'time' ) } - - ////////// Step-1 processes - - // Converting BAM to FASTQ - withName: 'S1:bam2fastq' { - cpus = 12 - memory = 2.GB - time = { check_max( 4.h * task.attempt, 'time' ) } - } - - // Primer disambiguation - withName: 'S1:disambiguate' { - cpus = 1 - memory = 1.GB - time = 20.m - } - - // Validate tags for demultiplexing - withName: 'S1:tag_validation' { - cpus = 1 - memory = 1.GB - time = 20.m - } - - // QC - PacBio single-end reads - // vsearch currently does not suppot multithreading for `--fastq_filter` - // see https://github.com/torognes/vsearch/issues/466 - withName: 'S1:qc_se' { - cpus = 2 // for pipes - memory = 1.GB - time = { check_max( 2.h * task.attempt, 'time' ) } - } - - // QC - Illumina paired-end reads - withName: 'S1:qc_pe' { - // max threads for fastp = 16 - cpus = 12 - memory = 20.GB - time = 4.h - } - - // Demultiplexing of PacBio reads (with LIMA) - withName: 'S1:demux' { - cpus = 12 - memory = { check_max( 3.GB * task.attempt, 'memory' ) } - time = { check_max( 1.h * task.attempt, 'time' ) } - } - - // Demultiplexing merged Illumina reads - withName: 'S1:demux_illumina' { - cpus = 16 - memory = 20.GB - time = 4.h - } - - // Demultiplexing non-merged Illumina reads - withName: 'S1:demux_illumina_notmerged' { - cpus = 10 - memory = 20.GB - time = 4.h - } - - // Merging of Illumina PE reads - withName: 'S1:merge_pe' { - cpus = 8 - memory = 20.GB - time = 4.h - } - - // Demultiplexing of Illumina reads (with cutadapt) - withName: 'S1:demux_illumina' { - cpus = 16 - memory = 20.GB - time = 4.h - } - - // Check primers - withName: 'S1:primer_check' { - cpus = 1 - memory = 1.GB - time = { check_max( 30.m * task.attempt, 'time' ) } - } - - // ITSx - withName: 'S1:itsx' { - cpus = 20 - memory = { check_max( 10.GB * task.attempt, 'memory' ) } - time = { check_max( 20.h * task.attempt, 'time' ) } - } - - // Collect all ITS parts extracted by ITSx - withName: 'S1:itsx_collect' { - cpus = 1 - memory = 1.GB - time = { check_max( 30.m * task.attempt, 'time' ) } - } - - // Sequence quality tables - withName: 'S1:seq_qual' { - cpus = 6 - memory = { check_max( 50.GB * task.attempt, 'memory' ) } - time = { check_max( 1.h * task.attempt, 'time' ) } - } - - // Homopolymer compression - withName: 'S1:homopolymer' { - cpus = 2 - memory = 2.GB - time = { check_max( 3.h * task.attempt, 'time' ) } - } - - // Reference-based chimera removal - withName: 'S1:CHIMERA_REMOVAL:chimera_ref' { - cpus = 4 - memory = { check_max( 3.GB * task.attempt, 'memory' ) } - time = { check_max( 1.h * task.attempt, 'time' ) } - } - - // Chimera rescue - withName: 'S1:CHIMERA_REMOVAL:chimera_rescue' { - cpus = 1 - memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { check_max( 1.h * task.attempt, 'time' ) } - } - - // De novo chimera search - withName: 'S1:CHIMERA_REMOVAL:chimera_denovo' { - cpus = 1 - memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { check_max( 1.h * task.attempt, 'time' ) } - } - - // Aggregate de novo chimeras - withName: 'S1:CHIMERA_REMOVAL:chimera_denovo_agg' { - cpus = 1 - memory = 1.GB - time = { check_max( 1.h * task.attempt, 'time' ) } - } - - // Pool sequences (for tag-jump removal and final sequence table creation) - withName: 'S1:pool_seqs' { - cpus = 3 - memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { check_max( 30.m * task.attempt, 'time' ) } - } - - // Dereplication or pre-clustering prior to tag-jump removal - withName: 'S1:tj_preclust' { - cpus = 12 - memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } - } - - // Tag-jump removal - withName: 'S1:tj' { - cpus = 1 - memory = { check_max( 2.GB * task.attempt, 'memory' ) } - time = { check_max( 30.m * task.attempt, 'time' ) } - } - - // Create sequence table - withName: 'S1:prep_seqtab' { - cpus = 8 - memory = { check_max( 50.GB * task.attempt, 'memory' ) } - time = { check_max( 30.m * task.attempt, 'time' ) } - } - - // Read count summary - withName: 'S1:read_counts' { - cpus = 4 - memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { check_max( 30.m * task.attempt, 'time' ) } - } - - - ////////// Step-2 processes - - // Aggregate sequences, remove de novo chimeras - withName: 'S2:aggregate_sequences' { - cpus = 12 - memory = { check_max( 40.GB * task.attempt, 'memory' ) } - time = { check_max( 1.h * task.attempt, 'time' ) } - } - - // Dereplication (only comression is multithreaded) - withName: 'S2:dereplication' { - cpus = 20 - memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { check_max( 5.h * task.attempt, 'time' ) } - } - - // 100% clustering with sequence length variation allowed (UNITE-style) - withName: 'S2:dereplication_unite' { - cpus = 20 - memory = { check_max( 50.GB * task.attempt, 'memory' ) } - time = { check_max( 24.h * task.attempt, 'time' ) } - } - - // Pre-clustering - withName: 'S2:linclust' { - cpus = 20 - memory = { check_max( 50.GB * task.attempt, 'memory' ) } - time = { check_max( 1.h * task.attempt, 'time' ) } - } - - // Bucketize sequences into clusters - withName: 'S2:bucketize' { - cpus = 6 - memory = { check_max( 50.GB * task.attempt, 'memory' ) } - time = { check_max( 2.h * task.attempt, 'time' ) } - } - - // UNOISE - withName: 'S2:CLUSTERING:unoise' { - cpus = 30 - memory = { check_max( 40.GB * task.attempt, 'memory' ) } - time = { check_max( 30.h * task.attempt, 'time' ) } - } - - // DADA2 - withName: 'S2:CLUSTERING:dada2' { - cpus = 30 - memory = { check_max( 60.GB * task.attempt, 'memory' ) } - time = { check_max( 30.h * task.attempt, 'time' ) } - } - - // VSEARCH clustering - withName: 'S2:CLUSTERING:cluster_vsearch' { - cpus = 30 - memory = { check_max( 80.GB * task.attempt, 'memory' ) } - time = { check_max( 30.h * task.attempt, 'time' ) } - } - - // SWARM clustering - withName: 'S2:CLUSTERING:cluster_swarm' { - cpus = 30 - memory = { check_max( 80.GB * task.attempt, 'memory' ) } - time = { check_max( 24.h * task.attempt, 'time' ) } - } - - // Bucketizing workflow - merge chunks into a single file - withName: 'S2:merge_buckets' { - cpus = 4 - memory = { check_max( 40.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } - } - - // Merge UC files - withName: 'S2:merge_uc' { - cpus = 4 - memory = { check_max( 40.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } - } - - // Summarize sequence abundance by OTU - withName: 'S2:summarize' { - cpus = 2 - memory = { check_max( 20.GB * task.attempt, 'memory' ) } - time = { check_max( 5.h * task.attempt, 'time' ) } - } - - // Post-clustering curation with LULU - withName: 'S2:lulu' { - cpus = 30 - memory = { check_max( 60.GB * task.attempt, 'memory' ) } - time = { check_max( 24.h * task.attempt, 'time' ) } - } - -} - - -// Function to ensure that resource requirements don't go beyond a maximum limit -def check_max(obj, type) { - if (type == 'memory') { - try { - if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit - else - return obj - } catch (all) { - println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'time') { - try { - if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration - else - return obj - } catch (all) { - println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'cpus') { - try { - return Math.min( obj, params.max_cpus as int ) - } catch (all) { - println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" - return obj - } - } -} - diff --git a/src/pipecraft-core/service_scripts/NextITS/conf/params.config b/src/pipecraft-core/service_scripts/NextITS/conf/params.config deleted file mode 100644 index 7d34acb3..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/conf/params.config +++ /dev/null @@ -1,307 +0,0 @@ -/* -This file includes default values for the NextITS pipeline parameters -*/ - - -params { - - step = "Step1" // default pipeline step to run - - //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Step-1 parameters - - // Input files - input = null // FASTQ file or directory - input_R1 = null - input_R2 = null - barcodes = null // FASTA file - - // Output - outdir = "${launchDir}/results" - tracedir = "${params.outdir}/pipeline_info" - - // Which sequencing platform was used? - seqplatform = "PacBio" - // seqplatform = "Illumina" - - // ITS part selector - its_region = "full" - // "full" = default (full-length ITS sequence, after trimming SSU and LSU regions by ITSx) - // "ITS1" or "ITS2" - // "none" = just trim primers - // "ITS1_5.8S_ITS2" = assemble near-full-length ITS from ITSx output (useful in the case if primers are too close to SSU or LSU, and ITSx is not able to detect full-length sequence) - // "SSU" or "LSU" - - // Quality control - qc_maxee = null // only for single-end reads - qc_maxeerate = 0.01 // only for single-end reads - qc_maxhomopolymerlen = 25 // max len of homopolymer regions (if >=, sequence will be removed) - qc_maxn = 4 - - qc_avgphred = null // Only for PE reads - qc_twocolor = false // reduced resolution Phred-scores (two-color Illumina chemistry) - - //// Conditional parameters are in separate configs - // if(qc_twocolor == true){ - // // params_illumina_2color.config - // qc_phredmin = 24 - // qc_phredperc = 30 - // qc_polyglen = 8 - // } else { - // // params_illumina_4color.config - // qc_phredmin = null - // qc_phredperc = null - // qc_polyglen = null - // } - - - // Is data demultiplexed? - // If false (default), input = 1 fastq file and 1 fasta file - // If true, input = multiple fastq files - demultiplexed = false - - // Demultiplexing - PacBio & LIMA - lima_barcodetype = "dual_symmetric" // "single", "dual", "dual_symmetric", "dual_asymmetric" - lima_minscore = 93 // minimum barcode score (93 is for 12bp-long barcodes) - lima_minendscore = 50 // only useful for asymmetric barcoding schemes with different barcodes in a pair - lima_minrefspan = 0.75 // min read span relative to the barcode length - lima_minscoringregions = 2 // for dual barcodes only (2 = requires both barcodes) - lima_windowsize = 70 // window size (in base pairs) - lima_minlen = 40 // minimum sequence length after clipping - lima_remove_unknown = false // remove unknown barcode combinations (in dual-barcoding modes) - - // Demultiplexing - Illumina & cutadapt - barcode_window = 30 - barcode_errors = 1 - barcode_overlap = 11 - - // Illimina pair-end read assembly - pe_minoverlap = 20 - pe_difflimit = 5 - pe_diffperclimit = 20 - pe_nlimit = 10 // disabled - pe_minlen = 30 - - // What to do with not merged reads (Illumina-only) - illumina_keep_notmerged = true - illumina_joinpadgap = "NNNNNNNNNN" - illumina_joinpadqual = "IIIIIIIIII" // quality score of 40 - - - // Primer checks - primer_forward = "TACACACCGCCCGTCG" // ITS9mun - primer_reverse = "CCTSCSCTTANTDATATGC" // ITS4ngsUni - primer_mismatches = 2 - // primer_mismatches_insertions = 1 - // primer_mismatches_deletions = 1 - primer_foverlap = primer_forward.length() - 2 - primer_roverlap = primer_reverse.length() - 2 - - // ITSx - ITSx_evalue = 1e-1 - ITSx_partial = 0 // off, otherwise specify min length cutoff for partial ITS sequences to keep - ITSx_tax = "all" - ITSx_complement = "F" // "F" (check single strand) or "T" (check both DNA strands for matches to HMM-profiles) - /// ITSx_singledomain = true .... optional arguments - ITSx_to_parquet = true // convert ITSx output (FASTA files) to Parquet - ITSx_chunk_size = 10000 // chunk size (number of dereplicated sequences per sample) for distributed ITSx processing; set to 0 to disable chunking - - // Primer trimming (for Illumina) - trim_minlen = 10 - - // Homopolymer compression - hp = true - hp_similarity = 0.999 - hp_iddef = 2 - - // Which chimera removal methods to use - chimera_methods = "ref,denovo" // null or "none" also supported - - // Reference-based chimera removal - chimera_db = "Eukaryome_1.9.3_241222_FullITS_100-800.udb" - chimera_rescueoccurrence = 2 - - // De novo chimera identification (UCHIME1) - chimeranov_abskew = 2.0 - chimeranov_dn = 1.4 - chimeranov_mindiffs = 3 - chimeranov_mindiv = 0.8 - chimeranov_minh = 0.28 - chimeranov_xn = 8.0 - - // Tag-jump removal - tj = true // run tag-jump removal - tj_id = 1 // depreplicate or pre-cluster: 1 = just dereplicate, < 1 (e.g., 0.99) = cluster at 99% similarity - tj_iddef = 2 - tj_f = 0.01 // UNCROSS parameter f - tj_p = 1 - - // Singleton removal - // singleton_minrelabundance = 1 // % of sample abundance // not implemented yet - TODO - - // Collapsing similar sequences - // coverage - - - //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Step-2 parameters - - // Path to the Step-1 results [input for Step-2] - data_path = "${launchDir}/Step1_Results" - - // Pool sample replicates (e.g., re-sequenced samples) in the final OTU table - merge_replicates = false - - // Filtering sequences (trimmed amplicons) by length - ampliconlen_min = null - ampliconlen_max = null - // if(ampliconlen_min != null | ampliconlen_max != null){ - // length_filtering = true - // } else { - // length_filtering = false - // } - - - // Default thresholds for singleton and de novo chimera removal - max_MEEP = 0.5 - max_ChimeraScore = 0.6 - recover_lowqsingletons = true - recover_denovochimeras = true - - // Number of chunks to split the dataset into prior clustering - chunking_n = null // number of chunks - chunking_id = 0.6 // minimum sequence identity for clustering - - - // Sequence denoising or pre-clustering ("none", "unoise", "dada2", "swarm_d1", "homopolymer") - preclustering = "none" - - // Denoising with UNOISE - unoise_alpha = 6.0 - unoise_minsize = 1 - - // Denoising with DADA2 - dada2_pooling = "global" // "global" or "byrun" (not implemented yet) - dada2_nbases = 1e6 - dada2_bandsize = 16 - dada2_detectsingletons = true - dada2_omegaA = 1e-20 - dada2_omegaC = 1e-40 - dada2_omegaP = 1e-4 - dada2_maxconsist = 10 - dada2_match = 4 - dada2_mismatch = -5 - dada2_gappenalty = -8 - - - // Sequence clustering method ("none" / "vsearch" / "swarm" / "shmatching") - clustering = "vsearch" - - // VSEARCH clustering - otu_id = 0.98 - otu_iddef = 2 // also for UNOISE - otu_qmask = "dust" // also for UNOISE - - // SWARM clustering - swarm_d = 1 - swarm_fastidious = true - swarm_d1boundary = 3 // min mass of large OTUs, only for Fastidious + d=1 - - // Alignment parameters - // NB. vsearch scores = 2 * usearch scores !! - // E.g., "20I/2E" = penalty 20 for opening internal gaps, and 2 for opening terminal gaps (left or right) - - alignment_penalties = "default" // alternatively, "UNITE" - - if(alignment_penalties == "UNITE"){ - - // Alternative dereplication as in UNITE - // Allow query sequences vary 4% in length at 100% similarity - unite_querycov = 0.96 - unite_targetcov = 0.96 - - // VSEARCH - vsearch_gapopen = "0I/0E" // penalties for gap opening (usearch, "0.0/0.0E") - vsearch_gapext = "2I/1E" // penalties for gap extension (usearch, "1.0/0.5E") - - } - if(alignment_penalties == "default"){ - - // VSEARCH - vsearch_gapopen = "20I/2E" - vsearch_gapext = "2I/1E" - } - - - // LULU - lulu = true - lulu_match = 95.0 // minimum similarity threshold (default, 84.0) - lulu_ratio = 1.0 // minimum abundance ratio (default, 1.0) - lulu_ratiotype = "min" // abundance ratio type - "min" or "avg" (default, "min") - lulu_relcooc = 0.95 // relative co-occurrence (default, 0.95) - lulu_maxhits = 0 // maximum number of hits (0 = unlimited; default, 10?) - - - //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Generic parameters - - - // GZIP compression level - gzip_compression = 7 - - // The file publishing method - storagemode = "rellink" // "symlink", "copy", "rellink", "link", or "move" - - - - // Help message flag - helpMsg = null - help = null - - // Generic Parameters - validate_params = true - // sleep_time = 5 - // help_all = false - - // Boilerplate options - version = false - email = null - email_on_fail = null - plaintext_email = false - - - // Max Job Request Parameters - max_cpus = 40 - max_memory = "132.GB" - max_time = "240.h" - // max_retry = 3 - // min_time = 60 - // max_downloads = 3 - - // Nextflow Configuration Parameters - // force = false - // cleanup_workdir = false - - // Nextflow Profile Parameters - // condadir = "${baseDir}/conda/envs" - // use_mamba = false - // registry = "quay" - // singularity_cache = "$HOME/.phylonext/singularity" - // singularity_pull_docker_container = false - // force_rebuild = false - - // nf-core - monochrome_logs = false - enable_conda = false - show_hidden_params = false - schema_ignore_params = '' - - // Parameters to ignore - // build_all = false - // include_tools = null - // verbose = null - // silent = null - - // nf-schema configuration - showHidden = false - -} - - diff --git a/src/pipecraft-core/service_scripts/NextITS/conf/params_illumina_2color.config b/src/pipecraft-core/service_scripts/NextITS/conf/params_illumina_2color.config deleted file mode 100644 index e2b6e47f..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/conf/params_illumina_2color.config +++ /dev/null @@ -1,11 +0,0 @@ - -// Conditinal parameters for Illumina PE reads -// if `params.qc_twocolor = true` - -// reduced resolution Phred-scores (two-color Illumina chemistry) -params { - qc_phredmin = 24 - qc_phredperc = 30 - qc_polyglen = 8 - -} diff --git a/src/pipecraft-core/service_scripts/NextITS/conf/params_illumina_4color.config b/src/pipecraft-core/service_scripts/NextITS/conf/params_illumina_4color.config deleted file mode 100644 index e9a70d56..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/conf/params_illumina_4color.config +++ /dev/null @@ -1,11 +0,0 @@ - -// Conditinal parameters for Illumina PE reads -// if `params.qc_twocolor = false` - -// reduced resolution Phred-scores (two-color Illumina chemistry) -params { - qc_phredmin = null - qc_phredperc = null - qc_polyglen = null - -} diff --git a/src/pipecraft-core/service_scripts/NextITS/conf/singularity.config b/src/pipecraft-core/service_scripts/NextITS/conf/singularity.config deleted file mode 100644 index 0186233e..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/conf/singularity.config +++ /dev/null @@ -1,16 +0,0 @@ -// Container specifications for the Singularity profile - -process { - - // Container from Singularity library - container = 'library://vmiks/nextits/nextits:1-2-0' - - // Container from Quay.io registry [not working yet] - // container = 'quay.io/vmikk/nextits:1.2.0' - - // Container from Docker Hub (will be converted to Singularity image) - // container = 'vmikk/nextits:1.2.0' - - // Local Singularity image file - // container = '/path/to/local/file/nextits-1-2-0.sif' -} diff --git a/src/pipecraft-core/service_scripts/NextITS/conf/test_step1.config b/src/pipecraft-core/service_scripts/NextITS/conf/test_step1.config deleted file mode 100644 index b89cf628..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/conf/test_step1.config +++ /dev/null @@ -1,40 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests for the Step-1 of the NextITS pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run vmikk/nextits -r main -profile test, --outdir - ----------------------------------------------------------------------------------------- -*/ - - -// Path to test data -testdata_base_path = "${projectDir}/test_data" - - -process { - resourceLimits = [ - cpus: 4, - memory: '8.GB', - time: '10.m' - ] -} - -params { - - input = "${testdata_base_path}/Test_Step1.fastq.gz" - barcodes = "${testdata_base_path}/Test_Step1_barcodes.fasta" - primer_forward = "TACACACCGCCCGTCG" // ITS9mun - primer_reverse = "CCTSCSCTTANTDATATGC" // ITS4ngsUni - chimera_db = "${testdata_base_path}/Test_ChimeraDB.udb" - step = "Step1" - its_region = "full" - ITSx_tax = "fungi" - outdir = "NextITS_Test_Step1" - tracedir = "${params.outdir}/pipeline_info" - - storagemode = "copy" -} diff --git a/src/pipecraft-core/service_scripts/NextITS/conf/test_step2.config b/src/pipecraft-core/service_scripts/NextITS/conf/test_step2.config deleted file mode 100644 index ce7fe2fb..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/conf/test_step2.config +++ /dev/null @@ -1,43 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests for the Step-2 of the NextITS pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run vmikk/nextits -r main -profile test2, --outdir - ----------------------------------------------------------------------------------------- -*/ - - -// Path to test data -testdata_base_path = "${projectDir}/test_data" - - -process { - resourceLimits = [ - cpus: 4, - memory: '8.GB', - time: '10.m' - ] -} - -params { - - data_path = "${testdata_base_path}/Test_S2" - step = "Step2" - outdir = "NextITS_Test_Step2" - tracedir = "${params.outdir}/pipeline_info" - merge_replicates = false - max_MEEP = 0.5 - max_ChimeraScore = 0.6 - preclustering = "unoise" - unoise_alpha = 6.0 - unoise_minsize = 1 - clustering = "vsearch" - otu_id = 0.98 - lulu = false - - storagemode = "copy" -} diff --git a/src/pipecraft-core/service_scripts/NextITS/containerfiles/NextITS.def b/src/pipecraft-core/service_scripts/NextITS/containerfiles/NextITS.def deleted file mode 100644 index 86c1a35d..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/containerfiles/NextITS.def +++ /dev/null @@ -1,306 +0,0 @@ -# NextITS - Singularity definition file, main container - -## To avoid GitHub API limits, use `SINGULARITYENV_GITHUB_PAT=[token]` with Singularity -## to specify the GITHUB_PAT token (the token will be available only during the `%post` section of the build) - - -## Build stage (Rust and Cargo), for runiq -Bootstrap: docker -From: rust:1.92.0-slim -Stage: build - -%post - # rustup --version # 1.28.2 - # rustc --version # 1.92.0 - # cargo --version # 1.92.0 - cargo install runiq sd - # runiq --version # 2.1.0 - # sd --version # 1.0.0 - -## Main stage -Bootstrap: docker -From: rocker/r-ver:4.5.2 -Stage: final - -%labels - Maintainer vladimir.mikryukov@ut.ee - R_Version 4.5.2 - NextITS_container_version 1.2.0 - -%runscript - exec "$@" - -%apprun R - exec R "${@}" - -%apprun Rscript - exec Rscript "${@}" - -%environment - export PATH="/opt/software/conda/bin:$PATH" - export CONDA_PREFIX="/opt/software/conda" - - # Source conda initialization if available (for interactive sessions) - if [ -f "/opt/software/conda/init.bash" ]; then - source "/opt/software/conda/init.bash" - fi - -## Install the required dependencies -%post - export R_VERSION=4.5.2 - - apt-get update -qq \ - && apt-get -y --no-install-recommends install \ - tar zip unzip pigz gzip zstd xz-utils bzip2 coreutils \ - curl wget git less gawk nano rename bc \ - ca-certificates locales \ - libtre-dev libtre5 zlib1g zlib1g-dev liblzma-dev libbz2-dev libcurl4-openssl-dev libglpk-dev libglpk40 \ - build-essential - - ## Configure default locale - # echo "C.UTF-8" >> /etc/locale.gen - # locale-gen C.UTF-8 - /usr/sbin/update-locale LANG=C.UTF-8 - export LC_ALL=C.UTF-8 - export LANG=C.UTF-8 - - ## Install additional R packages - install2.r --error --skipinstalled --ncpus -1 \ - remotes \ - optparse \ - R.utils \ - data.table \ - arrow \ - duckdb \ - BiocManager \ - plyr \ - dplyr \ - ggplot2 \ - doFuture \ - openxlsx \ - yaml - - R -e 'BiocManager::install("Biostrings", ask = FALSE)' \ - && R -e 'BiocManager::install("ShortRead", ask = FALSE)' \ - && R -e 'BiocManager::install("DECIPHER", ask = FALSE)' \ - && R -e 'BiocManager::install("dada2", ask = FALSE)' \ - && R -e 'BiocManager::install("phyloseq", ask = FALSE)' \ - && rm -rf /tmp/downloaded_packages - - install2.r --error --skipinstalled \ - geodist \ - phytools \ - ggdendro \ - gridExtra \ - && R -e 'ok <- tryCatch({ remotes::install_github("mikemc/speedyseq"); TRUE }, error=function(e){ message(e); FALSE }); \ - if (!ok || !requireNamespace("speedyseq", quietly=TRUE)) quit(status=1)' \ - && R -e 'ok <- tryCatch({ remotes::install_github("vmikk/metagMisc"); TRUE }, error=function(e){ message(e); FALSE }); \ - if (!ok || !requireNamespace("metagMisc", quietly=TRUE)) quit(status=1)' \ - && R -e 'ok <- tryCatch({ remotes::install_cran("qs", type = "source", configure.args = "--with-simd=AVX2"); TRUE }, error=function(e){ message(e); FALSE }); \ - if (!ok || !requireNamespace("qs", quietly=TRUE)) quit(status=1)' \ - && rm -rf /tmp/downloaded_packages - - ## Install conda - readonly conda_prefix="/opt/software/conda" - - mkdir -p /opt/software - cd /opt/software - curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh" - bash Miniforge3-Linux-x86_64.sh -u -b -p ${conda_prefix} - rm Miniforge3-Linux-x86_64.sh - ${conda_prefix}/bin/conda config --add channels bioconda - ${conda_prefix}/bin/mamba update -y --all - - ## Conda initialization script by Antoine Prouvost - # https://github.com/ds4dm/singularity-conda/blob/main/conda.def - # Singularity does all the environment sourcing as shell (only latter calls bash), which conda does not support. - # We put the content in a file, manually call bash, and source it. - { - echo 'eval "$(' "'${conda_prefix}/bin/conda' 'shell.bash' 'hook' 2> /dev/null" ')"' - echo 'if [ $? -eq 0 ]; then' - echo ' eval "$__conda_setup"' - echo 'else' - echo ' if [ -f ' "'${conda_prefix}/etc/profile.d/conda.sh'" ']; then' - echo ' .' "'${conda_prefix}/opt/mamba/etc/profile.d/conda.sh'" - echo ' else' - echo ' export PATH="/opt/software/conda/bin:$PATH"' - echo ' fi' - echo 'fi' - echo 'unset __conda_setup' - } >> ${conda_prefix}/init.bash - - ## Create conda environment and install software - ${conda_prefix}/bin/mamba install --quiet --yes \ - "lima>=2.13.0" \ - "pbtk>=3.5.0" \ - "vsearch>=2.30.3" \ - "swarm>=3.1.6" \ - "seqkit>=2.12.0" \ - "seqfu>=1.23.0" \ - "fastp>=1.0.1" \ - "blast>=2.17.0" \ - "bioawk" \ - "miller>=6.16.0" \ - "xsv>=0.13.0" \ - "bedtools>=2.31.1" \ - "parallel>=20251122" \ - "csvtk>=0.36.0" \ - "cutadapt>=5.2" \ - "itsx>=1.1.3" \ - "bbmap>=39.52" \ - "ripgrep>=15.1.0" \ - "fd-find>=10.3.0" \ - "mmseqs2" - - ## seqhasher - # https://github.com/vmikk/seqhasher - wget https://github.com/vmikk/seqhasher/releases/download/1.1.2/seqhasher - chmod +x seqhasher - mv seqhasher ${conda_prefix}/bin/ - - ## phredsort - # https://github.com/vmikk/phredsort - wget https://github.com/vmikk/phredsort/releases/download/1.4.0/phredsort - chmod +x phredsort - mv phredsort ${conda_prefix}/bin/ - - ## ucs - # https://github.com/vmikk/ucs - wget https://github.com/vmikk/ucs/releases/download/0.8.0/ucs - chmod +x ucs - mv ucs ${conda_prefix}/bin/ - - ## fqgrep v.0.4.4 - # https://github.com/indraniel/fqgrep - git clone --depth 1 https://github.com/indraniel/fqgrep - cd fqgrep - make - mv fqgrep ${conda_prefix}/bin/ - cd .. - rm -r fqgrep - - ## rush - # https://github.com/shenwei356/rush - wget https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_linux_amd64.tar.gz - tar -xzf rush_linux_amd64.tar.gz - mv rush ${conda_prefix}/bin/ - rm rush_linux_amd64.tar.gz - - ## brename - # https://github.com/shenwei356/brename - wget https://github.com/shenwei356/brename/releases/download/v2.14.0/brename_linux_amd64.tar.gz - tar -xzf brename_linux_amd64.tar.gz - mv brename ${conda_prefix}/bin/ - rm brename_linux_amd64.tar.gz - - ## MUMU - # https://github.com/frederic-mahe/mumu - git clone --depth 1 https://github.com/frederic-mahe/mumu.git \ - && cd ./mumu/ \ - && make && make check && make install - mv mumu ${conda_prefix}/bin/ - cd .. - rm -r mumu - - ## Get the updated ITSx databases ["Version 2", 5 April 2024, curated by Henrik Nilsson] - ## NB! Currently, there is no X.hmm profile (Apusozoa) - git clone --depth 1 https://github.com/USDA-ARS-GBRU/ITS_HMMs/ - ## Compress and index HMM flatfiles - find ITS_HMMs/ITSx_db/HMMs/ -name "*.hmm" | grep -v "N.hmm" \ - | ${conda_prefix}/bin/parallel -j1 "${conda_prefix}/bin/hmmpress {}" - ## Replace old profiles - rm ${conda_prefix}/bin/ITSx_db/HMMs/* - mv ITS_HMMs/ITSx_db/HMMs/* ${conda_prefix}/bin/ITSx_db/HMMs/ - rm -r ITS_HMMs - ## Activate Parabasalia model in ITSx - # grep "\#push(@profileSet,\"Y\")" ${conda_prefix}/bin/ITSx - sed -i '/#push(@profileSet,"Y")/s/#//' ${conda_prefix}/bin/ITSx - - ## DuckDB - # https://duckdb.org/docs/installation/?version=stable - curl -L https://github.com/duckdb/duckdb/releases/download/v1.4.3/duckdb_cli-linux-amd64.zip -o duckdb_cli-linux-amd64.zip \ - && unzip duckdb_cli-linux-amd64.zip -d ${conda_prefix}/bin/ \ - && rm duckdb_cli-linux-amd64.zip - - ## Clean up - apt-get autoremove -y \ - && apt-get autoclean -y \ - && rm -rf /var/lib/apt/lists/* \ - && ${conda_prefix}/bin/mamba clean --all --quiet --yes - - ## Activate environment - # source ${conda_prefix}/bin/activate ${conda_prefix} - - ## Ensure software is accessible to any user (NB! avoid slow recursive operations) - chmod 755 /opt/software - chmod a+rX /opt/software/conda/bin - chmod a+rX /opt/software/conda/lib - - ## Use bash to properly activate the conda environment - echo 'source' "'${conda_prefix}/init.bash'" | bash - - cd /opt/software - - -## Add binary from the stage one - runiq, sd -%files from build - /usr/local/cargo/bin/runiq /opt/software/conda/bin/ - /usr/local/cargo/bin/sd /opt/software/conda/bin/ - - -%test - echo "=== Testing R installation and packages ===" - R --quiet -e "stopifnot(getRversion() == '${R_VERSION}')" - - echo "Testing R package installations..." - R --quiet -e " - required_packages <- c('optparse', 'R.utils', 'data.table', 'arrow', 'duckdb', - 'plyr', 'dplyr', 'ggplot2', 'doFuture', 'openxlsx', 'yaml', - 'Biostrings', 'ShortRead', 'DECIPHER', 'dada2', 'phyloseq', - 'metagMisc', 'qs') - - for(pkg in required_packages) { - cat('Testing package:', pkg, '... ') - tryCatch({ - suppressPackageStartupMessages( - library(pkg, character.only = TRUE, quietly = TRUE, warn.conflicts = FALSE) - ) - cat('OK\n') - }, error = function(e) { - cat('FAILED\n') - stop('Package ', pkg, ' failed to load: ', e$message) - }) - } - cat('All R packages loaded successfully!\n') - " - - echo "=== Testing conda/mamba installed tools ===" - ## Source conda environment - source /opt/software/conda/init.bash - - ## Test conda/mamba tools - tools_conda="lima bam2fastq vsearch swarm seqkit seqfu fastp blastn bioawk mlr xsv bedtools parallel csvtk ITSx cutadapt bbduk.sh rg fd mmseqs" - for tool in $tools_conda; do - echo -n "Testing $tool... " - if command -v $tool >/dev/null 2>&1; then - echo "OK" - else - echo "FAILED - $tool not found in PATH" - exit 1 - fi - done - - echo "=== Testing manually installed tools ===" - ## Test manually installed tools - tools_manual="seqhasher phredsort ucs fqgrep rush brename mumu duckdb runiq sd" - for tool in $tools_manual; do - echo -n "Testing $tool... " - if command -v $tool >/dev/null 2>&1; then - echo "OK" - else - echo "FAILED - $tool not found in PATH" - exit 1 - fi - done - - echo "=== All tests passed! Container looks ready for use ===" diff --git a/src/pipecraft-core/service_scripts/NextITS/containerfiles/NextITS.dockerfile b/src/pipecraft-core/service_scripts/NextITS/containerfiles/NextITS.dockerfile deleted file mode 100644 index 99e1e4c2..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/containerfiles/NextITS.dockerfile +++ /dev/null @@ -1,270 +0,0 @@ -# NextITS - Dockerfile, main container -# Multi-stage build is used to compile Rust-based software -# Nextflow is included in the image - -## To build the image, run: -# docker build --tag nextits --file NextITS.dockerfile . -# -## To run tests during build: -# docker build --target test --tag nextits-test --file NextITS.dockerfile . - -## Build stage 1 (Rust and Cargo) -FROM rust:1.92.0-slim AS rust -RUN cargo install runiq sd - -## Build stage 2 - Main -FROM rocker/r-ver:4.5.2 AS main - -ENV LANG=C.UTF-8 -ENV LC_ALL=C.UTF-8 -ENV SHELL=/bin/bash -LABEL org.opencontainers.image.authors="vladimir.mikryukov@ut.ee" -LABEL org.opencontainers.image.version="1.2.0" - -RUN apt-get update -qq \ - && apt-get -y --no-install-recommends install \ - tar zip unzip pigz gzip zstd xz-utils bzip2 coreutils \ - curl wget git less gawk nano rename bc \ - ca-certificates locales \ - libtre-dev libtre5 zlib1g zlib1g-dev liblzma-dev libbz2-dev libcurl4-openssl-dev libglpk-dev libglpk40 \ - build-essential \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -## Install additional R packages -RUN install2.r --error --skipinstalled --ncpus -1 \ - remotes \ - optparse \ - R.utils \ - data.table \ - arrow \ - duckdb \ - BiocManager \ - plyr \ - dplyr \ - ggplot2 \ - doFuture \ - openxlsx \ - yaml - -RUN R -e 'BiocManager::install("Biostrings", ask = FALSE)' \ - && R -e 'BiocManager::install("ShortRead", ask = FALSE)' \ - && R -e 'BiocManager::install("DECIPHER", ask = FALSE)' \ - && R -e 'BiocManager::install("dada2", ask = FALSE)' \ - && R -e 'BiocManager::install("phyloseq", ask = FALSE)' \ - && rm -rf /tmp/downloaded_packages - -RUN install2.r --error --skipinstalled \ - geodist \ - phytools \ - ggdendro \ - gridExtra \ - && R -e 'ok <- tryCatch({ remotes::install_github("mikemc/speedyseq"); TRUE }, error=function(e){ message(e); FALSE }); \ - if (!ok || !requireNamespace("speedyseq", quietly=TRUE)) quit(status=1)' \ - && R -e 'ok <- tryCatch({ remotes::install_github("vmikk/metagMisc"); TRUE }, error=function(e){ message(e); FALSE }); \ - if (!ok || !requireNamespace("metagMisc", quietly=TRUE)) quit(status=1)' \ - && R -e 'ok <- tryCatch({ remotes::install_cran("qs", type = "source", configure.args = "--with-simd=AVX2"); TRUE }, error=function(e){ message(e); FALSE }); \ - if (!ok || !requireNamespace("qs", quietly=TRUE)) quit(status=1)' \ - && rm -rf /tmp/downloaded_packages - -## Install conda -RUN mkdir -p /opt/software \ - && cd /opt/software \ - && curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh" \ - && bash Miniforge3-Linux-x86_64.sh -u -b -p /opt/software/conda \ - && rm Miniforge3-Linux-x86_64.sh \ - && /opt/software/conda/bin/conda config --add channels bioconda \ - && /opt/software/conda/bin/mamba update -y --all \ - && /opt/software/conda/bin/mamba clean --all --yes - -## Create conda initialization script (for Singularity compatibility) -RUN cd /opt/software \ - && { \ - echo 'eval "$(' '"/opt/software/conda/bin/conda" "shell.bash" "hook" 2> /dev/null' ')"'; \ - echo 'if [ $? -eq 0 ]; then'; \ - echo ' eval "$__conda_setup"'; \ - echo 'else'; \ - echo ' if [ -f "/opt/software/conda/etc/profile.d/conda.sh" ]; then'; \ - echo ' . "/opt/software/conda/etc/profile.d/conda.sh"'; \ - echo ' else'; \ - echo ' export PATH="/opt/software/conda/bin:$PATH"'; \ - echo ' fi'; \ - echo 'fi'; \ - echo 'unset __conda_setup'; \ - } > /opt/software/conda/init.bash - -## Create conda environment and install software -RUN /opt/software/conda/bin/mamba install -y \ - "lima>=2.13.0" \ - "pbtk>=3.5.0" \ - "vsearch>=2.30.3" \ - "swarm>=3.1.6" \ - "seqkit>=2.12.0" \ - "seqfu>=1.23.0" \ - "fastp>=1.0.1" \ - "blast>=2.17.0" \ - "bioawk" \ - "miller>=6.16.0" \ - "xsv>=0.13.0" \ - "bedtools>=2.31.1" \ - "parallel>=20251122" \ - "csvtk>=0.36.0" \ - "cutadapt>=5.2" \ - "itsx>=1.1.3" \ - "bbmap>=39.52" \ - "ripgrep>=15.1.0" \ - "fd-find>=10.3.0" \ - "mmseqs2" \ - && /opt/software/conda/bin/conda clean --all --yes - - -## Install cutadapt (with dependencies) from pip - it fails with conda (Python 3.13 confilict) -# RUN /opt/software/conda/bin/pip install --no-cache-dir \ -# "dnaio>=1.2.3" "xopen>=2.0.2" "cutadapt>=5.1" - -## Add new tools (seqhasher, phredsort, ucs) -RUN cd /opt/software \ - && wget https://github.com/vmikk/seqhasher/releases/download/1.1.2/seqhasher \ - && chmod +x seqhasher \ - && mv seqhasher /opt/software/conda/bin/ \ - && wget https://github.com/vmikk/phredsort/releases/download/1.4.0/phredsort \ - && chmod +x phredsort \ - && mv phredsort /opt/software/conda/bin/ \ - && wget https://github.com/vmikk/ucs/releases/download/0.8.0/ucs \ - && chmod +x ucs \ - && mv ucs /opt/software/conda/bin/ - -## fqgrep -RUN git clone --depth 1 https://github.com/indraniel/fqgrep \ - && cd fqgrep \ - && make \ - && mv fqgrep /opt/software/conda/bin/ \ - && cd .. \ - && rm -r fqgrep - -## rush -RUN wget https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_linux_amd64.tar.gz \ - && tar -xzf rush_linux_amd64.tar.gz \ - && mv rush /opt/software/conda/bin/ \ - && rm rush_linux_amd64.tar.gz - -## brename -RUN wget https://github.com/shenwei356/brename/releases/download/v2.14.0/brename_linux_amd64.tar.gz \ - && tar -xzf brename_linux_amd64.tar.gz \ - && mv brename /opt/software/conda/bin/ \ - && rm brename_linux_amd64.tar.gz - -## MUMU -RUN git clone --depth 1 https://github.com/frederic-mahe/mumu.git \ - && cd ./mumu/ \ - && make && make check && make install \ - && mv mumu /opt/software/conda/bin/ \ - && cd .. \ - && rm -r mumu - -## Rust tools (from the Cargo-based stage) -COPY --from=rust /usr/local/cargo/bin/runiq /opt/software/conda/bin/runiq -COPY --from=rust /usr/local/cargo/bin/sd /opt/software/conda/bin/sd - -## Update ITSx databases -RUN cd /opt/software \ - && git clone --depth 1 https://github.com/USDA-ARS-GBRU/ITS_HMMs/ \ - && find ITS_HMMs/ITSx_db/HMMs/ -name "*.hmm" | grep -v "N.hmm" \ - | /opt/software/conda/bin/parallel -j1 "/opt/software/conda/bin/hmmpress {}" \ - && rm /opt/software/conda/bin/ITSx_db/HMMs/* \ - && mv ITS_HMMs/ITSx_db/HMMs/* /opt/software/conda/bin/ITSx_db/HMMs/ \ - && rm -r ITS_HMMs \ - && sed -i '/#push(@profileSet,"Y")/s/#//' /opt/software/conda/bin/ITSx - -## Install DuckDB -RUN cd /opt/software \ - && curl -L https://github.com/duckdb/duckdb/releases/download/v1.4.3/duckdb_cli-linux-amd64.zip -o duckdb_cli-linux-amd64.zip \ - && unzip duckdb_cli-linux-amd64.zip -d /opt/software/conda/bin/ \ - && rm duckdb_cli-linux-amd64.zip - -## Set up environment for both Docker and Singularity compatibility -ENV PATH="/opt/software/conda/bin:${PATH}" - -## Create non-privileged user -RUN groupadd -g 1000 nextits \ - && useradd -u 1000 -g 1000 -m -s /bin/bash nextits \ - && mkdir -p /home/nextits \ - && chown -R nextits:nextits /home/nextits \ - && mkdir -p /tmp/nextits \ - && chmod 1777 /tmp/nextits - -## Set software directory permissions -## (NB! avoid recursive operations on large conda env) -RUN chmod 755 /opt/software \ - && chmod 755 /opt/software/conda \ - && chmod a+rX /opt/software/conda/bin \ - && chmod a+rX /opt/software/conda/lib - -## Create entrypoint script that initializes conda properly -RUN echo '#!/bin/bash' > /opt/software/entrypoint.sh \ - && echo 'set -e' >> /opt/software/entrypoint.sh \ - && echo '# Try to source conda initialization if available' >> /opt/software/entrypoint.sh \ - && echo 'if [ -f "/opt/software/conda/init.bash" ]; then' >> /opt/software/entrypoint.sh \ - && echo ' source /opt/software/conda/init.bash' >> /opt/software/entrypoint.sh \ - && echo 'fi' >> /opt/software/entrypoint.sh \ - && echo 'exec "$@"' >> /opt/software/entrypoint.sh \ - && chmod +x /opt/software/entrypoint.sh - -## Switch to non-privileged user -USER nextits -## Change working directory (for compatiblity with Singularity) -WORKDIR /tmp/nextits -ENTRYPOINT ["/opt/software/entrypoint.sh"] - -## Test stage - run with: docker build --target test -FROM main AS test - -# Set environment variable for R version testing -ENV R_VERSION=4.5.2 - -RUN echo "=== Testing R installation and packages ===" \ - && R --quiet -e "stopifnot(getRversion() == '${R_VERSION}')" \ - && echo "Testing R package installations..." \ - && printf '%s\n' \ - 'required_packages <- c("optparse", "data.table", "arrow", "duckdb",' \ - ' "plyr", "dplyr", "ggplot2", "openxlsx", "yaml",' \ - ' "Biostrings", "DECIPHER", "dada2", "phyloseq",' \ - ' "metagMisc", "qs")' \ - '' \ - 'for(pkg in required_packages) {' \ - ' cat("Testing package:", pkg, "... ")' \ - ' tryCatch({' \ - ' suppressPackageStartupMessages(' \ - ' library(pkg, character.only = TRUE, quietly = TRUE, warn.conflicts = FALSE)' \ - ' )' \ - ' cat("OK\n")' \ - ' }, error = function(e) {' \ - ' cat("FAILED\n")' \ - ' stop("Package ", pkg, " failed to load: ", e$message)' \ - ' })' \ - '}' \ - 'cat("All R packages loaded successfully!\n")' \ - > test_packages.R \ - && Rscript test_packages.R \ - && rm test_packages.R \ - && echo "=== Testing conda/mamba installed tools ===" \ - && tools_conda="lima bam2fastq vsearch swarm seqkit seqfu fastp blastn bioawk mlr xsv bedtools parallel csvtk ITSx cutadapt bbduk.sh rg fd mmseqs" \ - && for tool in $tools_conda; do \ - echo -n "Testing $tool... " \ - && if command -v $tool >/dev/null 2>&1; then \ - echo "OK"; \ - else \ - echo "FAILED - $tool not found in PATH" && exit 1; \ - fi; \ - done \ - && echo "=== Testing manually installed tools ===" \ - && tools_manual="seqhasher phredsort ucs fqgrep rush brename mumu duckdb runiq sd" \ - && for tool in $tools_manual; do \ - echo -n "Testing $tool... " \ - && if command -v $tool >/dev/null 2>&1; then \ - echo "OK"; \ - else \ - echo "FAILED - $tool not found in PATH" && exit 1; \ - fi; \ - done \ - && echo "=== All tests passed! Container looks ready for use ===" diff --git a/src/pipecraft-core/service_scripts/NextITS/containerfiles/README.md b/src/pipecraft-core/service_scripts/NextITS/containerfiles/README.md deleted file mode 100644 index 49f95634..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/containerfiles/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# NextITS container images - -Reproducible computational environments are essential for scientific workflows. -NextITS provides pre-built container images to ensure consistent software versions and dependencies across different computing platforms, eliminating "it works on my machine" issues and enabling reproducible bioinformatics analyses. - -In general, NextITS will pull the required images automatically (e.g., when providing the `-profile singularity` or `-profile docker` flag to the Nextflow command). - -However, if you want to pull or build the container images manually, you can do so using the following instructions. - -## Pull pre-built images - -### Docker Hub - -```bash -# Pull specific version -docker pull vmikk/nextits:1.1.0 -``` - -### Singularity library - -```bash -# Pull specific version -singularity pull library://vmiks/nextits/nextits:1-1-0 -``` - diff --git a/src/pipecraft-core/service_scripts/NextITS/images/NextITS_logo.png b/src/pipecraft-core/service_scripts/NextITS/images/NextITS_logo.png deleted file mode 100644 index cdbdc26b..00000000 Binary files a/src/pipecraft-core/service_scripts/NextITS/images/NextITS_logo.png and /dev/null differ diff --git a/src/pipecraft-core/service_scripts/NextITS/images/NextITS_logo.svg b/src/pipecraft-core/service_scripts/NextITS/images/NextITS_logo.svg deleted file mode 100755 index 1b6b6dc2..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/images/NextITS_logo.svg +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/src/pipecraft-core/service_scripts/NextITS/main.nf b/src/pipecraft-core/service_scripts/NextITS/main.nf deleted file mode 100644 index 2da9cc32..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/main.nf +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env nextflow -/* -============================================================================ - NextITS: Pipeline to process eukaryotic ITS amplicons -============================================================================ - License: Apache-2.0 - Github : https://github.com/vmikk/NextITS - Website: https://Next-ITS.github.io/ ----------------------------------------------------------------------------- -*/ - -// NB!!: -// - provide absolute paths to the input data (e.g. --input and --barcodes) -// - File names should not contain period (.) characters (except for extensions) - -// Databases: -// - UDB for chimera identification - - -// Enable DSL2 syntax -nextflow.enable.dsl = 2 - -// Print the version and exit -if (params.version) { - ver = "NextITS " + workflow.manifest.version - if (workflow.commitId) { ver += " revision " + workflow.commitId.substring(0, 7) } - println "${ver}\n" - exit(0) -} - -// Note: nf-schema plugin handles --help automatically via configuration in nextflow.config - -// Show a custom help message and exit -if (params.helpMsg){ - include { helpMsg } from './modules/help_message.nf' - helpMsg() - exit(0) -} - - -// Enable topic channels -// nextflow.preview.topic = true // Nextflow < 25.04.0 - - - -// nf-schema functions for parameter validation -include { validateParameters } from 'plugin/nf-schema' - -// Include custom parameter summary function -include { paramSummary } from './modules/parameter_summary' - -// Include color utilities -include { getColors; colorize; colorizeMultiple; errorMsg; warningMsg; infoMsg; successMsg } from './modules/colors' - -// Include workflows -// NB! `include` statements are static, meaning they are resolved at compile time rather than at runtime! -include { S1 } from './workflows/STEP1.nf' -include { S2 } from './workflows/STEP2.nf' -include { seqstats } from './workflows/STEP1.nf' - - -//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE INPUTS - - - -// Print NextITS logo -def logoColors = getColors(params.monochrome_logs) - -def workflow_version = workflow.manifest.version ?: "unknown" -if (workflow.commitId) { workflow_version += " (${workflow.commitId.substring(0, 7)})" } - -def logo = """ -${logoColors.dim}----------------------------------------------------${logoColors.reset} - ${colorizeMultiple("Next", ['green', 'bold'], params.monochrome_logs)}${colorizeMultiple("ITS", ['purple', 'bold'], params.monochrome_logs)} ${logoColors.cyan}${workflow_version}${logoColors.reset} -${logoColors.green} SSU ${logoColors.purple}ITS1 ${logoColors.green}5.8S ${logoColors.purple}ITS2 ${logoColors.green}LSU ${logoColors.reset} -${logoColors.green} ▒▒▒▒▒▒▒▒▒${logoColors.purple}░░░░░░░░░${logoColors.green}▒▒▒▒▒${logoColors.purple}░░░░░░░░░░${logoColors.green}▒▒▒▒▒▒▒▒▒▒▒▒${logoColors.reset} -${logoColors.dim}----------------------------------------------------${logoColors.reset} -""" - -log.info logo - -// Print all parameters using nf-schema plugin -// include { paramsSummaryLog } from 'plugin/nf-schema' -// log.info paramsSummaryLog(workflow) // will print params from Step-1 and Step-2 simultaneously - - -// Additional runtime parameter validation -// These checks are performed after schema validation and handle -// conditional logic and file existence checks that cannot be expressed in JSON Schema - -// Additional parameter validation for Step-1 -if( params.step == "Step1" || params.step == "seqstats" ) { - - if (params.input == false && params.seqplatform == "PacBio") { - println( errorMsg("Please provide the input file with sequences in FASTQ.gz or BAM format with `--input` parameter.", params.monochrome_logs)) - exit(1) - } - if (params.input_R1 == false && params.input_R2 == false && params.seqplatform == "Illumina") { - println( errorMsg("Please provide input files with sequences in FASTQ.gz format with `--input_R1` and `--input_R2` parameters.", params.monochrome_logs)) - exit(1) - } - if (params.barcodes == false && params.demultiplexed == false) { - println( errorMsg("Please provide the file with sample barcodes in FASTA format with `--barcodes` parameter.", params.monochrome_logs)) - exit(1) - } -} - -if( params.step == "Step1" ) { - - // Reference-based chimera removal - if (params.chimera_methods && params.chimera_methods.toLowerCase().split(',').contains('ref')) { - if (!params.chimera_db || !file(params.chimera_db).exists()) { - println( errorMsg("For reference-based chimera removal, please provide the database in UDB format with `--chimera_db` parameter.", params.monochrome_logs)) - println( colorize(" See https://Next-ITS.github.io/installation/#databases for more information.", 'red', params.monochrome_logs)) - println( colorize("Alternatively, you can disable reference-based chimera removal with `--chimera_methods` parameter (set it to `none` or `denovo`).", 'red', params.monochrome_logs)) - exit(1) - } - if (!(params.chimera_db.toLowerCase().endsWith('.udb'))) { - println( errorMsg("The reference database file specified with `--chimera_db` parameter must be in UDB format.", params.monochrome_logs)) - println( colorize(" See https://Next-ITS.github.io/installation/#databases for more information.", 'red', params.monochrome_logs)) - exit 1 - } - } - - if (params.hp == true && params.seqplatform == "Illumina" && params.illumina_keep_notmerged == true) { - println( errorMsg("Homopolymer compression is not implemented for Illumina non-merged reads (add `--hp false` to your command).", params.monochrome_logs)) - exit(1) - } - if (params.seqplatform == "Illumina" && params.demultiplexed == true) { - println( errorMsg("Handling demultiplexed data for Illumina is not implemented yet.", params.monochrome_logs)) - exit(1) - } - - if (params.seqplatform == "Illumina" && params.illumina_keep_notmerged == true && params.its_region != "none") { - println( warningMsg("Unmerged Illumina reads are not compatible with ITSx. Amplicons will be primer-trimmed.", params.monochrome_logs)) - } - - // ITSx profiles validation - if (params.its_region != "none") { - - /* - Currently, the following regex pattern is used to pre-validate the `ITSx_tax` parameter (in schema): - "^(?:all| - (?:alveolata|bryophyta|bacillariophyta|amoebozoa|euglenozoa|fungi|chlorophyta|rhodophyta|phaeophyceae|marchantiophyta|metazoa|oomycota|haptophyceae|raphidophyceae|rhizaria|synurophyceae|tracheophyta|eustigmatophyceae|apusozoa|parabasalia) - (?:,\\s*(?:alveolata|bryophyta|bacillariophyta|amoebozoa|euglenozoa|fungi|chlorophyta|rhodophyta|phaeophyceae|marchantiophyta|metazoa|oomycota|haptophyceae|raphidophyceae|rhizaria|synurophyceae|tracheophyta|eustigmatophyceae|apusozoa|parabasalia))*)$" - - this forbids: - - mixing `all` with other values - - empty elements and trailing commas - - invalid values - */ - - def itsx_profiles = params.ITSx_tax - - // `ITSx_tax` must be a non-empty string (if specifying `--ITSx_tax ""`, Nextflow may coerce empty/flag to boolean) - if (itsx_profiles == null || itsx_profiles instanceof Boolean) { - println( errorMsg("Parameter --ITSx_tax must have a value (e.g. 'all' or 'fungi,rhizaria').", params.monochrome_logs) ) - exit(1) - } - if (itsx_profiles.toString().trim().isEmpty()) { - println( errorMsg("Parameter --ITSx_tax cannot be empty. Use 'all' or a comma-separated list of taxa.", params.monochrome_logs) ) - exit(1) - } - - // Allowed profiles - def ITSX_ALLOWED = [ - 'alveolata','bryophyta','bacillariophyta','amoebozoa','euglenozoa','fungi', - 'chlorophyta','rhodophyta','phaeophyceae','marchantiophyta','metazoa','oomycota', - 'haptophyceae','raphidophyceae','rhizaria','synurophyceae','tracheophyta', - 'eustigmatophyceae','apusozoa','parabasalia' - ] as Set - - // Parse the specified profile string - def itsx_items = itsx_profiles.toString().split(',', -1) as List - - // Empty-item validation (empty or whitespace-only tokens, incl. ",," and trailing commas) - def emptyIdx = [] - itsx_items.eachWithIndex { s, i -> - if (s == null || s.trim().isEmpty()) emptyIdx << i - } - if (emptyIdx) { - println( errorMsg("Parameter --ITSx_tax: empty entries are not allowed (check commas at positions: ${emptyIdx.join(', ')}).", params.monochrome_logs) ) - exit(1) - } - - // Disallow internal whitespaces - def whitespaces = itsx_items.findAll { s -> - def tr = s.toString().trim() - !(tr ==~ /\S+/) // after trimming, token must be all non-whitespace - } - if (whitespaces) { - println( errorMsg("Parameter --ITSx_tax: whitespace is not allowed in profile names.", params.monochrome_logs) ) - exit(1) - } - - // Detect duplicates - itsx_items = itsx_items.collect { it.trim() } - def dups = itsx_items.countBy { it }.findAll { k, v -> v > 1 }.keySet().toList() - if (dups) { - println( errorMsg("Parameter --ITSx_tax: duplicated profile names are not allowed: ${dups.join(', ')}", params.monochrome_logs) ) - exit(1) - } - - // Disallow mixing 'all' with specific profile names - if (itsx_items.size() > 1 && itsx_items.contains('all')) { - println( errorMsg("Parameter --ITSx_tax: do not combine 'all' with taxon-specific profile names.", params.monochrome_logs)) - exit(1) - } - - // Validate values against the allow-list (skip when it's exactly ['all']) - if (!(itsx_items.size() == 1 && itsx_items[0] == 'all')) { - def invalid_profiles = (itsx_items as Set) - ITSX_ALLOWED - if (invalid_profiles) { - println( errorMsg("Parameter --ITSx_tax: invalid profile names - ${invalid_profiles.join(', ')}", params.monochrome_logs) ) - println( colorize(" Supported profiles: `all` OR a comma-separated list of the following: ${ITSX_ALLOWED.join(', ')}", 'red', params.monochrome_logs)) - exit(1) - } - } - - // Currently, there is no X.hmm profile (Apusozoa) - if (itsx_items.contains('apusozoa')) { - println( errorMsg("Parameter --ITSx_tax: Apusozoa profile is not yet supported in ITSx.", params.monochrome_logs)) - exit(1) - } - - } // end of ITSx profiles validation - - -} // end of Step-1 parameter validation - - -// Additional parameter validation for Step-2 -if( params.step == "Step2" ) { - - if (params.preclustering == "none" && params.clustering == "none" && params.lulu == true){ - println errorMsg("LULU can not be applied when pre-clustering and clustering are set to 'none'", params.monochrome_logs) - exit(1) - } - - if (params.preclustering == "dada2" && params.dada2_pooling == "byrun" && - (params.chunking_n > 1 || params.chunking_n != null)){ - println errorMsg("By-sequencing-run pooling in DADA2 is not compatible with chunking.", params.monochrome_logs) - println( colorize("Set `--chunking_n` to 1 to disable chunking OR use `--dada2_pooling global`.", 'red', params.monochrome_logs)) - exit(1) - } - - -} // end of Step-2 parameter validation - - -// Run the workflow -workflow { - - // Print step-specific parameter summary - paramSummary(workflow, params) - validateParameters() - - if (params.step == "Step1") { - S1() - } - - if (params.step == "Step2") { - S2() - } - - if (params.step == "seqstats") { - seqstats() - } - -} - - -// On completion -workflow.onComplete { - println "Pipeline completed at : $workflow.complete" - println "Duration : ${workflow.duration}" - println "Execution status : ${workflow.success ? 'All done!' : 'Failed' }" -} - -// On error -workflow.onError { - println "Pipeline execution stopped with the following message: ${workflow.errorMessage}" -} diff --git a/src/pipecraft-core/service_scripts/NextITS/modules/Illumina_pe.nf b/src/pipecraft-core/service_scripts/NextITS/modules/Illumina_pe.nf deleted file mode 100644 index 34f33606..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/modules/Illumina_pe.nf +++ /dev/null @@ -1,484 +0,0 @@ - - - -// Quality filtering for pair-end reads -process qc_pe { - - label "main_container" - - // cpus 10 - - input: - path input_R1 - path input_R2 - - output: - path "QC_R1.fq.gz", emit: filtered_R1 - path "QC_R2.fq.gz", emit: filtered_R2 - - script: - filter_avgphred = params.qc_avgphred ? "--average_qual ${params.qc_avgphred}" : "--average_qual 0" - filter_phredmin = params.qc_phredmin ? "--qualified_quality_phred ${params.qc_phredmin}" : "" - filter_phredperc = params.qc_phredperc ? "--unqualified_percent_limit ${params.qc_phredperc}" : "" - filter_polyglen = params.qc_polyglen ? "--trim_poly_g --poly_g_min_len ${params.qc_polyglen}" : "" - """ - echo -e "QC\\n" - echo -e "Input R1: " ${input_R1} - echo -e "Input R2: " ${input_R2} - - ## If `filter_phredmin` && `filter_phredperc` are specified, - # Filtering based on percentage of unqualified bases - # how many percents of bases are allowed to be unqualified (Q < 24) - - fastp \ - --in1 ${input_R1} \ - --in2 ${input_R2} \ - --disable_adapter_trimming \ - --n_base_limit ${params.qc_maxn} \ - ${filter_avgphred} \ - ${filter_phredmin} \ - ${filter_phredperc} \ - ${filter_polyglen} \ - --length_required 100 \ - --thread ${task.cpus} \ - --html qc.html \ - --json qc.json \ - --out1 QC_R1.fq.gz \ - --out2 QC_R2.fq.gz - - echo -e "\\nQC finished" - """ -} - - - - - -// Demultiplexing with cutadapt - for Illumina PE reads (only not merged) -// NB. it's possible to use anchored adapters (e.g., -g ^file:barcodes.fa), -// but there could be a preceding nucleotides before the barcode, -// therefore, modified barcodes would be used here (e.g., XN{30}) -process demux_illumina_notmerged { - - label "main_container" - - publishDir "${out_1_demux}", mode: 'symlink' - // cpus 20 - - input: - path input_R1 - path input_R2 - path barcodes // barcodes_modified.fa (e.g., XN{30}) - - output: - path "*.fq.gz", emit: samples_demux - - script: - """ - echo -e "\nDemultiplexing not-merged reads" - - echo -e "Input R1: " ${input_R1} - echo -e "Input R2: " ${input_R2} - echo -e "Barcodes: " ${barcodes} - - ## First round - echo -e "\nRound 1:" - - cutadapt -g file:${barcodes} \ - -o round1-{name}.R1.fastq.gz \ - -p round1-{name}.R2.fastq.gz \ - --errors ${params.barcode_errors} \ - --overlap ${params.barcode_overlap} \ - --no-indels \ - --cores ${task.cpus} \ - ${input_R1} ${input_R2} \ - > cutadapt_round_1.log - - echo -e ".. round 1 finished" - - ## Second round - echo -e "\nRound 2:" - - cutadapt -g file:${barcodes} \ - -o round2-{name}.R2.fastq.gz \ - -p round2-{name}.R1.fastq.gz \ - --errors ${params.barcode_errors} \ - --overlap ${params.barcode_overlap} \ - --no-indels \ - --cores ${task.cpus} \ - round1-unknown.R2.fastq.gz round1-unknown.R1.fastq.gz \ - > cutadapt_round_2.log - - echo -e ".. round 2 finished" - - ## Remove empty files (no sequences) - echo -e "\nRemoving empty files" - find . -type f -name "round*.fastq.gz" -size -29c -print -delete - echo -e "..Done" - - ## Remove unknowns - echo -e "Removing unknowns" - rm round1-unknown.R{1,2}.fastq.gz - rm round2-unknown.R{1,2}.fastq.gz - - ## Combine sequences from round 1 and round 2 for each sample - echo -e "\nCombining sequences from round 1 and round 2 for each sample" - - mkdir -p Combined - - find . -name "round*.R1.fastq.gz" | sort | parallel -j1 \ - "cat {} >> Combined/{= s/round1-//; s/round2-// =}" - - find . -name "round*.R2.fastq.gz" | sort | parallel -j1 \ - "cat {} >> Combined/{= s/round1-//; s/round2-// =}" - - echo -e "..Done" - - ## Clean up - echo -e "..Removing temporary files" - find . -type f -name "round*.fastq.gz" -print -delete - - - - echo -e "\nDemultiplexing finished" - """ -} - - -// Trim primers of nonmerged PE reads -// + Estimate sequence qualities -process trim_primers_pe { - - label "main_container" - - publishDir "${out_3_trimPE}", mode: 'symlink' - // cpus 2 - - // Add sample ID to the log file - tag "${input.getSimpleName()}" - - input: - path input // tuple of size 2 - - output: - path "${input.getSimpleName()}_R1.fa.gz", emit: primertrimmed_fa_R1, optional: true - path "${input.getSimpleName()}_R2.fa.gz", emit: primertrimmed_fa_R2, optional: true - path "${input.getSimpleName()}_hash_table_R1.txt.gz", emit: hashes_R1, optional: true - path "${input.getSimpleName()}_hash_table_R2.txt.gz", emit: hashes_R2, optional: true - path "${input.getSimpleName()}_R1.fq.gz", emit: primertrimmed_fq_R1, optional: true - path "${input.getSimpleName()}_R2.fq.gz", emit: primertrimmed_fq_R2, optional: true - path "${input.getSimpleName()}_uc_R1.uc.gz", emit: ucR1, optional: true - path "${input.getSimpleName()}_uc_R2.uc.gz", emit: ucR2, optional: true - - script: - sampID="${input.getSimpleName()}" - - """ - echo -e "Input sample: " ${sampID} - echo -e "Forward primer: " ${params.primer_forward} - echo -e "Reverse primer: " ${params.primer_reverse} - - ## Reverse-complement primers - FR=\$(rc.sh ${params.primer_forward}) - RR=\$(rc.sh ${params.primer_reverse}) - - echo -e "Forward primer RC: " "\$FR" - echo -e "Reverse primer RC: " "\$RR" - - ## Discard sequences without both primers - echo -e "\nChecking primers" - - echo -e "..Forward strain" - - cutadapt \ - -a ${params.primer_forward}";required;min_overlap=${params.primer_foverlap}"..."\$RR"";required;min_overlap=${params.primer_roverlap}" \ - --errors ${params.primer_mismatches} \ - --cores ${task.cpus} \ - --action=none \ - --discard-untrimmed \ - -o for_R1.fastq.gz -p for_R2.fastq.gz \ - ${input[0]} ${input[1]} \ - > cutadapt_1.log - - - echo -e "..Reverse strain" - - cutadapt \ - -a ${params.primer_reverse}";required;min_overlap=${params.primer_roverlap}"..."\$FR"";required;min_overlap=${params.primer_foverlap}" \ - --errors ${params.primer_mismatches} \ - --cores ${task.cpus} \ - --action=none \ - --discard-untrimmed \ - -p rev_R1.fastq.gz -o rev_R2.fastq.gz \ - ${input[0]} ${input[1]} \ - > cutadapt_2.log - - # cutadapt \ - # -a FWDPRIMER...RCREVPRIMER \ - # -A REVPRIMER...RCFWDPRIMER \ - # --discard-untrimmed \ - # -o out.1.fastq.gz -p out.2.fastq.gz \ - # in.1.fastq.gz in.2.fastq.gz - - - echo -e "\nReorienting" - - if [ -s for_R1.fastq.gz ]; then - zcat for_R1.fastq.gz | seqkit replace -p "\\s.+" | gzip -7 > OK_R1.fastq.gz - zcat for_R2.fastq.gz | seqkit replace -p "\\s.+" | gzip -7 > OK_R2.fastq.gz - fi - - if [ -s rev_R1.fastq.gz ]; then - echo -e "..Adding sequences to the main pool" - zcat rev_R1.fastq.gz | seqkit replace -p "\\s.+" | gzip -7 >> OK_R1.fastq.gz - zcat rev_R2.fastq.gz | seqkit replace -p "\\s.+" | gzip -7 >> OK_R2.fastq.gz - - else - echo -e "..Probably all sequences are in forward orientation" - fi - - - echo -e "\nTrimming primers" - if [ -s OK_R1.fastq.gz]; then - - cutadapt \ - -a ${params.primer_forward}";required;min_overlap=${params.primer_foverlap}"..."\$RR"";required;min_overlap=${params.primer_roverlap}" \ - --errors ${params.primer_mismatches} \ - --cores ${task.cpus} \ - --action=trim \ - --discard-untrimmed \ - --minimum-length ${params.trim_minlen} \ - --output ${sampID}_R1.fq.gz --paired-output ${sampID}_R2.fq.gz \ - OK_R1.fastq.gz OK_R2.fastq.gz - - fi - - - ## Quality estimation and dereplication - - if [ -s ${sampID}_R1.fq.gz ]; then - - ## Estimate sequence quality (for the extracted region) - ## Sequence ID - Hash - Length - Average Phred score - echo -e "\nCreating sequence hash table with average sequence quality" - - seqkit fx2tab --length --avg-qual ${sampID}_R1.fq.gz \ - | hash_sequences.sh \ - | awk '{print \$1 "\t" \$6 "\t" \$4 "\t" \$5}' \ - > tmp_hash_table_R1.txt - - seqkit fx2tab --length --avg-qual ${sampID}_R2.fq.gz \ - | hash_sequences.sh \ - | awk '{print \$1 "\t" \$6 "\t" \$4 "\t" \$5}' \ - > tmp_hash_table_R2.txt - - echo -e "..Done" - - - ## Estimating MaxEE - echo -e "\nEstimating maximum number of expected errors per sequence" - - vsearch \ - --fastx_filter ${sampID}_R1.fq.gz \ - --fastq_qmax 93 \ - --eeout \ - --fastaout - \ - | seqkit seq --name \ - | sed 's/;ee=/\t/g' \ - > tmp_ee_R1.txt - - vsearch \ - --fastx_filter ${sampID}_R2.fq.gz \ - --fastq_qmax 93 \ - --eeout \ - --fastaout - \ - | seqkit seq --name \ - | sed 's/;ee=/\t/g' \ - > tmp_ee_R2.txt - - echo -e "..Done" - - - echo -e "\nMerging quality estimates" - - max_ee.R \ - tmp_hash_table_R1.txt \ - tmp_ee_R1.txt \ - ${sampID}_hash_table_R1.txt - - max_ee.R \ - tmp_hash_table_R2.txt \ - tmp_ee_R2.txt \ - ${sampID}_hash_table_R2.txt - - echo -e "..Done" - - - ## Independent dereplication of pair-end reads - echo -e "\nDereplicating R1 and R2 (independently)" - - seqkit fq2fa -w 0 ${sampID}_R1.fq.gz \ - | vsearch \ - --derep_fulllength - \ - --output - \ - --strand both \ - --fasta_width 0 \ - --threads 1 \ - --relabel_sha1 \ - --sizein --sizeout \ - --uc ${sampID}_uc_R1.uc \ - --quiet \ - | gzip -7 \ - > ${sampID}_R1.fa.gz - - seqkit fq2fa -w 0 ${sampID}_R2.fq.gz \ - | vsearch \ - --derep_fulllength - \ - --output - \ - --strand both \ - --fasta_width 0 \ - --threads 1 \ - --relabel_sha1 \ - --sizein --sizeout \ - --uc ${sampID}_uc_R2.uc \ - --quiet \ - | gzip -7 \ - > ${sampID}_R2.fa.gz - - - echo -e "..Done" - - ## Compress results - echo -e "Compressing result" - gzip -7 ${sampID}_hash_table_R1.txt - gzip -7 ${sampID}_hash_table_R2.txt - gzip -7 ${sampID}_uc_R1.uc - gzip -7 ${sampID}_uc_R2.uc - - - else - echo -e "\nNo sequences found after primer removal" - fi - - ## Clean up - if [ -f for_R1.fastq.gz ]; then rm for_R1.fastq.gz; fi - if [ -f for_R2.fastq.gz ]; then rm for_R2.fastq.gz; fi - if [ -f rev_R1.fastq.gz ]; then rm rev_R1.fastq.gz; fi - if [ -f rev_R2.fastq.gz ]; then rm rev_R2.fastq.gz; fi - if [ -f OK_R1.fastq.gz ]; then rm OK_R1.fastq.gz; fi - if [ -f OK_R2.fastq.gz ]; then rm OK_R2.fastq.gz; fi - - echo -e "..Done" - - """ -} - - - - -// Combine paired reads into single sequences -// by reverse-complementing the reverse read and inserting poly-N padding -// + Estimate sequence qualities (without N pads!) -process join_pe { - - label "main_container" - - // publishDir "${out_1_joinPE}", mode: "${params.storagemode}" - // cpus 2 - - // Add sample ID to the log file - tag "${input}" - - input: - val input // Sample name "(e.g., Barcode07_1__IS859)" - path all_samples - - output: - path "${input}_JoinedPE.fq.gz", emit: jj_FQ, optional: true - path "${input}_JoinedPE_hash_table.txt.gz", emit: jj_hashes, optional: true - - script: - sampID="${input}" - - """ - echo -e "Joining non-merged Illumina reads" - echo -e "Input sample: " ${sampID} - - echo -e "\\nJoining with N-pads" - vsearch \ - --fastq_join ${input}.R1.fastq.gz \ - --reverse ${input}.R2.fastq.gz \ - --join_padgap ${params.illumina_joinpadgap} \ - --join_padgapq ${params.illumina_joinpadqual} \ - --fastqout - \ - | seqkit replace -p "\\s.+" \ - | gzip -${params.gzip_compression} \ - > ${sampID}_JoinedPE.fq.gz - - ## Check if there are some sequences in the file - if [ -n "\$(find . -name ${sampID}_JoinedPE.fq.gz -prune -size +29c)" ]; then - - echo -e "\\nJoining without N-pads (for quality estimation)" - vsearch \ - --fastq_join ${input}.R1.fastq.gz \ - --reverse ${input}.R2.fastq.gz \ - --join_padgap "" \ - --join_padgapq "" \ - --fastqout - \ - | seqkit replace -p "\\s.+" \ - | gzip -${params.gzip_compression} \ - > tmp_for_qual.fq.gz - - - ## Estimate sequence quality (without N pads!) - ## Sequence ID - Hash - Length - Average Phred score - echo -e "\\nCreating sequence hash table with average sequence quality" - - seqkit fx2tab --length --avg-qual tmp_for_qual.fq.gz \ - | hash_sequences.sh \ - | awk '{print \$1 "\t" \$6 "\t" \$4 "\t" \$5}' \ - > tmp_hash_table.txt - - echo -e "..Done" - - ## Estimating MaxEE - echo -e "\\nEstimating maximum number of expected errors per sequence" - - vsearch \ - --fastx_filter tmp_for_qual.fq.gz \ - --fastq_qmax 93 \ - --eeout \ - --fastaout - \ - | seqkit seq --name \ - | sed 's/;ee=/\t/g' \ - > tmp_ee.txt - - echo -e "..Done" - - echo -e "\\nMerging quality estimates" - - max_ee.R \ - tmp_hash_table.txt \ - tmp_ee.txt \ - ${sampID}_JoinedPE_hash_table.txt - - echo -e "..Done" - - ## Compress results - gzip -${params.gzip_compression} ${sampID}_JoinedPE_hash_table.txt - - ## Clean up - rm tmp_for_qual.fq.gz - rm tmp_hash_table.txt - rm tmp_ee.txt - - else - echo -e "\\nIt looks like there are no joined reads" - fi - - ## Remove redundant symlinks - find -L . -name "*.fastq.gz" | grep -v ${input} | parallel -j1 "rm {}" - - """ -} - diff --git a/src/pipecraft-core/service_scripts/NextITS/modules/colors.nf b/src/pipecraft-core/service_scripts/NextITS/modules/colors.nf deleted file mode 100644 index ca6ccb42..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/modules/colors.nf +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Color utilities for NextITS pipeline - * Provides ANSI color codes and helper functions for terminal output - */ - -/** - * Get color codes map based on monochrome setting - * @param monochrome Whether to disable colors (from params.monochrome_logs) - * @return Map of color names to ANSI codes - */ -def getColors(boolean monochrome = false) { - return monochrome ? [:] : [ - // Basic colors - black: "\033[0;30m", - red: "\033[0;31m", - green: "\033[0;32m", - yellow: "\033[0;33m", - blue: "\033[0;34m", - purple: "\033[0;35m", - cyan: "\033[0;36m", - white: "\033[0;37m", - - // Bright colors - bright_black: "\033[0;90m", - bright_red: "\033[0;91m", - bright_green: "\033[0;92m", - bright_yellow: "\033[0;93m", - bright_blue: "\033[0;94m", - bright_purple: "\033[0;95m", - bright_cyan: "\033[0;96m", - bright_white: "\033[0;97m", - - // Text formatting - bold: "\033[1m", - dim: "\033[2m", - italic: "\033[3m", - underline: "\033[4m", - blink: "\033[5m", - reverse: "\033[7m", - - // Reset - reset: "\033[0m" - ] -} - -/** - * Apply color formatting to text - * @param text The text to colorize - * @param color The color name (e.g., 'red', 'green', 'bold') - * @param monochrome Whether to disable colors - * @return Formatted text string - */ -def colorize(String text, String color, boolean monochrome = false) { - def colors = getColors(monochrome) - if (!colors[color]) { - return text - } - return "${colors[color]}${text}${colors.reset}" -} - -/** - * Apply multiple color/format combinations to text - * @param text The text to colorize - * @param formats List of format names (e.g., ['red', 'bold']) - * @param monochrome Whether to disable colors - * @return Formatted text string - */ -def colorizeMultiple(String text, List formats, boolean monochrome = false) { - def colors = getColors(monochrome) - if (monochrome || !formats) { - return text - } - - def prefix = formats.findAll { colors[it] }.collect { colors[it] }.join('') - return "${prefix}${text}${colors.reset}" -} - -/** - * Create an error message with red coloring - * @param message The error message text - * @param monochrome Whether to disable colors - * @return Formatted error message - */ -def errorMsg(String message, boolean monochrome = false) { - return colorizeMultiple("ERROR: ${message}", ['red', 'bold'], monochrome) -} - -/** - * Create a warning message with yellow coloring - * @param message The warning message text - * @param monochrome Whether to disable colors - * @return Formatted warning message - */ -def warningMsg(String message, boolean monochrome = false) { - return colorizeMultiple("WARNING: ${message}", ['yellow', 'bold'], monochrome) -} - -/** - * Create an info message with cyan coloring - * @param message The info message text - * @param monochrome Whether to disable colors - * @return Formatted info message - */ -def infoMsg(String message, boolean monochrome = false) { - return colorize(message, 'cyan', monochrome) -} - -/** - * Create a success message with green coloring - * @param message The success message text - * @param monochrome Whether to disable colors - * @return Formatted success message - */ -def successMsg(String message, boolean monochrome = false) { - return colorizeMultiple(message, ['green', 'bold'], monochrome) -} diff --git a/src/pipecraft-core/service_scripts/NextITS/modules/dump_parameters.nf b/src/pipecraft-core/service_scripts/NextITS/modules/dump_parameters.nf deleted file mode 100644 index a27426d9..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/modules/dump_parameters.nf +++ /dev/null @@ -1,48 +0,0 @@ - -// Custom function to dump pipeline parameters to a TSV file - - -/* - * Flatten a nested map into dot-notated key/value pairs - * Example: [foo:[bar:1]] -> [[ "foo.bar", 1 ]] - */ -def flattenMap(Map m, String prefix = '') { - def out = [] - m.each { k, v -> - def key = prefix ? "${prefix}.${k}" : (k as String) - if( v instanceof Map ) - out.addAll( flattenMap((Map)v, key) ) - else - out << [ key, v ] - } - return out -} - -/* - * Make a value TSV-safe and readable - * - Converts collections to comma-separated lists - * - Normalizes tabs/newlines so the file stays valid TSV - */ -def tsvValue(Object v) { - if( v == null ) return 'null' - if( v instanceof Map ) return v.collect { kk, vv -> "${kk}:${tsvValue(vv)}" }.join(',') - if( v instanceof Collection ) return v.collect { tsvValue(it) }.join(',') - def s = v.toString() - return s.replace('\t',' ') - .replace('\r','\\r') - .replace('\n','\\n') -} - -/* - * Main function (exported in other files) - * Return a channel emitting one string per line: "param\tvalue" - */ -def dumpParamsTsv() { - assert params instanceof Map : 'params must be map-like' - def pairs = flattenMap((Map)params).sort { a, b -> a[0] <=> b[0] } - // If you want a header, uncomment the next line - // def lines = ['param\tvalue'] + pairs.collect { k,v -> "${k}\t${tsvValue(v)}" } - def lines = pairs.collect { k,v -> "${k}\t${tsvValue(v)}" } - // Emit each line on the channel (one item per line) - return channel.fromList(lines) // value/queue channel factory methods are standard :contentReference[oaicite:0]{index=0} -} diff --git a/src/pipecraft-core/service_scripts/NextITS/modules/help_message.nf b/src/pipecraft-core/service_scripts/NextITS/modules/help_message.nf deleted file mode 100644 index 663d82dc..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/modules/help_message.nf +++ /dev/null @@ -1,96 +0,0 @@ - - -// Pipeline help message -def helpMsg() { - log.info""" - ===================================================================== - NextITS v.${workflow.manifest.version} - ===================================================================== - - Pipeline Usage: - To run the pipeline, enter the following in the command line: - nextflow run vmikk/nextits -r ${workflow.manifest.version} --input ... --outdir ... - - Options: - REQUIRED: - --input File with single-end input sequences, PacBio (FASTQ or BAM) or a directory with pre-demultiplexed files - --input_R1 Files with paired-end input sequences, Illumina (FASTQ) - --input_R2 - --barcodes Barcodes for demultiplexing (FASTA) - --outdir The output directory where the results will be saved - - OPTIONAL: - --demultiplexed Boolean, input is multiplexed (true, single FASTQ file) or pre-demultiplexed (multiple FASTQ files) - --seqplatform Sequencing platform type - "PacBio" (default) or "Illumina" - --its_region ITS part selector - "full" (defalut), "ITS1", "ITS2", "none" (trims primers only), or "ITS1_5.8S_ITS2" - --primer_forward Forward primer sequence (default, ITS9mun) - --primer_reverse Reverse primer sequence (default, ITS4ngsUni) - --primer_mismatches - --primer_foverlap Min primer overlap (default, F primer length - 2) - --primer_roverlap Min primer overlap (default, R primer length - 2) - --qc_maxn Discard sequences with more than the specified number of N’s - --trim_minlen Min sequence length after primer trimming (default, 10) - --ITSx_tax ITSx taxonomy profile (default, "all") - --ITSx_evalue ITSx E-value cutoff threshold (default, 1e-1) - --ITSx_partial Keep partial ITS sequences (defalt, off), otherwise specify min length cutoff - --hp Homopolymer compression (default, true) - --hp_similarity Allowed sequence similarity for homopolymer compression (default, 0.999) - --hp_iddef Sequence similarity definition for homopolymer compression (default, 2) - - # Chimera identification - --chimera_db Database for reference-based chimera removal - --chimera_rescueoccurrence Min occurrence of chimeric sequences required to rescue them (default, 2) - --chimeranov_abskew De novo chimera identification `abskew` parameter (default, 2.0) - --chimeranov_dn De novo chimera identification `dn` parameter (default, 1.4) - --chimeranov_mindiffs De novo chimera identification `mindiffs` parameter (default, 3) - --chimeranov_mindiv De novo chimera identification `mindiv` parameter (default, 0.8) - --chimeranov_minh De novo chimera identification `minh` parameter (default, 0.28) - --chimeranov_xn De novo chimera identification `xn` parameter (default, 8.0) - - # Tag-jump removal - --tj_f Tag-jump filtering, UNCROSS parameter `f` (default, 0.01) - --tj_p Tag-jump filtering parameter `p` (default, 1) - --otu_id Sequence similarity for OTU clustering (default, 0.98) - --otu_iddef Sequence similarity definition for tag-jump removal step (default, 2) - - # PacBio-specific parameters - --lima_barcodetype Tag type ("single", "dual", "dual_symmetric", "dual_asymmetric") - --lima_minscore Minimum barcode score for demultiplexing (default, 93) - --lima_minendscore Minimum second barcode score (only for asymmetric and dual barcoding scheme; default, 50) - --lima_minrefspan Minimum read span relative to the barcode length (0-1; default, 0.75) - --lima_minscoringregions Number of barcodes scored required for demultiplexing using dual barcodes (default, 2 = requires both barcodes) - --lima_windowsize Window size for barcode lookup (default, 70 bp) - --lima_minlen Minimum sequence length after clipping barcodes (default, 40) - --qc_maxee Maximum number of expected errors (default, false) - --qc_maxeerate Maximum number of expected errors per base (default, 0.01) - --qc_maxhomopolymerlen Threshold for a homopolymer region length in a sequence (default, 25) - - # Illumina-specific parameters - --qc_avgphred Average Phred score for QC (default, false) - --qc_twocolor Enable two-color chemistry mode, e.g. for Illumina NovaSeq (default, false) - --qc_phredmin Two-color mode: min Phred score of qualified bases (default, 24) - --qc_phredperc Two-color mode: Percentage of bases allowed to be unqualified (default, 30) - --qc_polyglen Two-color mode: minimum length of polyG tail (default, 8) - --barcode_window Window size for barcode lookup (default, 30 bp) - --barcode_errors Maximum allowed number of errors in barcodes (default, 1) - --barcode_overlap Min overlap between read and barcode (default, 11) - --pe_minoverlap Min length to detect overlapped region of PE reads (default, 20) - --pe_difflimit Max number of mismatched bases in PE overlap (default, 5) - --pe_diffperclimit Max percentage of mismatched bases in PE overlap (default, 20) - --pe_minlen Min length of merged sequences (default, 30) - --illumina_keep_notmerged Keep not merged Illumina reads (default, true) - --illumina_joinpadgap Join not merged reads into one sequence using padding sequence string (default, NNNNNNNNNN) - --illumina_joinpadqual Join not merged reads into one sequence using padding quality string (default, IIIIIIIIII) - - # Miscellaneous parameters - --gzip_compression Compression level for GZIP (default, 7; 1 = fastest (worst compression), 9 = slowest (best)) - - NEXTFLOW-SPECIFIC: - -profile Configuration profile - -resume Execute the pipeline using the cached results (e.g., in case of ) - -work-dir Path to the directory where intermediate result files are stored - -qs Queue size (max number of processes that can be executed in parallel); e.g., 8 - -r Pipeline version to run (GitHub branch, tag, or SHA number) - """.stripIndent() -} - diff --git a/src/pipecraft-core/service_scripts/NextITS/modules/parameter_summary.nf b/src/pipecraft-core/service_scripts/NextITS/modules/parameter_summary.nf deleted file mode 100644 index d07b2d2f..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/modules/parameter_summary.nf +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Custom parameter summary function for NextITS pipeline - * Shows only step-specific parameters and relevant configuration - */ - -def parameterSummary(workflow, params) { - - // Color definitions - def colors = params.monochrome_logs ? [:] : [ - green: "\033[0;32m", blue: "\033[0;34m", yellow: "\033[0;33m", - cyan: "\033[0;36m", purple: "\033[0;35m", dim: "\033[2m", - bold: "\033[1m", reset: "\033[0m" - ] - - def summary = "" - summary += "${colors.bold}Parameters:${colors.reset}\n" - summary += " ${colors.green}step${colors.reset} : ${colors.cyan}${params.step}${colors.reset}\n" - - // Step-specific parameters - if (params.step == "Step1" || params.step == "seqstats") { - - if (params.seqplatform == "PacBio") { - if (params.input) { - summary += " ${colors.green}input${colors.reset} : ${colors.cyan}${params.input}${colors.reset}\n" - } - } else if (params.seqplatform == "Illumina") { - if (params.input_R1) { - summary += " ${colors.green}input_R1${colors.reset} : ${colors.cyan}${params.input_R1}${colors.reset}\n" - } - if (params.input_R2) { - summary += " ${colors.green}input_R2${colors.reset} : ${colors.cyan}${params.input_R2}${colors.reset}\n" - } - } - - if (params.barcodes && !params.demultiplexed) { - summary += " ${colors.green}barcodes${colors.reset} : ${colors.cyan}${params.barcodes}${colors.reset}\n" - } - - if (params.demultiplexed) { - summary += " ${colors.green}demultiplexed${colors.reset} : ${colors.cyan}${params.demultiplexed}${colors.reset}\n" - } - - summary += " ${colors.green}chimera_db${colors.reset} : ${colors.cyan}${params.chimera_db}${colors.reset}\n" - if(params.step == "Step1") { - summary += " ${colors.green}its_region${colors.reset} : ${colors.cyan}${params.its_region}${colors.reset}\n" - } - - } - else if (params.step == "Step2") { - summary += "\n${colors.bold}Step 2 inputs:${colors.reset}\n" - summary += " ${colors.green}data_path${colors.reset} : ${colors.cyan}${params.data_path}${colors.reset}\n" - summary += " ${colors.green}clustering${colors.reset} : ${colors.cyan}${params.clustering}${colors.reset}\n" - summary += " ${colors.green}preclustering${colors.reset} : ${colors.cyan}${params.preclustering}${colors.reset}\n" - } - - - // Common parameters - summary += "\n${colors.bold}Output:${colors.reset}\n" - summary += " ${colors.green}outdir${colors.reset} : ${colors.cyan}${params.outdir}${colors.reset}\n" - summary += " ${colors.green}workDir${colors.reset} : ${colors.cyan}${workflow.workDir}${colors.reset}\n" - - // Nextflow configuration - summary += "\n${colors.bold}Config:${colors.reset}\n" - summary += " ${colors.green}NextITS version${colors.reset} : ${colors.cyan}${workflow.manifest.version}${colors.reset}\n" - if(workflow.commitId) { - summary += " ${colors.green}NextITS revision${colors.reset} : ${colors.cyan}${workflow.commitId.substring(0, 7)}${colors.reset}\n" - } - summary += " ${colors.green}Profile${colors.reset} : ${colors.cyan}${workflow.profile}${colors.reset}\n" - summary += " ${colors.green}Container engine${colors.reset} : ${colors.cyan}${workflow.containerEngine ?: 'none'}${colors.reset}\n" - - if (workflow.container) { - summary += " ${colors.green}Container${colors.reset} : ${colors.cyan}${workflow.container}${colors.reset}\n" - } - - return summary -} - -// Export the function so it can be used in other files -workflow paramSummary { - take: - wf - prms - - main: - def summary = parameterSummary(wf, prms) - log.info summary -} diff --git a/src/pipecraft-core/service_scripts/NextITS/modules/version_parser.nf b/src/pipecraft-core/service_scripts/NextITS/modules/version_parser.nf deleted file mode 100644 index b93492ff..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/modules/version_parser.nf +++ /dev/null @@ -1,28 +0,0 @@ - -// Custom function to parse software versions and return a YAML string - -def software_versions_to_yaml(versions) { - - def workflow_info = Channel.of( - "NextITS:\n" + - " version: ${workflow.manifest.version}\n" + - (workflow.commitId ? " revision: ${workflow.commitId.substring(0,7)}\n" : "") + - "\nNextflow:\n" + - " version: ${nextflow.version}\n" - ) - - return workflow_info.mix( - versions - .unique() - .map { name, tool, version -> - [ name.tokenize(':')[-1], [ tool, version ] ] - } - .groupTuple() - .map { processName, toolInfo -> - def toolVersions = toolInfo.collect { tool, version -> " ${tool}: ${version}" }.join('\n') - "${processName}:\n${toolVersions}\n" - } - .map { it.trim() } - ) -} - diff --git a/src/pipecraft-core/service_scripts/NextITS/nextflow.config b/src/pipecraft-core/service_scripts/NextITS/nextflow.config deleted file mode 100644 index a723b6d3..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/nextflow.config +++ /dev/null @@ -1,459 +0,0 @@ -/* -============================================================================== - NextITS: Pipeline to process fungal ITS amplicons -============================================================================== - Default config options for all compute environments ------------------------------------------------------------------------------- -*/ - - -// Enable nf-schema for parameter validation -plugins { - id 'nf-schema@2.4.1' -} - -// Global default params, used in configs -includeConfig 'conf/params.config' - -// Configure validation behavior -validation { - parametersSchema = "${projectDir}/nextflow_schema.json" - monochromeLogs = false - failUnrecognisedParams = false - lenientMode = true - - // Help options - help { - enabled = true - command = "nextflow run vmikk/NextITS" - shortParameter = "help" - fullParameter = "helpFull" - showHiddenParameter = "showHidden" - showHidden = false - beforeText = "NextITS: Pipeline to process rRNA amplicons sequenced with PacBio" - afterText = """ -If you use NextITS for your analysis please cite: - -* NextITS pipeline - Mikryukov V, Anslan S, Tedersoo L. - NextITS: a pipeline for metabarcoding fungi and other eukaryotes with full-length ITS sequenced with PacBio. - https://github.com/vmikk/NextITS - -* Software dependencies - https://github.com/vmikk/NextITS/blob/main/CITATIONS.md - """ - } - - // Summary options - summary { - beforeText = "NextITS pipeline parameters:" - // afterText = "" - } -} - -// Conditional params -if (params.qc_twocolor == true) { - includeConfig 'conf/params_illumina_2color.config' -} else { - includeConfig 'conf/params_illumina_4color.config' -} - -// Ignore process selector warnings, -// could be disabled using the `devel` profile -nextflow.enable.configProcessNamesValidation = false - - -// Process-specific parameters -process { - - ////////// Step-1 processes - - // Converting BAM to FASTQ - withName: 'S1:bam2fastq'{ - cpus = 4 - } - - // QC - PacBio single-end reads - // vsearch currently does not suppot multithreading for `--fastq_filter` - // see https://github.com/torognes/vsearch/issues/466 - withName: 'S1:qc_se'{ - cpus = 1 - } - - // QC - Illumina paired-end reads - withName: 'S1:qc_pe'{ - // max threads for fastp = 16 - cpus = 8 - } - - // Primer disambiguation - withName: 'S1:disambiguate'{ - cpus = 1 - } - - // Validate tags for demultiplexing - withName: 'S1:tag_validation'{ - cpus = 1 - } - - // Demultiplexing of PacBio reads (with LIMA) - withName: 'S1:demux'{ - cpus = 8 - } - - // Demultiplexing merged Illumina reads - withName: 'S1:demux_illumina'{ - cpus = 8 - } - - // Demultiplexing non-merged Illumina reads - withName: 'S1:demux_illumina_notmerged'{ - cpus = 8 - } - - // Merging of Illumina PE reads - withName: 'S1:merge_pe'{ - cpus = 8 - } - - // Modify barcodes for cutadapt (Illumina only) - withName: 'S1:prep_barcodes'{ - cpus = 1 - } - - // Demultiplexing of Illumina reads (with cutadapt) - withName: 'S1:demux_illumina'{ - cpus = 8 - } - - // Check primers - withName: 'S1:primer_check'{ - cpus = 1 - } - - // ITSx - withName: 'S1:itsx'{ - cpus = 3 - } - - // Collect all ITS parts extracted by ITSx - withName: 'S1:itsx_collect'{ - cpus = 1 - } - - // Sequence quality tables - withName: 'S1:seq_qual'{ - cpus = 4 - memory = null - } - - // Homopolymer compression - withName: 'S1:homopolymer' { - cpus = 1 - } - - // Reference-based chimera removal - withName: 'S1:CHIMERA_REMOVAL:chimera_ref' { - cpus = 1 - } - - // Chimera rescue - withName: 'S1:CHIMERA_REMOVAL:chimera_rescue' { - cpus = 1 - } - - // De novo chimera search - withName: 'S1:CHIMERA_REMOVAL:chimera_denovo' { - cpus = 1 - } - - // Aggregate de novo chimeras - withName: 'S1:CHIMERA_REMOVAL:chimera_denovo_agg' { - cpus = 1 - } - - // Pool sequences (for tag-jump removal and final sequence table creation) - withName: 'S1:pool_seqs' { - cpus = 3 - } - - // Dereplication or pre-clustering prior to tag-jump removal - withName: 'S1:tj_preclust' { - cpus = 8 - } - - // Tag-jump removal - withName: 'S1:tj' { - cpus = 1 - } - - // Create sequence table - withName: 'S1:prep_seqtab' { - cpus = 4 - } - - // Read count summary - withName: 'S1:read_counts' { - cpus = 4 - } - - // Read count summary - // For a quick workflow for demultiplexing and estimation of the number of reads per sample - withName: 'S1:quick_stats' { - cpus = 4 - } - - - - ////////// Step-2 processes - - // Aggregate sequences, remove de novo chimeras - withName: 'S2:aggregate_sequences' { - cpus = 8 - } - - // Dereplication (currently, only single-threaded; only comression is multithreaded)) - withName: 'S2:dereplication' { - cpus = 8 - } - - // 100% clustering with sequence length variation allowed (UNITE-style) - withName: 'S2:dereplication_unite' { - cpus = 8 - } - - // Pre-clustering - withName: 'S2:linclust' { - cpus = 8 - } - - // Bucketize sequences into clusters - withName: 'S2:bucketize' { - cpus = 6 - } - - // UNOISE - withName: 'S2:CLUSTERING:unoise' { - cpus = 8 - } - - // DADA2 - withName: 'S2:CLUSTERING:dada2' { - cpus = 8 - } - - // VSEARCH clustering - withName: 'S2:CLUSTERING:cluster_vsearch' { - cpus = 8 - } - - // SWARM clustering - withName: 'S2:CLUSTERING:cluster_swarm' { - cpus = 8 - } - - // Bucketizing workflow - merge chunks into a single file - withName: 'S2:merge_buckets' { - cpus = 4 - } - - // Merge UC files - withName: 'S2:merge_uc' { - cpus = 4 - } - - // Summarize sequence abundance by OTU - withName: 'S2:summarize' { - cpus = 4 - } - - // Post-clustering curation with LULU - withName: 'S2:lulu' { - cpus = 8 - } - -} // end of process configs - - - - -profiles { - - // Test config (built-in data) - test { includeConfig 'conf/test_step1.config' } // Step-1 (default test = test for Step-1) - test1 { includeConfig 'conf/test_step1.config' } // Step-1 - test2 { includeConfig 'conf/test_step2.config' } // Step-2 - - // Docker-based profile - docker { - docker.enabled = true - conda.enabled = false - singularity.enabled = false - apptainer.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - docker.runOptions = '-u $(id -u):$(id -g)' - - // Container specifications are here - includeConfig 'conf/docker.config' - } - - // Singularity-based profile - singularity { - singularity.enabled = true - singularity.autoMounts = true - // singularity.ociAutoPull = true - conda.enabled = false - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false - - // Container specifications are here - includeConfig 'conf/singularity.config' - } - // Podman-based profile - podman { - podman.enabled = true - docker.enabled = false - conda.enabled = false - singularity.enabled = false - apptainer.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - - // Container specifications are the same as for Docker - includeConfig 'conf/docker.config' - } - - - - // Larger resource amount (e.g., CPUs) for execution on HPC - hpc { - includeConfig 'conf/hpc.config' - } - - // Profile for the UT HPC cluster (SLURM) - hpc_utslurm { - includeConfig 'conf/hpc_utslurm.config' - } - - // Enable process selector warnings - // see https://github.com/nextflow-io/nextflow/issues/2700#issuecomment-1383984109 - devel { - nextflow.enable.configProcessNamesValidation = true - } - -} // end of profiles - - - -// Export these variables to prevent local Python/R libraries from conflicting with those in the container -env { - PYTHONNOUSERSITE = 1 - R_PROFILE_USER = "/.Rprofile" - R_ENVIRON_USER = "/.Renviron" - JULIA_DEPOT_PATH = "/usr/local/share/julia" -} - -// Capture exit codes from upstream processes when piping -process.shell = ['/bin/bash', '-euo', 'pipefail'] - - -def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') -timeline { - enabled = true - file = "${params.tracedir}/execution_timeline_${trace_timestamp}.html" -} -report { - enabled = true - file = "${params.tracedir}/execution_report_${trace_timestamp}.html" -} -trace { - enabled = true - file = "${params.tracedir}/execution_trace_${trace_timestamp}.txt" -} -dag { - enabled = true - file = "${params.tracedir}/pipeline_dag_${trace_timestamp}.svg" -} - - -manifest { - name = 'NextITS' - author = 'Vladimir Mikryukov' // The author field is deprecated, keep it here for compatibility with Nextflow versions < 24.10.0 - - contributors = [ - [ - name: 'Vladimir Mikryukov', - affiliation: 'University of Tartu', - email: 'vladimir.mikryukov@ut.ee', - github: '@vmikk', - contribution: ["author", "maintainer"], - orcid: '0009-0006-0086-2470' - ], - [ - name: 'Sten Anslan', - affiliation: 'University of Tartu', - email: 'sten.anslan@ut.ee', - github: '@anslan', - contribution: ["contributor"], - orcid: '0000-0002-2299-454X' - ], - [ - name: 'Leho Tedersoo', - affiliation: 'University of Tartu', - email: 'leho.tedersoo@ut.ee', - contribution: ["contributor"], - orcid: '0000-0002-1635-1249' - ], - ] - - homePage = 'https://github.com/vmikk/NextITS' - docsUrl = 'https://Next-ITS.github.io/' - description = 'Pipeline to process long rRNA amplicons sequenced with PacBio' - license = 'Apache-2.0' - mainScript = 'main.nf' - nextflowVersion = '!>=25.04.0' - version = '1.1.0' - doi = '10.5281/zenodo.15074881' -} - - - -// Function to ensure that resource requirements don't go beyond -// a maximum limit -def check_max(obj, type) { - if (type == 'memory') { - try { - if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit - else - return obj - } catch (all) { - println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'time') { - try { - if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration - else - return obj - } catch (all) { - println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'cpus') { - try { - return Math.min( obj, params.max_cpus as int ) - } catch (all) { - println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" - return obj - } - } -} - diff --git a/src/pipecraft-core/service_scripts/NextITS/nextflow_schema.json b/src/pipecraft-core/service_scripts/NextITS/nextflow_schema.json deleted file mode 100644 index c19cf238..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/nextflow_schema.json +++ /dev/null @@ -1,771 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/vmikk/NextITS/main/nextflow_schema.json", - "title": "NextITS pipeline parameters", - "description": "Pipeline to process fungal ITS amplicons sequenced with PacBio", - "type": "object", - "$defs": { - "step_1_specific_parameters": { - "title": "Step-1-specific parameters", - "type": "object", - "properties": { - "input": { - "type": "string", - "format": "path", - "exists": true, - "mimetype": "text/csv", - "description": "Path to input data: either a single FASTQ or BAM file containing multiplexed sequences, or a directory containing pre-demultiplexed FASTQ files.", - "fa_icon": "fas fa-file-csv" - }, - "barcodes": { - "type": "string", - "fa_icon": "fas fa-barcode", - "description": "Path to the file with barcodes (in FASTA format) used for demultiplexing the input data." - }, - "demultiplexed": { - "type": "boolean", - "fa_icon": "fas fa-align-justify", - "description": "Whether input is multiplexed (`false`, single FASTQ file) or pre-demultiplexed (`true`, multiple FASTQ files)." - }, - "seqplatform": { - "type": "string", - "default": "PacBio", - "fa_icon": "fab fa-google-play", - "description": "Sequencing platform used", - "enum": ["PacBio", "Illumina"] - }, - "qc_maxee": { - "type": "number", - "description": "Maximum expected errors allowed in a sequence. Sequences with higher error rates will be discarded.", - "fa_icon": "fas fa-times" - }, - "qc_maxeerate": { - "type": "number", - "default": 0.01, - "description": "Maximum expected error rate per sequence (between 0 and 1). Sequences with higher error rates will be discarded.", - "fa_icon": "fas fa-times" - }, - "qc_maxhomopolymerlen": { - "type": "integer", - "default": 25, - "description": "Threshold for a homopolymer region length in a sequence.", - "fa_icon": "fas fa-redo" - }, - "qc_maxn": { - "type": "integer", - "default": 4, - "description": "Discard sequences with more than the specified number of ambiguous nucleotides (N's).", - "fa_icon": "fas fa-minus-square" - }, - "lima_barcodetype": { - "type": "string", - "default": "dual_symmetric", - "fa_icon": "fab fa-slack-hash", - "description": "Barcode configuration type used for multiplexing samples: 'single' if only one of the primers was tagged, 'dual_symmetric' if identical barcodes were used, 'dual_asymmetric' if different barcodes were used, 'dual' if both primers were tagged with a mixture of identical and different barcodes", - "enum": ["single", "dual", "dual_symmetric", "dual_asymmetric"] - }, - "lima_minscore": { - "type": "integer", - "default": 93, - "minimum": 0, - "maximum": 100, - "fa_icon": "fas fa-balance-scale-left", - "description": "Minimum barcode score (0-100) required for successful demultiplexing. Higher values mean stricter matching." - }, - "lima_minendscore": { - "type": "integer", - "default": 50, - "fa_icon": "fas fa-balance-scale-right", - "description": "Minimum second barcode score (only for asymmetric and dual barcoding scheme)", - "minimum": 0, - "maximum": 100 - }, - "lima_minrefspan": { - "type": "number", - "default": 0.75, - "fa_icon": "fas fa-random", - "description": "Minimum read span relative to the barcode length", - "minimum": 0, - "maximum": 1 - }, - "lima_minscoringregions": { - "type": "integer", - "default": 2, - "fa_icon": "fas fa-angle-down", - "description": "Number of barcodes scored required for demultiplexing using dual barcodes", - "minimum": 1, - "maximum": 2 - }, - "lima_windowsize": { - "type": "integer", - "default": 70, - "fa_icon": "fas fa-arrows-alt-h", - "description": "Window size for barcode lookup, in base pairs", - "minimum": 1 - }, - "lima_minlen": { - "type": "integer", - "default": 40, - "fa_icon": "fas fa-compress-alt", - "description": "Minimum sequence length after clipping barcodes.", - "minimum": 1 - }, - "lima_remove_unknown": { - "type": "boolean", - "fa_icon": "fas fa-compress-alt", - "description": "Remove unknown barcode combinations from the demultiplexed data (if false, unknown combinations will be named as tag IDs)." - }, - "primer_forward": { - "type": "string", - "fa_icon": "fas fa-arrow-right", - "description": "Sequence of the forward primer.", - "default": "TACACACCGCCCGTCG", - "help_text": "**Forward Primer:** `ITS9MUNngs` with sequence `TACACACCGCCCGTCG` \nRefer to [Tedersoo & Lindahl, 2016 DOI:`10.1111/1758-2229.12438`](https://ami-journals.onlinelibrary.wiley.com/doi/10.1111/1758-2229.12438) for more details on these primers. " - }, - "primer_reverse": { - "type": "string", - "fa_icon": "fas fa-arrow-left", - "description": "Sequence of the reverse primer.", - "default": "CCTSCSCTTANTDATATGC", - "help_text": "**Reverse Primer:** `ITS4ngsUni` with sequence `CCTSCSCTTANTDATATGC` \nRefer to [Tedersoo & Lindahl, 2016 DOI:`10.1111/1758-2229.12438`](https://ami-journals.onlinelibrary.wiley.com/doi/10.1111/1758-2229.12438) for more details on these primers. " - }, - "primer_mismatches": { - "type": "integer", - "default": 2, - "fa_icon": "fas fa-chevron-circle-down", - "description": "Allowed number of mismatches for primers." - }, - "primer_foverlap": { - "type": "integer", - "fa_icon": "fas fa-angle-right", - "description": "Minimum overlap for the forward primer.", - "default": null - }, - "primer_roverlap": { - "type": "integer", - "fa_icon": "fas fa-angle-left", - "description": "Minimum overlap for the reverse primer.", - "default": null - }, - "its_region": { - "type": "string", - "default": "full", - "enum": [ - "full", - "ITS1", - "ITS2", - "none", - "ITS1_5.8S_ITS2", - "SSU", - "LSU" - ], - "description": "Target ITS region to extract: 'full' for complete ITS region, 'ITS1' or 'ITS2' for specific spacers, 'ITS1_5.8S_ITS2' for the partial ITS (in case of HMM profiles are not able to detect the full ITS region), 'SSU' or 'LSU' for 18S or 28S ribosomal genes, or 'none' to skip extraction (in this case, only the primers will be trimmed)", - "fa_icon": "fas fa-arrows-alt-h" - }, - "ITSx_tax": { - "type": "string", - "default": "all", - "fa_icon": "fas fa-user-tag", - "description": "Taxonomic profile for ITS extraction", - "pattern": "^(?:all|(?:alveolata|bryophyta|bacillariophyta|amoebozoa|euglenozoa|fungi|chlorophyta|rhodophyta|phaeophyceae|marchantiophyta|metazoa|oomycota|haptophyceae|raphidophyceae|rhizaria|synurophyceae|tracheophyta|eustigmatophyceae|apusozoa|parabasalia)(?:,\\s*(?:alveolata|bryophyta|bacillariophyta|amoebozoa|euglenozoa|fungi|chlorophyta|rhodophyta|phaeophyceae|marchantiophyta|metazoa|oomycota|haptophyceae|raphidophyceae|rhizaria|synurophyceae|tracheophyta|eustigmatophyceae|apusozoa|parabasalia))*)$" - }, - "ITSx_evalue": { - "type": "number", - "default": 0.1, - "fa_icon": "fas fa-check-circle", - "description": "Min E-value threshold for ITSx" - }, - "ITSx_complement": { - "type": "string", - "default": "F", - "fa_icon": "fas fa-check-circle", - "description": "Check single strand (F, default) or both DNA strands (T) for matches to HMM-profiles", - "enum": [ - "F", - "T" - ] - }, - "ITSx_partial": { - "type": "integer", - "default": 0, - "fa_icon": "fas fa-arrows-alt-h", - "description": "Min length cutoff for partial ITS sequences to keep (0 = disabled, default)" - }, - "ITSx_to_parquet": { - "type": "boolean", - "default": true, - "fa_icon": "fas fa-database", - "description": "Convert ITSx output (FASTA files) to Parquet" - }, - "ITSx_chunk_size": { - "type": "integer", - "default": 10000, - "fa_icon": "fas fa-database", - "description": "Chunk size for distributed ITSx processing (number of dereplicated sequences per sample); set to 0 to disable chunking" - }, - "hp": { - "type": "boolean", - "default": true, - "description": "Enable homopolymer error correction in sequences" - }, - "hp_similarity": { - "type": "number", - "default": 0.999, - "minimum": 0.8, - "maximum": 1, - "description": "Sequence similarity threshold for pre-clustering during homopolymer error correction (0.8-1.0)" - }, - "hp_iddef": { - "type": "number", - "default": 2, - "minimum": 0, - "maximum": 4, - "description": "Identity definition for homopolymer compression" - }, - "chimera_methods": { - "type": "string", - "default": "ref,denovo", - "description": "Which chimera removal methods to use", - "enum": ["ref", "denovo", "ref,denovo", "denovo,ref", "none", "null"] - }, - "chimera_db": { - "type": "string", - "fa_icon": "fas fa-database", - "default": "Eukaryome_1.9.3_241222_FullITS_100-800.udb", - "description": "Database for refrence-based chimera removal" - }, - "chimera_rescueoccurrence": { - "type": "integer", - "default": 2, - "description": "Occurrence threshold for rescuing reference-based chimeras" - }, - "chimeranov_abskew": { - "type": "number", - "default": 2, - "description": "abskew parameter for de novo chimera removal" - }, - "chimeranov_dn": { - "type": "number", - "default": 1.4, - "description": "dn parameter for de novo chimera removal" - }, - "chimeranov_mindiffs": { - "type": "number", - "default": 3, - "description": "mindiffs parameter for de novo chimera removal" - }, - "chimeranov_mindiv": { - "type": "number", - "default": 0.8, - "description": "mindiv parameter for de novo chimera removal" - }, - "chimeranov_minh": { - "type": "number", - "default": 0.28, - "description": "minh parameter for de novo chimera removal" - }, - "chimeranov_xn": { - "type": "number", - "default": 8, - "description": "xn parameter for de novo chimera removal" - }, - "tj": { - "type": "boolean", - "default": true, - "description": "Run tag-jump removal" - }, - "tj_f": { - "type": "number", - "default": 0.01, - "minimum": 0, - "maximum": 1, - "description": "UNCROSS parameter f for tag-jump removal" - }, - "tj_p": { - "type": "number", - "default": 1, - "description": "Parameter p for tag-jump removal" - }, - "tj_id": { - "type": "number", - "default": 1, - "minimum": 0, - "maximum": 1, - "description": "Sequence similarity threshold for tag-jump removal (1 = dereplication, < 1 = pre-clustering)" - }, - "tj_iddef": { - "type": "integer", - "default": 2, - "minimum": 0, - "maximum": 4, - "description": "Sequence identity definition for pre-clustering prior tag-jump removal" - } - }, - "fa_icon": "fas fa-bullseye" - }, - "step_2_specific_parameters": { - "title": "Step-2-specific parameters", - "type": "object", - "properties": { - "data_path": { - "type": "string", - "format": "directory-path", - "fa_icon": "fas fa-folder-open", - "description": "Path to the Step-1 results", - "default": "Step1_Results" - }, - "merge_replicates": { - "type": "boolean", - "description": "Pool sample replicates (e.g., re-sequenced samples) in the final OTU table" - }, - "ampliconlen_min": { - "type": "integer", - "description": "Minimum amplicon length for filtering" - }, - "ampliconlen_max": { - "type": "integer", - "description": "Maximum amplicon length for filtering" - }, - "chunking_n": { - "type": "integer", - "description": "Number of chunks to split the dataset into prior clustering" - }, - "chunking_id": { - "type": "number", - "default": 0.6, - "description": "Minimum sequence identity for clustering" - }, - "preclustering": { - "type": "string", - "default": "none", - "enum": ["none", "unoise", "dada2", "swarm_d1", "homopolymer"], - "description": "Sequence denoising or pre-clustering method" - }, - "unoise_alpha": { - "type": "number", - "default": 6, - "description": "UNOISE alpha parameter" - }, - "unoise_minsize": { - "type": "integer", - "default": 1, - "description": "UNOISE minimum size parameter" - }, - "dada2_pooling": { - "type": "string", - "default": "byrun", - "enum": ["byrun", "global"], - "description": "DADA2 pooling strategy" - }, - "dada2_nbases": { - "type": "number", - "default": 1000000, - "description": "DADA2 number of bases" - }, - "dada2_bandsize": { - "type": "integer", - "default": 16, - "description": "DADA2 band size" - }, - "dada2_detectsingletons": { - "type": "boolean", - "default": true, - "description": "DADA2 detect singletons" - }, - "dada2_omegaA": { - "type": "number", - "default": 1e-20, - "description": "DADA2 omega A parameter" - }, - "dada2_omegaC": { - "type": "number", - "default": 1e-40, - "description": "DADA2 omega C parameter" - }, - "dada2_omegaP": { - "type": "number", - "default": 0.0001, - "description": "DADA2 omega P parameter" - }, - "dada2_maxconsist": { - "type": "integer", - "default": 10, - "description": "DADA2 maximum consistency" - }, - "dada2_match": { - "type": "integer", - "default": 4, - "description": "DADA2 match score" - }, - "dada2_mismatch": { - "type": "integer", - "default": -5, - "description": "DADA2 mismatch penalty" - }, - "dada2_gappenalty": { - "type": "integer", - "default": -8, - "description": "DADA2 gap penalty" - }, - "clustering": { - "type": "string", - "default": "vsearch", - "enum": ["none", "vsearch", "swarm", "shmatching"], - "description": "Sequence clustering method" - }, - "otu_id": { - "type": "number", - "default": 0.98, - "description": "Sequence similarity threshold for OTU clustering", - "minimum": 0, - "maximum": 1 - }, - "otu_iddef": { - "type": "integer", - "default": 2, - "description": "Sequence similarity definition type for OTU clustering", - "minimum": 1, - "maximum": 4 - }, - "otu_qmask": { - "type": "string", - "default": "dust", - "description": "Quality masking for OTU clustering" - }, - "swarm_d": { - "type": "integer", - "default": 1, - "description": "SWARM distance parameter" - }, - "swarm_fastidious": { - "type": "boolean", - "default": true, - "description": "Enable SWARM fastidious mode" - }, - "swarm_d1boundary": { - "type": "integer", - "default": 3, - "description": "SWARM boundary parameter for fastidious mode" - }, - "alignment_penalties": { - "type": "string", - "default": "default", - "enum": ["default", "UNITE"], - "description": "Alignment penalty configuration" - }, - "vsearch_gapopen": { - "type": "string", - "default": "20I/2E", - "description": "VSEARCH gap opening penalties" - }, - "vsearch_gapext": { - "type": "string", - "default": "2I/1E", - "description": "VSEARCH gap extension penalties" - }, - "max_MEEP": { - "type": "number", - "default": 0.5, - "description": "Maximum expected error as a percentage of read length (MEEP) threshold for singleton removal" - }, - "max_ChimeraScore": { - "type": "number", - "default": 0.6, - "description": "Maximum de novo chimera score" - }, - "recover_lowqsingletons": { - "type": "boolean", - "default": true, - "description": "Allow recovery of singletons" - }, - "recover_denovochimeras": { - "type": "boolean", - "default": true, - "description": "Allow recovery of de novo chimeras" - }, - "lulu": { - "type": "boolean", - "default": true, - "description": "Post-clustering curation using LULU algorithm" - }, - "lulu_match": { - "type": "integer", - "default": 95, - "description": "Minimum sequence similarity threshold for LULU" - }, - "lulu_ratio": { - "type": "integer", - "default": 1, - "description": "Minimum sequence abundance ratio for LULU" - }, - "lulu_ratiotype": { - "type": "string", - "default": "min", - "description": "Abundance ratio type for LULU", - "enum": ["min", "avg"] - }, - "lulu_relcooc": { - "type": "number", - "default": 0.95, - "description": "Relative co-occurrence threshold for LULU", - "minimum": 0, - "maximum": 1 - }, - "lulu_maxhits": { - "type": "integer", - "default": 0, - "description": "Maximum number of hits for LULU (0 = unlimited)" - }, - "sh": { - "type": "boolean", - "default": true, - "description": "Species-hypothesis (SH) matching [not implemented yet]" - }, - "sh_thresholds": { - "type": "string" - }, - "sh_coveragevariation": { - "type": "number", - "default": 0.96 - } - } - }, - "common_parameters": { - "title": "Common parameters for both steps", - "type": "object", - "properties": { - "outdir": { - "type": "string", - "format": "directory-path", - "description": "Path to the directory where the analysis results will be saved.", - "fa_icon": "fas fa-folder-open", - "default": "Step2_Results" - }, - "step": { - "type": "string", - "enum": ["Step1", "Step2", "seqstats"], - "description": "The step of the pipeline to run.", - "fa_icon": "fas fa-step-forward", - "default": "Step1" - }, - "storagemode": { - "type": "string", - "default": "rellink", - "fa_icon": "fas fa-boxes", - "enum": ["symlink", "copy", "move", "rellink", "link"], - "description": "How to store output files: 'symlink' for symbolic links, 'copy' to copy files, 'move' to move files, 'rellink' for relative links, 'link' for hard links" - }, - "gzip_compression": { - "type": "number", - "default": 7, - "minimum": 1, - "maximum": 9, - "fa_icon": "fas fa-file-archive", - "description": "Controls GZIP compression level in output files." - }, - "max_cpus": { - "type": "integer", - "default": 40, - "fa_icon": "fas fa-microchip", - "description": "Maximum number of CPUs that can be requested (for any single job).", - "help_text": "Use to set an upper-limit for the CPU requirement for each process. Should be an integer e.g. `--max_cpus 1`", - "minimum": 1, - "maximum": 200 - }, - "max_memory": { - "type": "string", - "default": "132.GB", - "fa_icon": "fas fa-memory", - "description": "Maximum amount of memory that can be requested (for any single job).", - "pattern": "^\\d+(\\.\\d+)?\\.?\\s*(K|M|G|T)?B$", - "help_text": "Use to set an upper-limit for the memory requirement for each process. Should be a string in the format integer-unit e.g. `--max_memory '8.GB'`" - }, - "max_time": { - "type": "string", - "default": "240.h", - "fa_icon": "fas fa-clock", - "description": "Maximum amount of time that can be requested (for any single job).", - "pattern": "^(\\d+\\.?\\s*(s|m|h|day)\\s*)+$", - "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`" - }, - "tracedir": { - "type": "string", - "fa_icon": "fas fa-folder-minus", - "description": "Directory to store pipeline execution logs" - }, - "version": { - "type": "boolean", - "description": "Display version and exit", - "fa_icon": "fas fa-question-circle", - "hidden": true - }, - "help": { - "type": "boolean", - "description": "Display help text", - "fa_icon": "fas fa-question-circle", - "hidden": true - }, - "helpMsg": { - "type": "boolean", - "fa_icon": "far fa-question-circle", - "description": "Custom help message" - }, - "monochrome_logs": { - "type": "boolean", - "description": "Do not use coloured log outputs", - "fa_icon": "fas fa-palette", - "hidden": true - }, - "validate_params": { - "type": "boolean", - "description": "Boolean whether to validate parameters against the schema at runtime", - "default": true, - "fa_icon": "fas fa-check-square", - "hidden": true - }, - "show_hidden_params": { - "type": "boolean", - "description": "Show all params when using `--help`", - "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters.", - "fa_icon": "fas fa-eye-slash" - }, - "enable_conda": { - "type": "boolean", - "fa_icon": "fas fa-adjust", - "hidden": true - }, - "email": { - "type": "string", - "description": "Email address for completion summary.", - "fa_icon": "fas fa-envelope", - "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.", - "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$", - "hidden": true - }, - "email_on_fail": { - "type": "string", - "description": "Email address for completion summary, only when pipeline fails.", - "fa_icon": "fas fa-exclamation-triangle", - "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$", - "help_text": "An email address to send a summary email to when the pipeline is completed - ONLY sent if the pipeline does not exit successfully.", - "hidden": true - }, - "plaintext_email": { - "type": "boolean", - "description": "Send plain-text email instead of HTML.", - "fa_icon": "fas fa-remove-format", - "hidden": true - }, - "schema_ignore_params": { - "type": "string" - } - }, - "required": ["step"], - "fa_icon": "fab fa-creative-commons-share" - }, - "illumina_specific_parameters": { - "title": "Illumina-specific parameters", - "type": "object", - "properties": { - "input_R1": { - "type": "string", - "description": "File with R1 reads for Illumina" - }, - "input_R2": { - "type": "string", - "description": "File with R2 reads for Illumina" - }, - "qc_avgphred": { - "type": "boolean", - "description": "Average Phred score (for Illumina PE reads only)" - }, - "qc_twocolor": { - "type": "boolean", - "description": "Reduced resolution Phred-scores (two-color Illumina chemistry)" - }, - "pe_minoverlap": { - "type": "integer", - "default": 20, - "description": "Paired-end reads minimum overlap (Illumina only)" - }, - "pe_diffperclimit": { - "type": "integer", - "default": 20, - "description": "Paired-end reads max percentage difference for the overlap (Illumina only)" - }, - "pe_difflimit": { - "type": "integer", - "default": 5, - "description": "Paired-end reads max difference for the overlap (Illumina only)" - }, - "pe_nlimit": { - "type": "integer", - "default": 10 - }, - "pe_minlen": { - "type": "integer", - "default": 30 - }, - "barcode_errors": { - "type": "integer", - "default": 1, - "description": "Number of allowed erros in demultiplexing (Illumina only)" - }, - "illumina_keep_notmerged": { - "type": "boolean", - "default": true - }, - "illumina_joinpadgap": { - "type": "string", - "default": "NNNNNNNNNN" - }, - "illumina_joinpadqual": { - "type": "string", - "default": "IIIIIIIIII" - }, - "barcode_window": { - "type": "integer", - "default": 30, - "description": "(Illumina only)" - }, - "barcode_overlap": { - "type": "integer", - "default": 11, - "description": "(Illumina only)" - }, - "trim_minlen": { - "type": "integer", - "default": 10 - }, - "qc_phredmin": { - "type": "integer", - "description": "Min Phred score threshold of unqualified bases (for two-color Illumina only)" - }, - "qc_phredperc": { - "type": "integer", - "description": "Max percentage of unqualified bases per read (for two-color Illumina only)" - }, - "qc_polyglen": { - "type": "integer", - "description": "Max poly-G length (for two-color Illumina only)" - } - }, - "description": "EXPERIMENTAL", - "fa_icon": "fas fa-flask" - } - }, - "allOf": [ - { - "$ref": "#/$defs/step_1_specific_parameters" - }, - { - "$ref": "#/$defs/step_2_specific_parameters" - }, - { - "$ref": "#/$defs/common_parameters" - }, - { - "$ref": "#/$defs/illumina_specific_parameters" - } - ] -} diff --git a/src/pipecraft-core/service_scripts/NextITS/subworkflows/chimera_removal_subworkflow.nf b/src/pipecraft-core/service_scripts/NextITS/subworkflows/chimera_removal_subworkflow.nf deleted file mode 100644 index 0a13786d..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/subworkflows/chimera_removal_subworkflow.nf +++ /dev/null @@ -1,296 +0,0 @@ -/* -============================================================================ - NextITS: Pipeline to process eukaryotic ITS amplicons -============================================================================ - License: Apache-2.0 - Github : https://github.com/vmikk/NextITS - Website: https://Next-ITS.github.io/ ----------------------------------------------------------------------------- -*/ - -// Subworkflow for chimera removal (reference-based and de novo) - -// Path to the output results -out_5_chim = params.outdir + "/05_Chimera" - -// Reference-based chimera removal -process chimera_ref { - - label "main_container" - - publishDir "${out_5_chim}", mode: "${params.storagemode}" - // cpus 1 - - // Add sample ID to the log file - tag "${input.getSimpleName().replaceAll(/_Homopolymer_compressed/, '')}" - - input: - path input - path DB - - output: - path "${input.getSimpleName().replaceAll(/_Homopolymer_compressed/, '')}_NoChimera.fa.gz", emit: nonchimeric, optional: true - path "${input.getSimpleName().replaceAll(/_Homopolymer_compressed/, '')}_Chimera.fa.gz", emit: chimeric, optional: true - - script: - sampID="${input.getSimpleName().replaceAll(/_Homopolymer_compressed/, '')}" - - """ - - ## Sample name will be added to the header of chimeric sequences - # sampID="\$(basename ${input} _Homopolymer_compressed.fa.gz)" - - ## Reference database based chimera filtering - echo -e "Reference-based chimera removal" - vsearch \ - --uchime_ref ${input} \ - --db ${DB} \ - --selfid \ - --fasta_width 0 \ - --threads ${task.cpus} \ - --sizein --sizeout \ - --chimeras chimeras.fasta \ - --nonchimeras nonchimeras.fasta \ - --borderline borderline.fasta - - # --selfid = ignore reference sequences that are 100% identical to the query - echo -e "..Done" - - - ## Add borderline sequences to non-chimeric sequences - if [ -e borderline.fasta ] - then - echo -e "\\nBorderline sequences were added to non-chimeric sequences" - cat borderline.fasta nonchimeras.fasta > nc_bo.fasta - mv nc_bo.fasta nonchimeras.fasta - rm borderline.fasta - fi - - ## Chimeric sequences - if [ -e chimeras.fasta ] - then - ## Add sample ID to the header and compress the file - sed 's/>.*/&;sample='"${sampID}"';/' chimeras.fasta \ - | gzip -${params.gzip_compression} \ - > "${sampID}_Chimera.fa.gz" - rm chimeras.fasta - else - echo -e "\\nNo chimeras detected" - fi - - ## Non-chimeric sequences - if [ -e nonchimeras.fasta ] - then - gzip -c nonchimeras.fasta > "${sampID}_NoChimera.fa.gz" - rm nonchimeras.fasta - else - echo "No non-chimeric sequences left" - fi - - """ -} - - -// Recovery of ref-based chimeric sequences with high occurrence -process chimera_rescue { - - label "main_container" - - publishDir "${out_5_chim}", mode: "${params.storagemode}" - // cpus 1 - - input: - path input - - output: - path "*_RescuedChimera.fa.gz", emit: rescuedchimeric, optional: true - - script: - """ - - ## Aggregate chimeric sequences from different samples - echo -e "\\nAggregating chimeric sequences" - find . -name "*_Chimera.fa.gz" \ - | parallel -j1 "zcat {}" \ - | seqkit fx2tab \ - | sed -r 's:\t+:\t:g' | sed 's/\t\$//g' \ - | gzip -${params.gzip_compression} > All_chimeras.txt.gz - echo -e "..Done" - - ### Inspect chimerae occurrence - ## Rescue sequences that were annotated as chimeric, - ## but have high occurrence within sequenceing run (e.g., occurrence > 2) - echo -e "\\nInspecting occurrence of chimeric sequences" - - chimera_rescue.R \ - "All_chimeras.txt.gz" \ - ${params.chimera_rescueoccurrence} \ - "Rescued_Chimeric_sequences.fa.gz" - - echo -e "..Done" - - ## Split rescured sequences by sample - if [ -e Rescued_Chimeric_sequences.fa.gz ] - then - echo -e "\\n..Splitting rescued sequences by sample" - seqkit split -i \ - --id-regexp ";sample=(.*);" \ - --threads ${task.cpus} \ - -w 0 \ - -O Rescued_by_sample \ - Rescued_Chimeric_sequences.fa.gz - - rename \ - --filename 's/^Rescued_Chimeric_sequences.id_//g ; s/^Rescued_Chimeric_sequences.part_//g ; s/.fa.gz/_RescuedChimera.fa.gz/' \ - \$(find Rescued_by_sample -name "*.fa.gz") - - mv Rescued_by_sample/*_RescuedChimera.fa.gz . - - echo -e "..Done" - fi - - ## Remove temporary files - rm All_chimeras.txt.gz - if [ -f Rescued_Chimeric_sequences.fa.gz ]; then rm Rescued_Chimeric_sequences.fa.gz; fi - - """ -} - - - -// De novo chimera identification -// NB! in uchime_denovo, sequences are compared on their plus strand only! -process chimera_denovo { - - label "main_container" - - publishDir "${out_5_chim}", mode: "${params.storagemode}" - // cpus 1 - - // Add sample ID to the log file - tag "${input.getSimpleName().replaceAll(/_Homopolymer_compressed/, '')}" - - input: - path input - - output: - path "${input.getSimpleName().replaceAll(/_Homopolymer_compressed/, '')}_DeNovoChim.txt", emit: denovochim, optional: true - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - - script: - sampID="${input.getSimpleName().replaceAll(/_Homopolymer_compressed/, '')}" - - """ - - ## Input order matters for chimera detection, - ## so sequences will be automatically sorted by decreasing abundance first - - echo -e "De novo chimera identification" - - vsearch \ - --uchime_denovo ${input} \ - --abskew ${params.chimeranov_abskew} \ - --dn ${params.chimeranov_dn} \ - --mindiffs ${params.chimeranov_mindiffs} \ - --mindiv ${params.chimeranov_mindiv} \ - --minh ${params.chimeranov_minh} \ - --xn ${params.chimeranov_xn} \ - --threads 1 \ - --qmask dust \ - --sizein --xsize \ - --fasta_width 0 \ - --fasta_score \ - --chimeras - \ - | seqkit seq --name \ - | sed 's/;+/;/g ; s/;/\t/g ; s/uchime_denovo=//' \ - | sed 's/\$/\t${sampID}/' \ - > ${sampID}_DeNovoChim.txt - - # --uchimeout uchimeout.txt - - ## Remove file, if empty - find . -maxdepth 1 -name ${sampID}_DeNovoChim.txt -size 0 -print -delete - - echo -e "..Done" - - """ -} - -// Aggregate de novo chimeras into a single file -process chimera_denovo_agg { - - label "main_container" - // cpus 1 - - input: - path input - - output: - path "DeNovo_Chimera.txt", emit: alldenovochim, optional: true - - script: - """ - echo -e "Aggregating de novo chimeric sequences" - - find . -name "*_DeNovoChim.txt" \ - | parallel -j1 "cat {}" \ - > DeNovo_Chimera.txt - - echo -e "..Done" - - """ -} - - - -// Chimera indetification, removal, and recovery -workflow CHIMERA_REMOVAL { - - take: - seqs - db - - main: - def methodsRaw = (params.chimera_methods ?: '').toString().toLowerCase() - def chim_rm = methodsRaw && methodsRaw != 'none' - def doRef = chim_rm && methodsRaw.split(',').contains('ref') - def doDenovo = chim_rm && methodsRaw.split(',').contains('denovo') - - if( doRef ) { - // Reference-based chimera identification and removal - chimera_ref(seqs, db) - nonchim = chimera_ref.out.nonchimeric - chim = chimera_ref.out.chimeric - - // Rescue chimeras per current logic - chimera_rescue(chim.collect()) - rescued = chimera_rescue.out.rescuedchimeric - } else { - nonchim = seqs - chim = Channel.empty() - rescued = Channel.empty() - } - - if( doDenovo ) { - - // De novo chimera identification - chimera_denovo(seqs) - - // Aggregate de novo chimeras into a single file - chimera_denovo_agg(chimera_denovo.out.denovochim.collect()) - dnvAgg = chimera_denovo_agg.out.alldenovochim - } else { - dnvAgg = Channel.empty() - } - - // Final sequences for downstream pooling - filtered = nonchim.concat(rescued).collect() - - emit: - filtered = filtered - chimeric = chim - rescued = rescued - denovo_agg = dnvAgg - -} // end of chimera_removal subworkflow - diff --git a/src/pipecraft-core/service_scripts/NextITS/subworkflows/clustering_subworkflow.nf b/src/pipecraft-core/service_scripts/NextITS/subworkflows/clustering_subworkflow.nf deleted file mode 100644 index d5e7f1bb..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/subworkflows/clustering_subworkflow.nf +++ /dev/null @@ -1,540 +0,0 @@ -/* -============================================================================ - NextITS: Pipeline to process eukaryotic ITS amplicons -============================================================================ - License: Apache-2.0 - Github : https://github.com/vmikk/NextITS - Website: https://Next-ITS.github.io/ ----------------------------------------------------------------------------- -*/ - -// Subworkflow for clustering sequences (with optional pre-clustering or denoising) - - -// Homopolymer correction (global, for pooled and dereplicated data) -process homopolymer { - - label "main_container" - - publishDir( - "${params.outdir}/02.Homopolymer", - mode: "${params.storagemode}", - enabled: params.chunking_n == null || params.chunking_n < 2 - ) - - // cpus 1 - - input: - path input - - output: - path "HomopolymerCompressed.fa.gz", emit: hp - path "HomopolymerCompressed.uc.gz", emit: hp_uc - - script: - """ - ## Run homopolyer correction globally - - echo -e "Running homopolymer correction" - - echo -e "\\nCompressing repeats" - zcat ${input} \ - | homopolymer_compression.sh \ - | gzip -2 \ - > homo_compressed.fa.gz - - echo -e "\\nAdditional dereplication" - vsearch \ - --derep_fulllength homo_compressed.fa.gz \ - --output - \ - --strand both \ - --fasta_width 0 \ - --threads 1 \ - --sizein --sizeout \ - --uc HomopolymerCompressed.uc \ - > homo_compressed_dereplicated.fa - - ## Substitute homopolymer-comressed sequences with uncompressed ones - ## (update size annotaions) - echo -e "\\nExtracting representative sequences" - - seqkit fx2tab ${input} > inp_tab.txt - seqkit fx2tab homo_compressed_dereplicated.fa > clust_tab.txt - - if [ -s inp_tab.txt ]; then - substitute_compressed_seqs.R \ - inp_tab.txt clust_tab.txt \ - HomopolymerCompressed_tmp.fa - - echo -e "..Done" - else - echo -e "..Input data looks empty, nothing to proceed with" - fi - - - ## Sort by number of reads - vsearch \ - --sortbysize HomopolymerCompressed_tmp.fa \ - --sizein --sizeout \ - --threads ${task.cpus} \ - --fasta_width 0 \ - --output - \ - | gzip -${params.gzip_compression} \ - > HomopolymerCompressed.fa.gz - - - #### combine_derep_and_hpcorrection.R - - echo -e "\\nHomopolymer correction finished\\n" - - ## Compress results - echo -e "\\nCompressing results" - gzip -${params.gzip_compression} HomopolymerCompressed.uc - - ## Remove temporary files - echo -e "\\nRemoving temporary files" - rm homo_compressed.fa.gz - rm homo_compressed_dereplicated.fa - rm HomopolymerCompressed_tmp.fa - rm inp_tab.txt - rm clust_tab.txt - """ -} - - -// Denoize sequences with UNOISE -process unoise { - - label "main_container" - - publishDir( - "${params.outdir}/02.UNOISE", - mode: "${params.storagemode}", - enabled: params.chunking_n == null || params.chunking_n < 2 - ) - - // cpus 8 - - input: - path input - - output: - path "UNOISE.fa.gz", emit: unoise - path "UNOISE.uc.gz", emit: unoise_uc - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - - script: - """ - echo -e "Denoizing sequences with UNOISE\\n" - - vsearch \ - --cluster_unoise ${input} \ - --unoise_alpha ${params.unoise_alpha} \ - --minsize ${params.unoise_minsize} \ - --iddef ${params.otu_iddef} \ - --qmask ${params.otu_qmask} \ - --gapopen ${params.vsearch_gapopen} \ - --gapext ${params.vsearch_gapext } \ - --threads ${task.cpus} \ - --fasta_width 0 \ - --sizein --sizeout \ - --centroids UNOISE.fa \ - --uc UNOISE.uc - - echo -e "..UNOISE done\\n" - - ## Compress results - echo -e "\\nCompressing UNOISE results" - parallel -j 1 \ - "pigz -p ${task.cpus} -${params.gzip_compression} {}" \ - ::: "UNOISE.fa" "UNOISE.uc" - - """ -} - - - -// Denoize sequences with DADA2 -process dada2 { - - label "main_container" - - publishDir( - "${params.outdir}/02.DADA2", - mode: "${params.storagemode}", - enabled: params.chunking_n == null || params.chunking_n < 2 - ) - - // cpus 8 - - input: - path input - - output: - path "DADA2_denoised.fa.gz", emit: dada - path "DADA2_denoised.uc.gz", emit: dada_uc - path "DADA2_UC.qs", emit: dada_ucr - path "DADA2_denoising_summary.txt", emit: dada_summary - // path "DADA2_ErrorRates_noqualErrfun.RData" - // path "DADA2_InferedSeqs_noqualErrfun.RData" - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('dada2'), eval('Rscript -e "cat(as.character(packageVersion(\'dada2\')))"'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - - script: - """ - echo -e "Denoizing sequences with DADA2\\n" - - ## DADA2 works with ACGT alphabet only - ## 1. So check if there are any sequences with ambiguities - ## 2. If any, remove them - ## 3. Sort by sequence abundance - ## 4. Convert FASTA to pseudo-FASTQ - ## 5. Denoise - - ## Remove sequences with ambiguities - echo -e "..Preparing sequences\\n" - zcat ${input} \ - | awk '{if (/^>/) {a = \$0} else {if (/^[ACGT]*\$/) {printf "%s\\n%s\\n", a, \$0}}}' \ - | vsearch --sortbysize - --output - --fasta_width 0 \ - | awk 'BEGIN {RS = ">" ; FS = "\\n"} NR > 1 {print "@"\$1"\\n"\$2"\\n+"\$1"\\n"gensub(/./, "I", "g", \$2)}' \ - | gzip -${params.gzip_compression} > no_ambigs.fq.gz - - echo -e "\\n\\n..Running DADA2\\n" - dada2_no_quals.R \ - --input no_ambigs.fq.gz \ - --nbases ${params.dada2_nbases} \ - --bandsize ${params.dada2_bandsize} \ - --detectsingletons ${params.dada2_detectsingletons} \ - --omegaA ${params.dada2_omegaA} \ - --omegaC ${params.dada2_omegaC} \ - --omegaP ${params.dada2_omegaP} \ - --maxconsist ${params.dada2_maxconsist} \ - --match ${params.dada2_match} \ - --mismatch ${params.dada2_mismatch} \ - --gappenalty ${params.dada2_gappenalty} \ - --threads ${task.cpus} - - echo -e "..Denoizing with DADA2 finished\\n" - """ -} - - - - -// Preclustering with SWARM and d1 -process precluster_swarm { - - label "main_container" - - publishDir( - "${params.outdir}/02.Preclustered_SWARM_d1", - mode: "${params.storagemode}", - enabled: params.chunking_n == null || params.chunking_n < 2 - ) - - // cpus 8 - - input: - path input - - output: - path "SWARM_representatives.fa.gz", emit: clust - path "SWARM.uc.gz", emit: clust_uc - path "SWARM.swarms.gz", emit: swarms - path "SWARM.struct.gz", emit: struct - path "SWARM.stats.gz", emit: stats - tuple val("${task.process}"), val('swarm'), eval('swarm --version 2>&1 | head -n 1 | sed "s/Swarm //"'), topic: versions - - script: - """ - echo -e "Pre-clustering sequences with SWARM d=1\\n" - echo -e "Note: sequences with ambiguous nucleotides will be excluded!\\n" - - ## Remove sequences with ambiguities - zcat ${input} \ - | awk '{if (/^>/) {a = \$0} else {if (/^[ACGT]*\$/) {printf "%s\\n%s\\n", a, \$0}}}' \ - | swarm \ - --differences 1 \ - --boundary ${params.swarm_d1boundary} \ - --fastidious \ - --threads ${task.cpus} \ - --usearch-abundance \ - --statistics-file SWARM.stats \ - --internal-structure SWARM.struct \ - --uclust-file SWARM.uc \ - --seeds SWARM_representatives.fa \ - > SWARM.swarms - - echo -e "\\n..Swarm pre-clustering finished\\n" - - ## Compress results - echo -e "\\n..Compressing results\\n" - parallel -j 1 \ - "pigz -p ${task.cpus} -${params.gzip_compression} {}" \ - ::: "SWARM_representatives.fa" "SWARM.uc" "SWARM.swarms" "SWARM.struct" "SWARM.stats" - - echo -e "..Done\\n" - """ -} - - - -// Cluster sequences with VSEARCH (fixed similarity threshold) -process cluster_vsearch { - - label "main_container" - - publishDir( - "${params.outdir}/03.Clustered_VSEARCH", - mode: "${params.storagemode}", - enabled: params.chunking_n == null || params.chunking_n < 2 - ) - - // cpus 8 - - input: - path input - - output: - path "Clustered.fa.gz", emit: clust - path "Clustered.uc.gz", emit: clust_uc - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - - script: - """ - echo -e "Clustering sequences with VSEARCH\\n" - - vsearch \ - --cluster_size ${input} \ - --id ${params.otu_id} \ - --iddef ${params.otu_iddef} \ - --qmask ${params.otu_qmask} \ - --gapopen ${params.vsearch_gapopen} \ - --gapext ${params.vsearch_gapext } \ - --threads ${task.cpus} \ - --sizein --sizeout \ - --strand both \ - --fasta_width 0 \ - --uc Clustered.uc \ - --centroids - \ - | gzip -${params.gzip_compression} > Clustered.fa.gz - - echo -e "..Done" - - ## Compress UC file - echo -e "\\nCompressing UC file" - pigz -p ${task.cpus} -${params.gzip_compression} Clustered.uc - - """ -} - - - -// Cluster sequences with SWARM (dynamic similarity threshold) -process cluster_swarm { - - label "main_container" - - publishDir( - "${params.outdir}/03.Clustered_SWARM", - mode: "${params.storagemode}", - enabled: params.chunking_n == null || params.chunking_n < 2 - ) - - // cpus 8 - - input: - path input - - output: - path "SWARM_representatives.fa.gz", emit: clust - path "SWARM.uc.gz", emit: clust_uc - path "SWARM.swarms.gz", emit: swarms - path "SWARM.struct.gz", emit: struct - path "SWARM.stats.gz", emit: stats - tuple val("${task.process}"), val('swarm'), eval('swarm --version 2>&1 | head -n 1 | sed "s/Swarm //"'), topic: versions - - exec: - fastidious = (params.swarm_fastidious.toBoolean() == true & params.swarm_d.toInteger() == 1) ? "--fastidious --boundary ${params.swarm_d1boundary}" : "" - // println("swarm_fastidious: ${params.swarm_fastidious}, swarm_d: ${params.swarm_d}") - // println("fastid option: ${fastidious}") - - script: - """ - echo -e "Clustering sequences with SWARM\\n" - echo -e "Note: sequences with ambiguous nucleotides will be excluded!\\n" - - ## Swarm works with ACGTU alphabet only - ## 1. So check if there are any sequences with ambiguities - ## 2. If any, remove them - ## 3. Cluster - - ## Count number of sequences with ambiguities (will go through the entire file) - # AMBIGS=\$(seqkit grep --count --by-seq --use-regexp --ignore-case --pattern "[RYSWKMBDHVN]" ${input}) - - ## Remove sequences with ambiguities - zcat ${input} \ - | awk '{if (/^>/) {a = \$0} else {if (/^[ACGT]*\$/) {printf "%s\\n%s\\n", a, \$0}}}' \ - | swarm \ - --differences ${params.swarm_d} \ - ${fastidious} \ - --threads ${task.cpus} \ - --usearch-abundance \ - --statistics-file SWARM.stats \ - --internal-structure SWARM.struct \ - --uclust-file SWARM.uc \ - --seeds SWARM_representatives.fa \ - > SWARM.swarms - - # --output-file SWARM.swarms # to avoid buffering, it's better to stream data into a file (with >) - # -r, --mothur # output using mothur-like format - - echo -e "\\n..Swarm clustering finished\\n" - - ## Compress results - echo -e "..Compressing results\\n" - parallel -j 1 \ - "pigz -p ${task.cpus} -${params.gzip_compression} {}" \ - ::: "SWARM_representatives.fa" "SWARM.uc" "SWARM.swarms" "SWARM.struct" "SWARM.stats" - - echo -e "..Done\\n" - """ -} - - -// Pre-clustering / denoising / clustering subworkflow -workflow CLUSTERING { - - take: - derep_ch - - main: - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Pre-clustering / denoising - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - // No pre-clustering or denoizing - if ( params.preclustering == "none" || params.preclustering == null ) { - denoise_ch = derep_ch - preclustuc_ch = file('NoPrecluster') - preclustaf_ch = file('NoPreclusterFASTA') - - // Denoise with UNOISE - } else if ( params.preclustering == "unoise" ) { - unoise(derep_ch) - denoise_ch = unoise.out.unoise - preclustuc_ch = unoise.out.unoise_uc - preclustaf_ch = denoise_ch - - // Denoise with DADA2 - } else if ( params.preclustering == "dada2" ) { - - // Denoise all dereplicated sequences - if(params.dada2_pooling == "global"){ - dada2(derep_ch) - denoise_ch = dada2.out.dada - preclustuc_ch = dada2.out.dada_uc - preclustaf_ch = denoise_ch - } - - // // Dereplicate and denoise by sequencing run - // if(params.dada2_pooling == "byrun"){ - // - // dereplication_byrun(ch_seqs) - // dada2(dereplication_byrun.out.dereps.flatten()) - // - // dada2pool( - // dereplication.out.derep_uc, - // dada2.out.dada_ucr.collect() - // ) - // - // /* - // denoise_ch = dada2pool.out.dada - // preclustuc_ch = dada2pool.out.dada_uc - // preclustaf_ch = file('NoPreclusterFASTA') - // - // */ - // } - - - // Precluster with SWARM - } else if ( params.preclustering == "swarm_d1" ){ - precluster_swarm(derep_ch) - denoise_ch = precluster_swarm.out.clust - preclustuc_ch = precluster_swarm.out.clust_uc - preclustaf_ch = denoise_ch - - // Global homopolymer correction - } else if ( params.preclustering == "homopolymer" ){ - homopolymer(derep_ch) - denoise_ch = homopolymer.out.hp - preclustuc_ch = homopolymer.out.hp_uc - preclustaf_ch = denoise_ch - } - - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Clustering - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - - // Greedy clustering with VSEARCH - if ( params.clustering == "vsearch" ) { - cluster_vsearch(denoise_ch) - cluster_ch = cluster_vsearch.out.clust - clustuc_ch = cluster_vsearch.out.clust_uc - - // Clustering with SWARM - } else if ( params.clustering == "swarm" ) { - - // If pre-clustering was already done with the same d, just take the previous results - if(params.preclustering == "swarm_d1" & params.swarm_d == 1){ - cluster_ch = precluster_swarm.out.clust - clustuc_ch = precluster_swarm.out.clust_uc - preclustuc_ch = file('NoPrecluster') - preclustaf_ch = file('NoPreclusterFASTA') - - // Otherwise, run SWARM - } else { - cluster_swarm(denoise_ch) - cluster_ch = cluster_swarm.out.clust - clustuc_ch = cluster_swarm.out.clust_uc - } - - // Do not cluster, use zOTUs from UNOISE - } else if ( params.preclustering == "unoise" & params.clustering == "none" ) { - cluster_ch = unoise.out.unoise - clustuc_ch = unoise.out.unoise_uc - preclustuc_ch = file('NoPrecluster') - preclustaf_ch = file('NoPreclusterFASTA') - - // Do not cluster, use ASVs from DADA2 - } else if ( params.preclustering == "dada2" & params.clustering == "none" ){ - - if(params.dada2_pooling == "global"){ - cluster_ch = dada2.out.dada - clustuc_ch = dada2.out.dada_uc - } - - preclustuc_ch = file('NoPrecluster') - preclustaf_ch = file('NoPreclusterFASTA') - - } else if ( params.preclustering == "none" & params.clustering == "none" ){ - println "No pre-clustering or clustering was done" - // This is done outside the clustering subworkflow - } - - - emit: - preclustuc_ch = preclustuc_ch // UC file for pre-clustering - preclustaf_ch = preclustaf_ch // FASTA file for pre-clustering - cluster_ch = cluster_ch // FASTA file for clustering - clustuc_ch = clustuc_ch // UC file for clustering - -} // end of subworkflow diff --git a/src/pipecraft-core/service_scripts/NextITS/subworkflows/itsx_subworkflow.nf b/src/pipecraft-core/service_scripts/NextITS/subworkflows/itsx_subworkflow.nf deleted file mode 100644 index 629eea07..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/subworkflows/itsx_subworkflow.nf +++ /dev/null @@ -1,564 +0,0 @@ -/* -============================================================================ - NextITS: Pipeline to process eukaryotic ITS amplicons -============================================================================ - License: Apache-2.0 - Github : https://github.com/vmikk/NextITS - Website: https://Next-ITS.github.io/ ----------------------------------------------------------------------------- -*/ - -// Subworkflow for ITSx processing -// (which splits large dereplicated FASTAs into chunks). -// The workflow is as follows: -// 1. Trim primers and dereplicate at sample level -// 2. Split the dereplicated primer-trimmed sequences (at sample level) into chunks while preserving metadata -// 3. Run ITSx on each chunk -// 4. Group results back by original sample ID and concatenate + convert ITSx output to Parquet - -// Path to the output results -out_3_itsx = params.outdir + "/03_ITSx" -out_3_itsxp = params.outdir + "/03_ITSx_PooledParts" - - -// Trim primers and dereplicate at sample level -process primer_trim { - - label "main_container" - - publishDir "${out_3_itsx}", mode: "${params.storagemode}" - // cpus 2 - - // Add sample ID to the log file - tag "${meta.id}" - - input: - tuple val(meta), path(fastq) - - output: - tuple val(meta), path("${meta.id}_derep.fasta.gz"), emit: derep, optional: true - tuple val(meta), path("${meta.id}_hash_table.txt.gz"), emit: hashes, optional: true - tuple val(meta), path("${meta.id}_uc.uc.gz"), emit: uc, optional: true - tuple val(meta), path("${meta.id}_primertrimmed_sorted.fq.gz"), emit: trimmed_seqs, optional: true - tuple val("${task.process}"), val('cutadapt'), eval('cutadapt --version'), topic: versions - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - tuple val("${task.process}"), val('phredsort'), eval('phredsort -v | sed "s/phredsort //"'), topic: versions - tuple val("${task.process}"), val('seqhasher'), eval('seqhasher -v | sed "s/SeqHasher //"'), topic: versions - tuple val("${task.process}"), val('parallel'), eval('parallel --version | head -n 1 | sed "s/GNU parallel //"'), topic: versions - tuple val("${task.process}"), val('brename'), eval('brename --help | head -n 4 | tail -1 | sed "s/Version: //"'), topic: versions - - script: - sampID="${meta.id}" - """ - echo -e "Primer trimming and dereplication at sample level\\n" - echo -e "Input sample: " ${sampID} - - ## Trim primers - echo -e "Trimming primers\\n" - - ## Reverse-complement rev primer - RR=\$(rc.sh ${params.primer_reverse}) - - cutadapt \ - -a ${params.primer_forward}";required;min_overlap=${params.primer_foverlap}"..."\$RR"";required;min_overlap=${params.primer_roverlap}" \ - --errors ${params.primer_mismatches} \ - --revcomp --rename "{id}" \ - --discard-untrimmed \ - --minimum-length ${params.trim_minlen} \ - --cores ${task.cpus} \ - --action trim \ - --output ${sampID}_primertrimmed.fq.gz \ - ${fastq} - - echo -e "..Done\\n" - - ## Check if there are sequences in the output - NUMSEQS=\$( seqkit stat --tabular --quiet ${sampID}_primertrimmed.fq.gz | awk -F'\t' 'NR==2 {print \$4}' ) - echo -e "Number of sequences after primer trimming: " \$NUMSEQS - if [ \$NUMSEQS -lt 1 ]; then - echo -e "\\nIt looks like no reads remained after trimming the primers\\n" - exit 0 - fi - - ## Estimate sequence quality and sort sequences by quality - echo -e "\\nSorting by sequence quality" - seqkit replace -p "\\s.+" ${sampID}_primertrimmed.fq.gz \ - | phredsort -i - -o - --metric meep --header avgphred,maxee,meep \ - | gzip -1 > ${sampID}_primertrimmed_sorted.fq.gz - echo -e "..Done" - - ## Hash sequences, add sample ID to the header - ## columns: Sample ID - Hash - PacBioID - AvgPhredScore - MaxEE - MEEP - Sequence - Quality - Length - ## Convert to Parquet format - echo -e "\\nCreating hash table" - seqhasher --hash sha1 --name ${sampID} ${sampID}_primertrimmed_sorted.fq.gz - \ - | seqkit fx2tab --length \ - | sed 's/;/\t/ ; s/;/\t/ ; s/ avgphred=/\t/ ; s/ maxee=/\t/ ; s/ meep=/\t/' \ - > ${sampID}_hash_table.txt - echo -e "..Done" - - ## Check the number of fields per record (should be 9!) - # awk '{print NF}' ${sampID}_hash_table.txt | sort | uniq -c - # awk 'NF > 9 {print \$0 }' ${sampID}_hash_table.txt - - ## Dereplicate at sample level (use quality-sorted sequences to make sure that the representative sequence is with the highest quality) - echo -e "\\nDereplicating at sample level" - seqkit fq2fa -w 0 ${sampID}_primertrimmed_sorted.fq.gz \ - | vsearch \ - --derep_fulllength - \ - --output - \ - --strand both \ - --fasta_width 0 \ - --threads 1 \ - --relabel_sha1 \ - --sizein --sizeout \ - --minseqlength ${params.trim_minlen} \ - --uc ${sampID}_uc.uc \ - --quiet \ - > ${sampID}_derep.fasta - - ## Remove temporary file - rm ${sampID}_primertrimmed.fq.gz - - ## Compress results - echo -e "\\nCompressing results" - parallel -j${task.cpus} "gzip -${params.gzip_compression} {}" ::: \ - ${sampID}_hash_table.txt \ - ${sampID}_uc.uc \ - ${sampID}_derep.fasta - - echo -e "..Done" - - """ -} - - -// Extract ITS region with ITSx -// NB. In input data, sequence header should not contain spaces! -process itsx { - - label "main_container" - - // No need to publish intermediate results for chunked workflow, as they will be concatenated later - publishDir "${out_3_itsx}", - mode: "${params.storagemode}", - enabled: params.ITSx_chunk_size == 0 - - // cpus 2 - - // Add sample ID to the log file - tag { meta.chunk_id != null ? "${meta.id}__chunk${meta.chunk_id}" : "${meta.id}" } - - input: - tuple val(meta), path(input) // FASTA file with dereplicated sequences - - output: - tuple val(meta), path( "${meta.id}*.full.fasta.gz"), emit: itsx_full, optional: true - tuple val(meta), path( "${meta.id}*.SSU.fasta.gz"), emit: itsx_ssu, optional: true - tuple val(meta), path( "${meta.id}*.ITS1.fasta.gz"), emit: itsx_its1, optional: true - tuple val(meta), path( "${meta.id}*.5_8S.fasta.gz"), emit: itsx_58s, optional: true - tuple val(meta), path( "${meta.id}*.ITS2.fasta.gz"), emit: itsx_its2, optional: true - tuple val(meta), path( "${meta.id}*.LSU.fasta.gz"), emit: itsx_lsu, optional: true - tuple val(meta), path( "${meta.id}*.positions.txt"), emit: itsx_positions, optional: true - tuple val(meta), path( "${meta.id}*.problematic.txt"), emit: itsx_problematic, optional: true - tuple val(meta), path( "${meta.id}*_no_detections.fasta.gz"), emit: itsx_nondetects, optional: true - tuple val(meta), path( "${meta.id}*.summary.txt"), emit: itsx_summary, optional: true - tuple val(meta), path( "${meta.id}*.extraction.results.gz"), emit: itsx_details, optional: true - tuple val(meta), path( "${meta.id}*.SSU.full_and_partial.fasta.gz"), emit: itsx_ssu_part, optional: true - tuple val(meta), path( "${meta.id}*.ITS1.full_and_partial.fasta.gz"), emit: itsx_its1_part, optional: true - tuple val(meta), path( "${meta.id}*.5_8S.full_and_partial.fasta.gz"), emit: itsx_58s_part, optional: true - tuple val(meta), path( "${meta.id}*.ITS2.full_and_partial.fasta.gz"), emit: itsx_its2_part, optional: true - tuple val(meta), path( "${meta.id}*.LSU.full_and_partial.fasta.gz"), emit: itsx_lsu_part, optional: true - tuple val("${task.process}"), val('ITSx'), eval('ITSx --help 2>&1 | head -n 3 | tail -n 1 | sed "s/Version: //"'), topic: versions - tuple val("${task.process}"), val('cutadapt'), eval('cutadapt --version'), topic: versions - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - tuple val("${task.process}"), val('phredsort'), eval('phredsort -v | sed "s/phredsort //"'), topic: versions - tuple val("${task.process}"), val('seqhasher'), eval('seqhasher -v | sed "s/SeqHasher //"'), topic: versions - tuple val("${task.process}"), val('parallel'), eval('parallel --version | head -n 1 | sed "s/GNU parallel //"'), topic: versions - tuple val("${task.process}"), val('brename'), eval('brename --help | head -n 4 | tail -1 | sed "s/Version: //"'), topic: versions - tuple val("${task.process}"), val('duckdb'), eval('duckdb --version | cut -d" " -f1 | sed "s/^v//"'), topic: versions - - script: - sampID="${meta.id}" - chunkPrefix="${meta.id}_chunk${meta.chunk_id}" - - // Allow inclusion of sequences that only find a single domain, given that they meet the given E-value and score thresholds, on with parameters 1e-9,0 by default - // singledomain = params.ITSx_singledomain ? "--allow_single_domain 1e-9,0" : "" - - """ - echo -e "Extraction of rRNA regions using ITSx\\n" - echo -e "Input sample: " ${sampID} - echo -e "Chunk ID: " ${meta.chunk_id} - - ## Check if input file is gz-compressed (by magic bytes `1f 8b`) - tmp_created=0 - tmpfile="" - if [[ -f "${input}" ]] && head -c 2 -- "${input}" | LC_ALL=C od -An -tx1 | tr -d ' \n' | grep -qi '^1f8b'; then - echo -e "Input file is gz-compressed, decompressing..." - tmpfile="\$(mktemp "tmp.decompressed.input.XXXXXX")" - gunzip -c -- "${input}" > "\$tmpfile" - tmp_created=1 - itsxinput="\$tmpfile" - itsxoutput="${sampID}" - else - itsxinput="${input}" - itsxoutput="${chunkPrefix}" - fi - - ## ITSx extraction - echo -e "\\nITSx extraction" - ITSx \ - -i "\$itsxinput" \ - --complement ${params.ITSx_complement} \ - --save_regions all \ - --graphical F \ - --detailed_results T \ - --positions T \ - --not_found T \ - -E ${params.ITSx_evalue} \ - -t ${params.ITSx_tax} \ - --partial ${params.ITSx_partial} \ - --cpu ${task.cpus} \ - --preserve T \ - -o "\$itsxoutput" - - echo -e "..Done" - - # ITSx.full.fasta - # ITSx.SSU.fasta - # ITSx.ITS1.fasta - # ITSx.5_8S.fasta - # ITSx.ITS2.fasta - # ITSx.LSU.fasta - # ITSx.positions.txt - # ITSx.problematic.txt - # ITSx_no_detections.fasta - # ITSx_no_detections.txt - # ITSx.summary.txt - # ITSx.extraction.results - # ITSx.SSU.full_and_partial.fasta - # ITSx.ITS1.full_and_partial.fasta - # ITSx.5_8S.full_and_partial.fasta - # ITSx.ITS2.full_and_partial.fasta - # ITSx.LSU.full_and_partial.fasta - - - ## If partial sequences were required, remove empty sequences - if [ \$(find . -type f -name "*.full_and_partial.fasta" | wc -l) -gt 0 ]; then - echo -e "Partial files found, removing empty sequences\\n." - - find . -name "*.full_and_partial.fasta" \ - | parallel -j${task.cpus} "seqkit seq -m 1 -w 0 {} > {.}_tmp.fasta" - - rm *.full_and_partial.fasta - brename -p "_tmp" -r "" -f "_tmp.fasta\$" - - fi - - ## Remove empty files (no sequences) - echo -e "\\nRemoving empty files" - find . -type f -name "*.fasta" -empty -print -delete - echo -e "..Done" - - ## Remove temporary file (if input file was gz-compressed) - if (( tmp_created )); then - rm -f -- "\$tmpfile" - fi - - ## Compress results - echo -e "\\nCompressing files" - - ## ITSx results (no symlinked derep input) - find . -type f -name "*.fasta" \ - | parallel -j${task.cpus} "gzip -${params.gzip_compression} {}" - - gzip -${params.gzip_compression} "\$itsxoutput".extraction.results - - echo -e "..Done" - """ -} - - -// Concatenate ITSx output from all chunks (per samples) -// Convert ITSx output to Parquet -process itsx_concatenate { - - label "main_container" - - publishDir "${out_3_itsx}", mode: "${params.storagemode}" - - tag "${meta.id}" - - input: - tuple val(meta), path(fasta_chunks, stageAs: "chunks/") // all files from ITSx for all chunks for each sample - - output: - path( "${meta.id}.full.fasta.gz"), emit: itsx_full, optional: true - path( "${meta.id}.SSU.fasta.gz"), emit: itsx_ssu, optional: true - path( "${meta.id}.ITS1.fasta.gz"), emit: itsx_its1, optional: true - path( "${meta.id}.5_8S.fasta.gz"), emit: itsx_58s, optional: true - path( "${meta.id}.ITS2.fasta.gz"), emit: itsx_its2, optional: true - path( "${meta.id}.LSU.fasta.gz"), emit: itsx_lsu, optional: true - path( "${meta.id}.positions.txt"), emit: itsx_positions, optional: true - path( "${meta.id}.problematic.txt"), emit: itsx_problematic, optional: true - path( "${meta.id}_no_detections.fasta.gz"), emit: itsx_nondetects, optional: true - path( "${meta.id}.summary.txt"), emit: itsx_summary, optional: true - path( "${meta.id}.extraction.results.gz"), emit: itsx_details, optional: true - path( "${meta.id}.SSU.full_and_partial.fasta.gz"), emit: itsx_ssu_part, optional: true - path( "${meta.id}.ITS1.full_and_partial.fasta.gz"), emit: itsx_its1_part, optional: true - path( "${meta.id}.5_8S.full_and_partial.fasta.gz"), emit: itsx_58s_part, optional: true - path( "${meta.id}.ITS2.full_and_partial.fasta.gz"), emit: itsx_its2_part, optional: true - path( "${meta.id}.LSU.full_and_partial.fasta.gz"), emit: itsx_lsu_part, optional: true - path( "parquet/*.parquet"), emit: parquet, optional: true - - script: - sampID="${meta.id}" - """ - - echo -e "Concatenating ITSx output from all chunks" - echo -e "Input sample: " ${sampID} - - shopt -s nullglob - - ## Concatenate ITSx FASTA outputs - echo -e "Concatenating:" - - full_files=( chunks/${sampID}_chunk*.full.fasta.gz ) - echo -e " - full ITS sequences: \${#full_files[@]}" - if [ \${#full_files[@]} -gt 0 ]; then - for f in "\${full_files[@]}"; do - echo -e " \$f" - done - cat "\${full_files[@]}" > ${sampID}.full.fasta.gz - fi - - ssu_files=( chunks/${sampID}_chunk*.SSU.fasta.gz ) - echo -e " - SSU sequences: \${#ssu_files[@]}" - if [ \${#ssu_files[@]} -gt 0 ]; then - for f in "\${ssu_files[@]}"; do - echo -e " \$f" - done - cat "\${ssu_files[@]}" > ${sampID}.SSU.fasta.gz - fi - - its1_files=( chunks/${sampID}_chunk*.ITS1.fasta.gz ) - echo -e " - ITS1 sequences: \${#its1_files[@]}" - if [ \${#its1_files[@]} -gt 0 ]; then - for f in "\${its1_files[@]}"; do - echo -e " \$f" - done - cat "\${its1_files[@]}" > ${sampID}.ITS1.fasta.gz - fi - - s58_files=( chunks/${sampID}_chunk*.5_8S.fasta.gz ) - echo -e " - 5.8S sequences: \${#s58_files[@]}" - if [ \${#s58_files[@]} -gt 0 ]; then - for f in "\${s58_files[@]}"; do - echo -e " \$f" - done - cat "\${s58_files[@]}" > ${sampID}.5_8S.fasta.gz - fi - - its2_files=( chunks/${sampID}_chunk*.ITS2.fasta.gz ) - echo -e " - ITS2 sequences: \${#its2_files[@]}" - if [ \${#its2_files[@]} -gt 0 ]; then - for f in "\${its2_files[@]}"; do - echo -e " \$f" - done - cat "\${its2_files[@]}" > ${sampID}.ITS2.fasta.gz - fi - - lsu_files=( chunks/${sampID}_chunk*.LSU.fasta.gz ) - echo -e " - LSU sequences: \${#lsu_files[@]}" - if [ \${#lsu_files[@]} -gt 0 ]; then - for f in "\${lsu_files[@]}"; do - echo -e " \$f" - done - cat "\${lsu_files[@]}" > ${sampID}.LSU.fasta.gz - fi - - nd_files=( chunks/${sampID}_chunk*_no_detections.fasta.gz ) - echo -e " - no detections sequences: \${#nd_files[@]}" - if [ \${#nd_files[@]} -gt 0 ]; then - for f in "\${nd_files[@]}"; do - echo -e " \$f" - done - cat "\${nd_files[@]}" > ${sampID}_no_detections.fasta.gz - fi - - ## Concatenate partial outputs if present - ssu_part_files=( chunks/${sampID}_chunk*.SSU.full_and_partial.fasta.gz ) - echo -e " - SSU partial sequences: \${#ssu_part_files[@]}" - if [ \${#ssu_part_files[@]} -gt 0 ]; then - for f in "\${ssu_part_files[@]}"; do - echo -e " \$f" - done - cat "\${ssu_part_files[@]}" > ${sampID}.SSU.full_and_partial.fasta.gz - fi - - its1_part_files=( chunks/${sampID}_chunk*.ITS1.full_and_partial.fasta.gz ) - echo -e " - ITS1 partial sequences: \${#its1_part_files[@]}" - if [ \${#its1_part_files[@]} -gt 0 ]; then - for f in "\${its1_part_files[@]}"; do - echo -e " \$f" - done - cat "\${its1_part_files[@]}" > ${sampID}.ITS1.full_and_partial.fasta.gz - fi - - s58_part_files=( chunks/${sampID}_chunk*.5_8S.full_and_partial.fasta.gz ) - echo -e " - 5.8S partial sequences: \${#s58_part_files[@]}" - if [ \${#s58_part_files[@]} -gt 0 ]; then - for f in "\${s58_part_files[@]}"; do - echo -e " \$f" - done - cat "\${s58_part_files[@]}" > ${sampID}.5_8S.full_and_partial.fasta.gz - fi - - its2_part_files=( chunks/${sampID}_chunk*.ITS2.full_and_partial.fasta.gz ) - echo -e " - ITS2 partial sequences: \${#its2_part_files[@]}" - if [ \${#its2_part_files[@]} -gt 0 ]; then - for f in "\${its2_part_files[@]}"; do - echo -e " \$f" - done - cat "\${its2_part_files[@]}" > ${sampID}.ITS2.full_and_partial.fasta.gz - fi - - lsu_part_files=( chunks/${sampID}_chunk*.LSU.full_and_partial.fasta.gz ) - echo -e " - LSU partial sequences: \${#lsu_part_files[@]}" - if [ \${#lsu_part_files[@]} -gt 0 ]; then - for f in "\${lsu_part_files[@]}"; do - echo -e " \$f" - done - cat "\${lsu_part_files[@]}" > ${sampID}.LSU.full_and_partial.fasta.gz - fi - - ## Concatenate text outputs - pos_files=( chunks/${sampID}_chunk*.positions.txt ) - echo -e " - positions: \${#pos_files[@]}" - if [ \${#pos_files[@]} -gt 0 ]; then - for f in "\${pos_files[@]}"; do - echo -e " \$f" - done - cat "\${pos_files[@]}" > ${sampID}.positions.txt - fi - - prob_files=( chunks/${sampID}_chunk*.problematic.txt ) - echo -e " - problematic sequences: \${#prob_files[@]}" - if [ \${#prob_files[@]} -gt 0 ]; then - for f in "\${prob_files[@]}"; do - echo -e " \$f" - done - cat "\${prob_files[@]}" > ${sampID}.problematic.txt - fi - - sum_files=( chunks/${sampID}_chunk*.summary.txt ) - echo -e " - ITSx summary reports: \${#sum_files[@]}" - if [ \${#sum_files[@]} -gt 0 ]; then - for f in "\${sum_files[@]}"; do - echo -e " \$f" - done - cat "\${sum_files[@]}" > ${sampID}.summary.txt - fi - - det_files=( chunks/${sampID}_chunk*.extraction.results.gz ) - echo -e " - ITSx extraction results: \${#det_files[@]}" - if [ \${#det_files[@]} -gt 0 ]; then - for f in "\${det_files[@]}"; do - echo -e " \$f" - done - cat "\${det_files[@]}" > ${sampID}.extraction.results.gz - fi - - echo -e "\\n" - - ## Convert ITSx output to Parquet - if [ ${params.ITSx_to_parquet} == true ]; then - - echo -e "\\nConverting ITSx output to Parquet" - mkdir -p parquet - - if [ -f ${sampID}.full.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.full.fasta.gz -o parquet/${sampID}.full.parquet - fi - - if [ -f ${sampID}.SSU.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.SSU.fasta.gz -o parquet/${sampID}.SSU.parquet - fi - - if [ -f ${sampID}.ITS1.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.ITS1.fasta.gz -o parquet/${sampID}.ITS1.parquet - fi - - if [ -f ${sampID}.5_8S.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.5_8S.fasta.gz -o parquet/${sampID}.5_8S.parquet - fi - - if [ -f ${sampID}.ITS2.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.ITS2.fasta.gz -o parquet/${sampID}.ITS2.parquet - fi - - if [ -f ${sampID}.LSU.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.LSU.fasta.gz -o parquet/${sampID}.LSU.parquet - fi - - echo -e "Parquet files created\\n" - - fi - - """ -} - - - - -// ITSx processing workflow -workflow ITSx { - - take: - seqs - - main: - - // Add metadata to the channel (fetch sample ID from the FASTQ file name) - ch_seqs = seqs.map { fastq -> - def sample_id = fastq.getSimpleName().replaceAll(/_PrimerChecked/, '') - def meta = [id: sample_id] - [meta, fastq] - } - - // Trim primers and dereplicate at sample level - primer_trim(ch_seqs) - - // Size of dereplicated input for ITSx - // if null, use default value (currently, 10000) - // if 0, use all sequences in one chunk - def chunk_size = (params.ITSx_chunk_size == null ? 10000 : params.ITSx_chunk_size as int) - - if( chunk_size == 0 ) { - // Single-chunk workflow (no data splitting) - // NB! here, fasta will be gz-compressed -> will be handled in the itsx process - chunks_ch = primer_trim.out.derep - .map { meta, fasta -> [ meta + [chunk_id: null], fasta ] } - } - else { - // Chunking mode: split the dereplicated primer-trimmed sequences (at sample level) into chunks while preserving metadata - // NB! here, fasta will be uncompressed - chunks_ch = primer_trim.out.derep - .flatMap { meta, fasta -> - def chunks = fasta.splitFasta(by: chunk_size, file: true, decompress: true, compress: false) - def result = [] - chunks.eachWithIndex { chunk_file, idx -> - result << [ meta + [chunk_id: idx], chunk_file ] - } - return result - } - } - - // Run ITSx - itsx(chunks_ch) - - - -} // end of ITSx workflow diff --git a/src/pipecraft-core/service_scripts/NextITS/test_data/Test_ChimeraDB.udb b/src/pipecraft-core/service_scripts/NextITS/test_data/Test_ChimeraDB.udb deleted file mode 100644 index c7556dd1..00000000 Binary files a/src/pipecraft-core/service_scripts/NextITS/test_data/Test_ChimeraDB.udb and /dev/null differ diff --git a/src/pipecraft-core/service_scripts/NextITS/test_data/Test_S2/Test/07_SeqTable/Seqs.parquet b/src/pipecraft-core/service_scripts/NextITS/test_data/Test_S2/Test/07_SeqTable/Seqs.parquet deleted file mode 100644 index caad0925..00000000 Binary files a/src/pipecraft-core/service_scripts/NextITS/test_data/Test_S2/Test/07_SeqTable/Seqs.parquet and /dev/null differ diff --git a/src/pipecraft-core/service_scripts/NextITS/test_data/Test_Step1.fastq.gz b/src/pipecraft-core/service_scripts/NextITS/test_data/Test_Step1.fastq.gz deleted file mode 100644 index 5238b0ee..00000000 Binary files a/src/pipecraft-core/service_scripts/NextITS/test_data/Test_Step1.fastq.gz and /dev/null differ diff --git a/src/pipecraft-core/service_scripts/NextITS/test_data/Test_Step1_barcodes.fasta b/src/pipecraft-core/service_scripts/NextITS/test_data/Test_Step1_barcodes.fasta deleted file mode 100644 index dcc3270d..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/test_data/Test_Step1_barcodes.fasta +++ /dev/null @@ -1,6 +0,0 @@ ->Test__A1 -ACAACACTCCGA ->Test__B1 -ACACCGCACAAT ->Test__B3 -ACAGTGCGTCCT diff --git a/src/pipecraft-core/service_scripts/NextITS/tests/test_step2_vsearch.yml b/src/pipecraft-core/service_scripts/NextITS/tests/test_step2_vsearch.yml deleted file mode 100644 index 4420b5df..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/tests/test_step2_vsearch.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: step2 vsearch - command: > - nextflow run . -profile test2,docker - --outdir out/step2_vsearch - -work-dir out/step2_vsearch_wd - --preclustering none - --clustering vsearch - --otu_id 0.98 - files: - - path: "out/step2_vsearch/01.Dereplicated/Dereplicated.fa.gz" - - path: "out/step2_vsearch/01.Dereplicated/Dereplicated.uc.gz" - - path: "out/step2_vsearch/03.Clustered_VSEARCH/Clustered.fa.gz" - - path: "out/step2_vsearch/03.Clustered_VSEARCH/Clustered.uc.gz" - - path: "out/step2_vsearch/04.PooledResults/UC_Pooled.parquet" - - path: "out/step2_vsearch/04.PooledResults/OTU_table_wide.txt.gz" - - path: "out/step2_vsearch/04.PooledResults/OTU_table_long.txt.gz" - - path: "out/step2_vsearch/04.PooledResults/OTUs.fa.gz" diff --git a/src/pipecraft-core/service_scripts/NextITS/workflows/STEP1.nf b/src/pipecraft-core/service_scripts/NextITS/workflows/STEP1.nf deleted file mode 100644 index f17e2fa8..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/workflows/STEP1.nf +++ /dev/null @@ -1,2646 +0,0 @@ -/* -============================================================================ - NextITS: Pipeline to process eukaryotic ITS amplicons -============================================================================ - License: Apache-2.0 - Github : https://github.com/vmikk/NextITS - Website: https://Next-ITS.github.io/ ----------------------------------------------------------------------------- -*/ - -// ---- Step-1 workflow ---- - - -// Include functions -include { software_versions_to_yaml } from '../modules/version_parser.nf' -include { dumpParamsTsv } from '../modules/dump_parameters.nf' -include { CHIMERA_REMOVAL } from '../subworkflows/chimera_removal_subworkflow.nf' - -if ( params.seqplatform == "Illumina" ){ - include { qc_pe; demux_illumina_notmerged; trim_primers_pe; join_pe } from '../modules/Illumina_pe.nf' -} - - -// Define output paths for different steps -out_0_bam = params.outdir + "/00_BAM2FASTQ" -out_1_demux = params.outdir + "/01_Demux" -out_1_joinPE = params.outdir + "/01_JoinedPE" -out_2_primer = params.outdir + "/02_PrimerCheck" -out_3_itsx = params.outdir + "/03_ITSx" -out_3_itsxp = params.outdir + "/03_ITSx_PooledParts" -out_3_trim = params.outdir + "/03_PrimerTrim" -out_3_trimPE = params.outdir + "/03_PrimerTrim_NotMerged" -out_4_homop = params.outdir + "/04_Homopolymer" -// out_5_chim = params.outdir + "/05_Chimera" -out_6_tj = params.outdir + "/06_TagJumpFiltration" -out_7_seq = params.outdir + "/07_SeqTable" -out_8_smr = params.outdir + "/08_RunSummary" -out_9_db = params.outdir + "/09_DB" -out_tracedir = params.tracedir - -// Sub-workflow-specific outputs -out_3_quickstats = params.outdir + "/03_Stats" - - -// Convert BAM to FASTQ -process bam2fastq { - - label "main_container" - publishDir "${out_0_bam}", mode: "${params.storagemode}" - - // cpus 2 - - input: - path input - path bam_index - - output: - path "*.fastq.gz", emit: fastq, optional: false - tuple val("${task.process}"), val('bam2fastq'), eval('bam2fastq --version | head -n 1 | sed "s/bam2fastq //"'), topic: versions - - script: - """ - echo -e "Converting BAM to FASTQ\\n" - echo -e "Input file: " ${input} - echo -e "BAM index: " ${bam_index} - - bam2fastq \ - -c ${params.gzip_compression} \ - --num-threads ${task.cpus} \ - ${input} - - echo -e "\\nConvertion finished" - """ -} - - -// Quality filtering for single-end reads -process qc_se { - - label "main_container" - - // cpus 10 - - // Add file ID to the log file - tag "${input.getSimpleName()}" - - input: - path input - - output: - path "${input.getSimpleName()}.fq.gz", emit: filtered, optional: true - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - - script: - filter_maxee = params.qc_maxee ? "--fastq_maxee ${params.qc_maxee}" : "" - filter_maxeerate = params.qc_maxeerate ? "--fastq_maxee_rate ${params.qc_maxeerate}" : "" - """ - echo -e "QC\\n" - echo -e "Input file: " ${input} - - ## We do not need to change the file name (output name should be the same as input) - ## Therefore, temporary rename input - mv ${input} inp.fq.gz - - vsearch \ - --fastq_filter inp.fq.gz \ - --fastq_qmax 93 \ - ${filter_maxee} \ - ${filter_maxeerate} \ - --fastq_maxns ${params.qc_maxn} \ - --threads ${task.cpus} \ - --fastqout - \ - | seqkit grep \ - --by-seq --ignore-case --invert-match --only-positive-strand --use-regexp -w 0 \ - --pattern '"(A{${params.qc_maxhomopolymerlen},}|C{${params.qc_maxhomopolymerlen},}|T{${params.qc_maxhomopolymerlen},}|G{${params.qc_maxhomopolymerlen},})"' \ - | gzip -${params.gzip_compression} \ - > "${input.getSimpleName()}.fq.gz" - - ## qc_maxhomopolymerlen - # e.g. "(A{26,}|C{26,}|T{26,}|G{26,})" - - echo -e "\\nQC finished" - """ -} - - -// Validate tags for demultiplexing -process tag_validation { - - label "main_container" - // cpus 1 - - publishDir "${out_1_demux}", pattern: "tag_names_renamed.tsv", mode: "${params.storagemode}" - - input: - path barcodes - - output: - path "barcodes_validated.fasta", emit: fasta - path "biosamples_asym.csv", emit: biosamples_asym, optional: true - path "biosamples_sym.csv", emit: biosamples_sym, optional: true - path "file_renaming.tsv", emit: file_renaming, optional: true - path "unknown_combinations.tsv", emit: unknown_combinations, optional: true - path "tag_names_renamed.tsv", emit: tag_names_renamed, optional: true - - script: - """ - echo -e "Valdidating demultiplexing tags\\n" - echo -e "Input file: " ${barcodes} - - ## Convert Windows-style line endings (CRLF) to Unix-style (LF) - LC_ALL=C sed -i 's/\r\$//g' ${barcodes} - - ## Perform tag validation - validate_tags.R \ - --tags ${barcodes} \ - --output barcodes_validated.fasta - - echo -e "\\nTag validation finished" - """ -} - - - -// Demultiplexing with LIMA - for PacBio reads -process demux { - - label "main_container" - - publishDir "${out_1_demux}", mode: "${params.storagemode}" // , saveAs: { filename -> "foo_$filename" } - // cpus 10 - - input: - path input_fastq - path barcodes - path biosamples_sym // for dual or asymmetric barcodes - path biosamples_asym // for dual or asymmetric barcodes - path file_renaming // for dual or asymmetric barcodes - path unknown_combinations // for dual or asymmetric barcodes - - output: - path "LIMA/*.fq.gz", emit: samples_demux - path "LIMA/lima.lima.report.gz", emit: lima_report - path "LIMA/lima.lima.counts", emit: lima_counts - path "LIMA/lima.lima.summary", emit: lima_summary - tuple val("${task.process}"), val('lima'), eval('lima --version | head -n 1 | sed "s/lima //"'), topic: versions - tuple val("${task.process}"), val('brename'), eval('brename --help | head -n 4 | tail -1 | sed "s/Version: //"'), topic: versions - - script: - """ - echo -e "Input file: " ${input_fastq} - echo -e "Barcodes: " ${barcodes} - - ## Directory for the results - mkdir -p LIMA - - echo -e "Validating data\n" - - ## Check if symmetric barcodes were provided in the `...` format - ## (if `biosamples_sym` does not exists, it means that it is a dummy file) - ## (if exists, it means that tags were split into sym and asym at the tag validation step) - if [[ ${params.lima_barcodetype} = "dual_symmetric" ]] && [ -e ${biosamples_sym} ] ; then - echo -e "\\nERROR: Symmetric tags are provided in '...' format.\\n" - echo -e "In the FASTA file, please include only one tag per sample, since these tags are identical.\\n" - exit 1 - fi - - ## Count the number of samples in Biosample files - only for `dual` and `dual_asymmetric` barcodes - if [[ ${params.lima_barcodetype} == "dual_asymmetric" ]] || [[ ${params.lima_barcodetype} == "dual" ]]; then - - if [ ! -e ${biosamples_asym} ]; then - - echo -e "\\nERROR: Tags are specified in wrong format" - echo -e "Use the '...' format in FASTA file.\\n" - exit 1 - - else - line_count_sym=\$(wc -l < ${biosamples_sym}) - line_count_asym=\$(wc -l < ${biosamples_asym}) - - echo -e "..Number of lines in symmetric file: " \$line_count_sym - echo -e "..Number of lines in asymmetric file: " \$line_count_asym - - ## Check the presence of dual barcode combinations - ## If line count is less than 2, it means there are no samples specified - if [[ ${params.lima_barcodetype} == "dual_asymmetric" ]] && [[ \$line_count_asym -lt 2 ]]; then - echo -e "\\nERROR: No asymmetric barcodes detected for demultiplexing.\\n" - return 1 - fi - - if [[ ${params.lima_barcodetype} == "dual" ]] && [[ \$line_count_asym -lt 2 ]]; then - echo -e "\\nWARNING: No asymmetric barcodes detected, consider using '--lima_barcodetype dual_symmetric'.\\n" - fi - - fi # end of missing asym biosamples - - fi # end of dual/asym validation - - - - ## Combine shared arguments into a single variable - ## Note the array syntax - that's because of LIMA parser error messages - ## (note also that it works in bash, but not in zsh) - common_args=("--ccs \ - --window-size ${params.lima_windowsize} \ - --min-length ${params.lima_minlen} \ - --min-score ${params.lima_minscore} \ - --min-ref-span ${params.lima_minrefspan} \ - --split-named \ - --num-threads ${task.cpus} \ - --log-level INFO \ - ${input_fastq} \ - ${barcodes}") - - - ## Demultiplex, depending on the barcode type selected - case ${params.lima_barcodetype} in - - "single") - echo -e "\\nDemultiplexing with LIMA (single barcode)" - lima --same --single-side \ - --log-file LIMA/_log.txt \ - \$common_args \ - "LIMA/lima.fq.gz" - ;; - - "dual_symmetric") - echo -e "\\nDemultiplexing with LIMA (dual symmetric barcodes)" - lima --same \ - --min-end-score ${params.lima_minendscore} \ - --min-scoring-regions ${params.lima_minscoringregions} \ - --log-file LIMA/_log.txt \ - \$common_args \ - "LIMA/lima.fq.gz" - ;; - - "dual_asymmetric") - echo -e "\\nDemultiplexing with LIMA (dual asymmetric barcodes)" - lima --different \ - --min-end-score ${params.lima_minendscore} \ - --min-scoring-regions ${params.lima_minscoringregions} \ - --biosample-csv ${biosamples_asym} \ - --log-file LIMA/_log.txt \ - \$common_args \ - "LIMA/lima.fq.gz" - ;; - - "dual") - mkdir -p LIMAs LIMAd - - if [[ \$line_count_sym -ge 2 ]]; then - echo -e "\\nDemultiplexing with LIMA (dual symmetric barcodes)" - lima --same \ - --min-end-score ${params.lima_minendscore} \ - --min-scoring-regions ${params.lima_minscoringregions} \ - --biosample-csv ${biosamples_sym} \ - --log-file LIMAs/_log.txt \ - \$common_args \ - "LIMAs/lima.fq.gz" - fi - - if [[ \$line_count_asym -ge 2 ]]; then - echo -e "\\nDemultiplexing with LIMA (dual asymmetric barcodes)" - lima --different \ - --min-end-score ${params.lima_minendscore} \ - --min-scoring-regions ${params.lima_minscoringregions} \ - --biosample-csv ${biosamples_asym} \ - --log-file LIMAd/_log.txt \ - \$common_args \ - "LIMAd/lima.fq.gz" - fi - ;; - esac - - - ## Combining symmetric and asymmetric files - if [ ${params.lima_barcodetype} = "dual" ]; then - - echo -e "\\nPooling of symmetric and asymmetric barcodes" - cd LIMA - find ../LIMAd -name "*.fq.gz" | parallel -j1 "ln -s {} ." - find ../LIMAs -name "*.fq.gz" | parallel -j1 "ln -s {} ." - cd .. - - fi - - - ## Rename barcode combinations into sample names - ## Only user-provided combinations whould be kept (based on `lima --biosample-csv`) - if [[ ${params.lima_barcodetype} == "dual_asymmetric" ]] || [[ ${params.lima_barcodetype} == "dual" ]]; then - - echo -e "\\n..Renaming files from tag IDs to sample names" - brename -p "(.+)" -r "{kv}" -k ${file_renaming} LIMA/ - - echo -e "\\n..Checking for unknown tag combinations" - echo -e "\\n...Number of unknowns detected:" - find LIMA -name "lima.*.fq.gz" | wc -l - - if [[ ${params.lima_remove_unknown} == "false" ]]; then - - if [ -s ${unknown_combinations} ]; then - echo -e "\\n...Renaming unknown combinations" - brename -p "(.+)" -r "{kv}" -k ${unknown_combinations} LIMA/ - else - echo -e "\\n...No unknown combinations require renaming" - fi - - echo -e "\\n...Number of unknowns remained:" - find LIMA -name "lima.*.fq.gz" | wc -l - - fi - - echo -e "\\n...Removing unknowns:" - find LIMA -name "lima.*.fq.gz" | parallel -j1 "echo {} && rm {}" - - fi # end of dual/asym renaming - - if [[ ${params.lima_barcodetype} == "dual_symmetric" ]] || [[ ${params.lima_barcodetype} == "single" ]]; then - - echo -e "\\n..Renaming demultiplexed files" - rename --filename \ - 's/^lima.//g; s/--.*\$/.fq.gz/' \ - \$(find LIMA -name "*.fq.gz") - - fi - - - ## Combine summary stats for dual barcodes (two LIMA runs) - if [[ ${params.lima_barcodetype} == "dual" ]]; then - - echo -e "\\n..Combining dual-barcode log files" - - if [ -f "LIMAd/lima.lima.summary" ]; then - echo -e "Asymmetric barcodes summary\\n\\n" >> LIMA/lima.lima.summary - cat LIMAd/lima.lima.summary >> LIMA/lima.lima.summary - - echo -e "Asymmetric barcodes counts\\n\\n" >> LIMA/lima.lima.counts - cat LIMAd/lima.lima.counts >> LIMA/lima.lima.counts - - echo -e "Asymmetric barcodes report\\n\\n" >> LIMA/lima.lima.report - cat LIMAd/lima.lima.report >> LIMA/lima.lima.report - fi - - if [ -f "LIMAs/lima.lima.summary" ]; then - echo -e "\\n\\nSymmetric barcodes summary\\n\\n" >> LIMA/lima.lima.summary - cat LIMAs/lima.lima.summary >> LIMA/lima.lima.summary - - echo -e "\\n\\nSymmetric barcodes counts\\n\\n" >> LIMA/lima.lima.counts - cat LIMAs/lima.lima.counts >> LIMA/lima.lima.counts - - ## Reports should be identical for symmetric and asymmetric barcodes, so no need to combine them - # echo -e "\\n\\nSymmetric barcodes report\\n\\n" >> LIMA/lima.lima.report - # cat LIMAs/lima.lima.report >> LIMA/lima.lima.report - fi - - fi # end of dual logs pooling - - - ## Compress logs - echo -e "..Compressing log file" - gzip -${params.gzip_compression} LIMA/lima.lima.report - - - ## LIMA defaults: - # SYMMETRIC : --ccs --min-score 0 --min-end-score 80 --min-ref-span 0.75 --same --single-end - # ASYMMETRIC : --ccs --min-score 80 --min-end-score 50 --min-ref-span 0.75 --different --min-scoring-regions 2 - - echo -e "\\nDemultiplexing finished" - """ -} - - -// Merge Illumina PE reads -process merge_pe { - - label "main_container" - - // publishDir "${out_1_demux}", mode: "${params.storagemode}" - // cpus 10 - - input: - path input_R1 - path input_R2 - - output: - path "Merged.fq.gz", emit: r12 - tuple path("NotMerged_R1.fq.gz"), path("NotMerged_R2.fq.gz"), emit: nm, optional: true - - script: - """ - echo -e "Merging Illumina pair-end reads\\n" - - ## By default, fastp modifies sequences header - ## e.g., `merged_150_15` means that 150bp are from read1, and 15bp are from read2 - ## But we'll preserve only sequence ID - - fastp \ - --in1 ${input_R1} \ - --in2 ${input_R2} \ - --merge --correction \ - --overlap_len_require ${params.pe_minoverlap} \ - --overlap_diff_limit ${params.pe_difflimit} \ - --overlap_diff_percent_limit ${params.pe_diffperclimit} \ - --length_required ${params.pe_minlen} \ - --disable_quality_filtering \ - --disable_adapter_trimming \ - --dont_eval_duplication \ - --compression 6 \ - --thread ${task.cpus} \ - --out1 NotMerged_R1.fq.gz \ - --out2 NotMerged_R2.fq.gz \ - --json log.json \ - --html log.html \ - --stdout \ - | seqkit seq --only-id \ - | gzip -${params.gzip_compression} \ - > Merged.fq.gz - - ## --merged_out Merged.fq.gz \ - ## --n_base_limit ${params.pe_nlimit} \ - - # --overlap_len_require the minimum length to detect overlapped region of PE reads - # --overlap_diff_limit the maximum number of mismatched bases to detect overlapped region of PE reads - # --overlap_diff_percent_limit the maximum percentage of mismatched bases to detect overlapped region of PE reads - ## NB: reads should meet these three conditions simultaneously! - - echo -e "..done" - """ -} - - -// Modify barcodes for cutadapt (restrict the search window) -process prep_barcodes { - - label "main_container" - - // publishDir "${out_1_demux}", mode: "${params.storagemode}" - // cpus 1 - - input: - path barcodes - - output: - path "barcodes_modified.fa", emit: barcodesm - - script: - """ - echo -e "Restricting the search window for barcode lookup" - echo -e "Provided barcodes: " ${barcodes} - - ## Add `XN{30}` to the barcodes - - sed -e '/^>/! s/^/XN{${params.barcode_window}}/' \ - ${barcodes} \ - > barcodes_modified.fa - - echo -e "..Done" - """ -} - - -// Demultiplexing with cutadapt - for Illumina SE reads -process demux_illumina { - - label "main_container" - - publishDir "${out_1_demux}", mode: "${params.storagemode}" - // cpus 10 - - input: - path input_fastq - path barcodes - - output: - path "*.fq.gz", emit: samples_demux - - script: - """ - echo -e "Input file: " ${input_fastq} - echo -e "Barcodes: " ${barcodes} - - echo -e "\nDemultiplexing with cutadapt:" - - ## Demultiplex with cutadapt - cutadapt \ - -g file:${barcodes} \ - --revcomp --rename "{header}" \ - --errors ${params.barcode_errors} \ - --overlap ${params.barcode_overlap} \ - --no-indels \ - --cores ${task.cpus} \ - --discard-untrimmed \ - --action none \ - -o "{name}.fq.gz" \ - ${input_fastq} \ - > cutadapt.log - - echo -e "\\n..done" - - ## Remove empty files (no sequences) - echo -e "\\nRemoving empty files" - find . -type f -name "*.fq.gz" -size -29c -print -delete - echo -e "..Done" - - echo -e "\\nDemultiplexing finished" - """ -} - - - -// Primer disambiguation -process disambiguate { - - label "main_container" - - // publishDir "${out_2_primer}", mode: "${params.storagemode}" - // cpus 1 - - output: - path "primer_F.fasta", emit: F - path "primer_R.fasta", emit: R - path "primer_Fr.fasta", emit: Fr - path "primer_Rr.fasta", emit: Rr - - script: - - """ - - ## Disambiguate forward primer - echo -e "Disambiguating forward primer" - disambiguate_primers.R \ - ${params.primer_forward} \ - primer_F.fasta - - ## Disambiguate reverse primer - echo -e "\\nDisambiguating reverse primer" - disambiguate_primers.R \ - ${params.primer_reverse} \ - primer_R.fasta - - ## Reverse-complement primers - echo -e "\\nReverse-complementing primers" - seqkit seq -r -p --seq-type dna primer_F.fasta > primer_Fr.fasta - seqkit seq -r -p --seq-type dna primer_R.fasta > primer_Rr.fasta - - """ -} - - -// Check primers + QC + Reorient sequences -// Count number of primer occurrences withnin a read, -// discard reads with > 1 primer occurrence -// NB. read names should not contain spaces! (because of bedtools) -process primer_check { - - label "main_container" - - publishDir "${out_2_primer}", mode: "${params.storagemode}" - - // cpus 1 - - // Add sample ID to the log file - tag "${input.getSimpleName()}" - - input: - path input - path primer_F - path primer_R - path primer_Fr - path primer_Rr - - output: - path "${input.getSimpleName()}_PrimerChecked.fq.gz", emit: fq_primer_checked, optional: true - path "${input.getSimpleName()}_PrimerArtefacts.fq.gz", emit: primerartefacts, optional: true - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - tuple val("${task.process}"), val('runiq'), eval('runiq --version | sed "s/runiq //"'), topic: versions - tuple val("${task.process}"), val('mlr'), eval('mlr --version | sed "s/mlr //"'), topic: versions - tuple val("${task.process}"), val('bedtools'), eval('bedtools --version | sed "s/bedtools v//"'), topic: versions - tuple val("${task.process}"), val('csvtk'), eval('csvtk version | sed "s/csvtk v//"'), topic: versions - tuple val("${task.process}"), val('cutadapt'), eval('cutadapt --version'), topic: versions - - script: - """ - echo -e "Input file: " ${input} - echo -e "Forward primer: " ${params.primer_forward} - echo -e "Reverse primer: " ${params.primer_reverse} - - ### Count number of pattern occurrences for each sequence - count_primers (){ - # \$1 = file with primers - - seqkit replace -p "\\s.+" ${input} \ - | seqkit locate \ - --max-mismatch ${params.primer_mismatches} \ - --only-positive-strand \ - --pattern-file "\$1" \ - --threads ${task.cpus} \ - | awk -vOFS='\\t' 'NR > 1 { print \$1 , \$5 , \$6 }' \ - | runiq - \ - | mlr --tsv \ - --implicit-tsv-header \ - --headerless-tsv-output \ - sort -f 1 -n 2 \ - | bedtools merge -i stdin - } - - echo -e "\\nCounting primers" - echo -e "..forward primer" - count_primers ${primer_F} > PF.txt - - echo -e "..rc-forward primer" - count_primers ${primer_Fr} >> PF.txt - - echo -e "..reverse primer" - count_primers ${primer_R} > PR.txt - - echo -e "..rc-reverse primer" - count_primers ${primer_Rr} >> PR.txt - - ## Sort by seqID and start position, remove overlapping regions, - ## Find duplicated records - echo -e "\\nLooking for multiple primer occurrences" - - echo -e "..Processing forward primers" - if [ -s PF.txt ]; then - - csvtk sort \ - -t -T -H -k 1:N -k 2:n \ - --num-cpus ${task.cpus} \ - PF.txt \ - | bedtools merge -i stdin \ - | awk '{ print \$1 }' \ - | runiq -i - \ - > multiprimer.txt - - else - echo -e "...No forward primer matches found (in both orientations)" - fi - - echo -e "..Processing reverse primers" - if [ -s PR.txt ]; then - - csvtk sort \ - -t -T -H -k 1:N -k 2:n \ - --num-cpus ${task.cpus} \ - PR.txt \ - | bedtools merge -i stdin \ - | awk '{ print \$1 }' \ - | runiq -i - \ - >> multiprimer.txt - - else - echo -e "...No reverse primer matches found (in both orientations)" - fi - - - ## If some artefacts are found - if [ -s multiprimer.txt ]; then - - ## Keep only uinque seqIDs - runiq multiprimer.txt > multiprimers.txt - rm multiprimer.txt - - echo -e "\\nNumber of artefacts found: " \$(wc -l < multiprimers.txt) - - echo -e "..Removing artefacts" - ## Remove primer artefacts - seqkit grep --invert-match \ - --threads ${task.cpus} \ - --pattern-file multiprimers.txt \ - --out-file no_multiprimers.fq.gz \ - ${input} - - ## Extract primer artefacts - echo -e "..Extracting artefacts" - seqkit grep \ - --threads ${task.cpus} \ - --pattern-file multiprimers.txt \ - --out-file "${input.getSimpleName()}_PrimerArtefacts.fq.gz" \ - ${input} - - echo -e "..done" - - else - - echo -e "\\nNo primer artefacts found" - ln -s ${input} no_multiprimers.fq.gz - - fi - echo -e "..Done" - - echo -e "\\nReorienting sequences" - - ## Reverse-complement rev primer - RR=\$(rc.sh ${params.primer_reverse}) - - ## Reorient sequences, discard sequences without both primers - cutadapt \ - -a ${params.primer_forward}";required;min_overlap=${params.primer_foverlap}"..."\$RR"";required;min_overlap=${params.primer_roverlap}" \ - --errors ${params.primer_mismatches} \ - --revcomp --rename "{header}" \ - --discard-untrimmed \ - --cores ${task.cpus} \ - --action none \ - --output ${input.getSimpleName()}_PrimerChecked.fq.gz \ - no_multiprimers.fq.gz - - echo -e "\\nAll done" - - ## Clean up - if [ -f no_multiprimers.fq.gz ]; then rm no_multiprimers.fq.gz; fi - - ## Remove empty file (no valid sequences) - echo -e "\\nRemoving empty files" - find . -type f -name ${input.getSimpleName()}_PrimerChecked.fq.gz -size -29c -print -delete - echo -e "..Done" - - """ -} - - -// Extract ITS region with ITSx -// NB. sequence header should not contain spaces! -process itsx { - - label "main_container" - - publishDir "${out_3_itsx}", mode: "${params.storagemode}" - // cpus 2 - - // Add sample ID to the log file - tag "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}" - - input: - path input - - output: - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}_hash_table.txt.gz", emit: hashes, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}_uc.uc.gz", emit: uc, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.full.fasta.gz", emit: itsx_full, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.SSU.fasta.gz", emit: itsx_ssu, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.ITS1.fasta.gz", emit: itsx_its1, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.5_8S.fasta.gz", emit: itsx_58s, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.ITS2.fasta.gz", emit: itsx_its2, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.LSU.fasta.gz", emit: itsx_lsu, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.positions.txt", optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.problematic.txt", optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}_no_detections.fasta.gz", emit: itsx_nondetects, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.summary.txt", emit: itsx_summary, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.extraction.results.gz", emit: itsx_details, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.SSU.full_and_partial.fasta.gz", emit: itsx_ssu_part, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.ITS1.full_and_partial.fasta.gz", emit: itsx_its1_part, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.5_8S.full_and_partial.fasta.gz", emit: itsx_58s_part, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.ITS2.full_and_partial.fasta.gz", emit: itsx_its2_part, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.LSU.full_and_partial.fasta.gz", emit: itsx_lsu_part, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}_primertrimmed_sorted.fq.gz", emit: trimmed_seqs, optional: true - path "parquet/*.parquet", emit: parquet, optional: true - tuple val("${task.process}"), val('ITSx'), eval('ITSx --help 2>&1 | head -n 3 | tail -n 1 | sed "s/Version: //"'), topic: versions - tuple val("${task.process}"), val('cutadapt'), eval('cutadapt --version'), topic: versions - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - tuple val("${task.process}"), val('phredsort'), eval('phredsort -v | sed "s/phredsort //"'), topic: versions - tuple val("${task.process}"), val('seqhasher'), eval('seqhasher -v | sed "s/SeqHasher //"'), topic: versions - tuple val("${task.process}"), val('parallel'), eval('parallel --version | head -n 1 | sed "s/GNU parallel //"'), topic: versions - tuple val("${task.process}"), val('brename'), eval('brename --help | head -n 4 | tail -1 | sed "s/Version: //"'), topic: versions - tuple val("${task.process}"), val('duckdb'), eval('duckdb --version | cut -d" " -f1 | sed "s/^v//"'), topic: versions - - script: - - sampID="${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}" - - // Allow inclusion of sequences that only find a single domain, given that they meet the given E-value and score thresholds, on with parameters 1e-9,0 by default - // singledomain = params.ITSx_singledomain ? "--allow_single_domain 1e-9,0" : "" - - """ - echo -e "Extraction of rRNA regions using ITSx\\n" - - ## Trim primers - echo -e "Trimming primers\\n" - - ## Reverse-complement rev primer - RR=\$(rc.sh ${params.primer_reverse}) - - cutadapt \ - -a ${params.primer_forward}";required;min_overlap=${params.primer_foverlap}"..."\$RR"";required;min_overlap=${params.primer_roverlap}" \ - --errors ${params.primer_mismatches} \ - --revcomp --rename "{id}" \ - --discard-untrimmed \ - --minimum-length ${params.trim_minlen} \ - --cores ${task.cpus} \ - --action trim \ - --output ${sampID}_primertrimmed.fq.gz \ - ${input} - - echo -e "..Done\\n" - - ## Check if there are sequences in the output - NUMSEQS=\$( seqkit stat --tabular --quiet ${sampID}_primertrimmed.fq.gz | awk -F'\t' 'NR==2 {print \$4}' ) - echo -e "Number of sequences after primer trimming: " \$NUMSEQS - if [ \$NUMSEQS -lt 1 ]; then - echo -e "\\nIt looks like no reads remained after trimming the primers\\n" - exit 0 - fi - - ## Estimate sequence quality and sort sequences by quality - echo -e "\\nSorting by sequence quality" - seqkit replace -p "\\s.+" ${sampID}_primertrimmed.fq.gz \ - | phredsort -i - -o - --metric meep --header avgphred,maxee,meep \ - | gzip -1 > ${sampID}_primertrimmed_sorted.fq.gz - echo -e "..Done" - - ## Hash sequences, add sample ID to the header - ## columns: Sample ID - Hash - PacBioID - AvgPhredScore - MaxEE - MEEP - Sequence - Quality - Length - ## Convert to Parquet format - echo -e "\\nCreating hash table" - seqhasher --hash sha1 --name ${sampID} ${sampID}_primertrimmed_sorted.fq.gz - \ - | seqkit fx2tab --length \ - | sed 's/;/\t/ ; s/;/\t/ ; s/ avgphred=/\t/ ; s/ maxee=/\t/ ; s/ meep=/\t/' \ - > ${sampID}_hash_table.txt - echo -e "..Done" - - ## Check the number of fields per record (should be 9!) - # awk '{print NF}' ${sampID}_hash_table.txt | sort | uniq -c - # awk 'NF > 9 {print \$0 }' ${sampID}_hash_table.txt - - ## Dereplicate at sample level (use quality-sorted sequences to make sure that the representative sequence is with the highest quality) - echo -e "\\nDereplicating at sample level" - seqkit fq2fa -w 0 ${sampID}_primertrimmed_sorted.fq.gz \ - | vsearch \ - --derep_fulllength - \ - --output - \ - --strand both \ - --fasta_width 0 \ - --threads 1 \ - --relabel_sha1 \ - --sizein --sizeout \ - --minseqlength ${params.trim_minlen} \ - --uc ${sampID}_uc.uc \ - --quiet \ - > derep.fasta - echo -e "..Done" - - ## ITSx extraction - echo -e "\\nITSx extraction" - ITSx \ - -i derep.fasta \ - --complement ${params.ITSx_complement} \ - --save_regions all \ - --graphical F \ - --detailed_results T \ - --positions T \ - --not_found T \ - -E ${params.ITSx_evalue} \ - -t ${params.ITSx_tax} \ - --partial ${params.ITSx_partial} \ - --cpu ${task.cpus} \ - --preserve T \ - -o ${sampID} - - echo -e "..Done" - - # ITSx.full.fasta - # ITSx.SSU.fasta - # ITSx.ITS1.fasta - # ITSx.5_8S.fasta - # ITSx.ITS2.fasta - # ITSx.LSU.fasta - # ITSx.positions.txt - # ITSx.problematic.txt - # ITSx_no_detections.fasta - # ITSx_no_detections.txt - # ITSx.summary.txt - # ITSx.extraction.results - # ITSx.SSU.full_and_partial.fasta - # ITSx.ITS1.full_and_partial.fasta - # ITSx.5_8S.full_and_partial.fasta - # ITSx.ITS2.full_and_partial.fasta - # ITSx.LSU.full_and_partial.fasta - - - ## If partial sequences were required, remove empty sequences - if [ \$(find . -type f -name "*.full_and_partial.fasta" | wc -l) -gt 0 ]; then - echo -e "Partial files found, removing empty sequences\\n." - - find . -name "*.full_and_partial.fasta" \ - | parallel -j${task.cpus} "seqkit seq -m 1 -w 0 {} > {.}_tmp.fasta" - - rm *.full_and_partial.fasta - brename -p "_tmp" -r "" -f "_tmp.fasta\$" - - fi - - - ## Remove empty files (no sequences) - echo -e "\\nRemoving empty files" - find . -type f -name "*.fasta" -empty -print -delete - echo -e "..Done" - - ## Remove temporary file - rm derep.fasta - rm ${sampID}_primertrimmed.fq.gz - - ## Compress results - echo -e "\\nCompressing files" - - parallel -j${task.cpus} "gzip -${params.gzip_compression} {}" ::: \ - ${sampID}_hash_table.txt \ - ${sampID}_uc.uc \ - *.fasta \ - ${sampID}.extraction.results - - ## Convert ITSx output to Parquet - if [ ${params.ITSx_to_parquet} == true ]; then - - echo -e "\\nConverting ITSx output to Parquet" - mkdir -p parquet - - if [ -f ${sampID}.full.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.full.fasta.gz -o parquet/${sampID}.full.parquet - fi - - if [ -f ${sampID}.SSU.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.SSU.fasta.gz -o parquet/${sampID}.SSU.parquet - fi - - if [ -f ${sampID}.ITS1.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.ITS1.fasta.gz -o parquet/${sampID}.ITS1.parquet - fi - - if [ -f ${sampID}.5_8S.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.5_8S.fasta.gz -o parquet/${sampID}.5_8S.parquet - fi - - if [ -f ${sampID}.ITS2.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.ITS2.fasta.gz -o parquet/${sampID}.ITS2.parquet - fi - - if [ -f ${sampID}.LSU.fasta.gz ]; then - ITSx_to_DuckDB.sh -i ${sampID}.LSU.fasta.gz -o parquet/${sampID}.LSU.parquet - fi - - echo -e "Parquet files created\\n" - - fi - - echo -e "..Done" - """ -} - -// Collect all ITS parts extracted by ITSx -process itsx_collect { - - label "main_container" - - publishDir "${out_3_itsxp}", mode: "${params.storagemode}" - // cpus 1 - - input: - path(itsx_full, stageAs: "full/*") - path(itsx_ssu, stageAs: "ssu/*") - path(itsx_its1, stageAs: "its1/*") - path(itsx_58s, stageAs: "58s/*") - path(itsx_its2, stageAs: "its2/*") - path(itsx_lsu, stageAs: "lsu/*") - path(itsx_ssu_part, stageAs: "ssu_partial/*") - path(itsx_its1_part, stageAs: "its1_partial/*") - path(itsx_58s_part, stageAs: "58s_partial/*") - path(itsx_its2_part, stageAs: "its2_partial/*") - path(itsx_lsu_part, stageAs: "lsu_partial/*") - - output: - path "ITS_Full.fasta.gz", emit: full, optional: true - path "SSU.fasta.gz", emit: ssu, optional: true - path "ITS1.fasta.gz", emit: its1, optional: true - path "5_8S.fasta.gz", emit: s58, optional: true - path "ITS2.fasta.gz", emit: its2, optional: true - path "LSU.fasta.gz", emit: lsu, optional: true - path "SSU_full_and_partial.fasta.gz", emit: ssu_part, optional: true - path "ITS1_full_and_partial.fasta.gz", emit: its1_part, optional: true - path "5_8S_full_and_partial.fasta.gz", emit: s58_part, optional: true - path "ITS2_full_and_partial.fasta.gz", emit: its2_part, optional: true - path "LSU_full_and_partial.fasta.gz", emit: lsu_part, optional: true - - script: - """ - # Check if each sub-dir has files with rRNA regions, then concatenate - - if [[ ! -n \$(find ./full -name NOFULL) ]]; then - echo -e "Pooling full ITS" - find full -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> ITS_Full.fasta.gz - fi - - if [[ ! -n \$(find ./ssu -name NOSSU) ]]; then - echo -e "Pooling SSU" - find ssu -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> SSU.fasta.gz - fi - - if [[ ! -n \$(find ./its1 -name NOITS1) ]]; then - echo -e "Pooling ITS1" - find its1 -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> ITS1.fasta.gz - fi - - if [[ ! -n \$(find ./58s -name NO58S) ]]; then - echo -e "Pooling 5.8S" - find 58s -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> 5_8S.fasta.gz - fi - - if [[ ! -n \$(find ./its2 -name NOITS2) ]]; then - echo -e "Pooling ITS2" - find its2 -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> ITS2.fasta.gz - fi - - if [[ ! -n \$(find ./lsu -name NOLSU) ]]; then - echo -e "Pooling LSU" - find lsu -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> LSU.fasta.gz - fi - - ##### Full and partial sequences ##### - - if [[ ! -n \$(find ./ssu_partial -name NOSSUPART) ]]; then - echo -e "Pooling SSU partial sequences" - find ssu_partial -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> SSU_full_and_partial.fasta.gz - fi - - if [[ ! -n \$(find ./its1_partial -name NOITS1PART) ]]; then - echo -e "Pooling ITS1 partial sequences" - find its1_partial -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> ITS1_full_and_partial.fasta.gz - fi - - if [[ ! -n \$(find ./58s_partial -name NO58SPART) ]]; then - echo -e "Pooling 5.8S partial sequences" - find 58s_partial -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> 5_8S_full_and_partial.fasta.gz - fi - - if [[ ! -n \$(find ./its2_partial -name NOITS2PART) ]]; then - echo -e "Pooling ITS2 partial sequences" - find its2_partial -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> ITS2_full_and_partial.fasta.gz - fi - - if [[ ! -n \$(find ./lsu_partial -name NOLSUPART) ]]; then - echo -e "Pooling LSU partial sequences" - find lsu_partial -name "*.fasta.gz" \ - | parallel -j1 "cat {}" >> LSU_full_and_partial.fasta.gz - fi - - echo -e "\\n..Done" - """ -} - - -// Trim primers (do not extract ITS) -// + Estimate sequence qualities -process trim_primers { - - label "main_container" - - publishDir "${out_3_trim}", mode: "${params.storagemode}" - // cpus 2 - - // Add sample ID to the log file - tag "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}" - - input: - path input - - output: - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}_hash_table.txt.gz", emit: hashes, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}_primertrimmed_sorted.fq.gz", emit: primertrimmed_fq, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}.fa.gz", emit: primertrimmed_fa, optional: true - path "${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}_uc.uc.gz", emit: uc, optional: true - tuple val("${task.process}"), val('cutadapt'), eval('cutadapt --version'), topic: versions - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - tuple val("${task.process}"), val('phredsort'), eval('phredsort -v | sed "s/phredsort //"'), topic: versions - tuple val("${task.process}"), val('seqhasher'), eval('seqhasher -v | sed "s/SeqHasher //"'), topic: versions - - script: - sampID="${input.getSimpleName().replaceAll(/_PrimerChecked/, '')}" - - """ - echo -e "Trimming primers\\n" - echo -e "Input sample: " ${sampID} - echo -e "Forward primer: " ${params.primer_forward} - echo -e "Reverse primer: " ${params.primer_reverse} - - ## Reverse-complement rev priver - RR=\$(rc.sh ${params.primer_reverse}) - echo -e "Reverse primer RC: " "\$RR" - - echo -e "\\nTrimming primers" - cutadapt \ - -a ${params.primer_forward}";required;min_overlap=${params.primer_foverlap}"..."\$RR"";required;min_overlap=${params.primer_roverlap}" \ - --errors ${params.primer_mismatches} \ - --revcomp --rename "{header}" \ - --cores ${task.cpus} \ - --action=trim \ - --discard-untrimmed \ - --minimum-length ${params.trim_minlen} \ - --output ${sampID}_primertrimmed.fq.gz \ - ${input} - - - if [ -n "\$(find . -name ${sampID}_primertrimmed.fq.gz -prune -size +29c)" ]; then - - ## Estimate sequence quality and sort sequences by quality - echo -e "\\nSorting by sequence quality" - seqkit replace -p "\\s.+" ${sampID}_primertrimmed.fq.gz \ - | phredsort -i - -o - --metric meep --header avgphred,maxee,meep \ - | gzip -${params.gzip_compression} \ - > ${sampID}_primertrimmed_sorted.fq.gz - echo -e "..Done" - - rm ${sampID}_primertrimmed.fq.gz - - ## Hash sequences, add sample ID to the header - ## columns: Sample ID - Hash - PacBioID - AvgPhredScore - MaxEE - MEEP - Sequence - Quality - Length - ## Convert to Parquet format - echo -e "\\nCreating hash table" - seqhasher --hash sha1 --name ${sampID} ${sampID}_primertrimmed_sorted.fq.gz - \ - | seqkit fx2tab --length \ - | sed 's/;/\t/ ; s/;/\t/ ; s/ avgphred=/\t/ ; s/ maxee=/\t/ ; s/ meep=/\t/' \ - > ${sampID}_hash_table.txt - echo -e "..Done" - - ## Compress results - echo -e "Compressing result" - gzip -${params.gzip_compression} ${sampID}_hash_table.txt - - ## Dereplicate at sample level - echo -e "\\nDereplicating at sample level" - seqkit fq2fa -w 0 ${sampID}_primertrimmed_sorted.fq.gz \ - | vsearch \ - --derep_fulllength - \ - --output - \ - --strand both \ - --fasta_width 0 \ - --threads 1 \ - --relabel_sha1 \ - --sizein --sizeout \ - --uc ${sampID}_uc.uc \ - --quiet \ - | gzip -${params.gzip_compression} \ - > ${sampID}.fa.gz - - echo -e "..Done" - - ## Compress UC file - gzip -${params.gzip_compression} ${sampID}_uc.uc - - else - - echo -e "\\nNo sequences found after primer removal" - if [ -f ${sampID}_primertrimmed.fq.gz ]; then rm ${sampID}_primertrimmed.fq.gz; fi - - fi - - echo -e "..Done" - - """ -} - - - -// Assemble near-full-length ITS from ITSx output -process assemble_its { - - label "main_container" - - publishDir "${out_3_itsx}", mode: "${params.storagemode}" - // cpus 1 - - // Add sample ID to the log file - tag "${ITS1.getSimpleName()}" - - input: - path ITS1 - path S58 - path ITS2 - - output: - path "${ITS1.getSimpleName()}_ITS1_58S_ITS2.fasta.gz", emit: itsnf, optional: true - // path "ITS1_58S.fasta.gz", emit: its1p, optional: true - // path "58S_ITS2.fasta.gz", emit: its2p, optional: true - - script: - sampID="${ITS1.getSimpleName()}" - - """ - echo -e "Checking if ITS1, 5.8S, and ITS2 parts are available" - - if [[ -f ${ITS1} && ${S58} && ${ITS2} ]]; then - - echo -e "\\n..All parts found" - - ## Prepare tables for ID matching - echo -e "\\n..Converting data to tabular format" - seqkit fx2tab ${ITS1} | sed 's/\t\$//g' | csvtk add-header -t -n id,ITS1 > tmp_1_ITS1.txt - seqkit fx2tab ${S58} | sed 's/\t\$//g' | csvtk add-header -t -n id,58S > tmp_1_s58.txt - seqkit fx2tab ${ITS2} | sed 's/\t\$//g' | csvtk add-header -t -n id,ITS2 > tmp_1_ITS2.txt - - ## Join ITS fragments - echo -e "\\n..Joining ITS fragments" - csvtk join -t -f "id" tmp_1_ITS1.txt tmp_1_s58.txt tmp_1_ITS2.txt > tmp_2_ITS1_58S_ITS2.txt - - ## Check joining results - NUMSEQS=\$(wc -l < tmp_2_ITS1_58S_ITS2.txt) - echo "...Number of joined sequences: " \$((NUMSEQS - 1)) - - if [ "\$NUMSEQS" -gt 1 ]; then - - ## Convert table back to fasta - ## Remove leading and trailing Ns - echo -e "\\n..Preparing fasta" - awk 'NR>1 { print \$1 "\t" \$2\$3\$4 }' tmp_2_ITS1_58S_ITS2.txt \ - | seqkit tab2fx -w 0 \ - | seqkit replace -p "^n+|n+\$" -r "" -is -w 0 \ - | gzip -${params.gzip_compression} > ${sampID}_ITS1_58S_ITS2.fasta.gz - - else - echo "...There are no sequences with all ITS parts present\\n" - echo -e "\\n..Skipping ITS assembly for this sample" - fi - - else - echo -e "\\n..Some or all parts are missing" - echo -e "\\n..Skipping ITS assembly for this sample" - fi - - """ -} - - - -// Merge tables with sequence qualities -process seq_qual { - - label "main_container" - - publishDir "${out_9_db}", mode: "${params.storagemode}" - // cpus 4 - - input: - path(input, stageAs: "hash_tables/*") - - output: - path "SeqQualities.parquet", emit: quals - tuple val("${task.process}"), val('duckdb'), eval('duckdb --version | cut -d" " -f1 | sed "s/^v//"'), topic: versions - - script: - def memoryArg = task.memory ? "-m ${task.memory.toMega()}.MB" : "" - """ - echo -e "Aggregating sequence qualities" - - merge_hash_tables.sh \ - -i ./hash_tables \ - -o SeqQualities.parquet \ - -t ${task.cpus} \ - ${memoryArg} - - echo -e "..Done" - """ -} - - -// Homopolymer compression -process homopolymer { - - label "main_container" - - publishDir "${out_4_homop}", mode: "${params.storagemode}" - // cpus 1 - - // Add sample ID to the log file - tag "${input.getSimpleName().replaceAll(/_ITS1_58S_ITS2/, '')}" - - input: - path input - - output: - path "${input.getSimpleName().replaceAll(/_ITS1_58S_ITS2/, '')}_Homopolymer_compressed.fa.gz", emit: hc, optional: true - path "${input.getSimpleName().replaceAll(/_ITS1_58S_ITS2/, '')}_uch.uc.gz", emit: uch, optional: true - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - - script: - sampID="${input.getSimpleName().replaceAll(/_ITS1_58S_ITS2/, '')}" - - """ - - ## Homopolymer compression - echo -e "Homopolymer compression" - - zcat ${input} \ - | homopolymer_compression.sh - \ - > homo_compressed.fa - - echo -e "..Done" - - ## Re-cluster homopolymer-compressed data - echo -e "\\nRe-clustering homopolymer-compressed data" - vsearch \ - --cluster_size homo_compressed.fa \ - --id ${params.hp_similarity} \ - --iddef ${params.hp_iddef} \ - --qmask "dust" \ - --strand "both" \ - --fasta_width 0 \ - --threads ${task.cpus} \ - --sizein --sizeout \ - --minseqlength 20 \ - --centroids homo_clustered.fa \ - --uc ${sampID}_uch.uc - echo -e "..Done" - - ## Check if clustering was succeful - ## (e.g., if all compressed sequences were too short, the file with be empty) - if [ -s homo_clustered.fa ]; then - - ## Compress UC file - gzip -${params.gzip_compression} ${sampID}_uch.uc - - ## Substitute homopolymer-comressed sequences with uncompressed ones - ## (update size annotaions) - echo -e "\\nExtracting sequences" - - seqkit fx2tab ${input} > inp_tab.txt - seqkit fx2tab homo_clustered.fa > clust_tab.txt - - if [ -s inp_tab.txt ]; then - substitute_compressed_seqs.R \ - inp_tab.txt clust_tab.txt res.fa - - echo -e "..Done" - else - echo -e "..Input data looks empty, nothing to proceed with" - fi - - if [ -s res.fa ]; then - gzip -c res.fa > ${sampID}_Homopolymer_compressed.fa.gz - fi - - ## Remove temporary files - rm homo_compressed.fa - rm homo_clustered.fa - rm inp_tab.txt - rm clust_tab.txt - rm res.fa - - else - echo -e "Clustering homopolymer-compressed sequences returned to results" - echo -e "(most likely, sequences were too short)\\n" - fi - - """ -} - - -// If no homopolymer compression is required, just dereplicate the samples -process just_derep { - - label "main_container" - - // publishDir "${out_4_homop}", mode: "${params.storagemode}" - // cpus 1 - - // Add sample ID to the log file - tag "${input.getSimpleName()}" - - input: - path input - - output: - path "${input.getSimpleName()}.fa.gz", emit: nhc, optional: true - path "${input.getSimpleName()}_uch.uc.gz", emit: ucnh, optional: true - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - - script: - sampID="${input.getSimpleName()}" - - """ - echo -e "Dereplicating sequences\\n" - - vsearch \ - --derep_fulllength ${input} \ - --output - \ - --strand both \ - --fasta_width 0 \ - --threads 1 \ - --sizein --sizeout \ - --uc ${sampID}_uc.uc \ - | gzip -${params.gzip_compression} \ - > ${sampID}.fa.gz - - """ -} - - -// Pool sequences from all samples and add sample ID into header (for OTU and "ASV" table creation) -process pool_seqs { - - label "main_container" - - // publishDir "${out_6_tj}", mode: "${params.storagemode}" - // cpus 2 - - input: - path(input, stageAs: 'sequences/*') - - output: - path "Seq_tab_not_filtered.txt.gz", emit: seqtabnf - path "Seq_not_filtered.fa.gz", emit: seqsnf - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - tuple val("${task.process}"), val('parallel'), eval('parallel --version | head -n 1 | sed "s/GNU parallel //"'), topic: versions - - script: - """ - - echo -e "\\nPooling and renaming sequences" - - ## If there is a sample ID in the header already, remove it - parallel -j 1 --group \ - --rpl '{/:} s:(.*/)?([^/.]+)(\\.[^/]+)*\$:\$2:' \ - "zcat {} \ - | sed -r '/^>/ s/;sample=[^;]*/;/g ; s/;;/;/g' \ - | sed 's/>.*/&;sample='{/:}';/ ; s/_NoChimera//g ; s/_RescuedChimera//g ; s/_JoinedPE//g ; s/_Homopolymer_compressed//g' \ - | sed 's/Rescued_Chimeric_sequences.part_//g' \ - | sed -r '/^>/ s/;;/;/g'" \ - ::: sequences/*.fa.gz \ - | vsearch --sortbysize - --sizein --sizeout --fasta_width 0 --output - \ - | sed -r '/^>/ s/;;/;/g' \ - | gzip -${params.gzip_compression} \ - > Seq_not_filtered.fa.gz - - echo "..Done" - - echo -e "\\nExtracting sequence count table" - seqkit seq --name Seq_not_filtered.fa.gz \ - | sed 's/;/\t/g; s/size=//; s/sample=// ; s/\t*\$//' \ - | csvtk -t cut -f 2,1,3 \ - | csvtk -t add-header -n "SampleID,SeqID,Abundance" \ - | gzip -${params.gzip_compression} \ - > Seq_tab_not_filtered.txt.gz - - echo "..Done" - - """ -} - - -// De-novo clustering of sequences for tag-jump removal -process tj_preclust { - - label "main_container" - - // publishDir "${out_6_tj}", mode: "${params.storagemode}" - // cpus 10 - - input: - path input - - output: - path "TJPreclust.uc.parquet", emit: preclust_uc_parquet - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - - script: - def derep = (params.tj_id as BigDecimal).compareTo(1G) == 0 // to handle floating point comparisons too - """ - echo -e "Pre-clustering sequences prior to tag-jump removal\\n" - - echo -e "Running dereplication\\n" - - vsearch \ - --derep_fulllength ${input} \ - --sizein --sizeout \ - --strand both \ - --fasta_width 0 \ - --threads 1 \ - --uc Dereplicated.uc \ - --output Dereplicated.fa - - echo -e "\\nCompressing files" - pigz -p ${task.cpus} -${params.gzip_compression} Dereplicated.uc - pigz -p ${task.cpus} -${params.gzip_compression} Dereplicated.fa - - ## Additional clustering (e.g., at 99% similarity) - if [[ ${derep} == false ]]; then - - echo -e "\\nAdditional clustering at ${params.tj_id} similarity threshold\\n" - - vsearch \ - --cluster_size Dereplicated.fa.gz \ - --id ${params.tj_id} \ - --iddef ${params.tj_iddef} \ - --sizein --sizeout \ - --qmask dust --strand plus \ - --maxrejects 128 --maxaccepts 1 \ - --fasta_width 0 \ - --threads ${task.cpus} \ - --uc Clustered.uc \ - --centroids Clustered.fa - - echo -e "\\nCompressing files" - pigz -p ${task.cpus} -${params.gzip_compression} Clustered.uc - pigz -p ${task.cpus} -${params.gzip_compression} Clustered.fa - - fi - - - ## Parse UC file - if [[ ${derep} == true ]]; then - - echo -e "\\nParsing UC file" - ucs --map-only --split-id --rm-dups \ - -i Dereplicated.uc.gz \ - -o TJPreclust.uc.parquet - - else - - echo -e "\\nParsing dereplicated UC file" - ucs --map-only --split-id --rm-dups \ - -i Dereplicated.uc.gz \ - -o Dereplicated.parquet - - echo -e "\\nParsing clustered UC file" - ucs --map-only --split-id --rm-dups \ - -i Clustered.uc.gz \ - -o Clustered.parquet - - echo -e "\\nCombining dereplication and clustering UC files" - merge_tj_memberships.sh \ - -d Dereplicated.parquet \ - -c Clustered.parquet \ - -o TJPreclust.uc.parquet \ - -t ${task.cpus} - - fi - - echo -e "\\n..Done" - """ -} - - - -// Tag-jump removal -process tj { - - label "main_container" - - publishDir "${out_6_tj}", mode: "${params.storagemode}" - // cpus 1 - - input: - path seqtab // seq table in long format - path precls // pre-clustered membership - - output: - path "Seq_tab_TagJumpFiltered.txt.gz", emit: seqtabtj - path "TagJump_scores.qs", emit: tjs - path "TagJump_plot.pdf" - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - tuple val("${task.process}"), val('ggplot2'), eval('Rscript -e "cat(as.character(packageVersion(\'ggplot2\')))"'), topic: versions - - script: - """ - - echo -e "Tag-jump removal" - - tag_jump_removal_longtab.R \ - --seqtab ${seqtab} \ - --precls ${precls} \ - -f ${params.tj_f} \ - -p ${params.tj_p} - - echo "..Done" - - """ -} - - - - -// Prepare a table with non-tag-jumped sequences -// Add quality estimate to singletons -// Add chimera-scores for putative de novo chimeras -process prep_seqtab { - - label "main_container" - - publishDir "${out_7_seq}", mode: "${params.storagemode}" - // cpus 4 - - input: - path seqtab // tag-jump filtered sequence table (long format) - path seqsnf // sequences in FASTA - path denovos // de novo chimera scores - path quals // quality scores - - output: - path "Seqs.parquet", emit: seq_pq - path "Seqs.txt.gz", emit: seq_tl // long table - path "Seqs.fa.gz", emit: seq_fa - // path "Seqs.RData", emit: seq_rd // deprecated - // path "Seq_tab.txt.gz", emit: seq_tw // wide table - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - tuple val("${task.process}"), val('arrow'), eval('Rscript -e "cat(as.character(packageVersion(\'arrow\')))"'), topic: versions - tuple val("${task.process}"), val('Biostrings'), eval('Rscript -e "cat(as.character(packageVersion(\'Biostrings\')))"'), topic: versions - - script: - """ - - echo -e "Sequence table creation" - - seq_table_assembly.R \ - --seqtab ${seqtab} \ - --fasta ${seqsnf} \ - --chimera ${denovos} \ - --quality ${quals} \ - --threads ${task.cpus} - - echo "..Done" - - """ -} - - - - - - - -// Run summary - count number of reads in the output of different processes -process read_counts { - - label "main_container" - - publishDir "${out_8_smr}", mode: "${params.storagemode}", pattern: "*.xlsx" - publishDir "${out_8_smr}/PerProcessStats", mode: "${params.storagemode}", pattern: "*.txt" - // cpus 4 - - input: - path(input_fastq, stageAs: "1_input/*") - path(qc, stageAs: "2_qc/*") - path(samples_demux, stageAs: "3_demux/*") - path(samples_primerch, stageAs: "4_primerch/*") - path(samples_primermult, stageAs: "4_primerartefacts/*") - path(samples_itsx_or_primertrim, stageAs: "5_itsxtrim/*") - path(homopolymers, stageAs: "5_homopolymers/*") - path(samples_chimref, stageAs: "6_chimref/*") - path(samples_chimdenovo, stageAs: "7_chimdenov/*") - path(chimera_recovered, stageAs: "8_chimrecov/*") - path(samples_tj) - path(seqtab) - - output: - path "Run_summary.xlsx", emit: xlsx - path "Counts_1.RawData.txt", emit: counts_1_raw - path "Counts_2.QC.txt", emit: counts_2_qc - path "Counts_3.Demux.txt", emit: counts_3_demux, optional: true - path "Counts_4.PrimerCheck.txt", emit: counts_4_primer, optional: true - path "Counts_4.PrimerArtefacts.txt", emit: counts_4_primerartef, optional: true - path "Counts_5.ITSx_or_PrimTrim.txt", emit: counts_5_itsx_ptrim, optional: true - path "Counts_5.Homopolymers.txt", emit: counts_5_homopolymers, optional: true - path "Counts_6.ChimRef_reads.txt", emit: counts_6_chimref_r, optional: true - path "Counts_6.ChimRef_uniqs.txt", emit: counts_6_chimref_u, optional: true - path "Counts_7.ChimDenov.txt", emit: counts_7_chimdenov, optional: true - path "Counts_8.ChimRecov_reads.txt", emit: counts_8_chimrecov_r, optional: true - path "Counts_8.ChimRecov_uniqs.txt", emit: counts_8_chimrecov_u, optional: true - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - tuple val("${task.process}"), val('parallel'), eval('parallel --version | head -n 1 | sed "s/GNU parallel //"'), topic: versions - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - - script: - - """ - echo -e "Summarizing run statistics\\n" - echo -e "Counting the number of reads in:\\n" - - - ## Count raw reads - echo -e "\\n..Raw data" - seqkit stat --basename --tabular --threads ${task.cpus} --quiet \ - 1_input/* > Counts_1.RawData.txt - - ## Count number of reads passed QC - echo -e "\\n..Sequenced passed QC" - seqkit stat --basename --tabular --threads ${task.cpus} --quiet \ - 2_qc/* > Counts_2.QC.txt - - ## Count demultiplexed reads - echo -e "\\n..Demultiplexed data" - seqkit stat --basename --tabular --threads ${task.cpus} --quiet \ - 3_demux/* > Counts_3.Demux.txt - - - ## Count primer-checked reads - echo -e "\\n..Primer-checked data" - if [ `find 4_primerch -name no_primerchecked 2>/dev/null` ] - then - echo -e "... No files found" - touch Counts_4.PrimerCheck.txt - else - seqkit stat --basename --tabular --threads ${task.cpus} --quiet \ - 4_primerch/* > Counts_4.PrimerCheck.txt - fi - - - ## Count primer-artefacts - echo -e "\\n..Primer-artefacts" - if [ `find 4_primerartefacts -name no_multiprimer 2>/dev/null` ] - then - echo -e "... No files found" - touch Counts_4.PrimerArtefacts.txt - else - seqkit stat --basename --tabular --threads ${task.cpus} --quiet \ - 4_primerartefacts/* > Counts_4.PrimerArtefacts.txt - fi - - - ## Count ITSx reads or primer-trimmed reads (if ITSx was not used) - ## Take number of reads into account (--sizein) - echo -e "\\n..ITSx- or primer-trimmed data" - if [ `find 5_itsxtrim \\( -name no_itsx -o -name no_primertrim \\) 2>/dev/null` ] - then - echo -e "... No files found" - touch Counts_5.ITSx_or_PrimTrim.txt - else - find 5_itsxtrim -name "*.fasta.gz" \ - | parallel -j ${task.cpus} "count_number_of_reads.sh {} {/.}" \ - | sed '1i SampleID\tNumReads' \ - > Counts_5.ITSx_or_PrimTrim.txt - fi - - - ## Count homopolymer-correction results - echo -e "\\n..Counting homopolymer-corrected reads" - if [ `find 5_homopolymers \\( -name no_homopolymer \\) 2>/dev/null` ] - then - echo -e "... No files found" - touch Counts_5.Homopolymers.txt - else - find 5_homopolymers -name "*.uc.gz" \ - | parallel -j ${task.cpus} "count_homopolymer_stats.sh {} {/.}" \ - | sed '1i SampleID\tQuery\tTarget' \ - > Counts_5.Homopolymers.txt - fi - - ## Count number of reads for reference-based chimeras - echo -e "\\n..Reference-based chimeras" - if [ `find 6_chimref -name no_chimref 2>/dev/null` ] - then - echo -e "... No files found" - touch Counts_6.ChimRef_reads.txt - touch Counts_6.ChimRef_uniqs.txt - else - - ## Count number of reads - find 6_chimref -name "*.fa.gz" \ - | parallel -j ${task.cpus} "count_number_of_reads.sh {} {/.}" \ - | sed '1i SampleID\tNumReads' \ - > Counts_6.ChimRef_reads.txt - - ## Count number of unique sequences - seqkit stat --basename --tabular --threads ${task.cpus} --quiet \ - 6_chimref/* > Counts_6.ChimRef_uniqs.txt - - fi - - - ## Number of de novo chimeras (read counts are not taken into account!) - echo -e "\\n..De novo chimeras" - if [ `find 7_chimdenov -name no_chimdenovo 2>/dev/null` ] - then - echo -e "... No files found" - touch Counts_7.ChimDenov.txt - else - cat 7_chimdenov/* > Counts_7.ChimDenov.txt - fi - - - ## Rescued chimeras - echo -e "\\n..Rescued chimeric sequences" - if [ `find 8_chimrecov -name no_chimrescued 2>/dev/null` ] - then - echo -e "... No files found" - touch Counts_8.ChimRecov_reads.txt - touch Counts_8.ChimRecov_uniqs.txt - else - - ## Count number of reads - echo -e "...Reads" - find 8_chimrecov -name "*.fa.gz" \ - | parallel -j ${task.cpus} "count_number_of_reads.sh {} {/.}" \ - | sed '1i SampleID\tNumReads' \ - > Counts_8.ChimRecov_reads.txt - - ## Count number of unique sequences - echo -e "...Unique sequences" - seqkit stat --basename --tabular --threads ${task.cpus} --quiet \ - 8_chimrecov/* > Counts_8.ChimRecov_uniqs.txt - - fi - - ## Summarize read counts - read_count_summary.R \ - --raw Counts_1.RawData.txt \ - --qc Counts_2.QC.txt \ - --demuxed Counts_3.Demux.txt \ - --primer Counts_4.PrimerCheck.txt \ - --primerartef Counts_4.PrimerArtefacts.txt \ - --itsx Counts_5.ITSx_or_PrimTrim.txt \ - --homopolymer Counts_5.Homopolymers.txt \ - --chimrefn Counts_6.ChimRef_reads.txt \ - --chimrefu Counts_6.ChimRef_uniqs.txt \ - --chimdenovo Counts_7.ChimDenov.txt \ - --chimrecovn Counts_8.ChimRecov_reads.txt \ - --chimrecovu Counts_8.ChimRecov_uniqs.txt \ - --tj ${samples_tj} \ - --seqtab ${seqtab} \ - --maxchim ${params.max_ChimeraScore} \ - --threads ${task.cpus} - - """ -} - -// Quick stats of demultiplexing and primer checking steps -// (for the `seqstats` sub-workflow) -process quick_stats { - - label "main_container" - - publishDir "${out_3_quickstats}", mode: "${params.storagemode}", pattern: "*.xlsx" - publishDir "${out_3_quickstats}/PerProcessStats", mode: "${params.storagemode}", pattern: "*.txt" - // cpus 5 - - input: - path(input_fastq, stageAs: "1_input/*") - path(qc, stageAs: "2_qc/*") - path(samples_demux, stageAs: "3_demux/*") - path(samples_primerch, stageAs: "4_primerch/*") - path(samples_primermult, stageAs: "4_primerartefacts/*") - - output: - path "Run_summary.xlsx", emit: xlsx - path "Counts_1.RawData.txt", emit: counts_1_raw - path "Counts_2.QC.txt", emit: counts_2_qc - path "Counts_3.Demux.txt", emit: counts_3_demux, optional: true - path "Counts_4.PrimerCheck.txt", emit: counts_4_primer, optional: true - path "Counts_4.PrimerArtefacts.txt", emit: counts_4_primerartef, optional: true - tuple val("${task.process}"), val('seqkit'), eval('seqkit version | sed "s/seqkit v//"'), topic: versions - tuple val("${task.process}"), val('parallel'), eval('parallel --version | head -n 1 | sed "s/GNU parallel //"'), topic: versions - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - - script: - - """ - echo -e "Summarizing run statistics\\n" - echo -e "Counting the number of reads in:\\n" - - - ## Count raw reads - echo -e "\\n..Raw data" - seqkit stat --basename --tabular --threads ${task.cpus} \ - 1_input/* > Counts_1.RawData.txt - - ## Count number of reads passed QC - echo -e "\\n..Sequenced passed QC" - seqkit stat --basename --tabular --threads ${task.cpus} \ - 2_qc/* > Counts_2.QC.txt - - ## Count demultiplexed reads - echo -e "\\n..Demultiplexed data" - seqkit stat --basename --tabular --threads ${task.cpus} \ - 3_demux/* > Counts_3.Demux.txt - - ## Count primer-checked reads - echo -e "\\n..Primer-checked data" - if [ `find 4_primerch -name no_primerchecked 2>/dev/null` ] - then - echo -e "... No files found" - touch Counts_4.PrimerCheck.txt - else - seqkit stat --basename --tabular --threads ${task.cpus} \ - 4_primerch/* > Counts_4.PrimerCheck.txt - fi - - ## Count primer-artefacts - echo -e "\\n..Primer-areifacts" - if [ `find 4_primerartefacts -name no_multiprimer 2>/dev/null` ] - then - echo -e "... No files found" - touch Counts_4.PrimerArtefacts.txt - else - seqkit stat --basename --tabular --threads ${task.cpus} \ - 4_primerartefacts/* > Counts_4.PrimerArtefacts.txt - fi - - ## Summarize read counts - quick_stats.R \ - --raw Counts_1.RawData.txt \ - --qc Counts_2.QC.txt \ - --demuxed Counts_3.Demux.txt \ - --primer Counts_4.PrimerCheck.txt \ - --primerartef Counts_4.PrimerArtefacts.txt \ - --threads ${task.cpus} - - """ -} - -// Auto documentation of analysis procedures -// (generate narrative description of methods with references) -process document_analysis_s1 { - - label "main_container" - - publishDir "${out_tracedir}", mode: 'copy', overwrite: true - // cpus 1 - - input: - path versions // "software_versions.yml" - path params // "pipeline_params.tsv" - - output: - path "README_Step1_Methods.txt", emit: docs - - - script: - """ - echo -e "Descriptive summary generation\\n" - - document_s1.R \ - ${versions} \ - ${params} \ - README_Step1_Methods.txt - - """ -} - - - - -// The default workflow - Step-1 -workflow S1 { - - // Primer disambiguation - disambiguate() - - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Demultiplex data - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - // Run demultiplexing - if( params.demultiplexed == false ){ - - // Input file with barcodes (FASTA) - ch_barcodes = Channel.value(params.barcodes) - - // Validate tags - tag_validation(ch_barcodes) - - // PacBio - if ( params.seqplatform == "PacBio" ) { - - // Input file with multiplexed reads (FASTQ.gz or BAM) - ch_input = Channel.value(params.input) - - // Check the extension of input - input_type = file(params.input).getExtension() =~ /bam|BAM/ ? "bam" : "oth" - // println("${input_type}") - - // If BAM is provided as input, convert it to FASTQ - if ( input_type == 'bam'){ - - // Add BAM index file - ch_input_pbi = ch_input + ".pbi" - - bam2fastq(ch_input, ch_input_pbi) - qc_se(bam2fastq.out.fastq) - - } else { - - // Initial QC - qc_se(ch_input) - - } - - // Demultiplexing with dual barcodes requires 4 additional files: - // - "biosamples" with symmertic/asymmetirc tag combinations - // - table for assigning sample names to demuxed files - // - and a table for renaming unknown combinations (if params.lima_remove_unknown == true) - // Create dummy files (for single or symmetic tags) if neccesary - ch_biosamples_sym = tag_validation.out.biosamples_sym.flatten().collect().ifEmpty(file("biosamples_sym")) - ch_biosamples_asym = tag_validation.out.biosamples_asym.flatten().collect().ifEmpty(file("biosamples_asym")) - ch_file_renaming = tag_validation.out.file_renaming.flatten().collect().ifEmpty(file("file_renaming")) - ch_unknown_combs = tag_validation.out.unknown_combinations.flatten().collect().ifEmpty(file("unknown_combinations")) - - // Demultiplexing - demux( - qc_se.out.filtered, - tag_validation.out.fasta, - ch_biosamples_sym, - ch_biosamples_asym, - ch_file_renaming, - ch_unknown_combs) - - // Check primers - primer_check( - demux.out.samples_demux.flatten(), - disambiguate.out.F, - disambiguate.out.R, - disambiguate.out.Fr, - disambiguate.out.Rr - ) - - } // end of PacBio-specific tasks - - // Illumina - if ( params.seqplatform == "Illumina" ) { - - // Input file with multiplexed pair-end reads (FASTQ.gz) - ch_inputR1 = Channel.value(params.input_R1) - ch_inputR2 = Channel.value(params.input_R2) - - // Initial QC - qc_pe(ch_inputR1, ch_inputR2) - - // PE assembly - merge_pe( - qc_pe.out.filtered_R1, - qc_pe.out.filtered_R2) - - // Modify barcodes (restict search window) - prep_barcodes(tag_validation.out.fasta) - - // Demultiplexing - demux_illumina( - merge_pe.out.r12, - prep_barcodes.out.barcodesm) - - ch_demux_merged = demux_illumina.out.samples_demux.flatten() - - // Illumina nonmerged PE reads sub-workflow (optional) - if(params.illumina_keep_notmerged == true){ - - // Demultiplexing non-merged reads - demux_illumina_notmerged( - merge_pe.out.nm, - prep_barcodes.out.barcodesm) - - // Channel of non-merged reads by sample (split into sample tuples) - // ch_R1 = demux_illumina_notmerged.out.demux_pe.... - - // Non-merged sample list - ch_nonmerged = demux_illumina_notmerged.out.samples_nonm_pe.splitText().map{it -> it.trim()} - - // Trim primers of nonmerged PE reads - // Estimate sequence qualities - // Dereplicate R1 and R2 independently - // trim_primers_pe(demux_illumina_notmerged.out.demux_pe.flatten()) - - // Join nonmerged reads with poly-N pads - join_pe( - ch_nonmerged, - demux_illumina_notmerged.out.demux_pe.flatten().collect() // all non-merged R1 and R2 files - ) - - // Add joined reads to the merged reads - ch_joined = join_pe.out.jj_FQ.flatten() - ch_demuxed = ch_demux_merged.concat(ch_joined) - - } else { // end of Illumina non-merged reads - - // Channel with demultiplexed reads - ch_demuxed = ch_demux_merged - - } - - // Check primers - primer_check( - ch_demuxed, - disambiguate.out.F, - disambiguate.out.R, - disambiguate.out.Fr, - disambiguate.out.Rr - ) - - } // end of Illumina-specific tasks - - } // end of demultiplexing - - - - // If samples were already demuliplexed - if( params.demultiplexed == true ){ - - // Input files with demultiplexed reads (FASTQ.gz) - ch_input = Channel.fromPath( params.input + '/*.{fastq.gz,fastq,fq.gz,fq}' ) - - // Check if the input channel is empty - ch_input - .ifEmpty { - error("ERROR: No FASTQ files found in the input directory: ${params.input}") - exit 1 - } - - // QC - qc_se(ch_input) - - // Check primers - primer_check( - qc_se.out.filtered, - disambiguate.out.F, - disambiguate.out.R, - disambiguate.out.Fr, - disambiguate.out.Rr - ) - - } // end of pre-demultiplexed branch - - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - ITS extraction or primer trimming - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - // Extract ITS - if(params.its_region == "full" || params.its_region == "ITS1" || params.its_region == "ITS2" || params.its_region == "SSU" || params.its_region == "LSU"){ - - // Run ITSx - itsx(primer_check.out.fq_primer_checked) - - // Merge tables with sequence qualities - seq_qual(itsx.out.hashes.collect()) - } - - // Trim the primers (instead of ITS extraction) - if(params.its_region == "none"){ - - // Trim primers with cutadapt - trim_primers(primer_check.out.fq_primer_checked) - - // Merge tables with sequence qualities - seq_qual(trim_primers.out.hashes.collect()) - } - - // Trim the primers, run ITSx, and assemble near-full-length ITS - if(params.its_region == "ITS1_5.8S_ITS2"){ - - // Run ITSx - itsx(primer_check.out.fq_primer_checked) - - // Assemble ITS1-5.8S-ITS2 from ITSx-extracted parts - if (params.ITSx_partial == 0) { - assemble_its( - itsx.out.itsx_its1, - itsx.out.itsx_58s, - itsx.out.itsx_its2) - } else { - assemble_its( - itsx.out.itsx_its1_part, - itsx.out.itsx_58s, - itsx.out.itsx_its2_part) - } - - // Merge tables with sequence qualities - seq_qual(itsx.out.hashes.collect()) - } - - - // Collect ITSx-extracted sequences - if(params.its_region == "full" || params.its_region == "ITS1" || params.its_region == "ITS2" || params.its_region == "SSU" || params.its_region == "LSU" || params.its_region == "ITS1_5.8S_ITS2"){ - - // Collect rRNA parts into separate channels - ch_cc_full = itsx.out.itsx_full.flatten().collect().ifEmpty(file("NOFULL")) - ch_cc_ssu = itsx.out.itsx_ssu.flatten().collect().ifEmpty(file("NOSSU")) - ch_cc_its1 = itsx.out.itsx_its1.flatten().collect().ifEmpty(file("NOITS1")) - ch_cc_58s = itsx.out.itsx_58s.flatten().collect().ifEmpty(file("NO58S")) - ch_cc_its2 = itsx.out.itsx_its2.flatten().collect().ifEmpty(file("NOITS2")) - ch_cc_lsu = itsx.out.itsx_lsu.flatten().collect().ifEmpty(file("NOLSU")) - - ch_cc_ssu_part = itsx.out.itsx_ssu_part.flatten().collect().ifEmpty(file("NOSSUPART")) - ch_cc_its1_part = itsx.out.itsx_its1_part.flatten().collect().ifEmpty(file("NOITS1PART")) - ch_cc_58s_part = itsx.out.itsx_58s_part.flatten().collect().ifEmpty(file("NO58SPART")) - ch_cc_its2_part = itsx.out.itsx_its2_part.flatten().collect().ifEmpty(file("NOITS2PART")) - ch_cc_lsu_part = itsx.out.itsx_lsu_part.flatten().collect().ifEmpty(file("NOLSUPART")) - - itsx_collect( - ch_cc_full, - ch_cc_ssu, - ch_cc_its1, - ch_cc_58s, - ch_cc_its2, - ch_cc_lsu, - ch_cc_ssu_part, - ch_cc_its1_part, - ch_cc_58s_part, - ch_cc_its2_part, - ch_cc_lsu_part - ) - - } // end of ITSx-extracted sequences - - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Homopolymer compression & chimera removal - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - // Homopolymer compression - if(params.hp == true){ - - // --Full-length ITS sequences - if(params.its_region == "full"){ - homopolymer(itsx.out.itsx_full) - } - // --ITS1 sequences - if(params.its_region == "ITS1"){ - if (params.ITSx_partial == 0) { - homopolymer(itsx.out.itsx_its1) - } else { - homopolymer(itsx.out.itsx_its1_part) - } - } - // --ITS2 sequences - if(params.its_region == "ITS2"){ - if (params.ITSx_partial == 0) { - homopolymer(itsx.out.itsx_its2) - } else { - homopolymer(itsx.out.itsx_its2_part) - } - } - // --SSU sequences - if(params.its_region == "SSU"){ - if (params.ITSx_partial == 0) { - homopolymer(itsx.out.itsx_ssu) - } else { - homopolymer(itsx.out.itsx_ssu_part) - } - } - // --LSU sequences - if(params.its_region == "LSU"){ - if (params.ITSx_partial == 0) { - homopolymer(itsx.out.itsx_lsu) - } else { - homopolymer(itsx.out.itsx_lsu_part) - } - } - - // --Primer-trimmed sequences - if(params.its_region == "none"){ - homopolymer(trim_primers.out.primertrimmed_fa) - } - // Near-full-length ITS - if(params.its_region == "ITS1_5.8S_ITS2"){ - homopolymer(assemble_its.out.itsnf) - } - - - } else { - // No homopolymer comression is required, - // Just dereplicate the data - - if(params.its_region == "full" || params.its_region == "ITS1" || params.its_region == "ITS2" || params.its_region == "SSU" || params.its_region == "LSU"){ - - // --Full-length ITS sequences - if(params.its_region == "full"){ - just_derep(itsx.out.itsx_full) - } - // --ITS1 sequences - if(params.its_region == "ITS1"){ - if (params.ITSx_partial == 0) { - just_derep(itsx.out.itsx_its1) - } else { - just_derep(itsx.out.itsx_its1_part) - } - } - // --ITS2 sequences - if(params.its_region == "ITS2"){ - if (params.ITSx_partial == 0) { - just_derep(itsx.out.itsx_its2) - } else { - just_derep(itsx.out.itsx_its2_part) - } - } - // --SSU sequences - if(params.its_region == "SSU"){ - if (params.ITSx_partial == 0) { - just_derep(itsx.out.itsx_ssu) - } else { - just_derep(itsx.out.itsx_ssu_part) - } - } - // --LSU sequences - if(params.its_region == "LSU"){ - if (params.ITSx_partial == 0) { - just_derep(itsx.out.itsx_lsu) - } else { - just_derep(itsx.out.itsx_lsu_part) - } - } - - } // end of ITS - - - } // end of homopolymer correction condition - - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Chimera removal - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - - // Chimera removal (optional) - ch_chimerabd = Channel.value(params.chimera_db) - - // Input depends on the selected workflow - if(params.hp == true){ - - ch_input_for_chim = homopolymer.out.hc - - } else { - - if(params.its_region == "none"){ - ch_input_for_chim = trim_primers.out.primertrimmed_fa - } else if(params.its_region == "ITS1_5.8S_ITS2"){ - ch_input_for_chim = assemble_its.out.itsnf - } else { - ch_input_for_chim = just_derep.out.nhc - } - - } - - CHIMERA_REMOVAL(ch_input_for_chim, ch_chimerabd) - - - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Data aggregation - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - // Pool sequences (for a final sequence table) - pool_seqs(CHIMERA_REMOVAL.out.filtered) - - // Tag-jump removal - if(params.tj == true){ - - // Pre-clustering prior to tag-jump removal - tj_preclust(pool_seqs.out.seqsnf) - - // Tag-jump removal - tj( - pool_seqs.out.seqtabnf, - tj_preclust.out.preclust_uc_parquet) - - ch_seqtab_after_tj = tj.out.seqtabtj - ch_tj_scores = tj.out.tjs - - } else { - - // Skip tag-jump removal - ch_seqtab_after_tj = pool_seqs.out.seqtabnf - ch_tj_scores = file("no_tj") - - } - - // Check optional channel with de novo chimera scores - ch_denovoscores = CHIMERA_REMOVAL.out.denovo_agg.ifEmpty(file('DeNovo_Chimera.txt')) - - // Create sequence table - prep_seqtab( - ch_seqtab_after_tj, // (optionally) tag-jump-filtered sequence table (long format) - pool_seqs.out.seqsnf, // Sequences in FASTA format - ch_denovoscores, // de novo chimera scores - seq_qual.out.quals // sequence qualities - ) - - - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Read count summary - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - // Initial data - Per-sample input channels - if( params.demultiplexed == false ){ - - if(params.seqplatform == "PacBio"){ - - // Input data and QC = single multiplexed file - ch_counts_1 = ch_input - ch_counts_2 = qc_se.out.filtered - - ch_all_demux = demux.out.samples_demux.flatten().collect() - } - - if(params.seqplatform == "Illumina"){ - ch_all_demux = demux_illumina.out.samples_demux.flatten().collect() - } - - } else { - - // Input data and QC = several demultiplexed files - ch_counts_1 = ch_input.flatten().collect() - ch_counts_2 = qc_se.out.filtered.flatten().collect() - - ch_all_demux = Channel.fromPath( params.input + '/*.{fastq.gz,fastq,fq.gz,fq}' ).flatten().collect() - } - - - // Primer-checked and multiprimer sequences - ch_all_primerchecked = primer_check.out.fq_primer_checked.flatten().collect().ifEmpty(file("no_primerchecked")) - ch_all_primerartefacts = primer_check.out.primerartefacts.flatten().collect().ifEmpty(file("no_multiprimer")) - - // ITSx and primer trimming channel - if(params.its_region == "full"){ - ch_all_trim = itsx.out.itsx_full.flatten().collect().ifEmpty(file("no_itsx")) - } - if(params.its_region == "ITS1"){ - if (params.ITSx_partial == 0) { - ch_all_trim = itsx.out.itsx_its1.flatten().collect().ifEmpty(file("no_itsx")) - } else { - ch_all_trim = itsx.out.itsx_its1_part.flatten().collect().ifEmpty(file("no_itsx")) - } - } - if(params.its_region == "ITS2"){ - if (params.ITSx_partial == 0) { - ch_all_trim = itsx.out.itsx_its2.flatten().collect().ifEmpty(file("no_itsx")) - } else { - ch_all_trim = itsx.out.itsx_its2_part.flatten().collect().ifEmpty(file("no_itsx")) - } - } - if(params.its_region == "SSU"){ - if (params.ITSx_partial == 0) { - ch_all_trim = itsx.out.itsx_ssu.flatten().collect().ifEmpty(file("no_itsx")) - } else { - ch_all_trim = itsx.out.itsx_ssu_part.flatten().collect().ifEmpty(file("no_itsx")) - } - } - if(params.its_region == "LSU"){ - if (params.ITSx_partial == 0) { - ch_all_trim = itsx.out.itsx_lsu.flatten().collect().ifEmpty(file("no_itsx")) - } else { - ch_all_trim = itsx.out.itsx_lsu_part.flatten().collect().ifEmpty(file("no_itsx")) - } - } - if(params.its_region == "ITS1_5.8S_ITS2"){ - ch_all_trim = assemble_its.out.itsnf.flatten().collect().ifEmpty(file("no_itsx")) - } - if(params.its_region == "none"){ - ch_all_trim = trim_primers.out.primertrimmed_fq.flatten().collect().ifEmpty(file("no_primertrim")) - } - - // Homopolymer-correction channel - if(params.hp == true){ - ch_homopolymers = homopolymer.out.uch.flatten().collect().ifEmpty(file("no_homopolymer")) - } else { - ch_homopolymers = file("no_homopolymer") - } - - // Chimeric channels - ch_chimref = CHIMERA_REMOVAL.out.chimeric.flatten().collect().ifEmpty(file("no_chimref")) - ch_chimdenovo = CHIMERA_REMOVAL.out.denovo_agg.flatten().collect().ifEmpty(file("no_chimdenovo")) - ch_chimrescued = CHIMERA_REMOVAL.out.rescued.flatten().collect().ifEmpty(file("no_chimrescued")) - - // Count reads and prepare summary stats for the run - // Currently, implemented only for PacBio - // For Illumina, need replace: - // `ch_input` -> `ch_inputR1` & `ch_inputR2` - // `qc_se` -> `qc_pe` - - if(params.seqplatform == "PacBio"){ - - read_counts( - ch_counts_1, // input data (single multiplexed file or several demultiplexed files) - ch_counts_2, // data that passed QC (single or several demuxed files) - ch_all_demux, // demultiplexed sequences per sample - ch_all_primerchecked, // primer-cheched sequences - ch_all_primerartefacts, // multiprimer artefacts - ch_all_trim, // ITSx-extracted or primer-trimmed sequences - ch_homopolymers, // Homopolymer stats - ch_chimref, // Reference-based chimeras - ch_chimdenovo, // De novo chimeras - ch_chimrescued, // Rescued chimeras - ch_tj_scores, // Tag-jump filtering scores - prep_seqtab.out.seq_pq // Final table with sequences (in Parquet format) - ) - - } // end of read_counts for PacBio - - - - // Dump the software versions to a file - ch_versions_yml = software_versions_to_yaml(Channel.topic('versions')) - .collectFile( - storeDir: "${params.tracedir}", - name: 'software_versions.yml', - sort: true, - newLine: true - ) - - // Dump the parameters to a file - ch_params_tsv = dumpParamsTsv() - .collectFile( - storeDir: "${params.tracedir}", - name: "pipeline_params.tsv", - sort: true, - newLine: true - ) - - // Document the analysis procedures - document_analysis_s1( - ch_versions_yml, - ch_params_tsv) - -} - - - - - - -// Quick workflow for demultiplexing and estimation of the number of reads per sample -// Only PacBio non-demultiplexed reads are supported -workflow seqstats { - - // Primer disambiguation - disambiguate() - - // Input file with barcodes (FASTA) - ch_barcodes = Channel.value(params.barcodes) - - // Input file with multiplexed reads (FASTQ.gz) - ch_input = Channel.value(params.input) - - // Initial QC - qc_se(ch_input) - - // Validate tags - tag_validation(ch_barcodes) - - // Tag-validation channels - ch_biosamples_sym = tag_validation.out.biosamples_sym.flatten().collect().ifEmpty(file("biosamples_sym")) - ch_biosamples_asym = tag_validation.out.biosamples_asym.flatten().collect().ifEmpty(file("biosamples_asym")) - ch_file_renaming = tag_validation.out.file_renaming.flatten().collect().ifEmpty(file("file_renaming")) - ch_unknown_combs = tag_validation.out.unknown_combinations.flatten().collect().ifEmpty(file("unknown_combinations")) - - // Demultiplexing - demux( - qc_se.out.filtered, - tag_validation.out.fasta, - ch_biosamples_sym, - ch_biosamples_asym, - ch_file_renaming, - ch_unknown_combs) - - // Check primers - primer_check( - demux.out.samples_demux.flatten(), - disambiguate.out.F, - disambiguate.out.R, - disambiguate.out.Fr, - disambiguate.out.Rr - ) - - // Prepare input channels - ch_all_demux = demux.out.samples_demux.flatten().collect() - ch_all_primerchecked = primer_check.out.fq_primer_checked.flatten().collect().ifEmpty(file("no_primerchecked")) - ch_all_primerartefacts = primer_check.out.primerartefacts.flatten().collect().ifEmpty(file("no_multiprimer")) - - // Count reads and prepare summary stats for the run - quick_stats( - ch_input, // input data - qc_se.out.filtered, // data that passed QC - ch_all_demux, // demultiplexed sequences per sample - ch_all_primerchecked, // primer-cheched sequences - ch_all_primerartefacts // primer artefacts - ) - -} // end of `seqstats` subworkflow - diff --git a/src/pipecraft-core/service_scripts/NextITS/workflows/STEP2.nf b/src/pipecraft-core/service_scripts/NextITS/workflows/STEP2.nf deleted file mode 100644 index 75ef3fa3..00000000 --- a/src/pipecraft-core/service_scripts/NextITS/workflows/STEP2.nf +++ /dev/null @@ -1,868 +0,0 @@ -/* -============================================================================ - NextITS: Pipeline to process eukaryotic ITS amplicons -============================================================================ - License: Apache-2.0 - Github : https://github.com/vmikk/NextITS - Website: https://Next-ITS.github.io/ ----------------------------------------------------------------------------- -*/ - -// ---- Step-2 workflow ---- - - -// Step-2 workflow: -// - Dereplicate sequences -// - (optionally) Denoize with UNOISE or DADA2 -// - (optionally) Cluster: -// * SWARM -// * VSEARCH -// - LULU (via MUMU implementation) -// - Prepare OTU table (wide, aggregate sequence abundance by ASV/OTU/Swarm cluster) - - - -// Enable DSL2 syntax -nextflow.enable.dsl = 2 - -include { software_versions_to_yaml } from '../modules/version_parser.nf' -include { CLUSTERING } from '../subworkflows/clustering_subworkflow.nf' -include { dumpParamsTsv } from '../modules/dump_parameters.nf' - -// Directory for storing pipeline information -out_tracedir = params.tracedir - - - - - - -// Aggregate sequences from all sequencing runs, remove de novo chimeras -process aggregate_sequences { - - label "main_container" - - // cpus 6 - - input: - path(inputs, stageAs: "?/*") - - output: - path "Seqs.fa.gz", emit: seqs - path "Seqs.parquet", emit: seqs_parquet - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - tuple val("${task.process}"), val('arrow'), eval('Rscript -e "cat(as.character(packageVersion(\'arrow\')))"'), topic: versions - tuple val("${task.process}"), val('Biostrings'), eval('Rscript -e "cat(as.character(packageVersion(\'Biostrings\')))"'), topic: versions - - script: - """ - echo -e "Aggregating sequences\\n" - - aggregate_sequences.R \ - --seqtabs . \ - --maxchim ${params.max_ChimeraScore} \ - --recoverdenovo ${params.recover_denovochimeras} \ - --output Seqs \ - --threads ${task.cpus} - - """ -} - -// Pool and dereplicate sequences from all sequencing runs -process dereplication { - - label "main_container" - - publishDir "${params.outdir}/01.Dereplicated", mode: "${params.storagemode}" - // cpus 8 - - input: - path seqs - - output: - path "Dereplicated.fa.gz", emit: derep - path "Dereplicated.uc.gz", emit: derep_uc - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - - script: - def minlen = params.ampliconlen_min ? "--minseqlength ${params.ampliconlen_min}" : "" - def maxlen = params.ampliconlen_max ? "--maxseqlength ${params.ampliconlen_max}" : "" - - // Calculate optimal number of threads - def maxPigzThreads = 8 // Maximum threads per pigz instance - def totalCPUs = task.cpus - - // Try to maximize CPUs per pigz while ensuring full CPU utilization - def pigzCPUs = Math.min(maxPigzThreads, Math.ceil(Math.sqrt(totalCPUs * 2)).intValue()) - def parallelJobs = Math.max(1, Math.floor(totalCPUs / pigzCPUs).intValue()) - - // Recalculate pigzCPUs to use all available CPUs - pigzCPUs = Math.min(maxPigzThreads, Math.floor(totalCPUs / parallelJobs).intValue()) - - """ - echo -e "Dereplicating sequences\\n" - - vsearch \ - --derep_fulllength ${seqs} \ - --output Dereplicated.fa \ - --strand both \ - ${minlen} ${maxlen} \ - --fasta_width 0 \ - --threads 1 \ - --sizein --sizeout \ - --uc Dereplicated.uc - - echo -e "..Dereplication finished\\n" - - ## Compress results - echo -e "\\nCompressing results" - parallel -j ${parallelJobs} \ - "pigz -p ${pigzCPUs} -${params.gzip_compression} {}" \ - ::: "Dereplicated.uc" "Dereplicated.fa" - - """ -} - - -// Pool sequences from all sequencing runs, -// Dereplicate allowing query sequences to vary in length at 100% similarity (by default, 4% length variation allowed) -process dereplication_unite { - - label "main_container" - - publishDir "${params.outdir}/01.Dereplicated", mode: "${params.storagemode}" - // cpus 8 - - input: - path(inputs, stageAs: "?/*") - - output: - path "Dereplicated.fa.gz", emit: derep - path "Dereplicated.uc.gz", emit: derep_uc - - script: - """ - echo -e "Dereplicating sequences\\n" - - ## NB. by default, UNITE uses `cluster_fast`, which sorts sequences by length - ## Here, we use `cluster_size`, which sorts by abundance - - find . -name "*.fa.gz" | parallel -j1 \ - "zcat {}" \ - | sed '/^>/ s/;sample=.*;/;/' \ - | vsearch \ - --cluster_size - \ - --id 1 \ - --iddef 2 \ - --query_cov ${params.unite_querycov} \ - --target_cov ${params.unite_targetcov} \ - --strand both \ - --sizein --sizeout \ - --threads ${task.cpus} \ - --uc Dereplicated.uc \ - --centroids Dereplicated.fa - - echo -e "..Dereplication finished" - - ## Compress results - echo -e "\\nCompressing results" - parallel -j ${task.cpus} "gzip -${params.gzip_compression} {}" \ - ::: "Dereplicated.uc" "Dereplicated.fa" - - """ -} - - -// Fast pre-clustering of the dataset (to split into chunks prior processing) -process linclust { - - label "main_container" - - input: - path input - - output: - path "DB_clu.tsv", emit: db_clu - tuple val("${task.process}"), val('mmseqs'), eval('mmseqs version'), topic: versions - - script: - """ - - ## Create DB - echo -e "..DB creation\\n" - - mmseqs createdb \ - --dbtype 2 \ - --createdb-mode 0 \ - --shuffle 0 \ - ${input} \ - mmseqs_db - - - ## Run (cascaded) clustering - echo -e "..Lin-Clustering\\n" - - mmseqs linclust \ - mmseqs_db \ - linclusters_db \ - tmplc \ - --min-seq-id ${params.chunking_id} \ - --cluster-mode 0 \ - --similarity-type 2 \ - -c 0.7 --cov-mode 0 \ - -k 15 \ - --kmer-per-seq 100 \ - --kmer-per-seq-scale 0.3 \ - --spaced-kmer-mode 0 \ - --mask 0 \ - --split-memory-limit 100G \ - --remove-tmp-files 1 \ - --threads ${task.cpus} - - ## Generate a TSV-formatted output of clustering - echo -e "..Generating TSV-formatted output of clustering\\n" - - mmseqs createtsv \ - mmseqs_db mmseqs_db \ - linclusters_db \ - DB_clu.tsv \ - --threads ${task.cpus} - - """ -} - -// Bucketize sequences into clusters -process bucketize { - - label "main_container" - - input: - path sequences - path clusters - - output: - path "bucket_*.fa.gz", emit: buckets - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - tuple val("${task.process}"), val('Biostrings'), eval('Rscript -e "cat(as.character(packageVersion(\'Biostrings\')))"'), topic: versions - - script: - numchunks = params.chunking_n ? "--numbuckets ${params.chunking_n}" : "" - """ - echo -e "..Bucketizing sequences\\n" - - bucketize_db.R \ - --db ${clusters} \ - --fasta ${sequences} \ - ${numchunks} \ - --summary bucket_summary.txt \ - --threads ${task.cpus} - - """ -} - - - - - -// Merge processed buckets (e.g., clustered sequences) -process merge_buckets { - - label "main_container" - - // Conditional publishing to match non-chunked directory structure - // (only enabled when chunking is used) - - // Pre-clustering results (if any) - publishDir "${params.outdir}/02.Homopolymer", - mode: "${params.storagemode}", - enabled: (params.chunking_n != null && params.chunking_n >= 2) && params.preclustering == "homopolymer", - pattern: "PreClustered.{fa,uc}.gz", - saveAs: { filename -> - switch(filename) { - case "PreClustered.fa.gz": return "HomopolymerCompressed.fa.gz" - case "PreClustered.uc.gz": return "HomopolymerCompressed.uc.gz" - default: return null - } - } - - publishDir "${params.outdir}/02.UNOISE", - mode: "${params.storagemode}", - enabled: (params.chunking_n != null && params.chunking_n >= 2) && params.preclustering == "unoise", - pattern: "PreClustered.{fa,uc}.gz", - saveAs: { filename -> - switch(filename) { - case "PreClustered.fa.gz": return "UNOISE.fa.gz" - case "PreClustered.uc.gz": return "UNOISE.uc.gz" - default: return null - } - } - - publishDir "${params.outdir}/02.DADA2", - mode: "${params.storagemode}", - enabled: (params.chunking_n != null && params.chunking_n >= 2) && params.preclustering == "dada2", - pattern: "PreClustered.{fa,uc}.gz", - saveAs: { filename -> - switch(filename) { - case "PreClustered.fa.gz": return "DADA2_denoised.fa.gz" - case "PreClustered.uc.gz": return "DADA2_denoised.uc.gz" - default: return null - } - } - - publishDir "${params.outdir}/02.Preclustered_SWARM_d1", - mode: "${params.storagemode}", - enabled: (params.chunking_n != null && params.chunking_n >= 2) && params.preclustering == "swarm_d1", - pattern: "PreClustered.{fa,uc}.gz", - saveAs: { filename -> - switch(filename) { - case "PreClustered.fa.gz": return "SWARM.fa.gz" - case "PreClustered.uc.gz": return "SWARM.uc.gz" - default: return null - } - } - - // Final clustering results - publish to clustering directory if clustering != "none" - publishDir "${params.outdir}/03.Clustered_VSEARCH", - mode: "${params.storagemode}", - enabled: (params.chunking_n != null && params.chunking_n >= 2) && params.clustering == "vsearch", - pattern: "Clustered.{fa,uc}.gz" - // No saveAs needed - files already have correct names for VSEARCH - - publishDir "${params.outdir}/03.Clustered_SWARM", - mode: "${params.storagemode}", - enabled: (params.chunking_n != null && params.chunking_n >= 2) && params.clustering == "swarm", - pattern: "Clustered.{fa,uc}.gz", - saveAs: { filename -> - switch(filename) { - case "Clustered.fa.gz": return "SWARM_representatives.fa.gz" - case "Clustered.uc.gz": return "SWARM.uc.gz" - default: return null - } - } - - - // Since there are name collisions, we need to stage files with unique names - input: - path(preclustuc_chunks, stageAs: "pre/?/*") // UC files for pre-clustering (optional) - path(preclustaf_chunks, stageAs: "pre/?/*") // FASTA files for pre-clustering (optional) - path(cluster_chunks, stageAs: "cls/?/*") // Sequence representatives - path(clustuc_chunks, stageAs: "ucs/?/*") // UC files for clustering - - output: - path "PreClustered.uc.gz", emit: preclustuc_ch, optional: true - path "PreClustered.fa.gz", emit: preclustaf_ch, optional: true - path "Clustered.fa.gz", emit: cluster_ch - path "Clustered.uc.gz", emit: clustuc_ch - - - script: - """ - echo -e "Merging buckets\\n" - - ## Pool sequence representatives - echo -e "..Pooling sequence representatives\\n" - find cls -name "*.fa.gz" \ - | parallel -j 1 "cat {}" \ - > Clustered.fa.gz - - ## Pool UC files - echo -e "..Pooling UC files\\n" - find ucs -name "*.uc.gz" \ - | parallel -j 1 "cat {}" \ - > Clustered.uc.gz - - ## Check if pre-clustering was performed - if [[ -e pre/1/NoPrecluster || -L "pre/1/NoPrecluster" ]]; then - echo -e "..Pre-clustering was not performed. Skipping pooling these data\\n" - else - echo -e "..Pre-clustering was performed\\n" - - echo -e "..Pooling pre-clustered UC files\\n" - find pre -name "*.uc.gz" \ - | parallel -j 1 "cat {}" \ - > PreClustered.uc.gz - - echo -e "..Pooling pre-clustered FASTA files\\n" - find pre -name "*.fa.gz" \ - | parallel -j 1 "cat {}" \ - | vsearch \ - --sortbysize - \ - --sizein --sizeout \ - --threads 1 \ - --fasta_width 0 \ - --output - \ - | pigz -p ${task.cpus} -${params.gzip_compression} \ - > PreClustered.fa.gz - fi - - echo -e "..Done\\n" - """ -} - - - -// Merge UC files -process merge_uc { - - label "main_container" - - publishDir "${params.outdir}/04.PooledResults", mode: "${params.storagemode}" - // cpus 4 - - input: - path(uc_derep) - path(uc_preclust) - path(uc_clust) - - output: - path "UC_Pooled.parquet", emit: uc - tuple val("${task.process}"), val('ucs'), eval('ucs --version | sed "s/ucs //"'), topic: versions - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('duckdb'), eval('Rscript -e "cat(as.character(packageVersion(\'duckdb\')))"'), topic: versions - - script: - """ - echo -e "Merging UC files\\n" - - ## Parse UC files from different steps, convert to parquet format - echo -e "..Parsing dereplicated UC file\\n" - ucs --input ${uc_derep} --output UC_derep.parquet - - if [ -f ${uc_preclust} ] && [ "${uc_preclust}" != "NoPrecluster" ]; then - echo -e "..Parsing pre-clustered UC file\\n" - ucs --input ${uc_preclust} --output UC_preclust.parquet - UCPRECLUST="UC_preclust.parquet" - else - UCPRECLUST="NoPrecluster" - fi - - if [ -f ${uc_clust} ]; then - echo -e "..Parsing clustered UC file\\n" - ucs --input ${uc_clust} --output UC_clust.parquet - fi - - ## Merge UC files into a single file - echo -e "..Merging UC files\\n" - merge_uc_files.R \ - --ucderep UC_derep.parquet \ - --ucpreclust \${UCPRECLUST} \ - --ucclust UC_clust.parquet \ - --output UC_Pooled.parquet - - """ -} - - -// Summarize sequence abundance by OTU -process summarize { - - label "main_container" - - publishDir "${params.outdir}/04.PooledResults", mode: "${params.storagemode}" - // cpus 4 - - input: - path(seqtab) - path(uc_parquet) - path(otus_fasta) - - output: - path "OTU_table_wide.txt.gz", emit: otutabwide - path "OTU_table_long.txt.gz", emit: otutablong - path "OTU_table_wide.RData", emit: otutabwider - path "OTU_table_long.RData", emit: otutablongr - path "OTUs.fa.gz", emit: seqs - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - tuple val("${task.process}"), val('arrow'), eval('Rscript -e "cat(as.character(packageVersion(\'arrow\')))"'), topic: versions - tuple val("${task.process}"), val('Biostrings'), eval('Rscript -e "cat(as.character(packageVersion(\'Biostrings\')))"'), topic: versions - - script: - """ - echo -e "Summarizing clustered data\\n" - - summarize_clustered_data.R \ - --seqtab ${seqtab} \ - --uc ${uc_parquet} \ - --otus ${otus_fasta} \ - --maxmeep ${params.max_MEEP} \ - --recoversinglet ${params.recover_lowqsingletons} \ - --mergesamples ${params.merge_replicates} \ - --threads ${task.cpus} - - """ -} - - -// Summarize dereplicated data -process summarize_dereplicated_data { - - label "main_container" - publishDir "${params.outdir}/04.PooledResults", mode: "${params.storagemode}" - // cpus 4 - - input: - path(seqtab) // Sequence tables in long format, parquet - path(uc_derep) // UC file from dereplication - path(fasta) // FASTA file with sequences - - output: - path "UC_Pooled.parquet", emit: uc - path "OTU_table_wide.txt.gz", emit: otutabwide - path "OTU_table_long.txt.gz", emit: otutablong - path "OTU_table_wide.RData", emit: otutabwider - path "OTU_table_long.RData", emit: otutablongr - path "OTUs.fa.gz", emit: seqs - tuple val("${task.process}"), val('ucs'), eval('ucs --version | sed "s/ucs //"'), topic: versions - tuple val("${task.process}"), val('R'), eval('Rscript -e "cat(R.version.string)" | sed "s/R version //" | cut -d" " -f1'), topic: versions - tuple val("${task.process}"), val('data.table'), eval('Rscript -e "cat(as.character(packageVersion(\'data.table\')))"'), topic: versions - tuple val("${task.process}"), val('arrow'), eval('Rscript -e "cat(as.character(packageVersion(\'arrow\')))"'), topic: versions - tuple val("${task.process}"), val('Biostrings'), eval('Rscript -e "cat(as.character(packageVersion(\'Biostrings\')))"'), topic: versions - - script: - """ - echo -e "Summarizing clustered data\\n" - - ## Parse UC file from dereplication - echo -e "..Parsing dereplicated UC file" - ucs --input ${uc_derep} --output UC_Pooled.parquet - - ## Summarize sequence abundance by OTU and sample - echo -e "\\n..Summarizing sequence abundance by OTU and sample\\n" - summarize_dereplicated_data.R \ - --seqtab ${seqtab} \ - --uc UC_Pooled.parquet \ - --seqs ${fasta} \ - --maxmeep ${params.max_MEEP} \ - --recoversinglet ${params.recover_lowqsingletons} \ - --mergesamples ${params.merge_replicates} \ - --threads ${task.cpus} - - """ -} - - -// Post-clustering curation -process lulu { - - label "main_container" - - publishDir "${params.outdir}/05.LULU", mode: "${params.storagemode}" - // cpus 8 - - input: - path otu_table - path sequences - - output: - path "OTU_table_LULU.txt.gz", emit: lulu - path "LULU_match_list.txt.gz", emit: matches - path "LULU_merging_statistics.txt.gz", emit: stats - path "OTUs_LULU.fa.gz", emit: fasta - tuple val("${task.process}"), val('mumu'), eval('mumu --version | head -n 1 | sed "s/mumu //"'), topic: versions - tuple val("${task.process}"), val('vsearch'), eval('vsearch --version 2>&1 | head -n 1 | sed "s/vsearch //g" | sed "s/,.*//g" | sed "s/^v//" | sed "s/_.*//"'), topic: versions - tuple val("${task.process}"), val('ripgrep'), eval('rg --version | head -1 | sed "s/ripgrep //"'), topic: versions - - script: - """ - echo -e "Post-clustering curation with MUMU (C++ implementation of LULU)\\n" - - ## If Clustered.fa.gz used as input - ## (but there are sequences excluded from the OTU table) - # echo -e "Removing size annotations from sequence headers" - # zcat ${sequences} \ - # | sed -r '/^>/ s/;size=[0-9]+//g' \ - # | gzip -${params.gzip_compression} > tmp_sequences.fa.gz - - - ## MUMU similarity threshold is specified as % (e.g., 84.0) - ## while VSEARCH requires a value in 0-1 range (e.g., 0.84) - - ## With bc - # VSID=\$(echo "scale=4; x = ${params.lulu_match} / 100; if(x<1) print 0; x" | bc) - - ## With awk - VSID=\$(awk -v a=${params.lulu_match} 'BEGIN { print(a/100) }') - - echo -e "VSEARCH similarity threshold: " "\$VSID" - - ## Prepare match list (+ remove size annotations) - echo -e "Preparing match list\\n" - vsearch \ - --usearch_global ${sequences} \ - --db ${sequences} \ - --self \ - --id "\$VSID" \ - --iddef 1 \ - --gapopen ${params.vsearch_gapopen} \ - --gapext ${params.vsearch_gapext } \ - --query_cov 0.9 \ - --userfields query+target+id \ - --maxaccepts 0 \ - --maxhits ${params.lulu_maxhits} \ - --threads ${task.cpus} \ - --userout LULU_match_list.txt - - # Input otu_table = tab-separated, samples in columns - # Input match_list = tab-separated, OTU pairwise similarity scores - - - echo -e "\\nUnpacking OTU table\\n" - gunzip --stdout ${otu_table} > tmp_OTU_table.txt - - echo -e "\\nRunning MUMU\\n" - mumu \ - --otu_table tmp_OTU_table.txt \ - --match_list LULU_match_list.txt \ - --new_otu_table OTU_table_LULU.txt \ - --log LULU_merging_statistics.txt \ - --threads ${task.cpus} \ - --minimum_match ${params.lulu_match} \ - --minimum_ratio ${params.lulu_ratio} \ - --minimum_ratio_type ${params.lulu_ratiotype} \ - --minimum_relative_cooccurence ${params.lulu_relcooc} - - echo -e "..Compressing LULU-curated OTU table\\n" - parallel -j 1 \ - "pigz -p ${task.cpus} -${params.gzip_compression} {}" \ - ::: "OTU_table_LULU.txt" "LULU_merging_statistics.txt" "LULU_match_list.txt" - - echo -e "..LULU done\\n" - - echo -e "\\nPreparing sequence subset\\n" - - echo -e "..Extracting OTU IDs\\n" - zcat OTU_table_LULU.txt.gz \ - | awk 'NR > 1 {print \$1}' \ - > curated_OTU_ids.txt - - echo -e "..Extracting sequences\\n" - rg -z -A 1 \ - -f curated_OTU_ids.txt \ - --context-separator "" \ - --threads ${task.cpus} \ - ${sequences} \ - | sed '/^\$/d' \ - | gzip -${params.gzip_compression} \ - > OTUs_LULU.fa.gz - - ## Remove temporary files - echo -e "\\nAll done!\\n" - echo -e "Removing temporary files\\n" - # rm tmp_sequences.fa.gz - rm tmp_OTU_table.txt curated_OTU_ids.txt - - """ -} - -// LULU merging statistics format: -// 1. `query_otu_name` - name of query OTU -// 2. `parent_otu_name` - name of potential parent OTU -// 3. `similarity_pct` - percentage of similarity (0 to 100) -// 4. `query_total_abundance` - total abundance of the query OTU (sum through all samples) -// 5. `parent_total_abundance` - total abundance of the potential parent OTU (sum through all samples) -// 6. `query_overlap_abundance` - overlap abundance of the query OTU (sum through all samples where the potential parent OTU is also present) -// 7. `parent_overlap_abundance` - overlap abundance of the potential parent OTU (sum through all samples where the query OTU is also present) -// 8. `query_incidence` - incidence of the query OTU (number of samples where the query OTU is present) -// 9. `parent_incidence` - incidence of the potential parent OTU (number of samples where the potential parent OTU is present) -// 10. `both_incidence` - incidence of the potential parent OTU (number of samples where both the potential parent OTU and the query OTU are present) -// 11. `smallest_abundance_ratio` - smallest abundance ratio (for each sample, compute the abundance of the potential parent OTU divided by the abundance of the query OTU) -// 12. `sum_abundance_ratios` - sum of the abundance ratios -// 13. `avg_abundance_ratio` - average value of abundance ratios -// 14. `smallest_nonnull_ratio` - smallest non-null abundance ratio (exclude ratios for samples where the query OTU is present but not the potential parent OTU) -// 15. `avg_nonnull_ratio` - average value of non-null abundance ratios (exclude ratios for samples where the query OTU is present but not the potential parent OTU) -// 16. `largest_ratio` - largest ratio value -// 17. `relative_cooccurrence` - relative co-occurence value (number of samples where both the potential parent OTU and the query OTU are present divided by the number of samples where the query OTU is present) -// 18. `status` - status: 'accepted' or 'rejected' -// The potential parent OTU is either accepted as a parent, or rejected - -// Auto documentation of analysis procedures -// (generate narrative description of methods) -process document_analysis_s2 { - - label "main_container" - - publishDir "${out_tracedir}", mode: 'copy', overwrite: true - // cpus 1 - - input: - path versions // "software_versions.yml" - path params // "pipeline_params.tsv" - - output: - path "README_Step2_Methods.txt", emit: docs - - - script: - """ - echo -e "Descriptive summary generation\n" - - document_s2.R \ - ${versions} \ - ${params} \ - README_Step2_Methods.txt - - """ -} - - - - -// Step-2 workflow -workflow S2 { - - // Find quality-filtered sequence tables - ch_seqtabs = Channel.fromPath( - params.data_path + "/**/07_SeqTable/Seqs.parquet", - checkIfExists: true).collect() - - // Aggregate sequences, remove de novo chimeras - aggregate_sequences(ch_seqtabs) - - // Pool and dereplicate all sequences - dereplication(aggregate_sequences.out.seqs) - derep_ch = dereplication.out.derep - derepuc_ch = dereplication.out.derep_uc - - - // // Pool and dereplicate all sequences - // if(params.alignment_penalties == "UNITE"){ - // // Clustering-based dereplication, allowing for a slight length variation of sequences - // dereplication_unite(ch_seqs) - // derep_ch = dereplication_unite.out.derep - // derepuc_ch = dereplication_unite.out.derep_uc - // } - // if(params.alignment_penalties == "default"){ - // // Fast, hash-based dereiplication - // dereplication(ch_seqs) - // derep_ch = dereplication.out.derep - // derepuc_ch = dereplication.out.derep_uc - // } - // - // NB. In case with large number of sequences, UNITE-style dereplication is extremly slow. - // Probably, it is possible to improve the speed, by using two steps: - // hash-based dereplication first, then additional round of clustering-based derep. - // But it would add extra complexity to manage and combine two UC files. - - // Prepare sequence table based on dereplicated sequences - // (no clustering, pre-clustering, or denoising) - if(params.preclustering == "none" & params.clustering == "none"){ - - summarize_dereplicated_data( - aggregate_sequences.out.seqs_parquet, - derepuc_ch, - derep_ch - ) - - } else { - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Clustering / pre-clustering / denoising with optional chunking - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - // No chunking (process all sequences at once) - if(params.chunking_n == null || params.chunking_n < 2){ - - CLUSTERING(derep_ch) - - preclustuc_ch = CLUSTERING.out.preclustuc_ch - cluster_ch = CLUSTERING.out.cluster_ch - clustuc_ch = CLUSTERING.out.clustuc_ch - - } else { - // Chunking (process sequences in N chunks) - - // Groupd sequences into clusters - linclust(derep_ch) - - // Bucketize sequence clusters into chunks - bucketize(derep_ch, linclust.out.db_clu) - buckets_ch = bucketize.out.buckets.flatten() - - // Run clustering/pre-clustering/denoising subworkflow - CLUSTERING(buckets_ch) - - // collect UC and FASTA files from all chunks - preclustuc_chunks = CLUSTERING.out.preclustuc_ch.collect() - preclustaf_chunks = CLUSTERING.out.preclustaf_ch.collect() - cluster_chunks = CLUSTERING.out.cluster_ch.collect() - clustuc_chunks = CLUSTERING.out.clustuc_ch.collect() - - // Merge buckets into a single file - merge_buckets( - preclustuc_chunks, - preclustaf_chunks, - cluster_chunks, - clustuc_chunks) - - cluster_ch = merge_buckets.out.cluster_ch - clustuc_ch = merge_buckets.out.clustuc_ch - preclustuc_ch = merge_buckets.out.preclustuc_ch.ifEmpty(file('NoPrecluster')) - - } - - - /* - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Result processing - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - */ - - - // Pool UC files - merge_uc( - derepuc_ch, // UC file with dereplication info - preclustuc_ch, // UC file with pre-clustering or denoising (optional) - clustuc_ch // UC file with OTU clustering info - ) - - // Summarize sequence abundances by OTU and sample - summarize( - aggregate_sequences.out.seqs_parquet, // Step-1 sequnece tables in long format with de novo chimeras removed - merge_uc.out.uc, // Combined UC files with sequence membership info - cluster_ch // FASTA file with OTUs - ) - - // Post-clustering curation with LULU - if ( params.lulu == true ) { - lulu( - summarize.out.otutabwide, - summarize.out.seqs - // cluster_ch // In the Clustered.fa.gz, there are seqs excluded from OTU table - ) - } - - } // end of preclustering == "none" & clustering == "none" - - - // Run statistics - // run_summary() - - // Dump the software versions to a file - ch_versions_yml = software_versions_to_yaml(Channel.topic('versions')) - .collectFile( - storeDir: "${params.tracedir}", - name: 'software_versions.yml', - sort: true, - newLine: true - ) - - // Dump the parameters to a file - ch_params_tsv = dumpParamsTsv() - .collectFile( - storeDir: "${params.tracedir}", - name: "pipeline_params.tsv", - sort: true, - newLine: true - ) - - // Auto documentation of analysis procedures - document_analysis_s2( - ch_versions_yml, - ch_params_tsv) - -} diff --git a/src/pipecraft-core/service_scripts/NextITS_Pipeline.sh b/src/pipecraft-core/service_scripts/NextITS_Pipeline.sh index 8302cce0..83a3c767 100755 --- a/src/pipecraft-core/service_scripts/NextITS_Pipeline.sh +++ b/src/pipecraft-core/service_scripts/NextITS_Pipeline.sh @@ -1,38 +1,13 @@ #!/bin/bash -export NXF_HOME="/Input/.nextflow" +export NXF_HOME="/opt/software/conda/bin" export NXF_ANSI_LOG="false" export NXF_LOG_COLOR="false" export NXF_ANSI="false" export TERM="dumb" -mkdir -p $NXF_HOME BASEDIR=$(pwd) -fix_permissions() { - # Try different possible locations for the NextITS scripts - for dir in \ - "/scripts/NextITS/bin" \ - "$NXF_HOME/assets/vmikk/NextITS/bin" \ - "/Input/.nextflow/assets/vmikk/NextITS/bin" \ - "$HOME/.nextflow/assets/vmikk/NextITS/bin" \ - "./work/*/vmikk/NextITS/bin" - do - if [ -d "$dir" ]; then - echo "Setting permissions for scripts in $dir" - find "$dir" -name "*.R" -exec chmod +x {} \; 2>/dev/null - find "$dir" -name "*.py" -exec chmod +x {} \; 2>/dev/null - find "$dir" -name "*.sh" -exec chmod +x {} \; 2>/dev/null - # Also fix line endings in case they're causing issues - find "$dir" -name "*.R" -exec sed -i 's/\r$//' {} \; 2>/dev/null - find "$dir" -name "*.py" -exec sed -i 's/\r$//' {} \; 2>/dev/null - find "$dir" -name "*.sh" -exec sed -i 's/\r$//' {} \; 2>/dev/null - fi - done -} - -fix_permissions - ls -la ## Run Step-1 for all sequencing runs @@ -51,7 +26,8 @@ find /Input/ -mindepth 1 -maxdepth 1 -type d \ ## Step-2 - standard VSEARCH clustering -stdbuf -oL -eL nextflow run /scripts/NextITS \ +stdbuf -oL -eL \ + nextflow run /opt/pipelines/NextITS/main.nf \ -resume \ --storagemode "copy" \ -params-file /scripts/NextFlowConfig.json \ diff --git a/src/pipecraft-core/service_scripts/submodules/NextITS_Step1.sh b/src/pipecraft-core/service_scripts/submodules/NextITS_Step1.sh index 26ea9932..31ba6b9b 100755 --- a/src/pipecraft-core/service_scripts/submodules/NextITS_Step1.sh +++ b/src/pipecraft-core/service_scripts/submodules/NextITS_Step1.sh @@ -38,7 +38,7 @@ ## Step-1 - with pre-demultiplexed data -run_nextflow run /scripts/NextITS \ +run_nextflow run /opt/pipelines/NextITS/main.nf \ -resume \ --step "Step1" \ --storagemode "copy" \