.gitignore

@@ -100,3 +100,12 @@ ENV/
 
 # Mac OS
 .DS_Store
+
+# nextflow analysis folders/files
+pipeline_trace*.txt*
+.nextflow*.log*
+report*.html*
+timeline*.html*
+/workflow/output/*
+/work/*
+/.nextflow/*

.gitlab-ci.yml

 before_script:
   - module add  python/3.6.1-2-anaconda
   - pip install --user pytest-pythonpath==0.7.1 pytest-cov==2.5.1
-  - module load  nextflow/0.31.0
+  - module load nextflow/0.31.0
   - ln -s /project/shared/bicf_workflow_ref/workflow_testdata/chipseq/*fastq.gz test_data/
 
 stages:
@@ -10,32 +10,37 @@ stages:
   - single
   - multiple
   - skip
+  - cleanup
 
 user_configuration:
   stage: unit
   script:
-  - pytest -m unit
   - pytest -m unit --cov=./workflow/scripts
 
+bash_tests:
+  stage: unit
+  script:
+    - module load singularity/3.0.2
+    - module load deeptools/2.5.0.1
+    - singularity run docker://bats/bats:v1.1.0 --tap workflow/tests/plot_profile.bats
+
 astrocyte:
   stage: astrocyte
   script:
-  - module load astrocyte/0.1.0
+  - module load astrocyte/0.3.1
   - module unload nextflow
   - cd ..
   - astrocyte_cli validate chipseq_analysis
-  artifacts:
-    expire_in: 2 days
+  after_script:
+    - rm -rf work/
 
 single_end_mouse:
   stage: single
   only:
     - master
   script:
-  - nextflow run workflow/main.nf --astrocyte true -resume
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --astrocyte true --ci true --dev true
   - pytest -m singleend
-  artifacts:
-    expire_in: 2 days
 
 paired_end_human:
   stage: single
@@ -44,39 +49,55 @@ paired_end_human:
   except:
     - master
   script:
-  - nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_ENCSR729LGA_PE.txt" --genome 'GRCh38' --pairedEnd true --astrocyte false -resume
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_ENCSR729LGA_PE.txt" --genome 'GRCh38' --pairedEnd true --astrocyte false --ci true --dev true
   - pytest -m pairedend
-  artifacts:
-    expire_in: 2 days
 
-single_end_diff:
-  stage: multiple
+single_end_single_control:
+  stage: single
   only:
     - branches
   except:
     - master
   script:
-  - nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_diff_SE.txt" --genome 'GRCm38' --astrocyte false -resume
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_single_contol_SE.txt" --genome 'GRCh38' --pairedEnd false --astrocyte false --ci true --dev true
+  - pytest -m singlecontrol
+
+single_end_diff:
+  stage: multiple
+  only:
+    - master
+  script:
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_diff_SE.txt" --genome 'GRCm38' --astrocyte false --ci true --dev true
+  - pytest -m singleend
   - pytest -m singlediff
-  artifacts:
-    expire_in: 2 days
 
 paired_end_diff:
   only:
+    - branches
+  except:
     - master
   stage: multiple
   script:
-  - nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_diff_PE.txt" --genome 'GRCh38' --pairedEnd true --astrocyte false -resume
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_diff_PE.txt" --genome 'GRCh38' --pairedEnd true --astrocyte false -with-dag flowchart.pdf --ci true --dev true
+  - pytest -m pairedend
   - pytest -m paireddiff
   artifacts:
-    expire_in: 2 days
+    name: "$CI_JOB_NAME"
+    when: always
+    paths:
+      - flowchart.pdf
+    expire_in: 7 days
 
 single_end_skip:
   stage: skip
   only:
     - master
   script:
-  - nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_diff_SE.txt" --genome 'GRCm38' --skipDiff true --skipMotif true --astrocyte false -resume
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_diff_SE.txt" --genome 'GRCm38' --skipDiff true --skipMotif true --skipPlotProfile true --astrocyte false --ci true --dev true
   - pytest -m singleskip_true
-  artifacts:
-    expire_in: 2 days
+
+cleanup_job:
+  stage: cleanup
+  script:
+    - cd $CI_BUILDS_DIR/$CI_RUNNER_SHORT_TOKEN/$CI_PROJECT_NAME
+    - rm -fr $CI_PIPELINE_ID/

.gitlab/merge_request_templates/merge_request.md

@@ -6,6 +6,7 @@ These are the most common things requested on pull requests (PRs).
  - [ ] This comment contains a description of changes (with reason)
  - [ ] If you've fixed a bug or added code that should be tested, add tests!
  - [ ] Documentation in `docs` is updated
+ - [ ] Replace dag.png with the most recent CI pipleine integrated_pe artifact
  - [ ] `CHANGELOG.md` is updated
  - [ ] `README.md` is updated
  - [ ] `LICENSE.md` is updated with new contributors

CHANGELOG.md

@@ -2,7 +2,35 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased]
+## [publish_1.1.3 ] - 2020-08-16
+### Updated
+- Updated astrocyte to 0.3.1
+
+### Fixed
+- Fixed missing gene names in annotation
+
+## [publish_1.1.2 ] - 2020-06-22
+- Add pipeline tracking
+
+## [publish_1.1.1 ] - 2020-04-23
+### Added
+- Add Nextflow to references.md
+- Add test data for test_pool_and_pseudoreplicate
+- Add PlotProfile Option
+- Add Python version to MultiQC
+- Add and Update tests
+- Use GTF files instead of TxDb and org libraries in Annotate Peaks
+- Make gtf and geneName files as param inputs
+- Add test data for single control and single replicate
+
+### Fixed
+- Fix references.md link in citation of README.md
+- Fix pool_and_psuedoreplicate.py to run single experiment
+- Fix xcor to increase file size for --random-source
+- Fix skip diff test for paired-end data
+- Fix xcor to get lowest non zero value above 50
+- Fix references to display in Multiqc report
+- Update astrocyte testing to 0.2.0
 
 ## [publish_1.0.6 ] - 2019-05-31
 ### Added

README.md

-# **CHIPseq Manual**
-## Version 1.0.6
-## May 31, 2019
+# **ChIP-seq Manual**
+## Version 1.1.2
+## June 21, 2020
 
 # BICF ChIP-seq Pipeline
 
-[![Build Status](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/master/build.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/master)
-[![Coverage Report](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/master/coverage.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/master)
-[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.24.0-brightgreen.svg
-)](https://www.nextflow.io/)
-[![Astrocyte](https://img.shields.io/badge/astrocyte-%E2%89%A50.1.0-blue.svg)](https://astrocyte-test.biohpc.swmed.edu/static/docs/index.html)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2648845.svg)](https://doi.org/10.5281/zenodo.2648845)
+|*master*|*dev*|
+|:-:|:-:|
+|[![pipeline status](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/master/pipeline.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/master)|[![pipeline status](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/dev/pipeline.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/dev)|
+|[![coverage report](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/master/coverage.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/master)|[![coverage report](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/dev/coverage.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/dev)|
+
+[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.31.0-brightgreen)](https://www.nextflow.io/)
+[![Astrocyte](https://img.shields.io/badge/astrocyte-%E2%89%A50.3.1-blue)](https://astrocyte-test.biohpc.swmed.edu/static/docs/index.html)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2648844.svg)](https://doi.org/10.5281/zenodo.2648844)
 
 
 ## Introduction
-BICF ChIPseq is a bioinformatics best-practice analysis pipeline used for ChIP-seq (chromatin immunoprecipitation sequencing) data analysis at [BICF](http://www.utsouthwestern.edu/labs/bioinformatics/) at [UT Southwestern Department of Bioinformatics](http://www.utsouthwestern.edu/departments/bioinformatics/).
+BICF ChIP-seq is a bioinformatics best-practice analysis pipeline used for ChIP-seq (chromatin immunoprecipitation sequencing) data analysis at [BICF](http://www.utsouthwestern.edu/labs/bioinformatics/) at [UT Southwestern Department of Bioinformatics](http://www.utsouthwestern.edu/departments/bioinformatics/).
 
 The pipeline uses [Nextflow](https://www.nextflow.io), a bioinformatics workflow tool. It pre-processes raw data from FastQ inputs, aligns the reads and performs extensive quality-control on the results.
 
@@ -58,7 +60,10 @@ $ git clone git@git.biohpc.swmed.edu:BICF/Astrocyte/chipseq_analysis.git
       - --designFile '/path/to/file/design.txt',
       - --genome 'GRCm38', 'GRCh38', or 'GRCh37' (if you need to use another genome contact the [BICF](mailto:BICF@UTSouthwestern.edu))
       - --pairedEnd 'true' or 'false' (where 'true' is PE and 'false' is SE; default 'false')
-      - --outDir (optional) path and folder name of the output data, example: /home2/s000000/Desktop/Chipseq_output (if not specficied will be under workflow/output/)
+      - --skipDiff 'true' or 'false' (where 'true' is skip differential peak and 'false' is do peak differential peak calling; default 'false')
+      - --skipMotif 'true' or 'false' (where 'true' is skip motif calling and 'false' is do motif calling; default 'false')
+      - --skipPlotProfile 'true' or 'false' (where 'true' is skip metageneplot for TSS and 'false' is do metageneplot for TSS; default 'false')
+      - --outDir (optional) path and folder name of the output data, example: /home2/s000000/Desktop/Chipseq_output (if not specified will be under workflow/output/)
 
 ## Pipeline
   + There are 11 steps to the pipeline
@@ -73,6 +78,7 @@ $ git clone git@git.biohpc.swmed.edu:BICF/Astrocyte/chipseq_analysis.git
     9. Annotate all peaks using ChipSeeker
     10. Calculate Differential Binding Activity with DiffBind (If more than 1 rep in more than 1 experiment)
     11. Use MEME-ChIP to find motifs in original peaks
+    12. Plot enrichment of signal around TSS
 
 See [FLOWCHART](docs/flowchart.pdf)
 
@@ -116,6 +122,8 @@ diffPeaks | normcount_peaksets.txt | Use only for replicated samples; peak set v
 diffPeaks | pca.pdf | Use only for replicated samples; PCA of peak location and peak intensity
 diffPeaks | *_diffbind.bed | Use only for replicated samples; bed file of peak locations between replicates
 diffPeaks | *_diffbind.csv | Use only for replicated samples; CSV file of peaks between replicates
+plotProfile | plotProfile.png | Plot profile of the TSS region
+plotProfile | computeMatrix.gz | Compute Matrix from deeptools to create custom plots other than plotProfile
 
 ## Common Quality Control Metrics
   + These are the list of files that should be reviewed before continuing on with the CHIPseq experiment. If your experiment fails any of these metrics, you should pause and re-evaluate whether the data should remain in the study.
@@ -130,25 +138,25 @@ diffPeaks | *_diffbind.csv | Use only for replicated samples; CSV file of peaks
 If you find an error, please let the [BICF](mailto:BICF@UTSouthwestern.edu) know and we will add it here.
 
 ## Citation
-Please cite individual programs and versions used [HERE](docs/references.txt), and the pipeline doi:[10.5281/zenodo.2648844](https://doi.org/10.5281/zenodo.2648844). Please cite in publications: Pipeline was developed by BICF from funding provided by Cancer Prevention and Research Institute of Texas (RP150596).
+Please cite individual programs and versions used [HERE](docs/references.md), and the pipeline doi:[10.5281/zenodo.2648844](https://doi.org/10.5281/zenodo.2648844). Please cite in publications: Pipeline was developed by BICF from funding provided by Cancer Prevention and Research Institute of Texas (RP150596).
 
 ## Programs and Versions
-  + python/3.6.1-2-anaconda [website](https://www.anaconda.com/download/#linux) [citation](docs/references.txt)
-  + trimgalore/0.4.1 [website](https://github.com/FelixKrueger/TrimGalore) [citation](docs/references.txt)
-  + cutadapt/1.9.1 [website](https://cutadapt.readthedocs.io/en/stable/index.html) [citation](docs/references.txt)
-  + bwa/intel/0.7.12 [website](http://bio-bwa.sourceforge.net/) [citation](docs/references.txt)
-  + samtools/1.6 [website](http://samtools.sourceforge.net/) [citation](docs/references.txt)
-  + sambamba/0.6.6 [website](http://lomereiter.github.io/sambamba/) [citation](docs/references.txt)
-  + bedtools/2.26.0 [website](https://bedtools.readthedocs.io/en/latest/) [citation](docs/references.txt)
-  + deeptools/2.5.0.1 [website](https://deeptools.readthedocs.io/en/develop/) [citation](docs/references.txt)
-  + phantompeakqualtools/1.2 [website](https://github.com/kundajelab/phantompeakqualtools) [citation](docs/references.txt)
-  + macs/2.1.0-20151222 [website](http://liulab.dfci.harvard.edu/MACS/) [citation](docs/references.txt)
-  + UCSC_userApps/v317 [website](https://genome.ucsc.edu/util.html) [citation](docs/references.txt)
-  + R/3.4.1 [website](https://www.r-project.org/) [citation](docs/references.txt)
+  + python/3.6.1-2-anaconda [website](https://www.anaconda.com/download/#linux) [citation](docs/references.md)
+  + trimgalore/0.4.1 [website](https://github.com/FelixKrueger/TrimGalore) [citation](docs/references.md)
+  + cutadapt/1.9.1 [website](https://cutadapt.readthedocs.io/en/stable/index.html) [citation](docs/references.md)
+  + bwa/intel/0.7.12 [website](http://bio-bwa.sourceforge.net/) [citation](docs/references.md)
+  + samtools/1.6 [website](http://samtools.sourceforge.net/) [citation](docs/references.md)
+  + sambamba/0.6.6 [website](http://lomereiter.github.io/sambamba/) [citation](docs/references.md)
+  + bedtools/2.26.0 [website](https://bedtools.readthedocs.io/en/latest/) [citation](docs/references.md)
+  + deeptools/2.5.0.1 [website](https://deeptools.readthedocs.io/en/develop/) [citation](docs/references.md)
+  + phantompeakqualtools/1.2 [website](https://github.com/kundajelab/phantompeakqualtools) [citation](docs/references.md)
+  + macs/2.1.0-20151222 [website](http://liulab.dfci.harvard.edu/MACS/) [citation](docs/references.md)
+  + UCSC_userApps/v317 [website](https://genome.ucsc.edu/util.html) [citation](docs/references.md)
+  + R/3.4.1 [website](https://www.r-project.org/) [citation](docs/references.md)
   + SPP/1.14
-  + meme/4.11.1-gcc-openmpi [website](http://meme-suite.org/doc/install.html?man_type=web) [citation](docs/references.txt)
-  + ChIPseeker [website](https://bioconductor.org/packages/release/bioc/html/ChIPseeker.html) [citation](docs/references.txt)
-  + DiffBind [website](https://bioconductor.org/packages/release/bioc/html/DiffBind.html) [citation](docs/references.txt)
+  + meme/4.11.1-gcc-openmpi [website](http://meme-suite.org/doc/install.html?man_type=web) [citation](docs/references.md)
+  + ChIPseeker [website](https://bioconductor.org/packages/release/bioc/html/ChIPseeker.html) [citation](docs/references.md)
+  + DiffBind [website](https://bioconductor.org/packages/release/bioc/html/DiffBind.html) [citation](docs/references.md)
 
 
 

astrocyte_pkg.yml

@@ -9,7 +9,7 @@
 # A unique identifier for the workflow package, text/underscores only
 name: 'chipseq_analysis_bicf'
 # Who wrote this?
-author: 'Holly Ruess, Spencer D. Barnes, Beibei Chen and Venkat Malladi'
+author: 'Holly Ruess, Spencer D. Barnes, Jeremy A. Mathews, Beibei Chen and Venkat Malladi'
 # A contact email address for questions
 email: 'bicf@utsouthwestern.edu'
 # A more informative title for the workflow package
@@ -52,6 +52,7 @@ workflow_modules:
   - 'R/3.3.2-gccmkl'
   - 'meme/4.11.1-gcc-openmpi'
   - 'pandoc/2.7'
+  - 'singularity/3.0.2'
 
 
 # A list of parameters used by the workflow, defining how to present them,
@@ -142,6 +143,15 @@ workflow_parameters:
     description: |
       Skip motif calling
 
+  - id: skipPlotProfile
+    type: select
+    required: true
+    choices:
+      - [ 'true', 'true']
+      - [ 'false', 'false']
+    description: |
+      Skip Plot Profile Analysis
+
   - id: astrocyte
     type: select
     choices:

docs/index.md

@@ -20,6 +20,7 @@ Report issues to the Bioinformatic Core Facility [BICF](mailto:BICF@UTSouthweste
     9. Annotate all peaks using ChipSeeker
     10. Calculate Differential Binding Activity with DiffBind (If more than 1 rep in more than 1 experiment)
     11. Use MEME-ChIP to find motifs in original peaks
+    12. Plot enrichment of signal around TSS
 
 
 ## Workflow Parameters
@@ -90,6 +91,8 @@ diffPeaks | normcount_peaksets.txt | Use only for replicated samples; peak set v
 diffPeaks | pca.pdf | Use only for replicated samples; PCA of peak location and peak intensity
 diffPeaks | *_diffbind.bed | Use only for replicated samples; bed file of peak locations between replicates
 diffPeaks | *_diffbind.csv | Use only for replicated samples; CSV file of peaks between replicates
+plotProfile | plotProfile.png | Plot profile of the TSS region
+plotProfile | computeMatrix.gz | Compute Matrix from deeptools to create custom plots other than plotProfile
 
 ## Common Quality Control Metrics
   + These are the list of files that should be reviewed before continuing on with the CHIPseq experiment. If your experiment fails any of these metrics, you should pause and re-evaluate whether the data should remain in the study.
@@ -123,3 +126,6 @@ Please cite in publications: Pipeline was developed by BICF from funding provide
   + meme/4.11.1-gcc-openmpi [website](http://meme-suite.org/doc/install.html?man_type=web) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
   + ChIPseeker [website](https://bioconductor.org/packages/release/bioc/html/ChIPseeker.html) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
   + DiffBind [website](https://bioconductor.org/packages/release/bioc/html/DiffBind.html) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + MultiQC [website](https://multiqc.info/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + BICFChip-seqAnalysisWorkflow [website](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + Nextflow [website](https://www.nextflow.io/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)

docs/references.md

@@ -52,6 +52,9 @@
   * Ewels P., Magnusson M., Lundin S. and Käller M. 2016. MultiQC: Summarize analysis results for multiple tools and samples in a single report. Bioinformatics 32(19): 3047–3048. doi:[10.1093/bioinformatics/btw354](https://dx.doi.org/10.1093/bioinformatics/btw354)
 
 17. **BICF ChIP-seq Analysis Workflow**:
-  * Spencer D. Barnes, Holly Ruess, Beibei Chen, & Venkat S. Malladi. (2019). BICF ChIP-seq Analysis Workflow (publish_1.0.5). Zenodo. doi:[10.5281/zenodo.2648844](https://doi.org/10.5281/zenodo.2648844)
+  * Spencer D. Barnes, Holly Ruess, Jeremy A. Mathews, Beibei Chen, and Venkat S. Malladi. 2020. BICF ChIP-seq Analysis Workflow (publish_1.1.3). Zenodo. doi:[10.5281/zenodo.3986942](https://doi.org/10.5281/zenodo.3986942)
+
+18. **Nextflow**:
+  * Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., and Notredame, C. 2017. Nextflow enables reproducible computational workflows. Nature biotechnology, 35(4), 316.
 
 Please cite in publications: Pipeline was developed by BICF from funding provided by **Cancer Prevention and Research Institute of Texas (RP150596)**.

test_data/design_single_contol_SE.txt

+sample_id	experiment_id	biosample	factor	treatment	replicate	control_id	fastq_read1
+ENCLB497XZB	ENCSR000DXB	Panc1	H3K4me3	None	1	ENCLB304SBJ	ENCFF001GBW.fastq.gz
+ENCLB304SBJ	ENCSR000DXC	Panc1	Control	None	1	ENCLB304SBJ	ENCFF001HWJ.fastq.gz

test_data/fetch_test_data.sh

@@ -25,3 +25,9 @@ wget https://www.encodeproject.org/files/ENCFF161HBP/@@download/ENCFF161HBP.fast
 wget https://www.encodeproject.org/files/ENCFF776KZU/@@download/ENCFF776KZU.fastq.gz
 wget https://www.encodeproject.org/files/ENCFF119KHM/@@download/ENCFF119KHM.fastq.gz
 echo "Done with Paired-end"
+
+echo "Downloading Single-end data set Human ENCSR000DXB and ENCSR000DXC"
+wget https://www.encodeproject.org/files/ENCFF001GBW/@@download/ENCFF001GBW.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF001GBV/@@download/ENCFF001GBV.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF001HWJ/@@download/ENCFF001HWJ.fastq.gz
+echo "Done with Single-end"

test_data/test_cross.qc

+Test.20.tagAlign.gz	18588987	0,20,33	0.211525291335199,0.211232019956852,0.211139666755398	35	0.2123067	1500	0.209429	1.01001	0.7284536       0

workflow/conf/biohpc.config

@@ -2,6 +2,7 @@ process {
   executor = 'slurm'
   queue = 'super'
   clusterOptions = '--hold'
+  beforeScript= 'ulimit -Ss unlimited'
 
   // Process specific configuration
   withName: checkDesignFile {
@@ -44,6 +45,10 @@ process {
     module = ['python/3.6.1-2-anaconda', 'macs/2.1.0-20151222', 'UCSC_userApps/v317', 'bedtools/2.26.0', 'phantompeakqualtools/1.2']
     queue = '128GB,256GB,256GBv1'
   }
+  withName: plotProfile {
+    module = ['deeptools/2.5.0.1']
+    cpus = 32
+  }
   withName: consensusPeaks {
     module = ['python/3.6.1-2-anaconda', 'bedtools/2.26.0']
     executor = 'local'
@@ -61,7 +66,7 @@ process {
     cpus = 32
   }
   withName: multiqcReport {
-    module = ['python/3.6.1-2-anaconda', 'pandoc/2.7', 'multiqc/1.7']
+    module = ['python/3.6.1-2-anaconda', 'pandoc/2.7', 'singularity/3.0.2']
     executor = 'local'
   }
 }
@@ -70,22 +75,28 @@ params {
   // Reference file paths on BioHPC
   genomes {
     'GRCh38' {
-      bwa = '/project/shared/bicf_workflow_ref/GRCh38'
+      bwa = '/project/shared/bicf_workflow_ref/human/GRCh38'
       genomesize = 'hs'
-      chromsizes = '/project/shared/bicf_workflow_ref/GRCh38/genomefile.txt'
-      fasta = '/project/shared/bicf_workflow_ref/GRCh38/genome.fa'
+      chromsizes = '/project/shared/bicf_workflow_ref/human/GRCh38/genomefile.txt'
+      fasta = '/project/shared/bicf_workflow_ref/human/GRCh38/genome.fa'
+      gtf = '/project/shared/bicf_workflow_ref/human/GRCh38/gencode.v25.chr_patch_hapl_scaff.annotation.gtf'
+      geneNames = '/project/shared/bicf_workflow_ref/human/GRCh38/genenames.txt'
     }
     'GRCh37' {
-      bwa = '/project/shared/bicf_workflow_ref/GRCh37'
+      bwa = '/project/shared/bicf_workflow_ref/human/GRCh37'
       genomesize = 'hs'
-      chromsizes = '/project/shared/bicf_workflow_ref/GRCh37/genomefile.txt'
-      fasta = '/project/shared/bicf_workflow_ref/GRCh37/genome.fa'
+      chromsizes = '/project/shared/bicf_workflow_ref/human/GRCh37/genomefile.txt'
+      fasta = '/project/shared/bicf_workflow_ref/human/GRCh37/genome.fa'
+      gtf = '/project/shared/bicf_workflow_ref/human/GRCh37/gencode.v19.chr_patch_hapl_scaff.annotation.gtf'
+      geneNames = '/project/shared/bicf_workflow_ref/human/GRCh37/genenames.txt'
     }
     'GRCm38' {
-      bwa = '/project/shared/bicf_workflow_ref/GRCm38'
+      bwa = '/project/shared/bicf_workflow_ref/mouse/GRCm38'
       genomesize = 'mm'
-      chromsizes = '/project/shared/bicf_workflow_ref/GRCm38/genomefile.txt'
-      fasta = '/project/shared/bicf_workflow_ref/GRCm38/genome.fa'
+      chromsizes = '/project/shared/bicf_workflow_ref/mouse/GRCm38/genomefile.txt'
+      fasta = '/project/shared/bicf_workflow_ref/mouse/GRCm38/genome.fa'
+      gtf = '/project/shared/bicf_workflow_ref/mouse/GRCm38/gencode.vM20.annotation.gtf'
+      geneNames = '/project/shared/bicf_workflow_ref/mouse/GRCm38/genenames.txt'
     }
   }
 }

workflow/main.nf

@@ -25,26 +25,46 @@ params.topPeakCount = 600
 params.astrocyte = false
 params.skipDiff = false
 params.skipMotif = false
+params.skipPlotProfile = false
 params.references = "$baseDir/../docs/references.md"
 params.multiqc =  "$baseDir/conf/multiqc_config.yaml"
+params.ci = false
+params.dev = false
+
 
 // Assign variables if astrocyte
 if (params.astrocyte) {
   print("Running under astrocyte")
   referenceLocation = "/project/shared/bicf_workflow_ref"
-  params.bwaIndex = "$referenceLocation/$params.genome"
-  params.chromSizes = "$referenceLocation/$params.genome/genomefile.txt"
-  params.fasta = "$referenceLocation/$params.genome/genome.fa"
-  if (params.genome == 'GRCh37' || params.genome == 'GRCh38') {
+  if (params.genome == 'GRCh37') {
+    params.bwaIndex = "$referenceLocation/human/$params.genome"
+    params.chromSizes = "$referenceLocation/human/$params.genome/genomefile.txt"
+    params.fasta = "$referenceLocation/human/$params.genome/genome.fa"
+    params.gtf = "$referenceLocation/human/$params.genome/gencode.v19.chr_patch_hapl_scaff.annotation.gtf"
+    params.geneNames = "$referenceLocation/human/$params.genome/genenames.txt"
     params.genomeSize = 'hs'
   } else if (params.genome == 'GRCm38') {
+    params.bwaIndex = "$referenceLocation/mouse/$params.genome"
+    params.chromSizes = "$referenceLocation/mouse/$params.genome/genomefile.txt"
+    params.fasta = "$referenceLocation/mouse/$params.genome/genome.fa"
+    params.gtf = "$referenceLocation/mouse/$params.genome/gencode.vM20.annotation.gtf"
+    params.geneNames = "$referenceLocation/mouse/$params.genome/genenames.txt"
     params.genomeSize = 'mm'
+  } else if (params.genome == 'GRCh38') {
+    params.bwaIndex = "$referenceLocation/human/$params.genome"
+    params.chromSizes = "$referenceLocation/human/$params.genome/genomefile.txt"
+    params.fasta = "$referenceLocation/human/$params.genome/genome.fa"
+    params.gtf = "$referenceLocation/human/$params.genome/gencode.v25.chr_patch_hapl_scaff.annotation.gtf"
+    params.geneNames = "$referenceLocation/human/$params.genome/genenames.txt"
+    params.genomeSize = 'hs'
   }
 } else {
     params.bwaIndex = params.genome ? params.genomes[ params.genome ].bwa ?: false : false
     params.genomeSize = params.genome ? params.genomes[ params.genome ].genomesize ?: false : false
     params.chromSizes = params.genome ? params.genomes[ params.genome ].chromsizes ?: false : false
     params.fasta = params.genome ? params.genomes[ params.genome ].fasta ?: false : false
+    params.gtf = params.genome ? params.genomes[ params.genome ].gtf ?: false : false
+    params.geneNames = params.genome ? params.genomes[ params.genome ].geneNames ?: false : false
 }
 
 
@@ -78,8 +98,36 @@ extendReadsLen = params.extendReadsLen
 topPeakCount = params.topPeakCount
 skipDiff = params.skipDiff
 skipMotif = params.skipMotif
+skipPlotProfile = params.skipPlotProfile
 references = params.references
 multiqc = params.multiqc
+gtfFile = params.gtf
+geneNames = params.geneNames
+
+/*
+ * trackStart: track start of pipeline
+ */
+
+process trackStart {
+  script:
+  """
+  hostname
+  ulimit -a
+
+  curl -H 'Content-Type: application/json' -X PUT -d '{ \
+      "sessionId": "${workflow.sessionId}", \
+      "pipeline": "chipseq_analysis", \
+      "start": "${workflow.start}", \
+      "astrocyte": ${params.astrocyte}, \
+      "status": "started", \
+      "nextflowVersion": "${workflow.nextflow.version}", \
+      "pipelineVersion": "1.1.2", \
+      "ci": ${params.ci}, \
+      "dev": ${params.dev}}' \
+  "https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking"
+  """
+}
+
 
 // Check design file for errors
 process checkDesignFile {
@@ -424,6 +472,7 @@ process callPeaksMACS {
   set sampleId, file('*.narrowPeak'), file('*.fc_signal.bw'), file('*.pvalue_signal.bw'), experimentId, biosample, factor, treatment, replicate, controlId into experimentPeaks
   file '*.xls' into callPeaksMACSsummit
   file('version_*.txt') into callPeaksMACSVersions
+  file("*.fc_signal.bw") into bigwigs
 
   script:
 
@@ -456,6 +505,29 @@ peaksDesign = experimentPeaks
               "$sampleId\t$peak\t$fcSignal\t$pvalueSignal\t$experimentId\t$biosample\t$factor\t$treatment\t$replicate\t$controlId\n"}
               .collectFile(name:'design_peak.tsv', seed:"sample_id\tpeaks\tfc_signal\tpvalue_signal\texperiment_id\tbiosample\tfactor\ttreatment\treplicate\tcontrol_id\n", storeDir:"$outDir/design")
 
+//plotProfile
+process plotProfile {
+  publishDir "$outDir/experimentQC", mode: 'copy'
+
+  input:
+
+  file bigWigList from bigwigs.collect()
+
+  output:
+
+  file '*.{png,gz}' into plotProfile
+
+  when:
+
+  !skipPlotProfile
+
+  script:
+  """
+  module load deeptools/2.5.0.1
+  bash $baseDir/scripts/plot_profile.sh -g $gtfFile
+  """
+}
+
 // Calculate Consensus Peaks
 process consensusPeaks {
 
@@ -504,7 +576,7 @@ process peakAnnotation {
 
   """
   module load R/3.3.2-gccmkl
-  Rscript $baseDir/scripts/annotate_peaks.R $designAnnotatePeaks $genome
+  Rscript $baseDir/scripts/annotate_peaks.R $designAnnotatePeaks $gtfFile $geneNames
   """
 
 }
@@ -575,7 +647,6 @@ process diffPeaks {
 
 // Generate Multiqc Report, gerernate Software Versions and references
 process multiqcReport {
-
   publishDir "$outDir/${task.process}", mode: 'copy'
 
   input:
@@ -608,11 +679,12 @@ process multiqcReport {
   """
   module load python/3.6.1-2-anaconda
   module load pandoc/2.7
-  module load multiqc/1.7
+  module load singularity/3.0.2
   echo $workflow.nextflow.version > version_nextflow.txt
-  multiqc --version > version_multiqc.txt
+  singularity exec /project/shared/bicf_workflow_ref/singularity_images/bicf-multiqc-2.0.0.img multiqc --version > version_multiqc.txt
+  python --version &> version_python.txt
   python3 $baseDir/scripts/generate_references.py -r $references -o software_references
   python3 $baseDir/scripts/generate_versions.py -o software_versions
-  multiqc -c $multiqc .
+  singularity exec /project/shared/bicf_workflow_ref/singularity_images/bicf-multiqc-2.0.0.img multiqc -c $multiqc .
   """
 }