863e467d · 1c18232a · 7dc50dc1 · b585c2ff · 144bd3f4 · 37906e07
--- a/.gitignore
+++ b/.gitignore
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Mac OS
+.DS_Store
+
+# nextflow analysis folders/files
+pipeline_trace*.txt*
+.nextflow*.log*
+report*.html*
+timeline*.html*
+/workflow/output/*
+/work/*
+/.nextflow/*
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
+before_script:
+  - module add  python/3.6.1-2-anaconda
+  - pip install --user pytest-pythonpath==0.7.1 pytest-cov==2.5.1
+  - module load nextflow/0.31.0
+  - ln -s /project/shared/bicf_workflow_ref/workflow_testdata/chipseq/*fastq.gz test_data/
+
+stages:
+  - unit
+  - astrocyte
+  - single
+  - multiple
+  - skip
+  - cleanup
+
+user_configuration:
+  stage: unit
+  script:
+  - pytest -m unit --cov=./workflow/scripts
+
+bash_tests:
+  stage: unit
+  script:
+    - module load singularity/3.0.2
+    - module load deeptools/2.5.0.1
+    - singularity run docker://bats/bats:v1.1.0 --tap workflow/tests/plot_profile.bats
+
+astrocyte:
+  stage: astrocyte
+  script:
+  - module load astrocyte/0.3.1
+  - module unload nextflow
+  - cd ..
+  - astrocyte_cli validate chipseq_analysis
+  after_script:
+    - rm -rf work/
+
+single_end_mouse:
+  stage: single
+  only:
+    - master
+  script:
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --astrocyte true --ci true --dev true
+  - pytest -m singleend
+
+paired_end_human:
+  stage: single
+  only:
+    - branches
+  except:
+    - master
+  script:
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_ENCSR729LGA_PE.txt" --genome 'GRCh38' --pairedEnd true --astrocyte false --ci true --dev true
+  - pytest -m pairedend
+
+single_end_single_control:
+  stage: single
+  only:
+    - branches
+  except:
+    - master
+  script:
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_single_contol_SE.txt" --genome 'GRCh38' --pairedEnd false --astrocyte false --ci true --dev true
+  - pytest -m singlecontrol
+
+single_end_diff:
+  stage: multiple
+  only:
+    - master
+  script:
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_diff_SE.txt" --genome 'GRCm38' --astrocyte false --ci true --dev true
+  - pytest -m singleend
+  - pytest -m singlediff
+
+paired_end_diff:
+  only:
+    - branches
+  except:
+    - master
+  stage: multiple
+  script:
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_diff_PE.txt" --genome 'GRCh38' --pairedEnd true --astrocyte false -with-dag flowchart.pdf --ci true --dev true
+  - pytest -m pairedend
+  - pytest -m paireddiff
+  artifacts:
+    name: "$CI_JOB_NAME"
+    when: always
+    paths:
+      - flowchart.pdf
+    expire_in: 7 days
+
+single_end_skip:
+  stage: skip
+  only:
+    - master
+  script:
+  - NXF_OPTS="-Dleveldb.mmap=false" nextflow run workflow/main.nf --designFile "$CI_PROJECT_DIR/test_data/design_diff_SE.txt" --genome 'GRCm38' --skipDiff true --skipMotif true --skipPlotProfile true --astrocyte false --ci true --dev true
+  - pytest -m singleskip_true
+
+cleanup_job:
+  stage: cleanup
+  script:
+    - cd $CI_BUILDS_DIR/$CI_RUNNER_SHORT_TOKEN/$CI_PROJECT_NAME
+    - rm -fr $CI_PIPELINE_ID/
--- a/.gitlab/merge_request_templates/merge_request.md
+++ b/.gitlab/merge_request_templates/merge_request.md
+
+Please fill in the appropriate checklist below (delete whatever is not relevant).
+These are the most common things requested on pull requests (PRs).
+
+## PR checklist
+ - [ ] This comment contains a description of changes (with reason)
+ - [ ] If you've fixed a bug or added code that should be tested, add tests!
+ - [ ] Documentation in `docs` is updated
+ - [ ] Replace dag.png with the most recent CI pipleine integrated_pe artifact
+ - [ ] `CHANGELOG.md` is updated
+ - [ ] `README.md` is updated
+ - [ ] `LICENSE.md` is updated with new contributors
--- a/test
+++ b/test
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [publish_1.1.3 ] - 2020-08-16
+### Updated
+- Updated astrocyte to 0.3.1
+
+### Fixed
+- Fixed missing gene names in annotation
+
+## [publish_1.1.2 ] - 2020-06-22
+- Add pipeline tracking
+
+## [publish_1.1.1 ] - 2020-04-23
+### Added
+- Add Nextflow to references.md
+- Add test data for test_pool_and_pseudoreplicate
+- Add PlotProfile Option
+- Add Python version to MultiQC
+- Add and Update tests
+- Use GTF files instead of TxDb and org libraries in Annotate Peaks
+- Make gtf and geneName files as param inputs
+- Add test data for single control and single replicate
+
+### Fixed
+- Fix references.md link in citation of README.md
+- Fix pool_and_psuedoreplicate.py to run single experiment
+- Fix xcor to increase file size for --random-source
+- Fix skip diff test for paired-end data
+- Fix xcor to get lowest non zero value above 50
+- Fix references to display in Multiqc report
+- Update astrocyte testing to 0.2.0
+
+## [publish_1.0.6 ] - 2019-05-31
+### Added
+- MIT LICENSE
+- Check for correct genome input
+- Check for correct output for motif serach
+
+### Fixed
+- Path to reference fasta file
+
+## [publish_1.0.5 ] - 2019-05-16
+### Fixed
+- Retitled documentation for skipDiff and skipMotif to be more clear
+- Add missing python module for motif search
+- Updated links for phantompeaktools and references
+- Fix callPeaks for single control experiments
+
+## [publish_1.0.3 ] - 2019-04-23
+### Fixed
+- Variable name for design file for Astrocyte
+- File path for design file to work for Astrocyte
+
+## [publish_1.0.0 ] - 2019-04-23
+Initial release of pipeline
--- a/LICENSE.md
+++ b/LICENSE.md
+MIT License
+
+Copyright (c) 2019, University of Texas Southwestern Medical Center.
+
+All rights reserved.
+
+Contributors: Spencer D. Barnes, Holly Ruess, Jeremy A. Mathews, Beibei Chen, Venkat S. Malladi
+
+Department: Bioinformatic Core Facility, Department of Bioinformatics
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--- a/README.md
+++ b/README.md
+# **ChIP-seq Manual**
+## Version 1.1.2
+## June 21, 2020
+
+# BICF ChIP-seq Pipeline
+
+|*master*|*dev*|
+|:-:|:-:|
+|[![pipeline status](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/master/pipeline.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/master)|[![pipeline status](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/dev/pipeline.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/dev)|
+|[![coverage report](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/master/coverage.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/master)|[![coverage report](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/badges/dev/coverage.svg)](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/commits/dev)|
+
+[![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.31.0-brightgreen)](https://www.nextflow.io/)
+[![Astrocyte](https://img.shields.io/badge/astrocyte-%E2%89%A50.3.1-blue)](https://astrocyte-test.biohpc.swmed.edu/static/docs/index.html)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2648844.svg)](https://doi.org/10.5281/zenodo.2648844)
+
+
+## Introduction
+BICF ChIP-seq is a bioinformatics best-practice analysis pipeline used for ChIP-seq (chromatin immunoprecipitation sequencing) data analysis at [BICF](http://www.utsouthwestern.edu/labs/bioinformatics/) at [UT Southwestern Department of Bioinformatics](http://www.utsouthwestern.edu/departments/bioinformatics/).
+
+The pipeline uses [Nextflow](https://www.nextflow.io), a bioinformatics workflow tool. It pre-processes raw data from FastQ inputs, aligns the reads and performs extensive quality-control on the results.
+
+This pipeline is primarily used with a SLURM cluster on the [BioHPC Cluster](https://portal.biohpc.swmed.edu/content/). However, the pipeline should be able to run on any system that supports Nextflow.
+
+Additionally, the pipeline is designed to work with [Astrocyte Workflow System](https://astrocyte.biohpc.swmed.edu/static/docs/index.html) using a simple web interface.
+
+Current version of the software and issue reports are at
+https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis
+
+To download the current version of the software
+```bash
+$ git clone git@git.biohpc.swmed.edu:BICF/Astrocyte/chipseq_analysis.git
+```
+
+## Input files
+##### 1) Fastq Files
+  + You will need the full path to the files for the Bash Scipt
+
+##### 2) Design File
+  + The Design file is a tab-delimited file with 8 columns for Single-End and 9 columns for Paired-End.  Letter, numbers, and underlines can be used in the names. However, the names can only begin with a letter. Columns must be as follows:
+      1. sample_id&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;a short, unique, and concise name used to label output files; will be used as a control_id if it is the control sample
+      2. experiment_id&nbsp;&nbsp;&nbsp;&nbsp;biosample_treatment_factor; same name given for all replicates of treatment. Will be used for the consensus header.
+      3. biosample&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;symbol for tissue type or cell line
+      4. factor&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;symbol for antibody target
+      5. treatment&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;symbol of treatment applied
+      6. replicate&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;a number, usually from 1-3 (i.e. 1)
+      7. control_id&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;sample_id name that is the control for this sample
+      8. fastq_read1&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;name of fastq file 1 for SE or PC data
+      9. fastq_read2&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;name of fastq file 2 for PE data
+
+
+  + See [HERE](test_data/design_ENCSR729LGA_PE.txt) for an example design file, paired-end
+  + See [HERE](test_data/design_ENCSR238SGC_SE.txt) for an example design file, single-end
+
+##### 3) Bash Script
+  + You will need to create a bash script to run the CHIPseq pipeline on [BioHPC](https://portal.biohpc.swmed.edu/content/)
+  + This pipeline has been optimized for the correct partition
+  + See [HERE](docs/CHIPseq.sh) for an example bash script
+  + The parameters that must be specified are:
+      - --reads '/path/to/files/name.fastq.gz'
+      - --designFile '/path/to/file/design.txt',
+      - --genome 'GRCm38', 'GRCh38', or 'GRCh37' (if you need to use another genome contact the [BICF](mailto:BICF@UTSouthwestern.edu))
+      - --pairedEnd 'true' or 'false' (where 'true' is PE and 'false' is SE; default 'false')
+      - --skipDiff 'true' or 'false' (where 'true' is skip differential peak and 'false' is do peak differential peak calling; default 'false')
+      - --skipMotif 'true' or 'false' (where 'true' is skip motif calling and 'false' is do motif calling; default 'false')
+      - --skipPlotProfile 'true' or 'false' (where 'true' is skip metageneplot for TSS and 'false' is do metageneplot for TSS; default 'false')
+      - --outDir (optional) path and folder name of the output data, example: /home2/s000000/Desktop/Chipseq_output (if not specified will be under workflow/output/)
+
+## Pipeline
+  + There are 11 steps to the pipeline
+    1. Check input files
+    2. Trim adaptors TrimGalore!
+    3. Aligned trimmed reads with bwa, and sorts/converts to bam with samtools
+    4. Mark duplicates with Sambamba, and filter reads with samtools
+    5. Quality metrics with deep tools
+    6. Calculate cross-correlation using PhantomPeakQualTools
+    7. Call peaks with MACS
+    8. Calculate consensus peaks
+    9. Annotate all peaks using ChipSeeker
+    10. Calculate Differential Binding Activity with DiffBind (If more than 1 rep in more than 1 experiment)
+    11. Use MEME-ChIP to find motifs in original peaks
+    12. Plot enrichment of signal around TSS
+
+See [FLOWCHART](docs/flowchart.pdf)
+
+## Output Files
+Folder | File | Description
+--- | --- | ---
+design | N/A | Inputs used for analysis; can ignore
+trimReads | *_trimming_report.txt | report detailing how many reads were trimmed
+trimReads | *_trimmed.fq.gz | trimmed fastq files used for analysis
+alignReads | *.srt.bam.flagstat.qc | QC metrics from the mapping process
+alignReads | *.srt.bam | sorted bam file
+filterReads | *.dup.qc | QC metrics of find duplicate reads (sambamba)
+filterReads | *.filt.nodup.bam | filtered bam file with duplicate reads removed
+filterReads | *.filt.nodup.bam.bai | indexed filtered bam file
+filterReads | *.filt.nodup.flagstat.qc | QC metrics of filtered bam file (mapping stats, samtools)
+filterReads | *.filt.nodup.pbc.qc | QC metrics of library complexity
+convertReads | *.filt.nodup.bedse.gz | bed alignment in BEDPE format
+convertReads | *.filt.nodup.tagAlign.gz | bed alignent in BEDPE format, same as bedse unless samples are paired-end
+multiqcReport | multiqc_report.html | Quality control report of NRF, PBC1, PBC2, NSC, and RSC. Also contains software versions and references to cite.
+experimentQC | coverage.pdf | plot to assess the sequencing depth of a given sample
+experimentQC | *_fingerprint.pdf | plot to determine if the antibody-treatment enriched sufficiently
+experimentQC | heatmeap_SpearmanCorr.pdf | plot of Spearman correlation between samples
+experimentQC | heatmeap_PearsonCorr.pdf | plot of Pearson correlation between samples
+experimentQC | sample_mbs.npz | array of multiple BAM summaries
+crossReads | *.cc.plot.pdf | Plot of cross-correlation to assess signal-to-noise ratios
+crossReads | *.cc.qc | cross-correlation metrics. File [HEADER](docs/xcor_header.txt)
+callPeaksMACS | pooled/*pooled.fc_signal.bw | bigwig data file; raw fold enrichment of sample/control
+callPeaksMACS | pooled/*pooled_peaks.xls | Excel file of peaks
+callPeaksMACS | pooled/*.pvalue_signal.bw | bigwig data file; sample/control signal adjusted for pvalue significance
+callPeaksMACS | pooled/*_pooled.narrowPeak | peaks file; see [HERE](https://genome.ucsc.edu/FAQ/FAQformat.html#format12) for ENCODE narrowPeak header format
+consensusPeaks | *.rejected.narrowPeak | peaks not supported by multiple testing (replicates and pseudo-replicates)
+consensusPeaks | *.replicated.narrowPeak | peaks supported by multiple testing (replicates and pseudo-replicates)
+peakAnnotation | *.chipseeker_annotation.tsv | annotated narrowPeaks file
+peakAnnotation | *.chipseeker_pie.pdf | pie graph of where narrow annotated peaks occur
+peakAnnotation | *.chipseeker_upsetplot.pdf | upsetplot showing the count of overlaps of the genes with different annotated location
+motifSearch | *_memechip/index.html | interactive HTML link of MEME output
+motifSearch | sorted-*.replicated.narrowPeak | Top 600 peaks sorted by p-value; input for motifSearch
+motifSearch | *_memechip/combined.meme | MEME identified motifs
+diffPeaks | heatmap.pdf | Use only for replicated samples; heatmap of relationship of peak location and peak intensity
+diffPeaks | normcount_peaksets.txt | Use only for replicated samples; peak set values of each sample
+diffPeaks | pca.pdf | Use only for replicated samples; PCA of peak location and peak intensity
+diffPeaks | *_diffbind.bed | Use only for replicated samples; bed file of peak locations between replicates
+diffPeaks | *_diffbind.csv | Use only for replicated samples; CSV file of peaks between replicates
+plotProfile | plotProfile.png | Plot profile of the TSS region
+plotProfile | computeMatrix.gz | Compute Matrix from deeptools to create custom plots other than plotProfile
+
+## Common Quality Control Metrics
+  + These are the list of files that should be reviewed before continuing on with the CHIPseq experiment. If your experiment fails any of these metrics, you should pause and re-evaluate whether the data should remain in the study.
+    1. multiqcReport/multiqc_report.html: follow the ChiP-seq standards [HERE](https://www.encodeproject.org/chip-seq/);
+    2. experimentQC/*_fingerprint.pdf: make sure the plots information is correct for your antibody/input. See [HERE](https://deeptools.readthedocs.io/en/develop/content/tools/plotFingerprint.html) for more details.
+    3. crossReads/*cc.plot.pdf: make sure your sample data has the correct signal intensity and location.  See [HERE](hhttps://ccg.epfl.ch//var/sib_april15/cases/landt12/strand_correlation.html) for more details.
+    4. crossReads/*.cc.qc: Column 9 (NSC) should be > 1.1 for experiment and < 1.1 for input. Column 10 (RSC) should be > 0.8 for experiment and < 0.8 for input. See [HERE](https://genome.ucsc.edu/encode/qualityMetrics.html) for more details.
+    5. experimentQC/coverage.pdf, experimentQC/heatmeap_SpearmanCorr.pdf, experimentQC/heatmeap_PearsonCorr.pdf: See [HERE](https://deeptools.readthedocs.io/en/develop/content/list_of_tools.html) for more details.
+
+
+## Common Errors
+If you find an error, please let the [BICF](mailto:BICF@UTSouthwestern.edu) know and we will add it here.
+
+## Citation
+Please cite individual programs and versions used [HERE](docs/references.md), and the pipeline doi:[10.5281/zenodo.2648844](https://doi.org/10.5281/zenodo.2648844). Please cite in publications: Pipeline was developed by BICF from funding provided by Cancer Prevention and Research Institute of Texas (RP150596).
+
+## Programs and Versions
+  + python/3.6.1-2-anaconda [website](https://www.anaconda.com/download/#linux) [citation](docs/references.md)
+  + trimgalore/0.4.1 [website](https://github.com/FelixKrueger/TrimGalore) [citation](docs/references.md)
+  + cutadapt/1.9.1 [website](https://cutadapt.readthedocs.io/en/stable/index.html) [citation](docs/references.md)
+  + bwa/intel/0.7.12 [website](http://bio-bwa.sourceforge.net/) [citation](docs/references.md)
+  + samtools/1.6 [website](http://samtools.sourceforge.net/) [citation](docs/references.md)
+  + sambamba/0.6.6 [website](http://lomereiter.github.io/sambamba/) [citation](docs/references.md)
+  + bedtools/2.26.0 [website](https://bedtools.readthedocs.io/en/latest/) [citation](docs/references.md)
+  + deeptools/2.5.0.1 [website](https://deeptools.readthedocs.io/en/develop/) [citation](docs/references.md)
+  + phantompeakqualtools/1.2 [website](https://github.com/kundajelab/phantompeakqualtools) [citation](docs/references.md)
+  + macs/2.1.0-20151222 [website](http://liulab.dfci.harvard.edu/MACS/) [citation](docs/references.md)
+  + UCSC_userApps/v317 [website](https://genome.ucsc.edu/util.html) [citation](docs/references.md)
+  + R/3.4.1 [website](https://www.r-project.org/) [citation](docs/references.md)
+  + SPP/1.14
+  + meme/4.11.1-gcc-openmpi [website](http://meme-suite.org/doc/install.html?man_type=web) [citation](docs/references.md)
+  + ChIPseeker [website](https://bioconductor.org/packages/release/bioc/html/ChIPseeker.html) [citation](docs/references.md)
+  + DiffBind [website](https://bioconductor.org/packages/release/bioc/html/DiffBind.html) [citation](docs/references.md)
+
+
+
+## Credits
+This example worklow is derived from original scripts kindly contributed by the Bioinformatic Core Facility ([BICF](https://www.utsouthwestern.edu/labs/bioinformatics/)), in the [Department of Bioinformatics](https://www.utsouthwestern.edu/departments/bioinformatics/).
--- a/astrocyte_pkg.yml
+++ b/astrocyte_pkg.yml
@@ -9,15 +9,15 @@
 # A unique identifier for the workflow package, text/underscores only
 name: 'chipseq_analysis_bicf'
 # Who wrote this?
-author: 'Beibei Chen'
+author: 'Holly Ruess, Spencer D. Barnes, Jeremy A. Mathews, Beibei Chen and Venkat Malladi'
 # A contact email address for questions
-email: 'biohpc-help@utsouthwestern.edu'
+email: 'bicf@utsouthwestern.edu'
 # A more informative title for the workflow package
 title: 'BICF ChIP-seq Analysis Workflow'
 # A summary of the workflow package in plain text
 description: |
  This is a workflow package for the BioHPC/BICF ChIP-seq workflow system.
-  It implements a simple ChIP-seq analysis workflow using deepTools, Diffbind, ChipSeeker and MEME-ChIP, visualization application.
+  It implements ChIP-seq analysis workflow and visualization application.

 # -----------------------------------------------------------------------------
 # DOCUMENTATION
@@ -27,20 +27,33 @@ description: |
 # web interface. These files are in the 'docs' subdirectory. The first file
 # listed will be used as a documentation index and is index.md by convention
 documentation_files:
-  - ['index.md', 'chipseq-analysis']
+  - 'index.md'

 # -----------------------------------------------------------------------------
 # NEXTFLOW WORKFLOW CONFIGURATION
 # -----------------------------------------------------------------------------

-# Remember - The workflow file is always named 'workflow/main.f'
+# Remember - The workflow file is always named 'workflow/main.nf'
 #            The workflow must publish all final output into $baseDir

-# A list of clueter environment modules that this workflow requires to run.
+# A list of cluster environment modules that this workflow requires to run.
 # Specify versioned module names to ensure reproducability.
 workflow_modules:
-  - 'deeptools/2.3.5'
+  - 'python/3.6.1-2-anaconda'
+  - 'trimgalore/0.4.1'
+  - 'bwa/intel/0.7.12'
+  - 'samtools/1.6'
+  - 'sambamba/0.6.6'
+  - 'bedtools/2.26.0'
+  - 'deeptools/2.5.0.1'
+  - 'phantompeakqualtools/1.2'
+  - 'macs/2.1.0-20151222'
+  - 'UCSC_userApps/v317'
+  - 'R/3.3.2-gccmkl'
  - 'meme/4.11.1-gcc-openmpi'
+  - 'pandoc/2.7'
+  - 'singularity/3.0.2'
+

 # A list of parameters used by the workflow, defining how to present them,
 # options etc in the web interface. For each parameter:
@@ -76,41 +89,77 @@ workflow_modules:

 workflow_parameters:

-  - id: bams
+  - id: reads
    type: files
    required: true
    description: |
-      Bam files of all samples
+      One or more input FASTQ files from a ChIP-seq expereiment and a design
+      file with the link bewetwen the same file name and sample id
+    regex: ".*(fastq|fq)*gz"
+    min: 2

-  - id: peaks
-    type: files
+  - id: pairedEnd
+    type: select
    required: true
+    choices:
+      - [ 'true', 'true']
+      - [ 'false', 'false']
    description: |
-      Peak files of all samples. Peaks should be sorted. Bed format
+      If Paired-end: True, if Single-end: False.

-  - id: design
-    type: files
+  - id: designFile
+    type: file
+    required: true
+    description: |
+      A design file listing sample id, fastq files, corresponding control id
+      and additional information about the sample.
+    regex: ".*txt"
+
+  - id: genome
+    type: select
+    required: true
+    choices:
+      - [ 'GRCh38', 'Human GRCh38']
+      - [ 'GRCh37', 'Human GRCh37']
+      - [ 'GRCm38', 'Mouse GRCm38']
+    description: |
+      Reference species and genome used for alignment and subsequent analysis.
+
+  - id: skipDiff
+    type: select
    required: true
-    regex: ".*(csv)"
+    choices:
+      - [ 'true', 'true']
+      - [ 'false', 'false']
    description: |
-      A design file listing pairs of sample name and sample group. Must be in csv format
-      Columns must include: SampleID,Tissue, Factor, Condition, Replicate, Peaks, bamReads, bamControl, ControlID, PeakCaller
+      Skip differential peak analysis

-  - id: genomepath
+  - id: skipMotif
    type: select
+    required: true
    choices:
-      - [ '/project/shared/bicf_workflow_ref/GRCh38', 'human GRCh38']
-      - [ '/project/shared/bicf_workflow_ref/GRCh37', 'human GRCh37']
-      - [ '/project/shared/bicf_workflow_ref/GRCm38', 'mouse GRCm38']
+      - [ 'true', 'true']
+      - [ 'false', 'false']
+    description: |
+      Skip motif calling
+
+  - id: skipPlotProfile
+    type: select
    required: true
+    choices:
+      - [ 'true', 'true']
+      - [ 'false', 'false']
    description: |
-      Reference genome for annotation
+      Skip Plot Profile Analysis

-  - id: toppeakcount
-    type: integer
+  - id: astrocyte
+    type: select
+    choices:
+      - [ 'true', 'true' ]
    required: true
+    default: 'true'
    description: |
-      The number of top peaks to use for motif discovery.
+      Ensure configuraton for astrocyte.


 # -----------------------------------------------------------------------------
@@ -121,7 +170,7 @@ workflow_parameters:
 #            The workflow must publish all final output into $baseDir

 # Name of the R module that the vizapp will run against
-vizapp_r_module: 'R/3.2.1-intel'
+vizapp_r_module: 'R/3.4.1-gccmkl'

 # List of any CRAN packages, not provided by the modules, that must be made
 # available to the vizapp
@@ -129,11 +178,6 @@ vizapp_cran_packages:
  - shiny
  - shinyFiles

-# # List of any Bioconductor packages, not provided by the modules, that must be made
-# available to the vizapp
-vizapp_bioc_packages:
-  - qusage
-#  - ballgown
-vizapp_github_packages:
-  - js229/Vennerable
-
+# List of any Bioconductor packages, not provided by the modules,
+# that must be made available to the vizapp
+vizapp_bioc_packages: []
--- a/docs/CHIPseq.sh
+++ b/docs/CHIPseq.sh
+#!/bin/bash
+
+#SBATCH --job-name=CHIPseq
+#SBATCH --partition=super
+#SBATCH --output=CHIPseq.%j.out
+#SBATCH --error=CHIPseq.%j.err
+
+module load nextflow/0.31.0
+module add  python/3.6.1-2-anaconda
+
+nextflow run workflow/main.nf \
+--reads '/path/to/*fastq.gz' \
+--designFile '/path/to/design.txt' \
+--genome 'GRCm38' \
+--pairedEnd 'true'
--- a/docs/design_example.txt
+++ b/docs/design_example.txt
+sample_id	experiment_id	biosample	factor	treatment	replicate	control_id	fastq_read1
+A1	A	tissueA	H3K27AC	None	1	B1	A1.fastq.gz
+A2	A	tissueA	H3K27AC	None	2	B2	A2.fastq.gz
+B1	B	tissueB	Input	None	1	B1	B1.fastq.gz
+B2	B	tissueB	Input	None	2	B2	B2.fastq.gz
--- a/docs/flowchart.pdf
+++ b/docs/flowchart.pdf
--- a/docs/images/phantompeakqualtools.png
+++ b/docs/images/phantompeakqualtools.png
--- a/docs/index.md
+++ b/docs/index.md
-# Astrocyte ChIPseq analysis Workflow Package
+# BICF ChIP-seq Analysis Workflow

-This SOP describes the analysis pipeline of downstream analysis of ChIP-seq sequencing data. This pipeline includes (1) Quality control, (2) Peak annotation, (3) Differential peak analysis, and (4) motif analysis. BAM files and SORTED peak BED files selected as input. For each file this workflow:
+## Introduction
+**ChIP-seq Analysis** is a bioinformatics best-practice analysis pipeline used for chromatin immunoprecipitation (ChIP-seq) data analysis.

-    1) Annotate all peaks using ChipSeeker
-    2) Qulity control and signal profiling with Deeptools 
-    3) Find differential expressed peaks using DiffBind
-    4) Annotate all differentially expressed peaks
-    5) Using MEME-ChIP in motif finding for both original peaks and differently expressed peaks
+The pipeline uses [Nextflow](https://www.nextflow.io), a bioinformatics workflow tool. It pre-processes raw data from FastQ inputs, aligns the reads and performs extensive quality-control on the results.

+Report issues to the Bioinformatic Core Facility [BICF](mailto:BICF@UTSouthwestern.edu)

-##Workflow Parameters
+### Pipeline Steps
+  + There are 11 steps to the pipeline
+    1. Check input files
+    2. Trim adaptors TrimGalore!
+    3. Aligned trimmed reads with bwa, and sorts/converts to bam with samtools
+    4. Mark duplicates with Sambamba, and filter reads with samtools
+    5. Quality metrics with deep tools
+    6. Calculate cross-correlation using PhantomPeakQualTools
+    7. Call peaks with MACS
+    8. Calculate consensus peaks
+    9. Annotate all peaks using ChipSeeker
+    10. Calculate Differential Binding Activity with DiffBind (If more than 1 rep in more than 1 experiment)
+    11. Use MEME-ChIP to find motifs in original peaks
+    12. Plot enrichment of signal around TSS

-    bam - Choose all ChIP-seq alignment files for analysis.
+
+## Workflow Parameters
+    1. One or more input FASTQ files from a ChIP-seq expereiment and a design file with the link bewetwen the same file name and sample id (required) - Choose all ChIP-seq fastq files for analysis.
+    2. In single-end sequencing, the sequencer reads a fragment from only one end to the other, generating the sequence of base pairs. In paired-end reading it starts at one read, finishes this direction at the specified read length, and then starts another round of reading from the opposite end of the fragment. (Paired-end: True, Single-end: False) (required)
+    3. A design file listing sample id, fastq files, corresponding control id and additional information about the sample.
    genome - Choose a genomic reference (genome).
-    peaks - Choose all the peak files for analysis. All peaks should be sorted by the user
-    toppeak - The number of top peaks used for motif analysis. Default is all
-    design - This file contains the experiment design information. CSV format
-
- The following columns are necessary, must be named as in template and can be in any order:
-
-    SampleID
-        The id of the sample. This will be the header in output files, please make sure it is concise
-    Tissue
-        Tissue of the sample
-    Factor
-        Factor of the experiment
-    Condition
-	    This is the group that will be used for pairwise differential expression analysis
-	Replicate
-	    Replicate id
-    Peaks
-        The file name of the peak file for this sample
-    bamReads
-        The file name of the IP BAM for this sample
-    bamControl
-        The file name of the control BAM for this sample
-    ContorlID
-        The id of the control sample
-    PeakCaller
-        The peak caller used
-	
-
-### Test Data
+    4. Reference species and genome used for alignment and subsequent analysis. (required)
+    5. Run differential peak analysis (required). Must have at least 2 replicates per experiment and at least 2 experiments.
+    6. Run motif calling (required). Top 600 peaks sorted by p-value.
+    7. Ensure configuration for astrocyte. (required; always true)

+## Design file

-### Credits
-This example worklow is derived from original scripts kindly contributed by the Bioinformatic Core Facility (BICF), Department of Bioinformatics
+  + The Design file is a tab-delimited file with 8 columns for Single-End and 9 columns for Paired-End.  Letter, numbers, and underlines can be used in the names. However, the names can only begin with a letter. Columns must be as follows:
+      1. sample_id&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;a short, unique, and concise name used to label output files; will be used as a control_id if it is the control sample
+      2. experiment_id&nbsp;&nbsp;&nbsp;&nbsp;biosample_treatment_factor; same name given for all replicates of treatment. Will be used for the consensus header.
+      3. biosample&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;symbol for tissue type or cell line
+      4. factor&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;symbol for antibody target
+      5. treatment&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;symbol of treatment applied
+      6. replicate&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;a number, usually from 1-3 (i.e. 1)
+      7. control_id&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;sample_id name that is the control for this sample
+      8. fastq_read1&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;name of fastq file 1 for SE or PC data
+      9. fastq_read2&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;name of fastq file 2 for PE data

-### References

-* ChipSeeker: http://bioconductor.org/packages/release/bioc/html/ChIPseeker.html
-* DiffBind: http://bioconductor.org/packages/release/bioc/html/DiffBind.html
-* Deeptools: https://deeptools.github.io/
-* MEME-ChIP: http://meme-suite.org/doc/meme-chip.html
+  + See [HERE](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/test_data/design_ENCSR729LGA_PE.txt) for an example design file, paired-end
+  + See [HERE](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/test_data/design_ENCSR238SGC_SE.txt) for an example design file, single-end
+
+## Output Files
+
+Folder | File | Description
+--- | --- | ---
+design | N/A | Inputs used for analysis; can ignore
+trimReads | *_trimming_report.txt | report detailing how many reads were trimmed
+trimReads | *_trimmed.fq.gz | trimmed fastq files used for analysis
+alignReads | *.srt.bam.flagstat.qc | QC metrics from the mapping process
+alignReads | *.srt.bam | sorted bam file
+filterReads | *.dup.qc | QC metrics of find duplicate reads (sambamba)
+filterReads | *.filt.nodup.bam | filtered bam file with duplicate reads removed
+filterReads | *.filt.nodup.bam.bai | indexed filtered bam file
+filterReads | *.filt.nodup.flagstat.qc | QC metrics of filtered bam file (mapping stats, samtools)
+filterReads | *.filt.nodup.pbc.qc | QC metrics of library complexity
+convertReads | *.filt.nodup.bedse.gz | bed alignment in BEDPE format
+convertReads | *.filt.nodup.tagAlign.gz | bed alignent in BEDPE format, same as bedse unless samples are paired-end
+multiqcReport | multiqc_report.html | Quality control report of NRF, PBC1, PBC2, NSC, and RSC. Also contains software versions and references to cite.
+experimentQC | coverage.pdf | plot to assess the sequencing depth of a given sample
+experimentQC | *_fingerprint.pdf | plot to determine if the antibody-treatment enriched sufficiently
+experimentQC | heatmeap_SpearmanCorr.pdf | plot of Spearman correlation between samples
+experimentQC | heatmeap_PearsonCorr.pdf | plot of Pearson correlation between samples
+experimentQC | sample_mbs.npz | array of multiple BAM summaries
+crossReads | *.cc.plot.pdf | Plot of cross-correlation to assess signal-to-noise ratios
+crossReads | *.cc.qc | cross-correlation metrics. File [HEADER](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/xcor_header.txt)
+callPeaksMACS | pooled/*pooled.fc_signal.bw | bigwig data file; raw fold enrichment of sample/control
+callPeaksMACS | pooled/*pooled_peaks.xls | Excel file of peaks
+callPeaksMACS | pooled/*.pvalue_signal.bw | bigwig data file; sample/control signal adjusted for pvalue significance
+callPeaksMACS | pooled/*_pooled.narrowPeak | peaks file; see [HERE](https://genome.ucsc.edu/FAQ/FAQformat.html#format12) for ENCODE narrowPeak header format
+consensusPeaks | *.rejected.narrowPeak | peaks not supported by multiple testing (replicates and pseudo-replicates)
+consensusPeaks | *.replicated.narrowPeak | peaks supported by multiple testing (replicates and pseudo-replicates)
+peakAnnotation | *.chipseeker_annotation.tsv | annotated narrowPeaks file
+peakAnnotation | *.chipseeker_pie.pdf | pie graph of where narrow annotated peaks occur
+peakAnnotation | *.chipseeker_upsetplot.pdf | upsetplot showing the count of overlaps of the genes with different annotated location
+motifSearch | *_memechip/index.html | interactive HTML link of MEME output
+motifSearch | sorted-*.replicated.narrowPeak | Top 600 peaks sorted by p-value; input for motifSearch
+motifSearch | *_memechip/combined.meme | MEME identified motifs
+diffPeaks | heatmap.pdf | Use only for replicated samples; heatmap of relationship of peak location and peak intensity
+diffPeaks | normcount_peaksets.txt | Use only for replicated samples; peak set values of each sample
+diffPeaks | pca.pdf | Use only for replicated samples; PCA of peak location and peak intensity
+diffPeaks | *_diffbind.bed | Use only for replicated samples; bed file of peak locations between replicates
+diffPeaks | *_diffbind.csv | Use only for replicated samples; CSV file of peaks between replicates
+plotProfile | plotProfile.png | Plot profile of the TSS region
+plotProfile | computeMatrix.gz | Compute Matrix from deeptools to create custom plots other than plotProfile
+
+## Common Quality Control Metrics
+  + These are the list of files that should be reviewed before continuing on with the CHIPseq experiment. If your experiment fails any of these metrics, you should pause and re-evaluate whether the data should remain in the study.
+    1. multiqcReport/multiqc_report.html: follow the ChiP-seq standards [HERE](https://www.encodeproject.org/chip-seq/);
+    2. experimentQC/*_fingerprint.pdf: make sure the plots information is correct for your antibody/input. See [HERE](https://deeptools.readthedocs.io/en/develop/content/tools/plotFingerprint.html) for more details.
+    3. crossReads/*cc.plot.pdf: make sure your sample data has the correct signal intensity and location.  See [HERE](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/phantompeaks.md) for more details.
+    4. crossReads/*.cc.qc: Column 9 (NSC) should be > 1.1 for experiment and < 1.1 for input. Column 10 (RSC) should be > 0.8 for experiment and < 0.8 for input. See [HERE](https://genome.ucsc.edu/encode/qualityMetrics.html) for more details.
+    5. experimentQC/coverage.pdf, experimentQC/heatmeap_SpearmanCorr.pdf, experimentQC/heatmeap_PearsonCorr.pdf: See [HERE](https://deeptools.readthedocs.io/en/develop/content/list_of_tools.html) for more details.


+
+### Credits
+This example worklow is derived from original scripts kindly contributed by the Bioinformatic Core Facility ([BICF](https://www.utsouthwestern.edu/labs/bioinformatics/)), in the [Department of Bioinformatics](https://www.utsouthwestern.edu/departments/bioinformatics/).
+
+Please cite in publications: Pipeline was developed by BICF from funding provided by Cancer Prevention and Research Institute of Texas (RP150596).
+
+### References
+
+  + python/3.6.1-2-anaconda [website](https://www.anaconda.com/download/#linux) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + trimgalore/0.4.1 [website](https://github.com/FelixKrueger/TrimGalore) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + cutadapt/1.9.1 [website](https://cutadapt.readthedocs.io/en/stable/index.html) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + bwa/intel/0.7.12 [website](http://bio-bwa.sourceforge.net/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + samtools/1.6 [website](http://samtools.sourceforge.net/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + sambamba/0.6.6 [website](http://lomereiter.github.io/sambamba/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + bedtools/2.26.0 [website](https://bedtools.readthedocs.io/en/latest/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + deeptools/2.5.0.1 [website](https://deeptools.readthedocs.io/en/develop/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + phantompeakqualtools/1.2 [website](https://github.com/kundajelab/phantompeakqualtools) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + macs/2.1.0-20151222 [website](http://liulab.dfci.harvard.edu/MACS/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + UCSC_userApps/v317 [website](https://genome.ucsc.edu/util.html) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + R/3.3.2-gccmkl [website](https://www.r-project.org/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + meme/4.11.1-gcc-openmpi [website](http://meme-suite.org/doc/install.html?man_type=web) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + ChIPseeker [website](https://bioconductor.org/packages/release/bioc/html/ChIPseeker.html) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + DiffBind [website](https://bioconductor.org/packages/release/bioc/html/DiffBind.html) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + MultiQC [website](https://multiqc.info/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + BICFChip-seqAnalysisWorkflow [website](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
+  + Nextflow [website](https://www.nextflow.io/) [citation](https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/blob/master/docs/references.md)
--- a/docs/phantompeaks.md
+++ b/docs/phantompeaks.md
+## Phantompeakqualtools
+Phantompeakqualtools plots the strand cross-correlation of aligned reads for each sample. In a strand cross-correlation plot, reads are shifted in the direction of the strand they map to by an increasing number of base pairs and the Pearson correlation between the per-position read count vectors for each strand is calculated. Two cross-correlation peaks are usually observed in a ChIP experiment, one corresponding to the read length ("phantom" peak) and one to the average fragment length of the library. The absolute and relative height of the two peaks are useful determinants of the success of a ChIP-seq experiment. A high-quality IP is characterized by a ChIP peak that is much higher than the "phantom" peak, while often very small or no such peak is seen in failed experiments.
+
+![Phantompeakqualtools](images/phantompeakqualtools.png)
+
+*Source: Landt SG et al, Genome Research (2012)*
+
+Normalized strand coefficient (NSC) is the normalized ratio between the fragment-length cross-correlation peak and the background cross-correlation. NSC values range from a minimum of 1 to larger positive numbers. 1.1 is the critical threshold. Datasets with NSC values much less than 1.1 (< 1.05) tend to have low signal to noise or few peaks (this could be biological eg. a factor that truly binds only a few sites in a particular tissue type OR it could be due to poor quality). ENCODE cutoff: NSC > 1.05.
+
+Relative strand correlation (RSC) is the ratio between the fragment-length peak and the read-length peak. RSC values range from 0 to larger positive values. 1 is the critical threshold. RSC values significantly lower than 1 (< 0.8) tend to have low signal to noise. The low scores can be due to failed and poor quality ChIP, low read sequence quality and hence lots of mismappings, shallow sequencing depth (significantly below saturation) or a combination of these. Like the NSC, datasets with few binding sites (< 200), which is biologically justifiable, also show low RSC scores. ENCODE cutoff: RSC > 0.8.
--- a/docs/references.md
+++ b/docs/references.md
+### References
+
+1. **python**:
+  * Anaconda (Anaconda Software Distribution, [https://anaconda.com](https://anaconda.com))
+
+2. **trimgalore**:
+  * trimgalore [https://github.com/FelixKrueger/TrimGalore](https://github.com/FelixKrueger/TrimGalore)
+
+3. **cutadapt**:
+  * Marcel, M. 2011. Cutadapt removes adapter sequences from high-throughput sequencing reads. EMBnet.journal 17(1):10-12. doi:[10.14806/ej.17.1.200](http://dx.doi.org/10.14806/ej.17.1.200)
+
+4. **bwa**:
+  * Li H., and R. Durbin. 2009. Fast and accurate short read alignment with Burrows-Wheeler Transform. Bioinformatics 25: 1754-60. doi:[10.1093/bioinformatics/btp324](http://dx.doi.org/10.1093/bioinformatics/btp324)
+
+5. **samtools**:
+  * Li H., B. Handsaker, A. Wysoker, T. Fennell, J. Ruan, N. Homer, G. Marth, G. Abecasis, R. Durbin, and 1000 Genome Project Data Processing Subgroup. 2009. The Sequence alignment/map (SAM) format and SAMtools. Bioinformatics 25: 2078-9. doi:[10.1093/bioinformatics/btp352](http://dx.doi.org/10.1093/bioinformatics/btp352)
+
+6. **sambamba**:
+  * Tarasov, A., A. J. Vilella, E. Cuppen, I. J. Nijman, and P. Prins. 2015 Sambamba: fast processing of NGS alignment formats. Bioinformatics 31(12): 2032-2034. doi:[10.1093/bioinformatics/btv098](http://dx.doi.org/10.1093/bioinformatics/btv098)
+
+7. **bedtools**:
+  * Quinlan, A. R., and I. M. Hall. 2010. BEDTools: a flexible suite of utilities for comparing genomic feautures. Bioinformatics 26(6): 841-842. doi:[10.1093/bioinformatics/btq033](http://dx.doi.org/10.1093/bioinformatics/btq033)
+
+8. **deeptools**:
+  * Ramírez, F., D. P. Ryan, B. Grüning, V. Bhardwaj, F. Kilpert, A. S. Richter, S. Heyne, F. Dündar, and T. Manke. 2016. deepTools2: a next generation web server for deep-sequencing data analysis. Nucleic Acids Research 44: W160-165. doi:[10.1093/nar/gkw257](http://dx.doi.org/10.1093/nar/gkw257)
+
+9. **phantompeakqualtools**:
+  * Landt S. G., G. K. Marinov, A. Kundaje, et al. 2012. ChIP-seq guidelines and practices of the ENCODE and modENCODE consortia. Genome Res 9: 1813-31. doi:[10.1101/gr.136184.111](http://dx.doi.org/10.1101/gr.136184.111)
+  * Kharchenko P. K., M. Y. Tolstorukov, and P. J. Park. 2008. Design and analysis of ChIP-seq experiments for DNA-binding proteins. Nat Biotechnol 26(12): 1351-1359. doi:[10.1038/nbt.1508](https://dx.doi.org/10.1038/nbt.1508)
+
+10. **macs**:
+  * Zhang Y., T. Liu, C. A. Meyer, J. Eeckhoute, D. S. Johnson, B. E. Bernstein, C. Nusbaum, R. M. Myers, M. Brown, W. Li, and X. S. Liu. 2008. Model-based Analysis of ChIP-Seq (MACS). Genome Biol 9: R137. doi:[10.1186/gb-2008-9-9-r137](https://dx.doi.org/10.1186/gb-2008-9-9-r137)
+
+11. **UCSC(bedGraphToBigWig)**:
+  * Kent W. J., A. S. Zweig, G. Barber, A. S. Hinrichs, and D. Karolchik. BigWig and BigBed: enabling browsing of large distributed data sets. Bioinformatics 26(17): 2204-2207. doi:[10.1093/bioinformatics/btq351](https://dx.doi.org/10.1093/bioinformatics/btq351)
+
+12. **R**:
+  * R Core Team 2014. R: A language and environment for statistical computing. R Foundation for Statistical Computing, Vienna, Austria. URL:[http://www.R-project.org/](http://www.R-project.org/).
+
+13. **meme**:
+  * Bailey T. L., M. Bodén, F. A. Buske, M. Frith, C. E. Grant, L. Clementi, J. Ren, W. W. Li, and W. S. Noble. 2009. MEME SUITE: tools for motif discovery and searching. Nucleic Acids Research 37: W202-W208. doi:[10.1093/nar/gkp335](https://dx.doi.org/10.1093/nar/gkp335)
+  * Machanick P., and T. L. Bailey. 2011. MEME-ChIP: motif analysis of large DNA datasets. Bioinformatics 27(12): 1696-1697. doi:[10.1093/bioinformatics/btr189](https://dx.doi.org/10.1093/bioinformatics/btr189)
+
+14. **ChIPseeker**:
+  * Yu G., L. Wang, and Q. He. 2015. ChIPseeker: an R/Bioconductor package for ChIP peak annotation, comparison and visualization. Bioinformatics 31(14): 2382-2383. doi:[10.1093/bioinformatics/btv145](https://dx.doi.org/10.1093/bioinformatics/btv145)
+
+15. **DiffBind**:
+  * Stark R., and G. Brown. 2011. DiffBind: differential binding analysis of ChIP-Seq peak data. [http://bioconductor.org/packages/release/bioc/vignettes/DiffBind/inst/doc/DiffBind.pdf](http://bioconductor.org/packages/release/bioc/vignettes/DiffBind/inst/doc/DiffBind.pdf). doi:[10.18129/B9.bioc.DiffBind](https://dx.doi.org/10.18129/B9.bioc.DiffBind)
+  * Ross-Innes C. S., R. Stark, A. E. Teschendorff, K. A. Holmes, H. R. Ali, M. J. Dunning,  G. D. Brown, O. Gojis, I. O. Ellis, A. R. Green, S. Ali, S. Chin, C. Palmieri, C. Caldas, and J. S. Carroll. 2012. Differential oestrogen receptor binding is associated with clinical outcome in breast cancer. Nature 481: 389-393. doi:[10.1038/nature10730](https://dx.doi.org/10.1038/nature10730)
+
+16. **MultiQc**:
+  * Ewels P., Magnusson M., Lundin S. and Käller M. 2016. MultiQC: Summarize analysis results for multiple tools and samples in a single report. Bioinformatics 32(19): 3047–3048. doi:[10.1093/bioinformatics/btw354](https://dx.doi.org/10.1093/bioinformatics/btw354)
+
+17. **BICF ChIP-seq Analysis Workflow**:
+  * Spencer D. Barnes, Holly Ruess, Jeremy A. Mathews, Beibei Chen, and Venkat S. Malladi. 2020. BICF ChIP-seq Analysis Workflow (publish_1.1.3). Zenodo. doi:[10.5281/zenodo.3986942](https://doi.org/10.5281/zenodo.3986942)
+
+18. **Nextflow**:
+  * Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., and Notredame, C. 2017. Nextflow enables reproducible computational workflows. Nature biotechnology, 35(4), 316.
+
+Please cite in publications: Pipeline was developed by BICF from funding provided by **Cancer Prevention and Research Institute of Texas (RP150596)**.
--- a/docs/xcor_header.txt
+++ b/docs/xcor_header.txt
+See https://github.com/crazyhottommy/phantompeakqualtools for more details
+
+COL1: Filename: tagAlign/BAM filename
+COL2: numReads: effective sequencing depth i.e. total number of mapped reads in input file
+COL3: estFragLen: comma separated strand cross-correlation peak(s) in decreasing order of correlation.
+	  The top 3 local maxima locations that are within 90% of the maximum cross-correlation value are output.
+	  In almost all cases, the top (first) value in the list represents the predominant fragment length.
+	  If you want to keep only the top value simply run
+	  sed -r 's/,[^\t]+//g' <outFile> > <newOutFile>
+COL4: corr_estFragLen: comma separated strand cross-correlation value(s) in decreasing order (col2 follows the same order)
+COL5: phantomPeak: Read length/phantom peak strand shift
+COL6: corr_phantomPeak: Correlation value at phantom peak
+COL7: argmin_corr: strand shift at which cross-correlation is lowest
+COL8: min_corr: minimum value of cross-correlation
+COL9: Normalized strand cross-correlation coefficient (NSC) = COL4 / COL8
+COL10: Relative strand cross-correlation coefficient (RSC) = (COL4 - COL8) / (COL6 - COL8)
+COL11: QualityTag: Quality tag based on thresholded RSC (codes: -2:veryLow,-1:Low,0:Medium,1:High,2:veryHigh)
--- a/env.md
+++ b/env.md
-## Create new env in specific folder
-```shell
-conda create -p /project/shared/bicf_workflow_ref/chipseq_bchen4/ -c r r-essentials
-#Add channels
-conda config --add channels conda-forge
-conda config --add channels r
-conda config --add channels bioconda
-pip install --user twobitreader
-conda install -c r r-xml
-```
-
-Install bioconductor in R console:
-```R
-source("http://bioconductor.org/biocLite.R")
-biocLite()
-biocLite(c("DiffBind","ChIPseeker"))
-```
\ No newline at end of file
--- a/index.md
+++ b/index.md
-this is a read me
--- a/nextflow.config
+++ b/nextflow.config
-process.executor='slurm'
-process.queue='super'
-process.time='5d'
-
-
--- a/pytest.ini
+++ b/pytest.ini
+[pytest]
+python_paths = workflow/scripts
No results found