pytest.ini

+[pytest]
+python_paths = workflow/scripts

test_data/design_ENCSR238SGC_SE.txt

+sample_id	experiment_id	biosample	factor	treatment	replicate	control_id	fastq_read1
+ENCLB144FDT	ENCSR238SGC	limb	H3K4me1	None	1	ENCLB304SBJ	ENCFF833BLU.fastq.gz
+ENCLB831RUI	ENCSR238SGC	limb	H3K4me1	None	2	ENCLB410VVO	ENCFF646LXU.fastq.gz
+ENCLB304SBJ	ENCSR687ALB	limb	Control	None	1	ENCLB304SBJ	ENCFF524CAC.fastq.gz
+ENCLB410VVO	ENCSR687ALB	limb	Control	None	2	ENCLB410VVO	ENCFF163AJI.fastq.gz

test_data/design_ENCSR729LGA_PE.txt

+sample_id	experiment_id	biosample	factor	treatment	replicate	control_id	fastq_read1	fastq_read2
+ENCLB637LZP	ENCSR729LGA	MCF-7	SP1	None	1	ENCLB678IDC	ENCFF957SQS.fastq.gz	ENCFF582IOZ.fastq.gz
+ENCLB568IYX	ENCSR729LGA	MCF-7	SP1	None	2	ENCLB336TVW	ENCFF330MCZ.fastq.gz	ENCFF293YFE.fastq.gz
+ENCLB678IDC	ENCSR217LRF	MCF-7	Control	None	1	ENCLB678IDC	ENCFF002DTU.fastq.gz	ENCFF002EFI.fastq.gz
+ENCLB336TVW	ENCSR217LRF	MCF-7	Control	None	2	ENCLB336TVW	ENCFF002EFG.fastq.gz	ENCFF002DTS.fastq.gz

test_data/design_diff_PE.txt

+sample_id	experiment_id	biosample	factor	treatment	replicate	control_id	fastq_read1	fastq_read2
+ENCLB637LZP	ENCSR729LGA	MCF-7	SP1	None	1	ENCLB678IDC	ENCFF957SQS.fastq.gz	ENCFF582IOZ.fastq.gz
+ENCLB568IYX	ENCSR729LGA	MCF-7	SP1	None	2	ENCLB336TVW	ENCFF330MCZ.fastq.gz	ENCFF293YFE.fastq.gz
+ENCLB678IDC	ENCSR217LRF	MCF-7	Control	None	1	ENCLB678IDC	ENCFF002DTU.fastq.gz	ENCFF002EFI.fastq.gz
+ENCLB336TVW	ENCSR217LRF	MCF-7	Control	None	2	ENCLB336TVW	ENCFF002EFG.fastq.gz	ENCFF002DTS.fastq.gz
+ENCLB552ACZ	ENCSR757EMK	MCF-7	SUZ12	None	1	ENCLB678IDC	ENCFF833EZX.fastq.gz	ENCFF161HBP.fastq.gz
+ENCLB872TQR	ENCSR757EMK	MCF-7	SUZ12	None	2	ENCLB336TVW	ENCFF776KZU.fastq.gz	ENCFF119KHM.fastq.gz

test_data/design_diff_SE.txt

+sample_id	experiment_id	biosample	factor	treatment	replicate	control_id	fastq_read1
+ENCLB144FDT	ENCSR238SGC	limb	H3K4me1	None	1	ENCLB304SBJ	ENCFF833BLU.fastq.gz
+ENCLB831RUI	ENCSR238SGC	limb	H3K4me1	None	2	ENCLB410VVO	ENCFF646LXU.fastq.gz
+ENCLB304SBJ	ENCSR687ALB	limb	Control	None	1	ENCLB304SBJ	ENCFF524CAC.fastq.gz
+ENCLB410VVO	ENCSR687ALB	limb	Control	None	2	ENCLB410VVO	ENCFF163AJI.fastq.gz
+ENCLB140BPV	ENCSR272GNQ	midbrain	H3K4me1	None	1	ENCLB841FLB	ENCFF278VQW.fastq.gz
+ENCLB785CNN	ENCSR272GNQ	midbrain	H3K4me1	None	2	ENCLB735ZEL	ENCFF466CFM.fastq.gz
+ENCLB841FLB	ENCSR842LMA	midbrain	Control	None	1	ENCLB841FLB	ENCFF914QXH.fastq.gz
+ENCLB735ZEL	ENCSR842LMA	midbrain	Control	None	2	ENCLB735ZEL	ENCFF942UQV.fastq.gz

test_data/design_single_contol_SE.txt

+sample_id	experiment_id	biosample	factor	treatment	replicate	control_id	fastq_read1
+ENCLB497XZB	ENCSR000DXB	Panc1	H3K4me3	None	1	ENCLB304SBJ	ENCFF001GBW.fastq.gz
+ENCLB304SBJ	ENCSR000DXC	Panc1	Control	None	1	ENCLB304SBJ	ENCFF001HWJ.fastq.gz

test_data/fetch_test_data.sh

+echo "Downloading Single-end data set Mouse ENCSR238SGC and ENCSR687ALB"
+wget https://www.encodeproject.org/files/ENCFF833BLU/@@download/ENCFF833BLU.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF646LXU/@@download/ENCFF646LXU.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF524CAC/@@download/ENCFF524CAC.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF163AJI/@@download/ENCFF163AJI.fastq.gz
+echo "Downloading Single-end data set Mouse ENCSR272GNQ and ENCSR842LMA"
+wget https://www.encodeproject.org/files/ENCFF278VQW/@@download/ENCFF278VQW.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF466CFM/@@download/ENCFF466CFM.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF914QXH/@@download/ENCFF914QXH.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF942UQV/@@download/ENCFF942UQV.fastq.gz
+echo "Done with Single-end"
+
+echo "Downloading Paired-end data set Human ENCSR729LGA and ENCSR217LRF"
+wget https://www.encodeproject.org/files/ENCFF957SQS/@@download/ENCFF957SQS.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF582IOZ/@@download/ENCFF582IOZ.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF330MCZ/@@download/ENCFF330MCZ.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF293YFE/@@download/ENCFF293YFE.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF002DTU/@@download/ENCFF002DTU.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF002EFI/@@download/ENCFF002EFI.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF002EFG/@@download/ENCFF002EFG.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF002DTS/@@download/ENCFF002DTS.fastq.gz
+echo "Downloading Paired-end data set Human ENCSR757EMK"
+wget https://www.encodeproject.org/files/ENCFF833EZX/@@download/ENCFF833EZX.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF161HBP/@@download/ENCFF161HBP.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF776KZU/@@download/ENCFF776KZU.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF119KHM/@@download/ENCFF119KHM.fastq.gz
+echo "Done with Paired-end"
+
+echo "Downloading Single-end data set Human ENCSR000DXB and ENCSR000DXC"
+wget https://www.encodeproject.org/files/ENCFF001GBW/@@download/ENCFF001GBW.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF001GBV/@@download/ENCFF001GBV.fastq.gz
+wget https://www.encodeproject.org/files/ENCFF001HWJ/@@download/ENCFF001HWJ.fastq.gz
+echo "Done with Single-end"

test_data/test_cross.qc

+Test.20.tagAlign.gz	18588987	0,20,33	0.211525291335199,0.211232019956852,0.211139666755398	35	0.2123067	1500	0.209429	1.01001	0.7284536       0

workflow/conf/biohpc.config

+process {
+  executor = 'slurm'
+  queue = 'super'
+  clusterOptions = '--hold'
+  beforeScript= 'ulimit -Ss unlimited'
+
+  // Process specific configuration
+  withName: checkDesignFile {
+    module = ['python/3.6.1-2-anaconda']
+    executor = 'local'
+  }
+  withName: trimReads {
+    module = ['python/3.6.1-2-anaconda', 'trimgalore/0.4.1']
+    cpus = 32
+  }
+  withName: alignReads{
+    module = ['python/3.6.1-2-anaconda', 'bwa/intel/0.7.12', 'samtools/1.6']
+    queue = '128GB,256GB,256GBv1'
+  }
+  withName: filterReads{
+    module = ['python/3.6.1-2-anaconda', 'samtools/1.6', 'sambamba/0.6.6', 'bedtools/2.26.0']
+    queue = '128GB,256GB,256GBv1'
+  }
+  withName: experimentQC {
+    module = ['python/3.6.1-2-anaconda', 'deeptools/2.5.0.1']
+    queue = '128GB,256GB,256GBv1'
+  }
+  withName: convertReads {
+    module = ['python/3.6.1-2-anaconda',  'samtools/1.6', 'bedtools/2.26.0']
+    queue = '128GB,256GB,256GBv1'
+  }
+  withName: crossReads {
+    module = ['python/3.6.1-2-anaconda', 'phantompeakqualtools/1.2']
+    cpus = 32
+  }
+  withName: defineExpDesignFiles {
+    module = ['python/3.6.1-2-anaconda']
+    executor = 'local'
+  }
+  withName: poolAndPsuedoReads {
+    module = ['python/3.6.1-2-anaconda']
+    executor = 'local'
+  }
+  withName: callPeaksMACS {
+    module = ['python/3.6.1-2-anaconda', 'macs/2.1.0-20151222', 'UCSC_userApps/v317', 'bedtools/2.26.0', 'phantompeakqualtools/1.2']
+    queue = '128GB,256GB,256GBv1'
+  }
+  withName: plotProfile {
+    module = ['deeptools/2.5.0.1']
+    cpus = 32
+  }
+  withName: consensusPeaks {
+    module = ['python/3.6.1-2-anaconda', 'bedtools/2.26.0']
+    executor = 'local'
+  }
+  withName: peakAnnotation {
+    module = ['R/3.3.2-gccmkl']
+    executor = 'local'
+  }
+  withName: diffPeaks {
+    module = ['R/3.3.2-gccmkl']
+    cpus = 32
+  }
+  withName: motifSearch {
+    module = ['python/3.6.1-2-anaconda', 'meme/4.11.1-gcc-openmpi', 'bedtools/2.26.0']
+    cpus = 32
+  }
+  withName: multiqcReport {
+    module = ['python/3.6.1-2-anaconda', 'pandoc/2.7', 'singularity/3.0.2']
+    executor = 'local'
+  }
+}
+
+params {
+  // Reference file paths on BioHPC
+  genomes {
+    'GRCh38' {
+      bwa = '/project/shared/bicf_workflow_ref/human/GRCh38'
+      genomesize = 'hs'
+      chromsizes = '/project/shared/bicf_workflow_ref/human/GRCh38/genomefile.txt'
+      fasta = '/project/shared/bicf_workflow_ref/human/GRCh38/genome.fa'
+      gtf = '/project/shared/bicf_workflow_ref/human/GRCh38/gencode.v25.chr_patch_hapl_scaff.annotation.gtf'
+      geneNames = '/project/shared/bicf_workflow_ref/human/GRCh38/genenames.txt'
+    }
+    'GRCh37' {
+      bwa = '/project/shared/bicf_workflow_ref/human/GRCh37'
+      genomesize = 'hs'
+      chromsizes = '/project/shared/bicf_workflow_ref/human/GRCh37/genomefile.txt'
+      fasta = '/project/shared/bicf_workflow_ref/human/GRCh37/genome.fa'
+      gtf = '/project/shared/bicf_workflow_ref/human/GRCh37/gencode.v19.chr_patch_hapl_scaff.annotation.gtf'
+      geneNames = '/project/shared/bicf_workflow_ref/human/GRCh37/genenames.txt'
+    }
+    'GRCm38' {
+      bwa = '/project/shared/bicf_workflow_ref/mouse/GRCm38'
+      genomesize = 'mm'
+      chromsizes = '/project/shared/bicf_workflow_ref/mouse/GRCm38/genomefile.txt'
+      fasta = '/project/shared/bicf_workflow_ref/mouse/GRCm38/genome.fa'
+      gtf = '/project/shared/bicf_workflow_ref/mouse/GRCm38/gencode.vM20.annotation.gtf'
+      geneNames = '/project/shared/bicf_workflow_ref/mouse/GRCm38/genenames.txt'
+    }
+  }
+}
+
+trace {
+  enabled = true
+  file = 'pipeline_trace.txt'
+  fields = 'task_id,native_id,process,name,status,exit,submit,start,complete,duration,realtime,%cpu,%mem,rss'
+}
+
+timeline {
+  enabled = true
+  file = 'timeline.html'
+}
+
+report {
+  enabled = true
+  file = 'report.html'
+}

workflow/conf/multiqc_config.yaml

+# Custom Logo
+custom_logo: 'bicf_logo.png'
+custom_logo_url: 'https://www.utsouthwestern.edu/labs/bioinformatics/'
+custom_logo_title: 'Bioinformatics Core Facility'
+
+report_header_info:
+    - Contact E-mail: 'bicf@utsouthwestern.edu'
+    - Application Type: 'ChIP-seq'
+    - Department: 'Bioinformatic Core Facility, Department of Bioinformatics'
+    - Contributors and Licensing: 'See https://doi.org/10.5281/zenodo.2648844'
+
+
+# Title to use for the report.
+title: BICF ChIP-seq Analysis Report
+
+report_comment: >
+  This report has been generated by the <a href="https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis/" target="_blank">BICF/chipseq_analysis</a>
+  pipeline.
+
+extra_fn_clean_exts:
+  - 'pbc.qc'
+  - 'cc.qc'
+
+fn_ignore_files:
+  - '*dedup.flagstat.qc'
+
+custom_data:
+  library_complexity:
+    file_format: 'tsv'
+    id: 'library_complexity'
+    contents: 'TotalReadPairs  DistinctReadPairs       OneReadPair     TwoReadPairs    NRF     PBC1    PBC2'
+    section_name: 'Library complexity'
+    plot_type: 'generalstats'
+    pconfig:
+        TotalReadPairs:
+          decimalPlaces: 0
+          shared_key: read_count
+        DistinctReadPairs:
+          decimalPlaces: 0
+          shared_key: read_count
+        NRF:
+          decimalPlaces: 2
+        PBC1:
+          decimalPlaces: 2
+        PBC2:
+          decimalPlaces: 2
+
+sp:
+    phantompeakqualtools/out:
+      fn: '*cc.qc'
+    library_complexity:
+      fn: '*pbc.qc'
+    macs2:
+      fn: '*_peaks.xls'
+
+
+report_section_order:
+    cutadapt:
+      order: -1000
+    Samtools:
+      order: -1100
+    Software_Versions:
+      order: -1200
+    Software_References:
+      order: -1300
+
+table_columns_placement:
+    library_complexity:
+      TotalReadPairs: 1100
+      DistinctReadPairs: 1200
+      NRF: 1300
+      PBC1: 1400
+      PBC2: 1500
+    phantompeakqualtools:
+      Estimated_Fragment_Length_bp: 1600
+      NSC: 1700
+      RSC: 1800
+
+table_columns_visible:
+  cutadapt:
+    percent_trimmed: False
+  library_complexity:
+    OneReadPair: False
+    TwoReadPairs: False
+
+table_cond_formatting_rules:
+    library_complexity_mqc-generalstats-library_complexity-NRF:
+      pass:
+        - gt: 0.8
+      warn:
+        - lt: 0.8
+      fail:
+        - lt: 0.5
+    library_complexity_mqc-generalstats-library_complexity-PBC1:
+      pass:
+        - gt: 0.8
+      warn:
+        - lt: 0.8
+      fail:
+        - lt: 0.5
+    library_complexity_mqc-generalstats-library_complexity-PBC2:
+      pass:
+        - gt: 3
+      warn:
+        - lt: 3
+      fail:
+        - lt: 1
+
+thousandsSep_format: ''

workflow/nextflow.config

+profiles {
+  standard {
+    includeConfig 'conf/biohpc.config'
+  }
+}
+
+trace {
+  enabled = true
+  file = 'pipeline_trace.txt'
+  fields = 'task_id,native_id,process,name,status,exit,submit,start,complete,duration,realtime,%cpu,%mem,rss'
+}
+
+timeline {
+  enabled = true
+  file = 'timeline.html'
+}
+
+report {
+  enabled = true
+  file = 'report.html'
+}
+
+
+manifest {
+  name = 'chipseq_analysis'
+  description = 'BICF ChIP-seq Analysis Workflow.'
+  homePage = 'https://git.biohpc.swmed.edu/BICF/Astrocyte/chipseq_analysis'
+  version = '1.1.2'
+  mainScript = 'main.nf'
+  nextflowVersion = '>=0.31.0'
+}

workflow/scripts/__init__.py

-