Skip to content
Snippets Groups Projects
Commit 43c8b517 authored by Gervaise Henry's avatar Gervaise Henry :cowboy:
Browse files

Containerize

parent dbb856c4
2 merge requests!63Develop,!62Resolve "Containerize"
Pipeline #6552 failed with stages
in 12 seconds
......@@ -2,12 +2,13 @@ before_script:
- module load astrocyte
- module load python/3.6.1-2-anaconda
- pip install --user pytest-pythonpath==0.7.1 pytest-cov==2.5.1
- module load singularity/3.0.2
- module load nextflow/19.09.0
- mkdir test_data/hu.v3s1r500
- mkdir test_data/mu.v3s1r500
- mkdir test_data/hu.v3s2r10k
- mkdir test_data/mu.v3s2r10k
- mkdir test_data/hu.v2s2r10k
- mkdir -p test_data/hu.v3s1r500
- mkdir -p test_data/mu.v3s1r500
- mkdir -p test_data/hu.v3s2r10k
- mkdir -p test_data/mu.v3s2r10k
- mkdir -p test_data/hu.v2s2r10k
- ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s1r500/* test_data/hu.v3s1r500/
- ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/mu.v3s1r500/* test_data/mu.v3s1r500/
- ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r10k/* test_data/hu.v3s2r10k/
......
......@@ -3,11 +3,14 @@
* Check Design File for spaces in name and file contents
* Attempt to preven thredding error (which appears to only happen on 256GBv1 nodes)
* Add option for 5' GEX chemistry
* Remove cellranger 3.0.1 as an option
* Add cellranger 2.2.0 as an option
**Background**
* Add Nextflow Tower integration into CI (GHH's profile)
* Add new layered config folders, including prepare for awsifying
* Update param to new standard
* Use docker containers
*Known Bugs*
* Vizapp does not yet work for Astrocyte
......
......@@ -83,11 +83,11 @@ To Run:
* eg: **--kitVersion '3GEXv3'**
* **--version**
* cellranger version
* --version (cellranger version) 2.1.1 can only read --kitVersion of two (2)
* --version (cellranger version) 2.1.1 and 2.2.0 can only read --kitVersion of 3GEXv2
* options:
* *'3.1.0'*
* *'3.0.2'*
* *'3.0.1'*
* *'2.2.0'*
* *'2.1.1'*
* eg: **--version '3.1.0'**
* **--outDir**
......
......@@ -12,4 +12,4 @@ process {
queue = 'default-'
cpus = 1
memory = '10 GB'
}
\ No newline at end of file
}
......@@ -51,3 +51,14 @@ params {
}
}
}
singularity {
enabled = true
cacheDir = '/project/BICF/BICF_Core/shared/gudmap/singularity_cache/'
}
env {
http_proxy = 'http://proxy.swmed.edu:3128'
https_proxy = 'http://proxy.swmed.edu:3128'
all_proxy = 'http://proxy.swmed.edu:3128'
}
......@@ -4,28 +4,24 @@ process {
clusterOptions = '--hold'
withLabel: checkDesignFile {
module = ['python/3.6.1-2-anaconda']
executor = 'local'
}
withLabel: count211 {
module = ['cellranger/2.1.1']
queue = '128GB,256GB,256GBv1,384GB'
}
withLabel: count301 {
module = ['cellranger/3.0.1']
withLabel: count220 {
queue = '128GB,256GB,256GBv1,384GB'
}
withLabel: count302 {
module = ['cellranger/3.0.2']
queue = '128GB,256GB,256GBv1,384GB'
}
withLabel: count310 {
module = ['cellranger/3.1.0']
queue = '128GB,256GB,256GBv1,384GB'
}
withLabel: versions {
module = ['python/3.6.1-2-anaconda','pandoc/2.7','multiqc/1.7']
executor = 'local'
}
withLabel: multiqc {
module = ['multiqc/1.7']
executor = 'local'
}
}
process {
executor = 'local'
withLabel: checkDesignFile {
module = ['python/3.6.1-2-anaconda']
}
withLabel: count211 {
module = ['cellranger/2.1.1']
}
withLabel: count301 {
module = ['cellranger/3.0.1']
}
withLabel: count302 {
module = ['cellranger/3.0.2']
}
withLabel: count310 {
module = ['cellranger/3.1.0']
}
withLabel: versions {
module = ['python/3.6.1-2-anaconda','pandoc/2.7','multiqc/1.7']
}
withLabel: multiqc {
module = ['multiqc/1.7']
}
}
......@@ -26,17 +26,23 @@ if (params.kitVersion == "3GEXv3" && params.version == '2.1.1') {
print("Cellranger Version 2.1.1 requires kitVersion 2")
System.exit(32)
}
if (params.kitVersion == "3GEXv3" && params.version == '2.2.0') {
print("Cellranger Version 2.2.0 requires kitVersion 2")
System.exit(32)
}
// Define variables if astrocyte (or from config)
if (params.astrocyte) {
print("Running under astrocyte")
params.genomeLocation = '/project/apps_database/cellranger/refdata-cellranger-'
if (params.kitVersion == "one") {
if (params.kitVersion == "3GEXv1") {
params.chemistryParam ='SC3Pv1'
} else if (params.kitVersion == "two") {
} else if (params.kitVersion == "3GEXv2") {
params.chemistryParam ='SC3Pv2'
} else if (params.kitVersion == "three") {
} else if (params.kitVersion == "3GEXv3") {
params.chemistryParam ='SC3Pv3'
} else if (params.kitVersion == "5GEX") {
params.chemistryParam ='fiveprime'
} else {
params.chemistryParam = 'auto'
}
......@@ -76,7 +82,7 @@ references = "${baseDir}/../docs/references.md"
*/
process checkDesignFile {
tag "${name}"
module 'python/3.6.1-2-anaconda'
container = 'bicf/python3:2.0.0'
input:
file designLocation
......@@ -108,22 +114,22 @@ samples = designPaths
// Duplicate variables
samples.into {
samples211
samples301
samples220
samples302
samples310
}
refLocation.into {
refLocation211
refLocation301
refLocation220
refLocation302
refLocation310
}
expectCells211 = expectCells
expectCells301 = expectCells
expectCells220 = expectCells
expectCells302 = expectCells
expectCells310 = expectCells
forceCells211 = forceCells
forceCells301 = forceCells
forceCells220 = forceCells
forceCells302 = forceCells
forceCells310 = forceCells
chemistryParam301 = chemistryParam
......@@ -137,7 +143,7 @@ process count211 {
queue '128GB,256GB,256GBv1,384GB'
tag "${sample}"
publishDir "${outDir}/${task.process}", mode: 'copy'
module 'cellranger/2.1.1'
container 'bicf/cellranger2.1.1:2.0.0'
input:
set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples211
......@@ -174,35 +180,35 @@ process count211 {
}
/*
* count211: run cellranger count version 3.0.1
* count220: run cellranger count version 2.2.0
*/
process count301 {
process count220 {
queue '128GB,256GB,256GBv1,384GB'
tag "${sample}"
publishDir "${outDir}/${task.process}", mode: 'copy'
module 'cellranger/3.0.1'
container 'bicf/cellranger2.2.0:2.0.0'
input:
set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples301
set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples220
file ref from refLocation301.first()
expectCells301
forceCells301
chemistryParam301
output:
file("**/outs/**") into outPaths301
file("*_metrics_summary.tsv") into metricsSummary301
file("**/outs/**") into outPaths220
file("*_metrics_summary.tsv") into metricsSummary220
when:
version == '3.0.1'
version == '2.2.0'
script:
if (forceCells301 == 0) {
if (forceCells220 == 0) {
"""
hostname
ulimit -a
bash ${baseDir}/scripts/filename_check.sh -r ${ref}
cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells301} --chemistry=${chemistryParam301}
cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells220} --chemistry=${chemistryParam220}
sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv
"""
}
......@@ -211,20 +217,20 @@ process count301 {
hostname
ulimit -a
bash ${baseDir}/scripts/filename_check.sh -r ${ref}
cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells301} --chemistry=${chemistryParam301}
cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells220} --chemistry=${chemistryParam220}
sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv
"""
}
}
/*
* count211: run cellranger count version 3.0.2
* count302: run cellranger count version 3.0.2
*/
process count302 {
queue '128GB,256GB,256GBv1,384GB'
tag "${sample}"
publishDir "${outDir}/${task.process}", mode: 'copy'
module 'cellranger/3.0.2'
container 'bicf/cellranger3.0.2:2.0.0'
input:
set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples302
......@@ -264,13 +270,13 @@ process count302 {
}
/*
* count211: run cellranger count version 3.1.0
* count310: run cellranger count version 3.1.0
*/
process count310 {
queue '128GB,256GB,256GBv1,384GB'
tag "${sample}"
publishDir "${outDir}/${task.process}", mode: 'copy'
module 'cellranger/3.1.0'
container 'bicf/cellranger3.1.0:2.0.0'
input:
set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples310
......@@ -313,7 +319,7 @@ process count310 {
*/
process versions {
tag "${name}"
module 'python/3.6.1-2-anaconda:pandoc/2.7:multiqc/1.7'
container 'bicf/python3:2.0.0'
input:
......
......@@ -13,6 +13,30 @@ profiles {
}
}
process {
withName:checkDesignFile {
container = 'bicf/python3:2.0.0'
}
withName:count211 {
container = 'bicf/cellranger2.1.1:2.0.0'
}
withName:count220 {
container = 'bicf/cellranger2.2.0:2.0.0'
}
withName:count302 {
container = 'bicf/cellranger3.0.2:2.0.0'
}
withName:count310 {
container = 'bicf/cellranger3.1.0:2.0.0'
}
withName:versions {
container = 'bicf/python3:2.0.0'
}
withName:multiqc {
container = 'bicf/multiqc:2.0.0'
}
}
trace {
enabled = true
file = 'pipeline_trace.txt'
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment