From 554ef719d7511bc849623353f36e6f141a06c4a4 Mon Sep 17 00:00:00 2001
From: "Gervaise H. Henry" <gervaise.henry@utsouthwestern.edu>
Date: Sun, 21 Jun 2020 14:11:18 -0500
Subject: [PATCH] De-containerize (remove cellranger 2.1.1 because no BioHPC
module)
---
.gitlab-ci.yml | 33 +++++---------------
astrocyte_pkg.yml | 1 -
workflow/conf/aws.config | 4 ---
workflow/conf/biohpc.config | 5 ---
workflow/conf/cluster.config | 3 --
workflow/main.nf | 59 +++---------------------------------
workflow/nextflow.config | 24 ---------------
7 files changed, 12 insertions(+), 117 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 598e601..4b44025 100755
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,8 +2,8 @@ before_script:
- module load astrocyte
- module load python/3.6.1-2-anaconda
- pip install --user pytest-pythonpath==0.7.1 pytest-cov==2.5.1
- - module load singularity/3.0.2
- module load nextflow/20.01.0
+ - module load singularity/3.0.2
- mkdir -p test_data/hu.v2s1r500
- mkdir -p test_data/hu.v3s1r500
- mkdir -p test_data/mu.v3s1r500
@@ -44,29 +44,8 @@ astrocyte_cli:
- master
- tags
script:
- - singularity run 'docker://bicf/cellranger2.1.1:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
- artifacts:
- name: "$CI_JOB_NAME"
- when: always
- paths:
- - test/outs/web_summary.html
- expire_in: 2 days
- retry:
- max: 0
- when:
- - always
-
-2.2.2_test:
- stage: container_test
- only:
- - branches
- except:
- refs:
- - develop
- - master
- - tags
- script:
- - singularity run 'docker://bicf/cellranger2.2.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
+ - module load cellranger/2.1.1
+ - cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -88,7 +67,8 @@ astrocyte_cli:
- master
- tags
script:
- - singularity run 'docker://bicf/cellranger3.0.2:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
+ - module load cellranger/3.0.2
+ - cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -110,7 +90,8 @@ astrocyte_cli:
- master
- tags
script:
- - singularity run 'docker://bicf/cellranger3.1.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
+ - module load cellranger/3.1.0
+ - cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
artifacts:
name: "$CI_JOB_NAME"
when: always
diff --git a/astrocyte_pkg.yml b/astrocyte_pkg.yml
index a58689f..e128e1f 100755
--- a/astrocyte_pkg.yml
+++ b/astrocyte_pkg.yml
@@ -150,7 +150,6 @@ workflow_parameters:
choices:
- ['3.1.0', '3.1.0']
- ['3.0.2', '3.0.2']
- - ['2.2.0', '2.2.0']
- ['2.1.1', '2.1.1']
required: true
description: |
diff --git a/workflow/conf/aws.config b/workflow/conf/aws.config
index 767a7d2..86a5b31 100644
--- a/workflow/conf/aws.config
+++ b/workflow/conf/aws.config
@@ -20,10 +20,6 @@ process {
cpus = 2
memory = '30 GB'
}
- withLabel: count220 {
- cpus = 2
- memory = '30 GB'
- }
withLabel: count302 {
cpus = 2
memory = '30 GB'
diff --git a/workflow/conf/biohpc.config b/workflow/conf/biohpc.config
index 56d5b65..1da174a 100644
--- a/workflow/conf/biohpc.config
+++ b/workflow/conf/biohpc.config
@@ -52,11 +52,6 @@ params {
}
}
-singularity {
- enabled = true
- cacheDir = '/project/BICF/BICF_Core/shared/gudmap/singularity_cache/'
-}
-
env {
http_proxy = 'http://proxy.swmed.edu:3128'
https_proxy = 'http://proxy.swmed.edu:3128'
diff --git a/workflow/conf/cluster.config b/workflow/conf/cluster.config
index 7322376..f5842a6 100644
--- a/workflow/conf/cluster.config
+++ b/workflow/conf/cluster.config
@@ -12,9 +12,6 @@ process {
withLabel: count211 {
queue = '128GB,256GB,256GBv1,384GB'
}
- withLabel: count220 {
- queue = '128GB,256GB,256GBv1,384GB'
- }
withLabel: count302 {
queue = '128GB,256GB,256GBv1,384GB'
}
diff --git a/workflow/main.nf b/workflow/main.nf
index 23b126d..4ceed7a 100755
--- a/workflow/main.nf
+++ b/workflow/main.nf
@@ -35,10 +35,6 @@ if (params.kitVersion == "3GEXv3" && params.version == '2.1.1') {
print("Cellranger Version 2.1.1 requires kitVersion 2")
System.exit(32)
}
-if (params.kitVersion == "3GEXv3" && params.version == '2.2.0') {
- print("Cellranger Version 2.2.0 requires kitVersion 2")
- System.exit(32)
-}
// Define variables if astrocyte (or from config)
if (params.astrocyte) {
@@ -118,6 +114,7 @@ process trackStart {
*/
process checkDesignFile {
tag "${name}"
+ module 'python/3.6.1-2-anaconda'
input:
file designLocation
@@ -149,26 +146,21 @@ samples = designPaths
// Duplicate variables
samples.into {
samples211
- samples220
samples302
samples310
}
refLocation.into {
refLocation211
- refLocation220
refLocation302
refLocation310
}
expectCells211 = expectCells
-expectCells220 = expectCells
expectCells302 = expectCells
expectCells310 = expectCells
forceCells211 = forceCells
-forceCells220 = forceCells
forceCells302 = forceCells
forceCells310 = forceCells
chemistryParam211 = chemistryParam
-chemistryParam220 = chemistryParam
chemistryParam302 = chemistryParam
chemistryParam310 = chemistryParam
@@ -179,6 +171,7 @@ process count211 {
tag "${sample}"
publishDir "${outDir}/${task.process}", mode: 'copy'
queue '128GB,256GB,256GBv1,384GB'
+ module 'cellranger/2.1.1'
input:
set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples211
@@ -217,50 +210,6 @@ process count211 {
}
}
-/*
- * count220: run cellranger count version 2.2.0
- */
-process count220 {
- queue '128GB,256GB,256GBv1,384GB'
- tag "${sample}"
- publishDir "${outDir}/${task.process}", mode: 'copy'
-
- input:
- set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples220
- file ref from refLocation220.first()
- expectCells220
- forceCells220
- chemistryParam220
-
- output:
- file("**/outs/**") into outPaths220
- file("*_metrics_summary.tsv") into metricsSummary220
-
- when:
- version == '2.2.0'
-
- script:
- if (forceCells220 == 0) {
- """
- hostname
- ulimit -u 16384
- ulimit -a
- bash ${baseDir}/scripts/filename_check.sh -r ${ref}
- cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells220} --chemistry=${chemistryParam220}
- sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv
- """
- }
- else {
- """
- hostname
- ulimit -u 16384
- ulimit -a
- bash ${baseDir}/scripts/filename_check.sh -r ${ref}
- cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells220} --chemistry=${chemistryParam220}
- sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv
- """
- }
-}
/*
* count302: run cellranger count version 3.0.2
@@ -269,6 +218,7 @@ process count302 {
tag "${sample}"
publishDir "${outDir}/${task.process}", mode: 'copy'
queue '128GB,256GB,256GBv1,384GB'
+ module 'cellranger/3.0.2'
input:
set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples302
@@ -314,6 +264,7 @@ process count310 {
tag "${sample}"
publishDir "${outDir}/${task.process}", mode: 'copy'
queue '128GB,256GB,256GBv1,384GB'
+ module 'cellranger/3.1.0'
input:
set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples310
@@ -379,7 +330,7 @@ process versions {
}
// Collect all metrics summaries reguardless of cellranger version
-metricsSummary = metricsSummary211.mix(metricsSummary220, metricsSummary302, metricsSummary310)
+metricsSummary = metricsSummary211.mix(metricsSummary302, metricsSummary310)
/*
* multiqc: create multiqc report
diff --git a/workflow/nextflow.config b/workflow/nextflow.config
index 4e2de2d..32b90f6 100644
--- a/workflow/nextflow.config
+++ b/workflow/nextflow.config
@@ -22,30 +22,6 @@ profiles {
}
}
-process {
- withName:checkDesignFile {
- container = 'bicf/python3:2.0.0'
- }
- withName:count211 {
- container = 'bicf/cellranger2.1.1:2.0.0'
- }
- withName:count220 {
- container = 'bicf/cellranger2.2.0:2.0.0'
- }
- withName:count302 {
- container = 'bicf/cellranger3.0.2:2.0.0'
- }
- withName:count310 {
- container = 'bicf/cellranger3.1.0:2.0.0'
- }
- withName:versions {
- container = 'bicf/python3:2.0.0'
- }
- withName:multiqc {
- container = 'bicf/multiqc:2.0.0'
- }
-}
-
trace {
enabled = true
file = 'pipeline_trace.txt'
--
GitLab