De-containerize (remove cellranger 2.1.1 because no BioHPC module)

.gitlab-ci.yml

@@ -2,8 +2,8 @@ before_script:
   - module load astrocyte
   - module load python/3.6.1-2-anaconda
   - pip install --user pytest-pythonpath==0.7.1 pytest-cov==2.5.1
-  - module load singularity/3.0.2
   - module load nextflow/20.01.0
+  - module load singularity/3.0.2
   - mkdir -p test_data/hu.v2s1r500
   - mkdir -p test_data/hu.v3s1r500
   - mkdir -p test_data/mu.v3s1r500
@@ -44,29 +44,8 @@ astrocyte_cli:
       - master
       - tags
   script:
-    - singularity run 'docker://bicf/cellranger2.1.1:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
-  artifacts:
-    name: "$CI_JOB_NAME"
-    when: always
-    paths:
-      - test/outs/web_summary.html
-    expire_in: 2 days
-  retry:
-    max: 0
-    when:
-      - always
-
-2.2.2_test:
-  stage: container_test
-  only:
-    - branches
-  except:
-    refs:
-      - develop
-      - master
-      - tags
-  script:
-    - singularity run 'docker://bicf/cellranger2.2.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
+    - module load cellranger/2.1.1
+    - cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -88,7 +67,8 @@ astrocyte_cli:
       - master
       - tags
   script:
-    - singularity run 'docker://bicf/cellranger3.0.2:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
+    - module load cellranger/3.0.2
+    - cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -110,7 +90,8 @@ astrocyte_cli:
       - master
       - tags
   script:
-    - singularity run 'docker://bicf/cellranger3.1.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
+    - module load cellranger/3.1.0
+    - cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
   artifacts:
     name: "$CI_JOB_NAME"
     when: always

astrocyte_pkg.yml

@@ -150,7 +150,6 @@ workflow_parameters:
     choices:
       - ['3.1.0', '3.1.0']
       - ['3.0.2', '3.0.2']
-      - ['2.2.0', '2.2.0']
       - ['2.1.1', '2.1.1']
     required: true
     description: |

workflow/conf/aws.config

@@ -20,10 +20,6 @@ process {
     cpus = 2
     memory = '30 GB'
   }
-  withLabel: count220 {
-    cpus = 2
-    memory = '30 GB'
-  }
   withLabel: count302 {
     cpus = 2
     memory = '30 GB'

workflow/conf/biohpc.config

@@ -52,11 +52,6 @@ params {
   }
 }
 
-singularity {
-  enabled = true
-  cacheDir = '/project/BICF/BICF_Core/shared/gudmap/singularity_cache/'
-}
-
 env {
   http_proxy = 'http://proxy.swmed.edu:3128'
   https_proxy = 'http://proxy.swmed.edu:3128'

workflow/conf/cluster.config

@@ -12,9 +12,6 @@ process {
   withLabel: count211 {
     queue = '128GB,256GB,256GBv1,384GB'
   }
-  withLabel: count220 {
-    queue = '128GB,256GB,256GBv1,384GB'
-  }
   withLabel: count302 {
     queue = '128GB,256GB,256GBv1,384GB'
   }

workflow/main.nf

@@ -35,10 +35,6 @@ if (params.kitVersion == "3GEXv3" && params.version == '2.1.1') {
   print("Cellranger Version 2.1.1 requires kitVersion 2")
   System.exit(32)	
 }
-if (params.kitVersion == "3GEXv3" && params.version == '2.2.0') {
-  print("Cellranger Version 2.2.0 requires kitVersion 2")
-  System.exit(32)	
-}
 
 // Define variables if astrocyte (or from config)
 if (params.astrocyte) {
@@ -118,6 +114,7 @@ process trackStart {
  */
 process checkDesignFile {
   tag "${name}"
+  module 'python/3.6.1-2-anaconda'
 
   input:
     file designLocation
@@ -149,26 +146,21 @@ samples = designPaths
 // Duplicate variables
 samples.into {
   samples211
-  samples220
   samples302
   samples310
 }
 refLocation.into {
   refLocation211
-  refLocation220
   refLocation302
   refLocation310
 }
 expectCells211 = expectCells
-expectCells220 = expectCells
 expectCells302 = expectCells
 expectCells310 = expectCells
 forceCells211 = forceCells
-forceCells220 = forceCells
 forceCells302 = forceCells
 forceCells310 = forceCells
 chemistryParam211 = chemistryParam
-chemistryParam220 = chemistryParam
 chemistryParam302 = chemistryParam
 chemistryParam310 = chemistryParam
 
@@ -179,6 +171,7 @@ process count211 {
   tag "${sample}"
   publishDir "${outDir}/${task.process}", mode: 'copy'
   queue '128GB,256GB,256GBv1,384GB'
+  module 'cellranger/2.1.1'
 
   input:
     set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples211
@@ -217,50 +210,6 @@ process count211 {
     }
 }
 
-/*
- * count220: run cellranger count version 2.2.0
- */
-process count220 {
-  queue '128GB,256GB,256GBv1,384GB'
-  tag "${sample}"
-  publishDir "${outDir}/${task.process}", mode: 'copy'
-
-  input:
-    set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples220
-    file ref from refLocation220.first()
-    expectCells220
-    forceCells220
-    chemistryParam220
-
-  output:
-    file("**/outs/**") into outPaths220
-    file("*_metrics_summary.tsv") into metricsSummary220
-
-  when:
-    version == '2.2.0'
-
-  script:
-    if (forceCells220 == 0) {
-      """
-      hostname
-      ulimit -u 16384
-      ulimit -a
-      bash ${baseDir}/scripts/filename_check.sh -r ${ref}
-      cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells220} --chemistry=${chemistryParam220}
-      sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv
-      """
-    } 
-    else {
-      """
-      hostname
-      ulimit -u 16384
-      ulimit -a
-      bash ${baseDir}/scripts/filename_check.sh -r ${ref}
-      cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells220} --chemistry=${chemistryParam220}
-      sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv
-      """
-    }
-}
 
 /*
  * count302: run cellranger count version 3.0.2
@@ -269,6 +218,7 @@ process count302 {
   tag "${sample}"
   publishDir "${outDir}/${task.process}", mode: 'copy'
   queue '128GB,256GB,256GBv1,384GB'
+  module 'cellranger/3.0.2'
 
   input:
     set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples302
@@ -314,6 +264,7 @@ process count310 {
   tag "${sample}"
   publishDir "${outDir}/${task.process}", mode: 'copy'
   queue '128GB,256GB,256GBv1,384GB'
+  module 'cellranger/3.1.0'
 
   input:
     set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples310
@@ -379,7 +330,7 @@ process versions {
 }
 
 // Collect all metrics summaries reguardless of cellranger version
-metricsSummary = metricsSummary211.mix(metricsSummary220, metricsSummary302, metricsSummary310)
+metricsSummary = metricsSummary211.mix(metricsSummary302, metricsSummary310)
 
 /*
  * multiqc: create multiqc report

workflow/nextflow.config

@@ -22,30 +22,6 @@ profiles {
   }
 }
 
-process {
-  withName:checkDesignFile {
-    container = 'bicf/python3:2.0.0'
-  }
-  withName:count211 {
-    container = 'bicf/cellranger2.1.1:2.0.0'
-  }
-  withName:count220 {
-    container = 'bicf/cellranger2.2.0:2.0.0'
-  }
-  withName:count302 {
-    container = 'bicf/cellranger3.0.2:2.0.0'
-  }
-  withName:count310 {
-    container = 'bicf/cellranger3.1.0:2.0.0'
-  }
-  withName:versions {
-    container = 'bicf/python3:2.0.0'
-  }
-  withName:multiqc {
-    container = 'bicf/multiqc:2.0.0'
-  }
-}
-
 trace {
   enabled = true
   file = 'pipeline_trace.txt'