Merge branch '30-v3.1.0' into 'develop'

Resolve "Add cellranger v 3.1.0"

Closes #30, #31, #32, and #33

See merge request !54

.gitlab-ci.yml

@@ -21,6 +21,9 @@ stages:
 
 astrocyte_check:
   stage: astrocyte
+  except:
+    changes:
+      - "**/*.md"
   script:
     - astrocyte_cli check ../cellranger_count
   artifacts:
@@ -36,16 +39,21 @@ simple_1:
     - branches
     - tags
   except:
-    - develop
-    - master
+    refs:
+      - develop
+      - master
+    changes:
+      - "**/*.md"
   script:
-  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.0.2'
-  - pytest -m count302
+  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.1.0'
+  - pytest -m count310
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
     paths:
       - .nextflow.log
+      - workflow/output/count310/sample1/outs/web_summary.html
+      - workflow/output/multiqc/run/multiqc_report.html
     expire_in: 2 days
   retry:
     max: 1
@@ -57,17 +65,22 @@ simple_2:
   only:
     - branches
   except:
-    - develop
-    - master
-    - tags
+    refs:
+      - develop
+      - master
+      - tags
+    changes:
+      - "**/*.md"
   script:
-  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.1'
-  - pytest -m count301
+  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.1.0'
+  - pytest -m count310
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
     paths:
       - .nextflow.log
+      - workflow/output/count310/sample1/outs/web_summary.html
+      - workflow/output/multiqc/run/multiqc_report.html
     expire_in: 2 days
   retry:
     max: 1
@@ -80,36 +93,47 @@ detailed_1:
     - develop
     - master
   except:
-    - tags
+    refs:
+      - tags
+    changes:
+      - "**/*.md"
   script:
-  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.0.2'
-  - pytest -m count302
+  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0'
+  - pytest -m count310
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
     paths:
       - .nextflow.log
+      - workflow/output/count310/sample1/outs/web_summary.html
+      - workflow/output/multiqc/run/multiqc_report.html
     expire_in: 2 days
   retry:
     max: 1
     when:
       - always
 
+
 detailed_2:
   stage: detailed
   only:
     - develop
     - master
   except:
-    - tags
+    refs:
+      - tags
+    changes:
+      - "**/*.md"
   script:
-  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.2'
+  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.0.2'
   - pytest -m count302
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
     paths:
       - .nextflow.log
+      - workflow/output/count302/sample1/outs/web_summary.html
+      - workflow/output/multiqc/run/multiqc_report.html
     expire_in: 2 days
   retry:
     max: 1
@@ -122,7 +146,36 @@ detailed_3:
     - develop
     - master
   except:
-    - tags
+    refs:
+      - tags
+    changes:
+      - "**/*.md"
+  script:
+  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.1'
+  - pytest -m count301
+  artifacts:
+    name: "$CI_JOB_NAME"
+    when: always
+    paths:
+      - .nextflow.log
+      - workflow/output/count301/sample1/outs/web_summary.html
+      - workflow/output/multiqc/run/multiqc_report.html
+    expire_in: 2 days
+  retry:
+    max: 1
+    when:
+      - always
+
+detailed_4:
+  stage: detailed
+  only:
+    - develop
+    - master
+  except:
+    refs:
+      - tags
+    changes:
+      - "**/*.md"
   script:
   - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v2s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v2s2r10k/design.csv" --genome 'GRCh38-1.2.0' --kitVersion 'two' --version '2.1.1'
   - pytest -m count211
@@ -131,6 +184,8 @@ detailed_3:
     when: always
     paths:
       - .nextflow.log
+      - workflow/output/count211/sample1/outs/web_summary.html
+      - workflow/output/multiqc/run/multiqc_report.html
     expire_in: 2 days
   retry:
     max: 1

CHANGELOG.md

 # v1.2.0 (in development)
 **User Facing**
+* Add Cellranger Version 3.1.0
+* Add CI Artifacts
 
 **Background**
 

README.md

@@ -46,8 +46,9 @@ To Run:
         * *'hg19-1.2.0'* = Human GRCh37 (hg19) release 84
         * *'mm10-3.0.0'* = Mouse GRCm38 (mm10) release 93
         * *'mm10-3.0.0'* = Mouse GRCm38 (mm10) release 84
-        * *'hg19_and_mm10-3.0.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 93
-        * *'hg19_and_mm10-1.2.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 84
+        * *'GRCh38_and_mm10-3.1.0'* = Human GRCh38 + Mouse GRCm38 (mm10) release 93
+        * *'hg19_and_mm10-3.0.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 93
+        * *'hg19_and_mm10-1.2.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 84
         * *'ercc92-1.2.0'* = ERCC.92 Spike-In
     * if --genome is used then --genomeLocationFull is not necessary
     * eg: **--genome 'GRCh38-3.0.0'**
@@ -83,16 +84,17 @@ To Run:
     * cellranger version
     * --version (cellranger version) 2.1.1 can only read --kitVersion of two (2)
     * options:
+        * *'3.1.0'*
         * *'3.0.2'*
         * *'3.0.1'*
         * *'2.1.1'*
-    * eg: **--version '3.0.2'**
+    * eg: **--version '3.1.0'**
   * **--outDir**
     * optional output directory for run
     * eg: **--outDir 'test'**
 * FULL EXAMPLE:
   ```
-  nextflow run workflow/main.nf --fastq '/project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r100k/*.fastq.gz' --designFile '/project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r100k/design.csv' --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.0.2' --outDir 'test'
+  nextflow run workflow/main.nf --fastq '/project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r100k/*.fastq.gz' --designFile '/project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r100k/design.csv' --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.1.0' --outDir 'test'
   ```
 * Design example:
 

astrocyte_pkg.yml

@@ -43,6 +43,7 @@ workflow_modules:
   - 'cellranger/2.1.1'
   - 'cellranger/3.0.1'
   - 'cellranger/3.0.2'
+  - 'cellranger/3.1.0'
   - 'bcl2fastq/2.17.1.14'
   - 'multiqc/1.7'
 
@@ -104,8 +105,9 @@ workflow_parameters:
       - ['hg19-1.2.0', 'Human GRCh37 (hg19) release 84']
       - ['mm10-3.0.0', 'Mouse GRCm38 (mm10) release 93']
       - ['mm10-1.2.0', 'Mouse GRCm38 (mm10) release 84']
-      - ['hg19_and_mm10-3.0.0', 'Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 93']
-      - ['hg19_and_mm10-1.2.0', 'Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 84']
+      - ['GRCh38_and_mm10-3.1.0', 'Human GRCh38 + Mouse GRCm38 (mm10) release 93']
+      - ['hg19_and_mm10-3.0.0', 'Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 93']
+      - ['hg19_and_mm10-1.2.0', 'Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 84']
       - ['ercc92-1.2.0', 'ERCC.92 Spike-In']
     required: true
     description: |
@@ -142,8 +144,9 @@ workflow_parameters:
 
   - id: version
     type: select
-    default: '3.0.2'
+    default: '3.1.0'
     choices:
+      - ['3.1.0', '3.1.0']
       - ['3.0.2', '3.0.2']
       - ['3.0.1', '3.0.1']
       - ['2.1.1', '2.1.1']

docs/index.md

@@ -36,8 +36,9 @@ To Run:
         * *'hg19-1.2.0'* = Human GRCh37 (hg19) release 84
         * *'mm10-3.0.0'* = Human GRCm38 (mm10) release 93
         * *'mm10-3.0.0'* = Human GRCm38 (mm10) release 84
-        * *'hg19_and_mm10-3.0.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 93
-        * *'hg19_and_mm10-1.2.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 84
+        * *'GRCh38-and-mm10-3.1.0'* = Human GRCh38 + Mouse GRCm38 (mm10) release 93
+        * *'hg19_and_mm10-3.0.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 93
+        * *'hg19_and_mm10-1.2.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 84
         * *'ercc92-1.2.0'* = ERCC.92 Spike-In
   * **expect cells**
         * Expected number of recovered cells.

workflow/conf/biohpc.config

@@ -18,6 +18,10 @@ process {
     module = ['cellranger/3.0.2']
     queue = '128GB,256GB,256GBv1,384GB'
   }
+  withLabel: count310 {
+    module = ['cellranger/3.1.0']
+    queue = '128GB,256GB,256GBv1,384GB'
+  }
   withLabel: versions {
     module = ['python/3.6.1-2-anaconda','pandoc/2.7','multiqc/1.7']
     executor = 'local'
@@ -49,6 +53,9 @@ params {
     'mm10-1.2.0' {
       loc = '/project/apps_database/cellranger/refdata-cellranger-'
     }
+    'GRCh38_and_mm10-3.1.0' {
+      loc = '/project/apps_database/cellranger/refdata-cellranger-'
+    }
     'hg19_and_mm10-3.0.0' {
       loc = '/project/apps_database/cellranger/refdata-cellranger-'
     }

workflow/main.nf

@@ -11,7 +11,7 @@ params.genome = 'GRCh38-3.0.0'
 params.expectCells = 10000
 params.forceCells = 0
 params.kitVersion = 'three'
-params.version = '3.0.2'
+params.version = '3.1.0'
 params.astrocyte = false
 params.outDir = "${baseDir}/output"
 params.multiqcConf = "${baseDir}/conf/multiqc_config.yaml"
@@ -100,20 +100,25 @@ samples.into {
   samples211
   samples301
   samples302
+  samples310
 }
 refLocation.into {
   refLocation211
   refLocation301
   refLocation302
+  refLocation310
 }
 expectCells211 = expectCells
 expectCells301 = expectCells
 expectCells302 = expectCells
+expectCells310 = expectCells
 forceCells211 = forceCells
 forceCells301 = forceCells
 forceCells302 = forceCells
+forceCells310 = forceCells
 chemistryParam301 = chemistryParam
 chemistryParam302 = chemistryParam
+chemistryParam310 = chemistryParam
 
 
 process count211 {
@@ -247,6 +252,50 @@ process count302 {
 }
 
 
+process count310 {
+
+  queue '128GB,256GB,256GBv1,384GB'
+  tag "${sample}"
+  publishDir "${outDir}/${task.process}", mode: 'copy'
+  module 'cellranger/3.1.0'
+
+  input:
+    set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples310
+    file ref from refLocation310.first()
+    expectCells310
+    forceCells310
+    chemistryParam310
+
+  output:
+    file("**/outs/**") into outPaths310
+    file("*_metrics_summary.tsv") into metricsSummary310
+
+  when:
+    version == '3.1.0'
+
+  script:
+    if (forceCells310 == 0) {
+      """
+      hostname
+      ulimit -a
+      bash ${baseDir}/scripts/filename_check.sh -r ${ref}
+      cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells310} --chemistry=${chemistryParam310}
+      sed -E 's/("([^"]*)")?,/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv
+      """
+    }
+    else {
+      """
+      hostname
+      ulimit -a
+      bash ${baseDir}/scripts/filename_check.sh -r ${ref}
+      cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells310} --chemistry=${chemistryParam310}
+      sed -E 's/("([^"]*)")?,/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv
+      """
+    }
+
+}
+
+
 process versions {
 
   tag "${name}"
@@ -272,7 +321,7 @@ process versions {
 }
 
 
-metricsSummary = metricsSummary211.mix(metricsSummary301, metricsSummary302)
+metricsSummary = metricsSummary211.mix(metricsSummary301, metricsSummary302, metricsSummary310)
 
 
 // Generate MultiQC Report

workflow/tests/test_check_design.py

@@ -19,3 +19,7 @@ def test_count301_design():
 @pytest.mark.count302
 def test_count302_design():
     assert os.path.exists(os.path.join(test_output_path, 'design.checked.csv'))
+
+@pytest.mark.count310
+def test_count310_design():
+    assert os.path.exists(os.path.join(test_output_path, 'design.checked.csv'))

workflow/tests/test_count.py

@@ -23,3 +23,8 @@ def test_count301_count():
 def test_count302_count():
     assert os.path.exists(os.path.join(test_output_path, 'count302', 'sample1_metrics_summary.tsv'))
     assert os.path.exists(os.path.join(test_output_path, 'count302', 'sample1', 'outs'))
+
+@pytest.mark.count310
+def test_count310_count():
+    assert os.path.exists(os.path.join(test_output_path, 'count310', 'sample1_metrics_summary.tsv'))
+    assert os.path.exists(os.path.join(test_output_path, 'count310', 'sample1', 'outs'))

workflow/tests/test_multiqc.py

@@ -19,3 +19,7 @@ def test_count301_multiqc():
 @pytest.mark.count302
 def test_count302_multiqc():
     assert os.path.exists(os.path.join(test_output_path, 'multiqc_report.html'))
+
+@pytest.mark.count310
+def test_count310_multiqc():
+    assert os.path.exists(os.path.join(test_output_path, 'multiqc_report.html'))

workflow/tests/test_versions.py

@@ -22,3 +22,8 @@ def test_count301_versions():
 def test_count302_versions():
     assert os.path.exists(os.path.join(test_output_path, 'versions_mqc.yaml'))
     assert os.path.exists(os.path.join(test_output_path, 'references_mqc.yaml'))
+
+@pytest.mark.count310
+def test_count310_versions():
+    assert os.path.exists(os.path.join(test_output_path, 'versions_mqc.yaml'))
+    assert os.path.exists(os.path.join(test_output_path, 'references_mqc.yaml'))