diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 791ef96be43a9ba90e9ef57e51a8e89a7f17091a..3701d03735232c14551766626e19eb94605ef3ee 100755 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,9 @@ stages: astrocyte_check: stage: astrocyte + except: + changes: + - "**/*.md" script: - astrocyte_cli check ../cellranger_count artifacts: @@ -36,16 +39,21 @@ simple_1: - branches - tags except: - - develop - - master + refs: + - develop + - master + changes: + - "**/*.md" script: - - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.0.2' - - pytest -m count302 + - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.1.0' + - pytest -m count310 artifacts: name: "$CI_JOB_NAME" when: always paths: - .nextflow.log + - workflow/output/count310/sample1/outs/web_summary.html + - workflow/output/multiqc/run/multiqc_report.html expire_in: 2 days retry: max: 1 @@ -57,17 +65,22 @@ simple_2: only: - branches except: - - develop - - master - - tags + refs: + - develop + - master + - tags + changes: + - "**/*.md" script: - - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.1' - - pytest -m count301 + - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.1.0' + - pytest -m count310 artifacts: name: "$CI_JOB_NAME" when: always paths: - .nextflow.log + - workflow/output/count310/sample1/outs/web_summary.html + - workflow/output/multiqc/run/multiqc_report.html expire_in: 2 days retry: max: 1 @@ -80,36 +93,47 @@ detailed_1: - develop - master except: - - tags + refs: + - tags + changes: + - "**/*.md" script: - - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.0.2' - - pytest -m count302 + - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0' + - pytest -m count310 artifacts: name: "$CI_JOB_NAME" when: always paths: - .nextflow.log + - workflow/output/count310/sample1/outs/web_summary.html + - workflow/output/multiqc/run/multiqc_report.html expire_in: 2 days retry: max: 1 when: - always + detailed_2: stage: detailed only: - develop - master except: - - tags + refs: + - tags + changes: + - "**/*.md" script: - - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.2' + - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.0.2' - pytest -m count302 artifacts: name: "$CI_JOB_NAME" when: always paths: - .nextflow.log + - workflow/output/count302/sample1/outs/web_summary.html + - workflow/output/multiqc/run/multiqc_report.html expire_in: 2 days retry: max: 1 @@ -122,7 +146,36 @@ detailed_3: - develop - master except: - - tags + refs: + - tags + changes: + - "**/*.md" + script: + - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.1' + - pytest -m count301 + artifacts: + name: "$CI_JOB_NAME" + when: always + paths: + - .nextflow.log + - workflow/output/count301/sample1/outs/web_summary.html + - workflow/output/multiqc/run/multiqc_report.html + expire_in: 2 days + retry: + max: 1 + when: + - always + +detailed_4: + stage: detailed + only: + - develop + - master + except: + refs: + - tags + changes: + - "**/*.md" script: - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v2s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v2s2r10k/design.csv" --genome 'GRCh38-1.2.0' --kitVersion 'two' --version '2.1.1' - pytest -m count211 @@ -131,6 +184,8 @@ detailed_3: when: always paths: - .nextflow.log + - workflow/output/count211/sample1/outs/web_summary.html + - workflow/output/multiqc/run/multiqc_report.html expire_in: 2 days retry: max: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b2eea50c1a70e16e7fd5679667615c80bdd3976..2311789bea4b10cca0fe405e2a3449942ea91115 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # v1.2.0 (in development) **User Facing** +* Add Cellranger Version 3.1.0 +* Add CI Artifacts **Background** diff --git a/README.md b/README.md index 6892ea715becad0d2669be7313747d5679e6c85f..504ea0089664ed4b2559a6a1e462251ed1eb91c7 100755 --- a/README.md +++ b/README.md @@ -46,8 +46,9 @@ To Run: * *'hg19-1.2.0'* = Human GRCh37 (hg19) release 84 * *'mm10-3.0.0'* = Mouse GRCm38 (mm10) release 93 * *'mm10-3.0.0'* = Mouse GRCm38 (mm10) release 84 - * *'hg19_and_mm10-3.0.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 93 - * *'hg19_and_mm10-1.2.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 84 + * *'GRCh38_and_mm10-3.1.0'* = Human GRCh38 + Mouse GRCm38 (mm10) release 93 + * *'hg19_and_mm10-3.0.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 93 + * *'hg19_and_mm10-1.2.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 84 * *'ercc92-1.2.0'* = ERCC.92 Spike-In * if --genome is used then --genomeLocationFull is not necessary * eg: **--genome 'GRCh38-3.0.0'** @@ -83,16 +84,17 @@ To Run: * cellranger version * --version (cellranger version) 2.1.1 can only read --kitVersion of two (2) * options: + * *'3.1.0'* * *'3.0.2'* * *'3.0.1'* * *'2.1.1'* - * eg: **--version '3.0.2'** + * eg: **--version '3.1.0'** * **--outDir** * optional output directory for run * eg: **--outDir 'test'** * FULL EXAMPLE: ``` - nextflow run workflow/main.nf --fastq '/project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r100k/*.fastq.gz' --designFile '/project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r100k/design.csv' --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.0.2' --outDir 'test' + nextflow run workflow/main.nf --fastq '/project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r100k/*.fastq.gz' --designFile '/project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r100k/design.csv' --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.1.0' --outDir 'test' ``` * Design example: diff --git a/astrocyte_pkg.yml b/astrocyte_pkg.yml index 0c7a0a05e2792dc8ead17dd738c0135d19ec911a..1a7cb7a0cf81e44d4f67198cc57b5ad91fdd3820 100755 --- a/astrocyte_pkg.yml +++ b/astrocyte_pkg.yml @@ -43,6 +43,7 @@ workflow_modules: - 'cellranger/2.1.1' - 'cellranger/3.0.1' - 'cellranger/3.0.2' + - 'cellranger/3.1.0' - 'bcl2fastq/2.17.1.14' - 'multiqc/1.7' @@ -104,8 +105,9 @@ workflow_parameters: - ['hg19-1.2.0', 'Human GRCh37 (hg19) release 84'] - ['mm10-3.0.0', 'Mouse GRCm38 (mm10) release 93'] - ['mm10-1.2.0', 'Mouse GRCm38 (mm10) release 84'] - - ['hg19_and_mm10-3.0.0', 'Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 93'] - - ['hg19_and_mm10-1.2.0', 'Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 84'] + - ['GRCh38_and_mm10-3.1.0', 'Human GRCh38 + Mouse GRCm38 (mm10) release 93'] + - ['hg19_and_mm10-3.0.0', 'Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 93'] + - ['hg19_and_mm10-1.2.0', 'Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 84'] - ['ercc92-1.2.0', 'ERCC.92 Spike-In'] required: true description: | @@ -142,8 +144,9 @@ workflow_parameters: - id: version type: select - default: '3.0.2' + default: '3.1.0' choices: + - ['3.1.0', '3.1.0'] - ['3.0.2', '3.0.2'] - ['3.0.1', '3.0.1'] - ['2.1.1', '2.1.1'] diff --git a/docs/index.md b/docs/index.md index b2c1ad6c49707a7c493a047c659b3c1cf42acd7f..29cf0d399120909478055abf5538241312f28a40 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,8 +36,9 @@ To Run: * *'hg19-1.2.0'* = Human GRCh37 (hg19) release 84 * *'mm10-3.0.0'* = Human GRCm38 (mm10) release 93 * *'mm10-3.0.0'* = Human GRCm38 (mm10) release 84 - * *'hg19_and_mm10-3.0.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 93 - * *'hg19_and_mm10-1.2.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm19) release 84 + * *'GRCh38-and-mm10-3.1.0'* = Human GRCh38 + Mouse GRCm38 (mm10) release 93 + * *'hg19_and_mm10-3.0.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 93 + * *'hg19_and_mm10-1.2.0'* = Human GRCh37 (hg19) + Mouse GRCm38 (mm10) release 84 * *'ercc92-1.2.0'* = ERCC.92 Spike-In * **expect cells** * Expected number of recovered cells. diff --git a/workflow/conf/biohpc.config b/workflow/conf/biohpc.config index 6e356466fb53801aa9c00876f55bfc6903798c50..a5ea037bbdc5558ff20dec18c238ffa4e3f4ed44 100755 --- a/workflow/conf/biohpc.config +++ b/workflow/conf/biohpc.config @@ -18,6 +18,10 @@ process { module = ['cellranger/3.0.2'] queue = '128GB,256GB,256GBv1,384GB' } + withLabel: count310 { + module = ['cellranger/3.1.0'] + queue = '128GB,256GB,256GBv1,384GB' + } withLabel: versions { module = ['python/3.6.1-2-anaconda','pandoc/2.7','multiqc/1.7'] executor = 'local' @@ -49,6 +53,9 @@ params { 'mm10-1.2.0' { loc = '/project/apps_database/cellranger/refdata-cellranger-' } + 'GRCh38_and_mm10-3.1.0' { + loc = '/project/apps_database/cellranger/refdata-cellranger-' + } 'hg19_and_mm10-3.0.0' { loc = '/project/apps_database/cellranger/refdata-cellranger-' } diff --git a/workflow/main.nf b/workflow/main.nf index e0e2f551111de61d27d892fa8ab0883c84ec4b4b..de45c73039d0b5e077bc8b29901ec8eb946329a5 100755 --- a/workflow/main.nf +++ b/workflow/main.nf @@ -11,7 +11,7 @@ params.genome = 'GRCh38-3.0.0' params.expectCells = 10000 params.forceCells = 0 params.kitVersion = 'three' -params.version = '3.0.2' +params.version = '3.1.0' params.astrocyte = false params.outDir = "${baseDir}/output" params.multiqcConf = "${baseDir}/conf/multiqc_config.yaml" @@ -100,20 +100,25 @@ samples.into { samples211 samples301 samples302 + samples310 } refLocation.into { refLocation211 refLocation301 refLocation302 + refLocation310 } expectCells211 = expectCells expectCells301 = expectCells expectCells302 = expectCells +expectCells310 = expectCells forceCells211 = forceCells forceCells301 = forceCells forceCells302 = forceCells +forceCells310 = forceCells chemistryParam301 = chemistryParam chemistryParam302 = chemistryParam +chemistryParam310 = chemistryParam process count211 { @@ -247,6 +252,50 @@ process count302 { } +process count310 { + + queue '128GB,256GB,256GBv1,384GB' + tag "${sample}" + publishDir "${outDir}/${task.process}", mode: 'copy' + module 'cellranger/3.1.0' + + input: + set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples310 + file ref from refLocation310.first() + expectCells310 + forceCells310 + chemistryParam310 + + output: + file("**/outs/**") into outPaths310 + file("*_metrics_summary.tsv") into metricsSummary310 + + when: + version == '3.1.0' + + script: + if (forceCells310 == 0) { + """ + hostname + ulimit -a + bash ${baseDir}/scripts/filename_check.sh -r ${ref} + cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells310} --chemistry=${chemistryParam310} + sed -E 's/("([^"]*)")?,/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv + """ + } + else { + """ + hostname + ulimit -a + bash ${baseDir}/scripts/filename_check.sh -r ${ref} + cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells310} --chemistry=${chemistryParam310} + sed -E 's/("([^"]*)")?,/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv + """ + } + +} + + process versions { tag "${name}" @@ -272,7 +321,7 @@ process versions { } -metricsSummary = metricsSummary211.mix(metricsSummary301, metricsSummary302) +metricsSummary = metricsSummary211.mix(metricsSummary301, metricsSummary302, metricsSummary310) // Generate MultiQC Report diff --git a/workflow/tests/test_check_design.py b/workflow/tests/test_check_design.py index 28968b0dbf1ab4bca46df91ec1d7e23812b47e9d..c22850c9954dad11dbcc1fa229ea9d038c4e167d 100644 --- a/workflow/tests/test_check_design.py +++ b/workflow/tests/test_check_design.py @@ -19,3 +19,7 @@ def test_count301_design(): @pytest.mark.count302 def test_count302_design(): assert os.path.exists(os.path.join(test_output_path, 'design.checked.csv')) + +@pytest.mark.count310 +def test_count310_design(): + assert os.path.exists(os.path.join(test_output_path, 'design.checked.csv')) diff --git a/workflow/tests/test_count.py b/workflow/tests/test_count.py index b1bafe439b74fbe84e88f891e28e7880d653b1c3..3144d1edfad6752b7ff5d623e2005ed17264dc6a 100644 --- a/workflow/tests/test_count.py +++ b/workflow/tests/test_count.py @@ -23,3 +23,8 @@ def test_count301_count(): def test_count302_count(): assert os.path.exists(os.path.join(test_output_path, 'count302', 'sample1_metrics_summary.tsv')) assert os.path.exists(os.path.join(test_output_path, 'count302', 'sample1', 'outs')) + +@pytest.mark.count310 +def test_count310_count(): + assert os.path.exists(os.path.join(test_output_path, 'count310', 'sample1_metrics_summary.tsv')) + assert os.path.exists(os.path.join(test_output_path, 'count310', 'sample1', 'outs')) diff --git a/workflow/tests/test_multiqc.py b/workflow/tests/test_multiqc.py index d625e4088219cc3f875d1e55e7d17d3153e017e7..041ad232e843ef95ef0191cd20737392ceb44160 100644 --- a/workflow/tests/test_multiqc.py +++ b/workflow/tests/test_multiqc.py @@ -19,3 +19,7 @@ def test_count301_multiqc(): @pytest.mark.count302 def test_count302_multiqc(): assert os.path.exists(os.path.join(test_output_path, 'multiqc_report.html')) + +@pytest.mark.count310 +def test_count310_multiqc(): + assert os.path.exists(os.path.join(test_output_path, 'multiqc_report.html')) diff --git a/workflow/tests/test_versions.py b/workflow/tests/test_versions.py index 07a233c459c087051441780f1d0a42d00f6c6d65..b299fb9950d11f13f562bc7302d2fd923cdd3e0e 100644 --- a/workflow/tests/test_versions.py +++ b/workflow/tests/test_versions.py @@ -22,3 +22,8 @@ def test_count301_versions(): def test_count302_versions(): assert os.path.exists(os.path.join(test_output_path, 'versions_mqc.yaml')) assert os.path.exists(os.path.join(test_output_path, 'references_mqc.yaml')) + +@pytest.mark.count310 +def test_count310_versions(): + assert os.path.exists(os.path.join(test_output_path, 'versions_mqc.yaml')) + assert os.path.exists(os.path.join(test_output_path, 'references_mqc.yaml'))