diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df02f25e8f5fb5fd9b1e39e370b161bff189479d..a22e859f34169ebf0451cd6229f8756d9b9d2b6d 100755 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,12 +2,15 @@ before_script: - module load astrocyte - module load python/3.6.1-2-anaconda - pip install --user pytest-pythonpath==0.7.1 pytest-cov==2.5.1 + - module load singularity/3.0.2 - module load nextflow/19.09.0 - - mkdir test_data/hu.v3s1r500 - - mkdir test_data/mu.v3s1r500 - - mkdir test_data/hu.v3s2r10k - - mkdir test_data/mu.v3s2r10k - - mkdir test_data/hu.v2s2r10k + - mkdir -p test_data/hu.v2s1r500 + - mkdir -p test_data/hu.v3s1r500 + - mkdir -p test_data/mu.v3s1r500 + - mkdir -p test_data/hu.v3s2r10k + - mkdir -p test_data/mu.v3s2r10k + - mkdir -p test_data/hu.v2s2r10k + - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v2s1r500/* test_data/hu.v2s1r500/ - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s1r500/* test_data/hu.v3s1r500/ - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/mu.v3s1r500/* test_data/mu.v3s1r500/ - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r10k/* test_data/hu.v3s2r10k/ @@ -15,12 +18,13 @@ before_script: - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v2s2r10k/* test_data/hu.v2s2r10k/ stages: - - astrocyte - - simple - - detailed + - astrocyte_test + - container_test + - reference_test + - multiSample_test -astrocyte_check: - stage: astrocyte +astrocyte_cli: + stage: astrocyte_test script: - astrocyte_cli check . artifacts: @@ -30,33 +34,30 @@ astrocyte_check: when: - always -simple_1: - stage: simple +2.1.1_test: + stage: container_test only: - branches - - tags except: refs: - develop - master + - tags script: - - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s1r500/*.fastq.gz" --designFile "test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion '3GEXv3' --version '3.1.0' - - pytest -m count310 + - singularity run 'docker://bicf/cellranger2.1.1:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2 artifacts: name: "$CI_JOB_NAME" when: always paths: - - .nextflow.log - - workflow/output/count310/sample1/outs/web_summary.html - - workflow/output/multiqc/run/multiqc_report.html + - test/outs/web_summary.html expire_in: 2 days retry: max: 1 when: - always -simple_2: - stage: simple +2.2.2_test: + stage: container_test only: - branches except: @@ -65,63 +66,81 @@ simple_2: - master - tags script: - - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/mu.v3s1r500/*.fastq.gz" --designFile "test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion '3GEXv3' --version '3.1.0' - - pytest -m count310 + - singularity run 'docker://bicf/cellranger2.2.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2 artifacts: name: "$CI_JOB_NAME" when: always paths: - - .nextflow.log - - workflow/output/count310/sample1/outs/web_summary.html - - workflow/output/multiqc/run/multiqc_report.html + - test/outs/web_summary.html expire_in: 2 days retry: max: 1 when: - always -detailed_1: - stage: detailed +3.0.2_test: + stage: container_test only: - - develop - - master + - branches except: refs: + - develop + - master - tags script: - - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0' - - pytest -m count310 + - singularity run 'docker://bicf/cellranger3.0.2:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2 artifacts: name: "$CI_JOB_NAME" when: always paths: - - .nextflow.log - - workflow/output/count310/sample1/outs/web_summary.html - - workflow/output/multiqc/run/multiqc_report.html + - test/outs/web_summary.html expire_in: 2 days retry: max: 1 when: - always - -detailed_2: - stage: detailed +3.1.0_test: + stage: container_test only: - - develop - - master + - branches except: - refs: + refs: + - develop + - master - tags script: - - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.0.2' - - pytest -m count302 + - singularity run 'docker://bicf/cellranger3.1.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2 + artifacts: + name: "$CI_JOB_NAME" + when: always + paths: + - develop + - master + - test/outs/web_summary.html + expire_in: 2 days + retry: + max: 1 + when: + - always + +GRCh38-3.0.0: + stage: reference_test + only: + refs: + - develop + - master + except: + - tags + script: + - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s1r500/*.fastq.gz" --designFile "test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion '3GEXv3' --version '3.1.0' + - pytest -m count310 artifacts: name: "$CI_JOB_NAME" when: always paths: - .nextflow.log - - workflow/output/count302/sample1/outs/web_summary.html + - workflow/output/count310/sample1/outs/web_summary.html - workflow/output/multiqc/run/multiqc_report.html expire_in: 2 days retry: @@ -129,23 +148,23 @@ detailed_2: when: - always -detailed_3: - stage: detailed +mm10-3.0.0: + stage: reference_test only: - - develop - - master - except: refs: - - tags + - develop + - master + except: + - tags script: - - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/mu.v3s2r10k/*.fastq.gz" --designFile "test_data/mu.v3s2r10k/design.csv" --genome 'mm10-3.0.0' --kitVersion '3GEXv3' --version '3.0.1' - - pytest -m count301 + - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/mu.v3s1r500/*.fastq.gz" --designFile "test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion '3GEXv3' --version '3.1.0' + - pytest -m count310 artifacts: name: "$CI_JOB_NAME" when: always paths: - .nextflow.log - - workflow/output/count301/sample1/outs/web_summary.html + - workflow/output/count310/sample1/outs/web_summary.html - workflow/output/multiqc/run/multiqc_report.html expire_in: 2 days retry: @@ -153,23 +172,22 @@ detailed_3: when: - always -detailed_4: - stage: detailed +2Samples: + stage: multiSample_test only: - - develop - - master - except: - refs: + refs: + - master - tags script: - - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v2s2r10k/*.fastq.gz" --designFile "test_data/hu.v2s2r10k/design.csv" --genome 'GRCh38-1.2.0' --kitVersion '3GEXv2' --version '2.1.1' - - pytest -m count211 + - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0' + - pytest -m count310 artifacts: name: "$CI_JOB_NAME" when: always paths: - .nextflow.log - - workflow/output/count211/sample1/outs/web_summary.html + - workflow/output/count310/sample1/outs/web_summary.html + - workflow/output/count310/sample2/outs/web_summary.html - workflow/output/multiqc/run/multiqc_report.html expire_in: 2 days retry: diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md new file mode 100644 index 0000000000000000000000000000000000000000..2ffc8b3ca4ebacc6b452a57aebdaa59906ce9b78 --- /dev/null +++ b/.gitlab/issue_templates/Bug.md @@ -0,0 +1,21 @@ +# Summary + + +# Steps to reproduce + + +# Observed bug behavior + + +# Expected behavior + + +# Relevant logs and/or screenshots + + +# Potential fixes + + + +/label ~bug ~"To Do" +/cc @ghenry diff --git a/.gitlab/merge_request_templates/Merge_Request.md b/.gitlab/merge_request_templates/Merge_Request.md new file mode 100644 index 0000000000000000000000000000000000000000..153c0b6de9e6c69b31ff5f0a627b3fb7d5278649 --- /dev/null +++ b/.gitlab/merge_request_templates/Merge_Request.md @@ -0,0 +1,17 @@ +Please fill in the appropriate checklist below (delete those which are not relevant). +These are the most common things requested on pull requests. + +## PR checklist + - [ ] This comment contains a description of changes (with reason) + - [ ] If you've fixed a bug or added code that should be tested, add tests! + - [ ] Documentation in `docs` is updated + - [ ] `CHANGELOG.md` is updated + - [ ] `README.md` is updated + - [ ] `LICENSE.md` is updated with new contributors + + +* [ ] **Close issue**\ +Closes # + +/cc @ghenry +/assign @ghenry diff --git a/CHANGELOG.md b/CHANGELOG.md index 6004b61479ec52d3128d3969f7e936b9d0204bd4..186cd6d02e9d8f26c9247b3fc6c90b7559564104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,15 @@ * Check Design File for spaces in name and file contents * Attempt to preven thredding error (which appears to only happen on 256GBv1 nodes) * Add option for 5' GEX chemistry +* Remove cellranger 3.0.1 as an option +* Add cellranger 2.2.0 as an option **Background** * Add Nextflow Tower integration into CI (GHH's profile) * Add new layered config folders, including prepare for awsifying * Update param to new standard +* Use docker containers +* Update CI *Known Bugs* * Vizapp does not yet work for Astrocyte diff --git a/README.md b/README.md index 85f6b7dc49ac4254a028c4d60e45abeefcfd81bd..ab2141e93077c9523590335cb541c70794fb94a5 100755 --- a/README.md +++ b/README.md @@ -83,11 +83,11 @@ To Run: * eg: **--kitVersion '3GEXv3'** * **--version** * cellranger version - * --version (cellranger version) 2.1.1 can only read --kitVersion of two (2) + * --version (cellranger version) 2.1.1 and 2.2.0 can only read --kitVersion of 3GEXv2 * options: * *'3.1.0'* * *'3.0.2'* - * *'3.0.1'* + * *'2.2.0'* * *'2.1.1'* * eg: **--version '3.1.0'** * **--outDir** diff --git a/workflow/conf/aws.config b/workflow/conf/aws.config index 6875c65e8ee30121798dd052822a363950ac19cc..e3612374c3997d368e808f7f9e0f31099a8dadda 100644 --- a/workflow/conf/aws.config +++ b/workflow/conf/aws.config @@ -12,4 +12,4 @@ process { queue = 'default-' cpus = 1 memory = '10 GB' -} \ No newline at end of file +} diff --git a/workflow/conf/biohpc.config b/workflow/conf/biohpc.config index 73042192e0ecd639910316421a8a2e198f5ff7cd..56d5b65c6196d6bf9a841561bd85330cd758f775 100644 --- a/workflow/conf/biohpc.config +++ b/workflow/conf/biohpc.config @@ -51,3 +51,14 @@ params { } } } + +singularity { + enabled = true + cacheDir = '/project/BICF/BICF_Core/shared/gudmap/singularity_cache/' +} + +env { + http_proxy = 'http://proxy.swmed.edu:3128' + https_proxy = 'http://proxy.swmed.edu:3128' + all_proxy = 'http://proxy.swmed.edu:3128' +} diff --git a/workflow/conf/cluster.config b/workflow/conf/cluster.config index 2526210647cbbc6bf1ccb4dfb6757e55320b97d0..6abbd5a6b727ce8ec62573e83dcd83ee80414025 100644 --- a/workflow/conf/cluster.config +++ b/workflow/conf/cluster.config @@ -4,28 +4,24 @@ process { clusterOptions = '--hold' withLabel: checkDesignFile { - module = ['python/3.6.1-2-anaconda'] + executor = 'local' } withLabel: count211 { - module = ['cellranger/2.1.1'] queue = '128GB,256GB,256GBv1,384GB' } - withLabel: count301 { - module = ['cellranger/3.0.1'] + withLabel: count220 { queue = '128GB,256GB,256GBv1,384GB' } withLabel: count302 { - module = ['cellranger/3.0.2'] queue = '128GB,256GB,256GBv1,384GB' } withLabel: count310 { - module = ['cellranger/3.1.0'] queue = '128GB,256GB,256GBv1,384GB' } withLabel: versions { - module = ['python/3.6.1-2-anaconda','pandoc/2.7','multiqc/1.7'] + executor = 'local' } withLabel: multiqc { - module = ['multiqc/1.7'] + executor = 'local' } } diff --git a/workflow/conf/local.config b/workflow/conf/local.config index 556476fe2256d196b301a6324ce7b6ab22725dea..a1e4055c4636c8282f85c0327f9742017b060ee2 100755 --- a/workflow/conf/local.config +++ b/workflow/conf/local.config @@ -1,25 +1,3 @@ process { executor = 'local' - - withLabel: checkDesignFile { - module = ['python/3.6.1-2-anaconda'] - } - withLabel: count211 { - module = ['cellranger/2.1.1'] - } - withLabel: count301 { - module = ['cellranger/3.0.1'] - } - withLabel: count302 { - module = ['cellranger/3.0.2'] - } - withLabel: count310 { - module = ['cellranger/3.1.0'] - } - withLabel: versions { - module = ['python/3.6.1-2-anaconda','pandoc/2.7','multiqc/1.7'] - } - withLabel: multiqc { - module = ['multiqc/1.7'] - } } diff --git a/workflow/main.nf b/workflow/main.nf index c9215a4f7df779a050f0405cc59c37cacf1b07a5..1194038abc4c462c12397439f676af9d2334cf47 100755 --- a/workflow/main.nf +++ b/workflow/main.nf @@ -26,17 +26,23 @@ if (params.kitVersion == "3GEXv3" && params.version == '2.1.1') { print("Cellranger Version 2.1.1 requires kitVersion 2") System.exit(32) } +if (params.kitVersion == "3GEXv3" && params.version == '2.2.0') { + print("Cellranger Version 2.2.0 requires kitVersion 2") + System.exit(32) +} // Define variables if astrocyte (or from config) if (params.astrocyte) { print("Running under astrocyte") params.genomeLocation = '/project/apps_database/cellranger/refdata-cellranger-' - if (params.kitVersion == "one") { + if (params.kitVersion == "3GEXv1") { params.chemistryParam ='SC3Pv1' - } else if (params.kitVersion == "two") { + } else if (params.kitVersion == "3GEXv2") { params.chemistryParam ='SC3Pv2' - } else if (params.kitVersion == "three") { + } else if (params.kitVersion == "3GEXv3") { params.chemistryParam ='SC3Pv3' + } else if (params.kitVersion == "5GEX") { + params.chemistryParam ='fiveprime' } else { params.chemistryParam = 'auto' } @@ -76,7 +82,7 @@ references = "${baseDir}/../docs/references.md" */ process checkDesignFile { tag "${name}" - module 'python/3.6.1-2-anaconda' + container = 'bicf/python3:2.0.0' input: file designLocation @@ -108,25 +114,26 @@ samples = designPaths // Duplicate variables samples.into { samples211 - samples301 + samples220 samples302 samples310 } refLocation.into { refLocation211 - refLocation301 + refLocation220 refLocation302 refLocation310 } expectCells211 = expectCells -expectCells301 = expectCells +expectCells220 = expectCells expectCells302 = expectCells expectCells310 = expectCells forceCells211 = forceCells -forceCells301 = forceCells +forceCells220 = forceCells forceCells302 = forceCells forceCells310 = forceCells -chemistryParam301 = chemistryParam +chemistryParam211 = chemistryParam +chemistryParam220 = chemistryParam chemistryParam302 = chemistryParam chemistryParam310 = chemistryParam @@ -137,13 +144,14 @@ process count211 { queue '128GB,256GB,256GBv1,384GB' tag "${sample}" publishDir "${outDir}/${task.process}", mode: 'copy' - module 'cellranger/2.1.1' + container 'bicf/cellranger2.1.1:2.0.0' input: set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples211 file ref from refLocation211.first() expectCells211 forceCells211 + chemistryParam211 output: file("**/outs/**") into outPaths211 @@ -158,7 +166,7 @@ process count211 { hostname ulimit -a bash ${baseDir}/scripts/filename_check.sh -r ${ref} - cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells211} + cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells211} --chemistry=${chemistryParam211} sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv """ } @@ -167,42 +175,42 @@ process count211 { hostname ulimit -a bash ${baseDir}/scripts/filename_check.sh -r ${ref} - cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells211} + cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells211} --chemistry=${chemistryParam211} sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv """ } } /* - * count211: run cellranger count version 3.0.1 + * count220: run cellranger count version 2.2.0 */ -process count301 { +process count220 { queue '128GB,256GB,256GBv1,384GB' tag "${sample}" publishDir "${outDir}/${task.process}", mode: 'copy' - module 'cellranger/3.0.1' + container 'bicf/cellranger2.2.0:2.0.0' input: - set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples301 - file ref from refLocation301.first() - expectCells301 - forceCells301 - chemistryParam301 + set sample, file("${sample}_S1_L00?_R1_001.fastq.gz"), file("${sample}_S1_L00?_R2_001.fastq.gz") from samples220 + file ref from refLocation220.first() + expectCells220 + forceCells220 + chemistryParam220 output: - file("**/outs/**") into outPaths301 - file("*_metrics_summary.tsv") into metricsSummary301 + file("**/outs/**") into outPaths220 + file("*_metrics_summary.tsv") into metricsSummary220 when: - version == '3.0.1' + version == '2.2.0' script: - if (forceCells301 == 0) { + if (forceCells220 == 0) { """ hostname ulimit -a bash ${baseDir}/scripts/filename_check.sh -r ${ref} - cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells301} --chemistry=${chemistryParam301} + cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells220} --chemistry=${chemistryParam220} sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv """ } @@ -211,20 +219,20 @@ process count301 { hostname ulimit -a bash ${baseDir}/scripts/filename_check.sh -r ${ref} - cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells301} --chemistry=${chemistryParam301} + cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells220} --chemistry=${chemistryParam220} sed -E 's/("([^"]*)")?(,|\$)/\\2\t/g' ${sample}/outs/metrics_summary.csv | tr -d "," | sed "s/^/${sample}\t/" > ${sample}_metrics_summary.tsv """ } } /* - * count211: run cellranger count version 3.0.2 + * count302: run cellranger count version 3.0.2 */ process count302 { queue '128GB,256GB,256GBv1,384GB' tag "${sample}" publishDir "${outDir}/${task.process}", mode: 'copy' - module 'cellranger/3.0.2' + container 'bicf/cellranger3.0.2:2.0.0' input: set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples302 @@ -264,13 +272,13 @@ process count302 { } /* - * count211: run cellranger count version 3.1.0 + * count310: run cellranger count version 3.1.0 */ process count310 { queue '128GB,256GB,256GBv1,384GB' tag "${sample}" publishDir "${outDir}/${task.process}", mode: 'copy' - module 'cellranger/3.1.0' + container 'bicf/cellranger3.1.0:2.0.0' input: set sample, file("${sample}_S?_L001_R1_001.fastq.gz"), file("${sample}_S?_L001_R2_001.fastq.gz") from samples310 @@ -313,7 +321,7 @@ process count310 { */ process versions { tag "${name}" - module 'python/3.6.1-2-anaconda:pandoc/2.7:multiqc/1.7' + container 'bicf/python3:2.0.0' input: @@ -333,7 +341,7 @@ process versions { } // Collect all metrics summaries reguardless of cellranger version -metricsSummary = metricsSummary211.mix(metricsSummary301, metricsSummary302, metricsSummary310) +metricsSummary = metricsSummary211.mix(metricsSummary220, metricsSummary302, metricsSummary310) /* * multiqc: create multiqc report diff --git a/workflow/nextflow.config b/workflow/nextflow.config index 1cfa59d2b5ac321f6af1d87fab074b3e82eaa981..3e15f5769a1da8e676852a504655cf8845fd6db1 100644 --- a/workflow/nextflow.config +++ b/workflow/nextflow.config @@ -13,6 +13,30 @@ profiles { } } +process { + withName:checkDesignFile { + container = 'bicf/python3:2.0.0' + } + withName:count211 { + container = 'bicf/cellranger2.1.1:2.0.0' + } + withName:count220 { + container = 'bicf/cellranger2.2.0:2.0.0' + } + withName:count302 { + container = 'bicf/cellranger3.0.2:2.0.0' + } + withName:count310 { + container = 'bicf/cellranger3.1.0:2.0.0' + } + withName:versions { + container = 'bicf/python3:2.0.0' + } + withName:multiqc { + container = 'bicf/multiqc:2.0.0' + } +} + trace { enabled = true file = 'pipeline_trace.txt'