diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a22e859f34169ebf0451cd6229f8756d9b9d2b6d..7b2ddfc833b5cfe83c447a33353e42ed7fdf0f5a 100755
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -110,7 +110,7 @@ astrocyte_cli:
- master
- tags
script:
- - singularity run 'docker://bicf/cellranger3.1.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
+ - singularity run 'docker://bicf/cellranger3.1.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2 --ci true
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -134,7 +134,7 @@ GRCh38-3.0.0:
- tags
script:
- nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s1r500/*.fastq.gz" --designFile "test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion '3GEXv3' --version '3.1.0'
- - pytest -m count310
+ - pytest -m count310 --ci true
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -158,7 +158,7 @@ mm10-3.0.0:
- tags
script:
- nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/mu.v3s1r500/*.fastq.gz" --designFile "test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion '3GEXv3' --version '3.1.0'
- - pytest -m count310
+ - pytest -m count310 --ci true
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -179,7 +179,7 @@ mm10-3.0.0:
- master
- tags
script:
- - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0'
+ - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0' --ci true
- pytest -m count310
artifacts:
name: "$CI_JOB_NAME"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d53ae5aaf0b232fe95dca9a9bb193be4def70669..af20e29ec90e047ae2ac11d6fead4ce02f6bb158 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,4 @@
-# v2.0.0
+# v2.0.4
**User Facing**
* Check Design File for spaces in name and file contents
* Attempt to preven thredding error (which appears to only happen on 256GBv1 nodes)
@@ -12,6 +12,7 @@
* Update param to new standard
* Use docker containers
* Update CI
+* Add pipeline tracking tool
*Known Bugs*
* Vizapp does not yet work for Astrocyte
diff --git a/workflow/conf/cluster.config b/workflow/conf/cluster.config
index 6abbd5a6b727ce8ec62573e83dcd83ee80414025..7322376199a795c72f973f2bf14ae75c1c2c6e7b 100644
--- a/workflow/conf/cluster.config
+++ b/workflow/conf/cluster.config
@@ -3,6 +3,9 @@ process {
queue = '32GB'
clusterOptions = '--hold'
+ withLabel: trackStart {
+ executor = 'local'
+ }
withLabel: checkDesignFile {
executor = 'local'
}
diff --git a/workflow/conf/local.config b/workflow/conf/local.config
index ec805b0156837e24cc6b9d7e1c6c1b599074d208..9ca703b1d7f806b19fde6bd88a4fe2e7caeed1af 100755
--- a/workflow/conf/local.config
+++ b/workflow/conf/local.config
@@ -1,14 +1,3 @@
process {
executor = 'local'
-}
-
-singularity {
- enabled = true
- cacheDir = '/project/BICF/BICF_Core/shared/gudmap/singularity_cache/'
-}
-
-env {
- http_proxy = 'http://proxy.swmed.edu:3128'
- https_proxy = 'http://proxy.swmed.edu:3128'
- all_proxy = 'http://proxy.swmed.edu:3128'
}
\ No newline at end of file
diff --git a/workflow/main.nf b/workflow/main.nf
index 8e13e950f1b35ee0cfe9d4f20bd5404b58a07e53..6a181b0de06a2fe35f7190bf5e22f4ed25d5e61b 100755
--- a/workflow/main.nf
+++ b/workflow/main.nf
@@ -10,9 +10,9 @@ main.nf
// Define Input variables
params.name = "run"
-params.fastq = "${baseDir}/../test_data/*.fastq.gz"
-params.designFile = "${baseDir}/../test_data/design.csv"
-params.genome = 'GRCh38-3.0.0'
+params.fastq = "test_data/mu.v3s1r500/*.fastq.gz"
+params.designFile = "test_data/mu.v3s1r500/design.csv"
+params.genome = 'mm10-3.0.0'
params.genomeLocation = '/project/apps_database/cellranger/refdata-cellranger-'
params.expectCells = 10000
params.forceCells = 0
@@ -80,48 +80,26 @@ references = "${baseDir}/../docs/references.md"
/*
* trackStart: track start of pipeline
*/
+params.ci = false
process trackStart {
script:
"""
hostname
ulimit -a
export https_proxy=\${http_proxy}
-
- aws dynamodb put-item \
- --table-name pipeline.tracking \
- --item '{ \
- "sessionId": {"S": "${workflow.sessionId}"}, \
- "pipeline": {"S": "cellranger_count"}, \
- "projectDir": {"S": "${workflow.projectDir}"}, \
- "start": {"S": "${workflow.start}"}, \
- "astrocyte": {"BOOL": ${params.astrocyte}}, \
- "status": {"S": "started"}}'
+
+ curl -H 'Content-Type: application/json' -X PUT -d '{ \
+ "sessionId": "${workflow.sessionId}", \
+ "pipeline": "cellranger_count", \
+ "start": "${workflow.start}", \
+ "astrocyte": ${params.astrocyte}, \
+ "status": "started", \
+ "nextflowVersion": "${workflow.nextflow.version}",
+ "ci": ${params.ci}}' \
+ "https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking"
"""
}
-errorScript = """
- singularity run 'docker://bicf/awscli:1.1' \
- aws dynamodb update-item \
- --table-name pipeline.tracking \
- --key '{ \
- "sessionId": {"S": "${workflow.sessionId}"}, \
- "projectDir": {"S": "${workflow.projectDir}"}}' \
- --update-expression 'SET #status = :status' \
- --expression-attribute-names '{"#status": "status"}' \
- --expression-attribute-values '{":status": {"S":"error"}}'
- """
-completeScript = """
- singularity run 'docker://bicf/awscli:1.1' \
- aws dynamodb update-item \
- --table-name pipeline.tracking \
- --key '{ \
- "sessionId": {"S": "${workflow.sessionId}"}, \
- "projectDir": {"S": "${workflow.projectDir}"}}' \
- --update-expression 'SET #status = :status' \
- --expression-attribute-names '{"#status": "status"}' \
- --expression-attribute-values '{":status": {"S":"complete"}}'
-"""
-
/*
* checkDesignFile: check design file for errors
*/
@@ -417,11 +395,3 @@ process multiqc {
multiqc -c ${multiqcConf} .
"""
}
-
-workflow.onError {
- errorResult = errorScript.execute().text
-}
-
-workflow.onComplete {
- completeResult = completeScript.execute().text
-}
diff --git a/workflow/nextflow.config b/workflow/nextflow.config
index 20ffa629e3b984589f7905579edc001e2bbe6ce4..b05454d76d7f73ef9e519d33530d746af19af762 100644
--- a/workflow/nextflow.config
+++ b/workflow/nextflow.config
@@ -14,9 +14,6 @@ profiles {
}
process {
- withName:trackStart {
- container = 'bicf/awscli:1.1'
- }
withName:checkDesignFile {
container = 'bicf/python3:2.0.0'
}
@@ -65,6 +62,6 @@ manifest {
homePage = 'https://git.biohpc.swmed.edu/BICF/Astrocyte/cellranger_count'
description = 'This pipeline is a wrapper for the cellranger count tool from 10x Genomics. It takes fastq files from 10x Genomics Single Cell Gene Expression libraries, performs alignment, filtering, barcode counting, and UMI counting. It uses the Chromium cellular barcodes to generate gene-barcode matrices, determine clusters, and perform gene expression analysis.'
mainScript = 'main.nf'
- version = 'publish_2.0.0'
+ version = 'publish_2.0.4'
nextflowVersion = '>=0.31.0'
}