From 49d27e424e44513ae978e1bfddadef7c5c11b745 Mon Sep 17 00:00:00 2001
From: "Gervaise H. Henry" <gervaise.henry@utsouthwestern.edu>
Date: Sat, 25 Apr 2020 16:20:50 -0500
Subject: [PATCH] Convert tracking to curl
---
.gitlab-ci.yml | 8 ++---
CHANGELOG.md | 3 +-
workflow/conf/cluster.config | 3 ++
workflow/conf/local.config | 11 -------
workflow/main.nf | 58 +++++++++---------------------------
workflow/nextflow.config | 5 +---
6 files changed, 24 insertions(+), 64 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a22e859..7b2ddfc 100755
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -110,7 +110,7 @@ astrocyte_cli:
- master
- tags
script:
- - singularity run 'docker://bicf/cellranger3.1.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
+ - singularity run 'docker://bicf/cellranger3.1.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2 --ci true
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -134,7 +134,7 @@ GRCh38-3.0.0:
- tags
script:
- nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s1r500/*.fastq.gz" --designFile "test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion '3GEXv3' --version '3.1.0'
- - pytest -m count310
+ - pytest -m count310 --ci true
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -158,7 +158,7 @@ mm10-3.0.0:
- tags
script:
- nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/mu.v3s1r500/*.fastq.gz" --designFile "test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion '3GEXv3' --version '3.1.0'
- - pytest -m count310
+ - pytest -m count310 --ci true
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -179,7 +179,7 @@ mm10-3.0.0:
- master
- tags
script:
- - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0'
+ - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0' --ci true
- pytest -m count310
artifacts:
name: "$CI_JOB_NAME"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d53ae5a..af20e29 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,4 @@
-# v2.0.0
+# v2.0.4
**User Facing**
* Check Design File for spaces in name and file contents
* Attempt to preven thredding error (which appears to only happen on 256GBv1 nodes)
@@ -12,6 +12,7 @@
* Update param to new standard
* Use docker containers
* Update CI
+* Add pipeline tracking tool
*Known Bugs*
* Vizapp does not yet work for Astrocyte
diff --git a/workflow/conf/cluster.config b/workflow/conf/cluster.config
index 6abbd5a..7322376 100644
--- a/workflow/conf/cluster.config
+++ b/workflow/conf/cluster.config
@@ -3,6 +3,9 @@ process {
queue = '32GB'
clusterOptions = '--hold'
+ withLabel: trackStart {
+ executor = 'local'
+ }
withLabel: checkDesignFile {
executor = 'local'
}
diff --git a/workflow/conf/local.config b/workflow/conf/local.config
index ec805b0..9ca703b 100755
--- a/workflow/conf/local.config
+++ b/workflow/conf/local.config
@@ -1,14 +1,3 @@
process {
executor = 'local'
-}
-
-singularity {
- enabled = true
- cacheDir = '/project/BICF/BICF_Core/shared/gudmap/singularity_cache/'
-}
-
-env {
- http_proxy = 'http://proxy.swmed.edu:3128'
- https_proxy = 'http://proxy.swmed.edu:3128'
- all_proxy = 'http://proxy.swmed.edu:3128'
}
\ No newline at end of file
diff --git a/workflow/main.nf b/workflow/main.nf
index 8e13e95..6a181b0 100755
--- a/workflow/main.nf
+++ b/workflow/main.nf
@@ -10,9 +10,9 @@ main.nf
// Define Input variables
params.name = "run"
-params.fastq = "${baseDir}/../test_data/*.fastq.gz"
-params.designFile = "${baseDir}/../test_data/design.csv"
-params.genome = 'GRCh38-3.0.0'
+params.fastq = "test_data/mu.v3s1r500/*.fastq.gz"
+params.designFile = "test_data/mu.v3s1r500/design.csv"
+params.genome = 'mm10-3.0.0'
params.genomeLocation = '/project/apps_database/cellranger/refdata-cellranger-'
params.expectCells = 10000
params.forceCells = 0
@@ -80,48 +80,26 @@ references = "${baseDir}/../docs/references.md"
/*
* trackStart: track start of pipeline
*/
+params.ci = false
process trackStart {
script:
"""
hostname
ulimit -a
export https_proxy=\${http_proxy}
-
- aws dynamodb put-item \
- --table-name pipeline.tracking \
- --item '{ \
- "sessionId": {"S": "${workflow.sessionId}"}, \
- "pipeline": {"S": "cellranger_count"}, \
- "projectDir": {"S": "${workflow.projectDir}"}, \
- "start": {"S": "${workflow.start}"}, \
- "astrocyte": {"BOOL": ${params.astrocyte}}, \
- "status": {"S": "started"}}'
+
+ curl -H 'Content-Type: application/json' -X PUT -d '{ \
+ "sessionId": "${workflow.sessionId}", \
+ "pipeline": "cellranger_count", \
+ "start": "${workflow.start}", \
+ "astrocyte": ${params.astrocyte}, \
+ "status": "started", \
+ "nextflowVersion": "${workflow.nextflow.version}",
+ "ci": ${params.ci}}' \
+ "https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking"
"""
}
-errorScript = """
- singularity run 'docker://bicf/awscli:1.1' \
- aws dynamodb update-item \
- --table-name pipeline.tracking \
- --key '{ \
- "sessionId": {"S": "${workflow.sessionId}"}, \
- "projectDir": {"S": "${workflow.projectDir}"}}' \
- --update-expression 'SET #status = :status' \
- --expression-attribute-names '{"#status": "status"}' \
- --expression-attribute-values '{":status": {"S":"error"}}'
- """
-completeScript = """
- singularity run 'docker://bicf/awscli:1.1' \
- aws dynamodb update-item \
- --table-name pipeline.tracking \
- --key '{ \
- "sessionId": {"S": "${workflow.sessionId}"}, \
- "projectDir": {"S": "${workflow.projectDir}"}}' \
- --update-expression 'SET #status = :status' \
- --expression-attribute-names '{"#status": "status"}' \
- --expression-attribute-values '{":status": {"S":"complete"}}'
-"""
-
/*
* checkDesignFile: check design file for errors
*/
@@ -417,11 +395,3 @@ process multiqc {
multiqc -c ${multiqcConf} .
"""
}
-
-workflow.onError {
- errorResult = errorScript.execute().text
-}
-
-workflow.onComplete {
- completeResult = completeScript.execute().text
-}
diff --git a/workflow/nextflow.config b/workflow/nextflow.config
index 20ffa62..b05454d 100644
--- a/workflow/nextflow.config
+++ b/workflow/nextflow.config
@@ -14,9 +14,6 @@ profiles {
}
process {
- withName:trackStart {
- container = 'bicf/awscli:1.1'
- }
withName:checkDesignFile {
container = 'bicf/python3:2.0.0'
}
@@ -65,6 +62,6 @@ manifest {
homePage = 'https://git.biohpc.swmed.edu/BICF/Astrocyte/cellranger_count'
description = 'This pipeline is a wrapper for the cellranger count tool from 10x Genomics. It takes fastq files from 10x Genomics Single Cell Gene Expression libraries, performs alignment, filtering, barcode counting, and UMI counting. It uses the Chromium cellular barcodes to generate gene-barcode matrices, determine clusters, and perform gene expression analysis.'
mainScript = 'main.nf'
- version = 'publish_2.0.0'
+ version = 'publish_2.0.4'
nextflowVersion = '>=0.31.0'
}
--
GitLab