Convert tracking to curl

.gitlab-ci.yml

@@ -110,7 +110,7 @@ astrocyte_cli:
       - master
       - tags
   script:
-    - singularity run 'docker://bicf/cellranger3.1.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2
+    - singularity run 'docker://bicf/cellranger3.1.0:2.0.0' cellranger count --id=test --transcriptome=/project/apps_database/cellranger/refdata-cellranger-GRCh38-3.0.0 --fastqs=./test_data/hu.v2s1r500 --sample=pbmc_1k_v2 --chemistry=SC3Pv2 --ci true
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -134,7 +134,7 @@ GRCh38-3.0.0:
     - tags
   script:
   - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s1r500/*.fastq.gz" --designFile "test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion '3GEXv3' --version '3.1.0'
-  - pytest -m count310
+  - pytest -m count310 --ci true
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -158,7 +158,7 @@ mm10-3.0.0:
     - tags
   script:
   - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/mu.v3s1r500/*.fastq.gz" --designFile "test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion '3GEXv3' --version '3.1.0'
-  - pytest -m count310
+  - pytest -m count310 --ci true
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -179,7 +179,7 @@ mm10-3.0.0:
       - master
       - tags
   script:
-  - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0'
+  - nextflow -q run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0' --ci true
   - pytest -m count310
   artifacts:
     name: "$CI_JOB_NAME"

CHANGELOG.md

-# v2.0.0
+# v2.0.4
 **User Facing**
 * Check Design File for spaces in name and file contents
 * Attempt to preven thredding error (which appears to only happen on 256GBv1 nodes)
@@ -12,6 +12,7 @@
 * Update param to new standard
 * Use docker containers
 * Update CI
+* Add pipeline tracking tool
 
 *Known Bugs*
 * Vizapp does not yet work for Astrocyte

workflow/conf/cluster.config

@@ -3,6 +3,9 @@ process {
   queue = '32GB'
   clusterOptions = '--hold'
 
+  withLabel: trackStart {
+    executor = 'local'
+  }
   withLabel: checkDesignFile {
     executor = 'local'
   }

workflow/conf/local.config

 process {
   executor = 'local'
-}
-
-singularity {
-  enabled = true
-  cacheDir = '/project/BICF/BICF_Core/shared/gudmap/singularity_cache/'
-}
-
-env {
-  http_proxy = 'http://proxy.swmed.edu:3128'
-  https_proxy = 'http://proxy.swmed.edu:3128'
-  all_proxy = 'http://proxy.swmed.edu:3128'
 }
\ No newline at end of file

workflow/main.nf

@@ -10,9 +10,9 @@ main.nf
 
 // Define Input variables
 params.name = "run"
-params.fastq = "${baseDir}/../test_data/*.fastq.gz"
-params.designFile = "${baseDir}/../test_data/design.csv"
-params.genome = 'GRCh38-3.0.0'
+params.fastq = "test_data/mu.v3s1r500/*.fastq.gz"
+params.designFile = "test_data/mu.v3s1r500/design.csv"
+params.genome = 'mm10-3.0.0'
 params.genomeLocation = '/project/apps_database/cellranger/refdata-cellranger-'
 params.expectCells = 10000
 params.forceCells = 0
@@ -80,48 +80,26 @@ references = "${baseDir}/../docs/references.md"
 /*
  * trackStart: track start of pipeline
  */
+params.ci = false
 process trackStart {
   script:
   """
   hostname
   ulimit -a
   export https_proxy=\${http_proxy}
-  
-  aws dynamodb put-item \
-    --table-name pipeline.tracking \
-    --item '{ \
-      "sessionId": {"S": "${workflow.sessionId}"}, \
-      "pipeline": {"S": "cellranger_count"}, \
-      "projectDir": {"S": "${workflow.projectDir}"}, \
-      "start": {"S": "${workflow.start}"}, \
-      "astrocyte": {"BOOL": ${params.astrocyte}}, \
-      "status": {"S": "started"}}'
+ 
+  curl -H 'Content-Type: application/json' -X PUT -d '{ \
+      "sessionId": "${workflow.sessionId}", \
+      "pipeline": "cellranger_count", \
+      "start": "${workflow.start}", \
+      "astrocyte": ${params.astrocyte}, \
+      "status": "started", \
+      "nextflowVersion": "${workflow.nextflow.version}",
+      "ci": ${params.ci}}' \
+  "https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking"
   """
 }
 
-errorScript = """
-  singularity run 'docker://bicf/awscli:1.1' \
-    aws dynamodb update-item \
-      --table-name pipeline.tracking \
-      --key '{ \
-        "sessionId": {"S": "${workflow.sessionId}"}, \
-        "projectDir": {"S": "${workflow.projectDir}"}}' \
-      --update-expression 'SET #status = :status'  \
-      --expression-attribute-names '{"#status": "status"}' \
-      --expression-attribute-values '{":status": {"S":"error"}}'
-  """
-completeScript = """
-  singularity run 'docker://bicf/awscli:1.1' \
-    aws dynamodb update-item \
-      --table-name pipeline.tracking \
-      --key '{ \
-        "sessionId": {"S": "${workflow.sessionId}"}, \
-        "projectDir": {"S": "${workflow.projectDir}"}}' \
-      --update-expression 'SET #status = :status'  \
-      --expression-attribute-names '{"#status": "status"}' \
-      --expression-attribute-values '{":status": {"S":"complete"}}'
-"""
-
 /*
  * checkDesignFile: check design file for errors
  */
@@ -417,11 +395,3 @@ process multiqc {
     multiqc -c ${multiqcConf} .
     """
 }
-
-workflow.onError {
-  errorResult = errorScript.execute().text
-}
-
-workflow.onComplete {
-  completeResult = completeScript.execute().text
-}

workflow/nextflow.config

@@ -14,9 +14,6 @@ profiles {
 }
 
 process {
-  withName:trackStart {
-    container = 'bicf/awscli:1.1'
-  }
   withName:checkDesignFile {
     container = 'bicf/python3:2.0.0'
   }
@@ -65,6 +62,6 @@ manifest {
   homePage = 'https://git.biohpc.swmed.edu/BICF/Astrocyte/cellranger_count'
   description = 'This pipeline is a wrapper for the cellranger count tool from 10x Genomics. It takes fastq files from 10x Genomics Single Cell Gene Expression libraries, performs alignment, filtering, barcode counting, and UMI counting. It uses the Chromium cellular barcodes to generate gene-barcode matrices, determine clusters, and perform gene expression analysis.'
   mainScript = 'main.nf'
-  version = 'publish_2.0.0'
+  version = 'publish_2.0.4'
   nextflowVersion = '>=0.31.0'
 }