Remove all tracking with AWS resources

.gitlab-ci.yml

@@ -767,7 +767,7 @@ integration_se:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./SE_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png -with-report ./SE_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \;
     - pytest -m completionMultiqc --filename SE_multiqc_data.json
   artifacts:
@@ -792,7 +792,7 @@ integration_pe:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./PE_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png -with-report ./PE_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \;
     - pytest -m completionMultiqc --filename PE_multiqc_data.json
   artifacts:
@@ -819,7 +819,7 @@ override_inputBag:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true --track false -with-report ./inputBagOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false -with-report ./inputBagOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./inputBagOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename inputBagOverride_multiqc_data.json
@@ -844,7 +844,7 @@ override_fastq:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6  --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true --track false -with-report ./fastqOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6  --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false -with-report ./fastqOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./fastqOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename fastqOverride_multiqc_data.json
@@ -869,7 +869,7 @@ override_species:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true --dev false --ci true --track false -with-report ./speciesOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true -with-report ./speciesOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./speciesOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename speciesOverride_multiqc_data.json
@@ -894,7 +894,7 @@ override_stranded:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true --dev false --ci true --track false -with-report ./strandedOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true -with-report ./strandedOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./strandedOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./strandedOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename strandedOverride_multiqc_data.json
@@ -919,7 +919,7 @@ override_spike:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true --dev false --ci true --track false -with-report ./spikeOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true -with-report ./spikeOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./spikeOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./spikeOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename spikeOverride_multiqc_data.json

CHANGELOG.md

@@ -2,6 +2,7 @@
 **User Facing**
 * Corrected spelling of inferred (#125)
 * Add param for seqwho reference (#127)
+* Remove tracking param and all tracking code
 
 **Background**
 * Corrected file search parameters due to name inconsistency (#129)

README.md

@@ -72,9 +72,6 @@ To Run:
     * eg: `--strandedForce 'unstranded'`
   * `--spikeForce` forces the spike-in to be "false", or "true", it bypasses a metadata mismatch error
     * eg: `--spikeForce 'true'`
-* Tracking parameters ([Tracking Site](http://bicf.pipeline.tracker.s3-website-us-east-1.amazonaws.com/)):
-  * `--ci` boolean (default = false)
-  * `--dev` boolean (default = true)
 
 FULL EXAMPLE:
 -------------

rna-seq.nf

@@ -27,7 +27,6 @@ params.seqwhoRef = "https://cloud.biohpc.swmed.edu/index.php/s/sP48taKmymSkJBM/d
 params.outDir = "${baseDir}/output"
 params.upload = false
 params.email = ""
-params.track = false
 
 // Define override input variable
 params.refSource = "biohpc"
@@ -38,10 +37,6 @@ params.speciesForce = ""
 params.strandedForce = ""
 params.spikeForce = ""
 
-// Define tracking input variables
-params.ci = false
-params.dev = true
-
 // Parse input variables
 deriva = Channel
   .fromPath(params.deriva)
@@ -123,47 +118,6 @@ script_deleteEntry_uploadQC = Channel.fromPath("${baseDir}/workflow/scripts/dele
 script_deleteEntry_uploadQC_fail = Channel.fromPath("${baseDir}/workflow/scripts/delete_entry.py")
 script_deleteEntry_uploadProcessedFile = Channel.fromPath("${baseDir}/workflow/scripts/delete_entry.py")
 
-/*
- * trackStart: track start of pipeline
- */
-process trackStart {
-  script:
-    """
-    hostname
-    ulimit -a
-
-    curl -H 'Content-Type: application/json' -X PUT -d \
-      '{ \
-        "sessionId": "${workflow.sessionId}", \
-        "pipeline": "gudmap.rbk_rnaseq", \
-        "start": "${workflow.start}", \
-        "repRID": "${repRID}", \
-        "astrocyte": false, \
-        "status": "started", \
-        "nextflowVersion": "${workflow.nextflow.version}", \
-        "pipelineVersion": "${workflow.manifest.version}", \
-        "ci": ${params.ci}, \
-        "dev": ${params.dev} \
-      }' \
-      "https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking"
-
-    if [ ${params.track} == true ]
-    then
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "repRID": "${repRID}", \
-          "PipelineVersion": "${workflow.manifest.version}", \
-          "Server": "${params.source}", \
-          "Queued": "NA", \
-          "CheckedOut": "NA", \
-          "Started": "${workflow.start}" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
-    """
-}
-
 log.info """\
 ====================================
 BICF RNA-seq Pipeline for GUDMAP/RBK
@@ -176,15 +130,11 @@ ERCC Reference Version : ${params.refERCCVersion}
 Reference source       : ${params.refSource}
 Output Directory       : ${params.outDir}
 Upload                 : ${upload}
-Track                  : ${params.track}
 ------------------------------------
 Nextflow Version       : ${workflow.nextflow.version}
 Pipeline Version       : ${workflow.manifest.version}
 Session ID             : ${workflow.sessionId}
 ------------------------------------
-CI                     : ${params.ci}
-Development            : ${params.dev}
-------------------------------------
 """
 
 /*
@@ -2198,13 +2148,6 @@ process aggrQC {
     echo -e "LOG: running multiqc" >> ${repRID}.aggrQC.log
     multiqc -c ${multiqcConfig} . -n ${repRID}.multiqc.html
     cp ${repRID}.multiqc_data/multiqc_data.json ${repRID}.multiqc_data.json
-
-    if [ ${params.track} == true ]
-    then
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        @./${repRID}.multiqc_data.json \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/qc"
-    fi
     """
 }
 
@@ -2359,16 +2302,6 @@ process uploadExecutionRun {
     fi
 
     echo "\${executionRun_rid}" > executionRunRID.csv
-
-    if [ ${params.track} == true ]
-    then
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "ExecutionRunRID": "'\${executionRun_rid}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
     """
 }
 
@@ -2686,17 +2619,6 @@ process finalizeExecutionRun {
 
     rid=\$(python3 ${script_uploadExecutionRun_finalizeExecutionRun} -r ${repRID} -w \${workflow} -g \${genome} -i ${inputBagRID} -s Success -d 'Run Successful' -o ${source} -c \${cookie} -u ${executionRunRID})
     echo LOG: execution run RID marked as successful - \${rid} >> ${repRID}.finalizeExecutionRun.log
-
-    if [ ${params.track} == true ]
-    then
-    dt=`date +%FT%T.%3N%:z`
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "Complete": "'\${dt}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
     """
 }
 
@@ -2794,18 +2716,6 @@ process failPreExecutionRun {
     fi
 
     echo "\${rid}" > executionRunRID.csv
-
-    if [ ${params.track} == true ]
-    then
-    dt=`date +%FT%T.%3N%:z`
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "ExecutionRunRID": "'\${rid}'", \
-          "Failure": "'\${dt}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
   """
 }
 // Extract execution run RID into channel
@@ -2895,18 +2805,6 @@ process failPreExecutionRun_seqwho {
     fi
 
     echo "\${rid}" > executionRunRID.csv
-
-    if [ ${params.track} == true ]
-    then
-    dt=`date +%FT%T.%3N%:z`
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "ExecutionRunRID": "'\${rid}'", \
-          "Failure": "'\${dt}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
   """
 }
 // Extract execution run RID into channel
@@ -2998,18 +2896,6 @@ process failExecutionRun {
       rid=\$(python3 ${script_uploadExecutionRun_failExecutionRun} -r ${repRID} -w \${workflow} -g \${genome} -i ${inputBagRID} -s Error -d "\${pipelineError_details}" -o ${source} -c \${cookie} -u ${executionRunRID})
       echo LOG: execution run RID marked as error - \${rid} >> ${repRID}.failExecutionRun.log
     fi
-
-    if [ ${params.track} == true ]
-    then
-      dt=`date +%FT%T.%3N%:z`
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "ExecutionRunRID": "'\${rid}'", \
-          "Failure": "'\${dt}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
   """
 }