Merge branch 'develop' into '131-add.force.seqtype'

.gitlab-ci.yml

@@ -255,7 +255,7 @@ seqwho:
     - seqwhoImg=$(cat nextflow.config | grep -oP "container = \K.*" | tr -d "'" | sort | uniq | grep seqwho | cut -d"/" -f2 | cut -d":" -f1)
     - seqwhoVar=$(cat nextflow.config | grep -oP "container = \K.*" | tr -d "'" | sort | uniq | grep seqwho | cut -d"/" -f2 | cut -d":" -f2)
     - echo ${dir}${seqwhoImg}_${seqwhoVar}.sif
-    - wget -O SeqWho.ix https://cloud.biohpc.swmed.edu/index.php/s/eeNWqZz8jqN5zWY/download
+    - wget -O SeqWho.ix https://cloud.biohpc.swmed.edu/index.php/s/sP48taKmymSkJBM/download
     - mkdir -p SeqWho_call_plots/test_data/fastq/small/
     - singularity run ${dir}${seqwhoImg}_${seqwhoVar}.sif seqwho.py -f test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -x SeqWho.ix
     - pytest -m seqwho
@@ -767,7 +767,7 @@ integration_se:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./SE_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png -with-report ./SE_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \;
     - pytest -m completionMultiqc --filename SE_multiqc_data.json
   artifacts:
@@ -792,7 +792,7 @@ integration_pe:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./PE_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png -with-report ./PE_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \;
     - pytest -m completionMultiqc --filename PE_multiqc_data.json
   artifacts:
@@ -819,7 +819,7 @@ override_inputBag:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true --track false -with-report ./inputBagOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false -with-report ./inputBagOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./inputBagOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename inputBagOverride_multiqc_data.json
@@ -844,7 +844,7 @@ override_fastq:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6  --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true --track false -with-report ./fastqOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6  --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false -with-report ./fastqOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./fastqOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename fastqOverride_multiqc_data.json
@@ -869,7 +869,7 @@ override_species:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true --dev false --ci true --track false -with-report ./speciesOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true -with-report ./speciesOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./speciesOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename speciesOverride_multiqc_data.json
@@ -894,7 +894,7 @@ override_stranded:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true --dev false --ci true --track false -with-report ./strandedOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true -with-report ./strandedOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./strandedOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./strandedOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename strandedOverride_multiqc_data.json
@@ -919,7 +919,7 @@ override_spike:
   script:
     - hostname
     - ulimit -a
-    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true --dev false --ci true --track false -with-report ./spikeOverride_report.html
+    - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true -with-report ./spikeOverride_report.html
     - find . -type f -name "multiqc_data.json" -exec cp {} ./spikeOverride_multiqc_data.json \;
     - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./spikeOverride_multiqc.html \;
     - pytest -m completionMultiqc --filename spikeOverride_multiqc_data.json

CHANGELOG.md

 # v.2.0.1 (indev)
 **User Facing**
 * Corrected spelling of inferred (#125)
+ce
 * Add seqtype force parameter (#131)
+* Add param for seqwho reference (#127)
+* Remove tracking param and all tracking code
+
 
 **Background**
 * Corrected file search parameters due to name inconsistency (#129)
@@ -12,6 +16,9 @@
 * Remove "fail" integration CI tests (#133 addendum)
 * Add licence header to nf file,  R and python scriptsand pytest scripts
 * Stop using RCB and RMB (default) with deriva.core insert calls to match server update (#134)
+* Replace default seqwho reference location to one provided by source lab (#127)
+* Update NF Tower access token to BICF user
+* Disable NF Tower
 
 # v2.0.0
 **User Facing**

README.md

@@ -38,6 +38,7 @@ To Run:
   * `--refMoVersion` mouse reference version ***(optional, default = 38.p6.vM25)***
   * `--refHuVersion` human reference version ***(optional, default = 38.p13.v36)***
   * `--refERCCVersion` human reference version ***(optional, default = 92)***
+  * `--seqwhoRef` seqwho reference location ***(optional, default = https://cloud.biohpc.swmed.edu/index.php/s/sP48taKmymSkJBM/download)***
   * `--upload` option to not upload output back to the data-hub ***(optional, default = false)***
     * **true** = upload outputs to the data-hub
     * **false** = do *NOT* upload outputs to the data-hub
@@ -73,9 +74,6 @@ To Run:
     * eg: `--strandedForce 'unstranded'`
   * `--spikeForce` forces the spike-in to be "false", or "true", it bypasses a metadata mismatch error
     * eg: `--spikeForce 'true'`
-* Tracking parameters ([Tracking Site](http://bicf.pipeline.tracker.s3-website-us-east-1.amazonaws.com/)):
-  * `--ci` boolean (default = false)
-  * `--dev` boolean (default = true)
 
 FULL EXAMPLE:
 -------------

nextflow.config

@@ -110,8 +110,9 @@ process {
 }
 
 tower {
-  accessToken = '3ade8f325d4855434b49aa387421a44c63e3360f'
-  enabled = true
+  accessToken = 'eyJ0aWQiOiAzODIyfS42YWFjOWMwZDM4Mjc1NmY0MGJiY2QxNWRkOTZlYTU4MDlmY2NkZmRj'
+  workspaceId = '84512216752206'
+  enabled = false
 }
 
 manifest {

rna-seq.nf

@@ -23,10 +23,10 @@ params.source = "dev"
 params.refMoVersion = "38.p6.vM25"
 params.refHuVersion = "38.p13.v36"
 params.refERCCVersion = "92"
+params.seqwhoRef = "https://cloud.biohpc.swmed.edu/index.php/s/sP48taKmymSkJBM/download"
 params.outDir = "${baseDir}/output"
 params.upload = false
 params.email = ""
-params.track = false
 
 // Define override input variable
 params.refSource = "biohpc"
@@ -38,10 +38,6 @@ params.strandedForce = ""
 params.spikeForce = ""
 params.seqtypeForce = ""
 
-// Define tracking input variables
-params.ci = false
-params.dev = true
-
 // Parse input variables
 deriva = Channel
   .fromPath(params.deriva)
@@ -68,6 +64,7 @@ repRID = params.repRID
 refMoVersion = params.refMoVersion
 refHuVersion = params.refHuVersion
 refERCCVersion = params.refERCCVersion
+seqwhoRef = params.seqwhoRef
 outDir = params.outDir
 logsDir = "${outDir}/Logs"
 upload = params.upload
@@ -123,47 +120,6 @@ script_deleteEntry_uploadQC = Channel.fromPath("${baseDir}/workflow/scripts/dele
 script_deleteEntry_uploadQC_fail = Channel.fromPath("${baseDir}/workflow/scripts/delete_entry.py")
 script_deleteEntry_uploadProcessedFile = Channel.fromPath("${baseDir}/workflow/scripts/delete_entry.py")
 
-/*
- * trackStart: track start of pipeline
- */
-process trackStart {
-  script:
-    """
-    hostname
-    ulimit -a
-
-    curl -H 'Content-Type: application/json' -X PUT -d \
-      '{ \
-        "sessionId": "${workflow.sessionId}", \
-        "pipeline": "gudmap.rbk_rnaseq", \
-        "start": "${workflow.start}", \
-        "repRID": "${repRID}", \
-        "astrocyte": false, \
-        "status": "started", \
-        "nextflowVersion": "${workflow.nextflow.version}", \
-        "pipelineVersion": "${workflow.manifest.version}", \
-        "ci": ${params.ci}, \
-        "dev": ${params.dev} \
-      }' \
-      "https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking"
-
-    if [ ${params.track} == true ]
-    then
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "repRID": "${repRID}", \
-          "PipelineVersion": "${workflow.manifest.version}", \
-          "Server": "${params.source}", \
-          "Queued": "NA", \
-          "CheckedOut": "NA", \
-          "Started": "${workflow.start}" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
-    """
-}
-
 log.info """\
 ====================================
 BICF RNA-seq Pipeline for GUDMAP/RBK
@@ -176,15 +132,11 @@ ERCC Reference Version : ${params.refERCCVersion}
 Reference source       : ${params.refSource}
 Output Directory       : ${params.outDir}
 Upload                 : ${upload}
-Track                  : ${params.track}
 ------------------------------------
 Nextflow Version       : ${workflow.nextflow.version}
 Pipeline Version       : ${workflow.manifest.version}
 Session ID             : ${workflow.sessionId}
 ------------------------------------
-CI                     : ${params.ci}
-Development            : ${params.dev}
-------------------------------------
 """
 
 /*
@@ -716,6 +668,7 @@ process seqwho {
   tag "${repRID}"
 
   input:
+    val seqwhoRef
     path (fastq) from fastqs_seqwho
     val ends from endsManual_seqwho
     val speciesMeta from speciesMeta_seqwho
@@ -736,7 +689,7 @@ process seqwho {
     ulimit -a >> ${repRID}.seqwho.log
 
     # get seqwho index
-    wget -O SeqWho.ix https://cloud.biohpc.swmed.edu/index.php/s/eeNWqZz8jqN5zWY/download
+    wget -O SeqWho.ix ${seqwhoRef}
     echo -e "LOG: seqwho index downloaded" >> ${repRID}.seqwho.log
 
     # run seqwho
@@ -2209,13 +2162,6 @@ process aggrQC {
     echo -e "LOG: running multiqc" >> ${repRID}.aggrQC.log
     multiqc -c ${multiqcConfig} . -n ${repRID}.multiqc.html
     cp ${repRID}.multiqc_data/multiqc_data.json ${repRID}.multiqc_data.json
-
-    if [ ${params.track} == true ]
-    then
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        @./${repRID}.multiqc_data.json \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/qc"
-    fi
     """
 }
 
@@ -2370,16 +2316,6 @@ process uploadExecutionRun {
     fi
 
     echo "\${executionRun_rid}" > executionRunRID.csv
-
-    if [ ${params.track} == true ]
-    then
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "ExecutionRunRID": "'\${executionRun_rid}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
     """
 }
 
@@ -2697,17 +2633,6 @@ process finalizeExecutionRun {
 
     rid=\$(python3 ${script_uploadExecutionRun_finalizeExecutionRun} -r ${repRID} -w \${workflow} -g \${genome} -i ${inputBagRID} -s Success -d 'Run Successful' -o ${source} -c \${cookie} -u ${executionRunRID})
     echo LOG: execution run RID marked as successful - \${rid} >> ${repRID}.finalizeExecutionRun.log
-
-    if [ ${params.track} == true ]
-    then
-    dt=`date +%FT%T.%3N%:z`
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "Complete": "'\${dt}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
     """
 }
 
@@ -2805,18 +2730,6 @@ process failPreExecutionRun {
     fi
 
     echo "\${rid}" > executionRunRID.csv
-
-    if [ ${params.track} == true ]
-    then
-    dt=`date +%FT%T.%3N%:z`
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "ExecutionRunRID": "'\${rid}'", \
-          "Failure": "'\${dt}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
   """
 }
 // Extract execution run RID into channel
@@ -2906,18 +2819,6 @@ process failPreExecutionRun_seqwho {
     fi
 
     echo "\${rid}" > executionRunRID.csv
-
-    if [ ${params.track} == true ]
-    then
-    dt=`date +%FT%T.%3N%:z`
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "ExecutionRunRID": "'\${rid}'", \
-          "Failure": "'\${dt}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
   """
 }
 // Extract execution run RID into channel
@@ -3009,18 +2910,6 @@ process failExecutionRun {
       rid=\$(python3 ${script_uploadExecutionRun_failExecutionRun} -r ${repRID} -w \${workflow} -g \${genome} -i ${inputBagRID} -s Error -d "\${pipelineError_details}" -o ${source} -c \${cookie} -u ${executionRunRID})
       echo LOG: execution run RID marked as error - \${rid} >> ${repRID}.failExecutionRun.log
     fi
-
-    if [ ${params.track} == true ]
-    then
-      dt=`date +%FT%T.%3N%:z`
-      curl -H 'Content-Type: application/json' -X PUT -d \
-        '{ \
-          "ID": "${workflow.sessionId}", \
-          "ExecutionRunRID": "'\${rid}'", \
-          "Failure": "'\${dt}'" \
-        }' \
-        "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
-    fi
   """
 }