diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4519e674866692619c2439246a2430c2d305acf5..21b094892729ce9518c6b74ec6cab0979a436aa3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -255,7 +255,7 @@ seqwho: - seqwhoImg=$(cat nextflow.config | grep -oP "container = \K.*" | tr -d "'" | sort | uniq | grep seqwho | cut -d"/" -f2 | cut -d":" -f1) - seqwhoVar=$(cat nextflow.config | grep -oP "container = \K.*" | tr -d "'" | sort | uniq | grep seqwho | cut -d"/" -f2 | cut -d":" -f2) - echo ${dir}${seqwhoImg}_${seqwhoVar}.sif - - wget -O SeqWho.ix https://cloud.biohpc.swmed.edu/index.php/s/eeNWqZz8jqN5zWY/download + - wget -O SeqWho.ix https://cloud.biohpc.swmed.edu/index.php/s/sP48taKmymSkJBM/download - mkdir -p SeqWho_call_plots/test_data/fastq/small/ - singularity run ${dir}${seqwhoImg}_${seqwhoVar}.sif seqwho.py -f test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -x SeqWho.ix - pytest -m seqwho @@ -767,7 +767,7 @@ integration_se: script: - hostname - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./SE_report.html + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png -with-report ./SE_report.html - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \; - pytest -m completionMultiqc --filename SE_multiqc_data.json artifacts: @@ -792,7 +792,7 @@ integration_pe: script: - hostname - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./PE_report.html + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png -with-report ./PE_report.html - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \; - pytest -m completionMultiqc --filename PE_multiqc_data.json artifacts: @@ -819,7 +819,7 @@ override_inputBag: script: - hostname - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true --track false -with-report ./inputBagOverride_report.html + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false -with-report ./inputBagOverride_report.html - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_multiqc_data.json \; - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./inputBagOverride_multiqc.html \; - pytest -m completionMultiqc --filename inputBagOverride_multiqc_data.json @@ -844,7 +844,7 @@ override_fastq: script: - hostname - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true --track false -with-report ./fastqOverride_report.html + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false -with-report ./fastqOverride_report.html - find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_multiqc_data.json \; - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./fastqOverride_multiqc.html \; - pytest -m completionMultiqc --filename fastqOverride_multiqc_data.json @@ -869,7 +869,7 @@ override_species: script: - hostname - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true --dev false --ci true --track false -with-report ./speciesOverride_report.html + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true -with-report ./speciesOverride_report.html - find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_multiqc_data.json \; - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./speciesOverride_multiqc.html \; - pytest -m completionMultiqc --filename speciesOverride_multiqc_data.json @@ -894,7 +894,7 @@ override_stranded: script: - hostname - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true --dev false --ci true --track false -with-report ./strandedOverride_report.html + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true -with-report ./strandedOverride_report.html - find . -type f -name "multiqc_data.json" -exec cp {} ./strandedOverride_multiqc_data.json \; - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./strandedOverride_multiqc.html \; - pytest -m completionMultiqc --filename strandedOverride_multiqc_data.json @@ -919,7 +919,7 @@ override_spike: script: - hostname - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true --dev false --ci true --track false -with-report ./spikeOverride_report.html + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true -with-report ./spikeOverride_report.html - find . -type f -name "multiqc_data.json" -exec cp {} ./spikeOverride_multiqc_data.json \; - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./spikeOverride_multiqc.html \; - pytest -m completionMultiqc --filename spikeOverride_multiqc_data.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c24a82f7f38b86e05b0f5eaf7e2c855a9eac854c..76ca7d646ddf2ab51c859c7fca459ec710de15b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,11 @@ # v.2.0.1 (indev) **User Facing** * Corrected spelling of inferred (#125) +ce * Add seqtype force parameter (#131) +* Add param for seqwho reference (#127) +* Remove tracking param and all tracking code + **Background** * Corrected file search parameters due to name inconsistency (#129) @@ -12,6 +16,9 @@ * Remove "fail" integration CI tests (#133 addendum) * Add licence header to nf file, R and python scriptsand pytest scripts * Stop using RCB and RMB (default) with deriva.core insert calls to match server update (#134) +* Replace default seqwho reference location to one provided by source lab (#127) +* Update NF Tower access token to BICF user +* Disable NF Tower # v2.0.0 **User Facing** diff --git a/README.md b/README.md index 80c4c65491e98e1c0c107ff113a4a77c0e2c705c..7e5abc805b2f91b1c545b1e79e118585216be5a2 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ To Run: * `--refMoVersion` mouse reference version ***(optional, default = 38.p6.vM25)*** * `--refHuVersion` human reference version ***(optional, default = 38.p13.v36)*** * `--refERCCVersion` human reference version ***(optional, default = 92)*** + * `--seqwhoRef` seqwho reference location ***(optional, default = https://cloud.biohpc.swmed.edu/index.php/s/sP48taKmymSkJBM/download)*** * `--upload` option to not upload output back to the data-hub ***(optional, default = false)*** * **true** = upload outputs to the data-hub * **false** = do *NOT* upload outputs to the data-hub @@ -73,9 +74,6 @@ To Run: * eg: `--strandedForce 'unstranded'` * `--spikeForce` forces the spike-in to be "false", or "true", it bypasses a metadata mismatch error * eg: `--spikeForce 'true'` -* Tracking parameters ([Tracking Site](http://bicf.pipeline.tracker.s3-website-us-east-1.amazonaws.com/)): - * `--ci` boolean (default = false) - * `--dev` boolean (default = true) FULL EXAMPLE: ------------- diff --git a/nextflow.config b/nextflow.config index fbfb2f154ec529822550560fe2fac158219212e0..83f93fbe198b27d01a5352c292bcf0013f0ccde9 100644 --- a/nextflow.config +++ b/nextflow.config @@ -110,8 +110,9 @@ process { } tower { - accessToken = '3ade8f325d4855434b49aa387421a44c63e3360f' - enabled = true + accessToken = 'eyJ0aWQiOiAzODIyfS42YWFjOWMwZDM4Mjc1NmY0MGJiY2QxNWRkOTZlYTU4MDlmY2NkZmRj' + workspaceId = '84512216752206' + enabled = false } manifest { diff --git a/rna-seq.nf b/rna-seq.nf index bbe62c7307c72b0cbd0c198c014647a363111f93..3094d2865cf38a41b30476d66593b8bea015593d 100644 --- a/rna-seq.nf +++ b/rna-seq.nf @@ -23,10 +23,10 @@ params.source = "dev" params.refMoVersion = "38.p6.vM25" params.refHuVersion = "38.p13.v36" params.refERCCVersion = "92" +params.seqwhoRef = "https://cloud.biohpc.swmed.edu/index.php/s/sP48taKmymSkJBM/download" params.outDir = "${baseDir}/output" params.upload = false params.email = "" -params.track = false // Define override input variable params.refSource = "biohpc" @@ -38,10 +38,6 @@ params.strandedForce = "" params.spikeForce = "" params.seqtypeForce = "" -// Define tracking input variables -params.ci = false -params.dev = true - // Parse input variables deriva = Channel .fromPath(params.deriva) @@ -68,6 +64,7 @@ repRID = params.repRID refMoVersion = params.refMoVersion refHuVersion = params.refHuVersion refERCCVersion = params.refERCCVersion +seqwhoRef = params.seqwhoRef outDir = params.outDir logsDir = "${outDir}/Logs" upload = params.upload @@ -123,47 +120,6 @@ script_deleteEntry_uploadQC = Channel.fromPath("${baseDir}/workflow/scripts/dele script_deleteEntry_uploadQC_fail = Channel.fromPath("${baseDir}/workflow/scripts/delete_entry.py") script_deleteEntry_uploadProcessedFile = Channel.fromPath("${baseDir}/workflow/scripts/delete_entry.py") -/* - * trackStart: track start of pipeline - */ -process trackStart { - script: - """ - hostname - ulimit -a - - curl -H 'Content-Type: application/json' -X PUT -d \ - '{ \ - "sessionId": "${workflow.sessionId}", \ - "pipeline": "gudmap.rbk_rnaseq", \ - "start": "${workflow.start}", \ - "repRID": "${repRID}", \ - "astrocyte": false, \ - "status": "started", \ - "nextflowVersion": "${workflow.nextflow.version}", \ - "pipelineVersion": "${workflow.manifest.version}", \ - "ci": ${params.ci}, \ - "dev": ${params.dev} \ - }' \ - "https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking" - - if [ ${params.track} == true ] - then - curl -H 'Content-Type: application/json' -X PUT -d \ - '{ \ - "ID": "${workflow.sessionId}", \ - "repRID": "${repRID}", \ - "PipelineVersion": "${workflow.manifest.version}", \ - "Server": "${params.source}", \ - "Queued": "NA", \ - "CheckedOut": "NA", \ - "Started": "${workflow.start}" \ - }' \ - "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track" - fi - """ -} - log.info """\ ==================================== BICF RNA-seq Pipeline for GUDMAP/RBK @@ -176,15 +132,11 @@ ERCC Reference Version : ${params.refERCCVersion} Reference source : ${params.refSource} Output Directory : ${params.outDir} Upload : ${upload} -Track : ${params.track} ------------------------------------ Nextflow Version : ${workflow.nextflow.version} Pipeline Version : ${workflow.manifest.version} Session ID : ${workflow.sessionId} ------------------------------------ -CI : ${params.ci} -Development : ${params.dev} ------------------------------------- """ /* @@ -716,6 +668,7 @@ process seqwho { tag "${repRID}" input: + val seqwhoRef path (fastq) from fastqs_seqwho val ends from endsManual_seqwho val speciesMeta from speciesMeta_seqwho @@ -736,7 +689,7 @@ process seqwho { ulimit -a >> ${repRID}.seqwho.log # get seqwho index - wget -O SeqWho.ix https://cloud.biohpc.swmed.edu/index.php/s/eeNWqZz8jqN5zWY/download + wget -O SeqWho.ix ${seqwhoRef} echo -e "LOG: seqwho index downloaded" >> ${repRID}.seqwho.log # run seqwho @@ -2209,13 +2162,6 @@ process aggrQC { echo -e "LOG: running multiqc" >> ${repRID}.aggrQC.log multiqc -c ${multiqcConfig} . -n ${repRID}.multiqc.html cp ${repRID}.multiqc_data/multiqc_data.json ${repRID}.multiqc_data.json - - if [ ${params.track} == true ] - then - curl -H 'Content-Type: application/json' -X PUT -d \ - @./${repRID}.multiqc_data.json \ - "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/qc" - fi """ } @@ -2370,16 +2316,6 @@ process uploadExecutionRun { fi echo "\${executionRun_rid}" > executionRunRID.csv - - if [ ${params.track} == true ] - then - curl -H 'Content-Type: application/json' -X PUT -d \ - '{ \ - "ID": "${workflow.sessionId}", \ - "ExecutionRunRID": "'\${executionRun_rid}'" \ - }' \ - "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track" - fi """ } @@ -2697,17 +2633,6 @@ process finalizeExecutionRun { rid=\$(python3 ${script_uploadExecutionRun_finalizeExecutionRun} -r ${repRID} -w \${workflow} -g \${genome} -i ${inputBagRID} -s Success -d 'Run Successful' -o ${source} -c \${cookie} -u ${executionRunRID}) echo LOG: execution run RID marked as successful - \${rid} >> ${repRID}.finalizeExecutionRun.log - - if [ ${params.track} == true ] - then - dt=`date +%FT%T.%3N%:z` - curl -H 'Content-Type: application/json' -X PUT -d \ - '{ \ - "ID": "${workflow.sessionId}", \ - "Complete": "'\${dt}'" \ - }' \ - "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track" - fi """ } @@ -2805,18 +2730,6 @@ process failPreExecutionRun { fi echo "\${rid}" > executionRunRID.csv - - if [ ${params.track} == true ] - then - dt=`date +%FT%T.%3N%:z` - curl -H 'Content-Type: application/json' -X PUT -d \ - '{ \ - "ID": "${workflow.sessionId}", \ - "ExecutionRunRID": "'\${rid}'", \ - "Failure": "'\${dt}'" \ - }' \ - "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track" - fi """ } // Extract execution run RID into channel @@ -2906,18 +2819,6 @@ process failPreExecutionRun_seqwho { fi echo "\${rid}" > executionRunRID.csv - - if [ ${params.track} == true ] - then - dt=`date +%FT%T.%3N%:z` - curl -H 'Content-Type: application/json' -X PUT -d \ - '{ \ - "ID": "${workflow.sessionId}", \ - "ExecutionRunRID": "'\${rid}'", \ - "Failure": "'\${dt}'" \ - }' \ - "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track" - fi """ } // Extract execution run RID into channel @@ -3009,18 +2910,6 @@ process failExecutionRun { rid=\$(python3 ${script_uploadExecutionRun_failExecutionRun} -r ${repRID} -w \${workflow} -g \${genome} -i ${inputBagRID} -s Error -d "\${pipelineError_details}" -o ${source} -c \${cookie} -u ${executionRunRID}) echo LOG: execution run RID marked as error - \${rid} >> ${repRID}.failExecutionRun.log fi - - if [ ${params.track} == true ] - then - dt=`date +%FT%T.%3N%:z` - curl -H 'Content-Type: application/json' -X PUT -d \ - '{ \ - "ID": "${workflow.sessionId}", \ - "ExecutionRunRID": "'\${rid}'", \ - "Failure": "'\${dt}'" \ - }' \ - "https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track" - fi """ }