Skip to content
Snippets Groups Projects
Commit daea36d6 authored by Gervaise Henry's avatar Gervaise Henry :cowboy:
Browse files

Remove all tracking with AWS resources

parent 5c4a68d9
Branches
Tags
2 merge requests!95Update rna-seq.nf,!94Resolve "Move seqwho reference"
Pipeline #10567 passed with stages
in 6 hours, 54 minutes, and 39 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 7
− 7
View file @ daea36d6
......@@ -767,7 +767,7 @@ integration_se:
script:
- hostname
- ulimit -a
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./SE_report.html
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png -with-report ./SE_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \;
- pytest -m completionMultiqc --filename SE_multiqc_data.json
artifacts:
......@@ -792,7 +792,7 @@ integration_pe:
script:
- hostname
- ulimit -a
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./PE_report.html
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png -with-report ./PE_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \;
- pytest -m completionMultiqc --filename PE_multiqc_data.json
artifacts:
......@@ -819,7 +819,7 @@ override_inputBag:
script:
- hostname
- ulimit -a
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true --track false -with-report ./inputBagOverride_report.html
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false -with-report ./inputBagOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_multiqc_data.json \;
- find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./inputBagOverride_multiqc.html \;
- pytest -m completionMultiqc --filename inputBagOverride_multiqc_data.json
......@@ -844,7 +844,7 @@ override_fastq:
script:
- hostname
- ulimit -a
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true --track false -with-report ./fastqOverride_report.html
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false -with-report ./fastqOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_multiqc_data.json \;
- find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./fastqOverride_multiqc.html \;
- pytest -m completionMultiqc --filename fastqOverride_multiqc_data.json
......@@ -869,7 +869,7 @@ override_species:
script:
- hostname
- ulimit -a
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true --dev false --ci true --track false -with-report ./speciesOverride_report.html
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true -with-report ./speciesOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_multiqc_data.json \;
- find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./speciesOverride_multiqc.html \;
- pytest -m completionMultiqc --filename speciesOverride_multiqc_data.json
......@@ -894,7 +894,7 @@ override_stranded:
script:
- hostname
- ulimit -a
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true --dev false --ci true --track false -with-report ./strandedOverride_report.html
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true -with-report ./strandedOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./strandedOverride_multiqc_data.json \;
- find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./strandedOverride_multiqc.html \;
- pytest -m completionMultiqc --filename strandedOverride_multiqc_data.json
......@@ -919,7 +919,7 @@ override_spike:
script:
- hostname
- ulimit -a
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true --dev false --ci true --track false -with-report ./spikeOverride_report.html
- nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true -with-report ./spikeOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./spikeOverride_multiqc_data.json \;
- find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./spikeOverride_multiqc.html \;
- pytest -m completionMultiqc --filename spikeOverride_multiqc_data.json
......
CHANGELOG.md
+ 1
− 0
View file @ daea36d6
......@@ -2,6 +2,7 @@
**User Facing**
* Corrected spelling of inferred (#125)
* Add param for seqwho reference (#127)
* Remove tracking param and all tracking code
**Background**
* Corrected file search parameters due to name inconsistency (#129)
......
README.md
+ 0
− 3
View file @ daea36d6
......@@ -72,9 +72,6 @@ To Run:
* eg: `--strandedForce 'unstranded'`
* `--spikeForce` forces the spike-in to be "false", or "true", it bypasses a metadata mismatch error
* eg: `--spikeForce 'true'`
* Tracking parameters ([Tracking Site](http://bicf.pipeline.tracker.s3-website-us-east-1.amazonaws.com/)):
* `--ci` boolean (default = false)
* `--dev` boolean (default = true)
FULL EXAMPLE:
-------------
......
rna-seq.nf
+ 0
− 114
View file @ daea36d6
......@@ -27,7 +27,6 @@ params.seqwhoRef = "https://cloud.biohpc.swmed.edu/index.php/s/sP48taKmymSkJBM/d
params.outDir = "${baseDir}/output"
params.upload = false
params.email = ""
params.track = false
// Define override input variable
params.refSource = "biohpc"
......@@ -38,10 +37,6 @@ params.speciesForce = ""
params.strandedForce = ""
params.spikeForce = ""
// Define tracking input variables
params.ci = false
params.dev = true
// Parse input variables
deriva = Channel
.fromPath(params.deriva)
......@@ -123,47 +118,6 @@ script_deleteEntry_uploadQC = Channel.fromPath("${baseDir}/workflow/scripts/dele
script_deleteEntry_uploadQC_fail = Channel.fromPath("${baseDir}/workflow/scripts/delete_entry.py")
script_deleteEntry_uploadProcessedFile = Channel.fromPath("${baseDir}/workflow/scripts/delete_entry.py")
/*
* trackStart: track start of pipeline
*/
process trackStart {
script:
"""
hostname
ulimit -a
curl -H 'Content-Type: application/json' -X PUT -d \
'{ \
"sessionId": "${workflow.sessionId}", \
"pipeline": "gudmap.rbk_rnaseq", \
"start": "${workflow.start}", \
"repRID": "${repRID}", \
"astrocyte": false, \
"status": "started", \
"nextflowVersion": "${workflow.nextflow.version}", \
"pipelineVersion": "${workflow.manifest.version}", \
"ci": ${params.ci}, \
"dev": ${params.dev} \
}' \
"https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking"
if [ ${params.track} == true ]
then
curl -H 'Content-Type: application/json' -X PUT -d \
'{ \
"ID": "${workflow.sessionId}", \
"repRID": "${repRID}", \
"PipelineVersion": "${workflow.manifest.version}", \
"Server": "${params.source}", \
"Queued": "NA", \
"CheckedOut": "NA", \
"Started": "${workflow.start}" \
}' \
"https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
fi
"""
}
log.info """\
====================================
BICF RNA-seq Pipeline for GUDMAP/RBK
......@@ -176,15 +130,11 @@ ERCC Reference Version : ${params.refERCCVersion}
Reference source : ${params.refSource}
Output Directory : ${params.outDir}
Upload : ${upload}
Track : ${params.track}
------------------------------------
Nextflow Version : ${workflow.nextflow.version}
Pipeline Version : ${workflow.manifest.version}
Session ID : ${workflow.sessionId}
------------------------------------
CI : ${params.ci}
Development : ${params.dev}
------------------------------------
"""
/*
......@@ -2198,13 +2148,6 @@ process aggrQC {
echo -e "LOG: running multiqc" >> ${repRID}.aggrQC.log
multiqc -c ${multiqcConfig} . -n ${repRID}.multiqc.html
cp ${repRID}.multiqc_data/multiqc_data.json ${repRID}.multiqc_data.json
if [ ${params.track} == true ]
then
curl -H 'Content-Type: application/json' -X PUT -d \
@./${repRID}.multiqc_data.json \
"https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/qc"
fi
"""
}
......@@ -2359,16 +2302,6 @@ process uploadExecutionRun {
fi
echo "\${executionRun_rid}" > executionRunRID.csv
if [ ${params.track} == true ]
then
curl -H 'Content-Type: application/json' -X PUT -d \
'{ \
"ID": "${workflow.sessionId}", \
"ExecutionRunRID": "'\${executionRun_rid}'" \
}' \
"https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
fi
"""
}
......@@ -2686,17 +2619,6 @@ process finalizeExecutionRun {
rid=\$(python3 ${script_uploadExecutionRun_finalizeExecutionRun} -r ${repRID} -w \${workflow} -g \${genome} -i ${inputBagRID} -s Success -d 'Run Successful' -o ${source} -c \${cookie} -u ${executionRunRID})
echo LOG: execution run RID marked as successful - \${rid} >> ${repRID}.finalizeExecutionRun.log
if [ ${params.track} == true ]
then
dt=`date +%FT%T.%3N%:z`
curl -H 'Content-Type: application/json' -X PUT -d \
'{ \
"ID": "${workflow.sessionId}", \
"Complete": "'\${dt}'" \
}' \
"https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
fi
"""
}
......@@ -2794,18 +2716,6 @@ process failPreExecutionRun {
fi
echo "\${rid}" > executionRunRID.csv
if [ ${params.track} == true ]
then
dt=`date +%FT%T.%3N%:z`
curl -H 'Content-Type: application/json' -X PUT -d \
'{ \
"ID": "${workflow.sessionId}", \
"ExecutionRunRID": "'\${rid}'", \
"Failure": "'\${dt}'" \
}' \
"https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
fi
"""
}
// Extract execution run RID into channel
......@@ -2895,18 +2805,6 @@ process failPreExecutionRun_seqwho {
fi
echo "\${rid}" > executionRunRID.csv
if [ ${params.track} == true ]
then
dt=`date +%FT%T.%3N%:z`
curl -H 'Content-Type: application/json' -X PUT -d \
'{ \
"ID": "${workflow.sessionId}", \
"ExecutionRunRID": "'\${rid}'", \
"Failure": "'\${dt}'" \
}' \
"https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
fi
"""
}
// Extract execution run RID into channel
......@@ -2998,18 +2896,6 @@ process failExecutionRun {
rid=\$(python3 ${script_uploadExecutionRun_failExecutionRun} -r ${repRID} -w \${workflow} -g \${genome} -i ${inputBagRID} -s Error -d "\${pipelineError_details}" -o ${source} -c \${cookie} -u ${executionRunRID})
echo LOG: execution run RID marked as error - \${rid} >> ${repRID}.failExecutionRun.log
fi
if [ ${params.track} == true ]
then
dt=`date +%FT%T.%3N%:z`
curl -H 'Content-Type: application/json' -X PUT -d \
'{ \
"ID": "${workflow.sessionId}", \
"ExecutionRunRID": "'\${rid}'", \
"Failure": "'\${dt}'" \
}' \
"https://9ouc12dkwb.execute-api.us-east-2.amazonaws.com/prod/db/track"
fi
"""
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment