From 04536845eecaed36c8d11580189e2c72b2e38e6c Mon Sep 17 00:00:00 2001
From: "Gervaise H. Henry" <gervaise.henry@utsouthwestern.edu>
Date: Mon, 14 Dec 2020 16:30:27 -0600
Subject: [PATCH] Debug process file deriva upload & no retry for integration
---
.gitlab-ci.yml | 10 +++++-----
workflow/rna-seq.nf | 26 +++++---------------------
2 files changed, 10 insertions(+), 26 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d6fdd26..29b3c61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -494,7 +494,7 @@ integration_se:
- SE_multiqc_data.json
expire_in: 7 days
retry:
- max: 1
+ max: 0
when:
- always
@@ -519,7 +519,7 @@ integration_pe:
- PE_multiqc_data.json
expire_in: 7 days
retry:
- max: 1
+ max: 0
when:
- always
@@ -542,7 +542,7 @@ override_inputBag:
- inputBagOverride_PE_multiqc_data.json
expire_in: 7 days
retry:
- max: 1
+ max: 0
when:
- always
@@ -564,7 +564,7 @@ override_fastq:
- fastqOverride_PE_multiqc_data.json
expire_in: 7 days
retry:
- max: 1
+ max: 0
when:
- always
@@ -586,7 +586,7 @@ override_species:
- speciesOverride_PE_multiqc_data.json
expire_in: 7 days
retry:
- max: 1
+ max: 0
when:
- always
diff --git a/workflow/rna-seq.nf b/workflow/rna-seq.nf
index 401e00a..09d35d6 100644
--- a/workflow/rna-seq.nf
+++ b/workflow/rna-seq.nf
@@ -1480,7 +1480,6 @@ qcRID_fl.splitCsv(sep: ",", header: false).separate(
qcRID
)
-
/*
*ouputBag: create ouputBag
*/
@@ -1519,28 +1518,11 @@ process outputBag {
cookie=\$(cat credential.json | grep -A 1 '\\"${source}\\": {' | grep -o '\\"cookie\\": \\".*\\"')
cookie=\${cookie:20:-1}
- deriva-upload-cli --catalog 2 --token \${cookie} ${source} ./deriva --purge-state
-
- fileBam=\$(basename -a ${bam})
- md5Bam=\$(md5sum ./\${fileBam} | awk '{ print \$1 }')
- fileBigwig=\$(basename -a ${bigwig})
- md5Bigwig=\$(md5sum ./\${fileBigwig} | awk '{ print \$1 }')
- fileCounts=\$(basename -a ${counts})
- md5Counts=\$(md5sum ./\${fileCounts} | awk '{ print \$1 }')
- urlBam=\$(curl -s https://${source}/ermrest/catalog/2/entity/RNASeq:Processed_File/File_MD5=\${md5Bam})
- urlBam=\$(echo \${urlBam} | grep -o '\\"File_URL\\":\\".*\\",\\"File_Name')
- urlBam=\${urlBam:12:-12}
- urlBigwig=\$(curl -s https://${source}/ermrest/catalog/2/entity/RNASeq:Processed_File/File_MD5=\${md5Bigwig})
- urlBigwig=\$(echo \${urlBigwig} | grep -o '\\"File_URL\\":\\".*\\",\\"File_Name')
- urlBigwig=\${urlBigwig:12:-12}
- urlCounts=\$(curl -s https://${source}/ermrest/catalog/2/entity/RNASeq:Processed_File/File_MD5=\${md5Counts})
- urlCounts=\$(echo \${urlCounts} | grep -o '\\"File_URL\\":\\".*\\",\\"File_Name')
- urlCounts=\${urlCounts:12:-12}
- echo \${urlBam} > url.txt
- echo \${urlBigwig} >> url.txt
- echo \${urlCounts} >> url.txt
+ deriva-upload-cli --catalog 2 --token \${cookie} ${source} ./deriva
+ echo LOG: processed files uploaded >> ${repRID}.outputBag.log
deriva-download-cli --catalog 2 --token \${cookie} ${source} ${executionRunExportConfig} . rid=${executionRunRID}
+ echo LOG: execution run bag downloaded >> ${repRID}.outputBag.log
echo -e "### Run Details" >> runDetails.md
echo -e "**Workflow URL:** https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq" >> runDetails.md
@@ -1558,6 +1540,7 @@ process outputBag {
echo -e "**Genome Assembly Version:** \${genome} patch \${patch}" >> runDetails.md
echo -e "**Annotation Version:** GENCODE release \${annotation}" >> runDetails.md
echo -e "**Run ID:** ${repRID}" >> runDetails.md
+ echo LOG: runDetails.md created >> ${repRID}.outputBag.log
unzip Execution_Run_${executionRunRID}.zip
mv Execution_Run_${executionRunRID} ${repRID}_Output_Bag
@@ -1568,6 +1551,7 @@ process outputBag {
cp ${multiqcJSON} \${loc}
bdbag ./${repRID}_Output_Bag/ --update --archiver zip --debug
+ echo LOG: output bag created >> ${repRID}.outputBag.log
"""
}
--
GitLab