From 0883570c2c65e642300f9e782dc94d32a1163232 Mon Sep 17 00:00:00 2001
From: "Gervaise H. Henry" <gervaise.henry@utsouthwestern.edu>
Date: Fri, 11 Sep 2020 18:41:08 -0500
Subject: [PATCH] Change datahub ref ci form get to ls
---
.gitlab-ci.yml | 13 ++++++++-----
workflow/scripts/extractRefData.py | 12 +++++++++++-
2 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cff1572..1fb4b3d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -239,8 +239,10 @@ humanDataHub:
- GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
- query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
- curl --request GET ${query} > refQuery.json
- - refURL=$(python ./workflow/scripts/extractRefData.py)
- - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} get ${refURL}
+ - refURL=$(python ./workflow/scripts/extractRefData.py --return URL)
+ - refMD5=$(python ./workflow/scripts/extractRefData.py --return MD5)
+ - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls /hatrac/resources/rnaseq/pipeline/reference_genome/ | grep -o ${refMD5})
+ - if [ "${test}" == "" ]; then exit 1; fi
mousenDataHub:
stage: reference
@@ -260,9 +262,10 @@ mousenDataHub:
- GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
- query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
- curl --request GET ${query} > refQuery.json
- - refURL=$(python ./workflow/scripts/extractRefData.py)
- - pip install --user deriva==1.3.0
- - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} get ${refURL}
+ - refURL=$(python ./workflow/scripts/extractRefData.py --return URL)
+ - refMD5=$(python ./workflow/scripts/extractRefData.py --return MD5)
+ - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls /hatrac/resources/rnaseq/pipeline/reference_genome/ | grep -o ${refMD5})
+ - if [ "${test}" == "" ]; then exit 1; fi
integration_se:
diff --git a/workflow/scripts/extractRefData.py b/workflow/scripts/extractRefData.py
index 7c9dfa9..8f58751 100644
--- a/workflow/scripts/extractRefData.py
+++ b/workflow/scripts/extractRefData.py
@@ -5,10 +5,20 @@ import pandas as pd
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
+def get_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-r', '--return',help="The parameter to return (URL or MD5).",required=True)
+ args = parser.parse_args()
+ return args
+
def main():
+ args = get_args()
refQuery=pd.read_json("refQuery.json")
if refQuery["File_URL"].count() == 1:
- print(refQuery["File_URL"].values[0])
+ if args.return == "URL":
+ print(refQuery["File_URL"].values[0])
+ elif args.return == "MD5":
+ print(refQuery["File_MD5"].values[0])
else:
raise Exception("Multple references found: \n%s" %
refQuery["RID"])
--
GitLab