diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cff15723a1d70f9950813acc90c7002d07aa07a0..1fb4b3d1e235b7ee8a2090de4c90ad37d10b7260 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -239,8 +239,10 @@ humanDataHub: - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}) - curl --request GET ${query} > refQuery.json - - refURL=$(python ./workflow/scripts/extractRefData.py) - - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} get ${refURL} + - refURL=$(python ./workflow/scripts/extractRefData.py --return URL) + - refMD5=$(python ./workflow/scripts/extractRefData.py --return MD5) + - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls /hatrac/resources/rnaseq/pipeline/reference_genome/ | grep -o ${refMD5}) + - if [ "${test}" == "" ]; then exit 1; fi mousenDataHub: stage: reference @@ -260,9 +262,10 @@ mousenDataHub: - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}) - curl --request GET ${query} > refQuery.json - - refURL=$(python ./workflow/scripts/extractRefData.py) - - pip install --user deriva==1.3.0 - - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} get ${refURL} + - refURL=$(python ./workflow/scripts/extractRefData.py --return URL) + - refMD5=$(python ./workflow/scripts/extractRefData.py --return MD5) + - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls /hatrac/resources/rnaseq/pipeline/reference_genome/ | grep -o ${refMD5}) + - if [ "${test}" == "" ]; then exit 1; fi integration_se: diff --git a/workflow/scripts/extractRefData.py b/workflow/scripts/extractRefData.py index 7c9dfa9d2a67c433e40fc015087dac9a2dfc1164..8f587514e23ed76dfe8d95de657884c08c74dcbc 100644 --- a/workflow/scripts/extractRefData.py +++ b/workflow/scripts/extractRefData.py @@ -5,10 +5,20 @@ import pandas as pd import warnings warnings.simplefilter(action='ignore', category=FutureWarning) +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-r', '--return',help="The parameter to return (URL or MD5).",required=True) + args = parser.parse_args() + return args + def main(): + args = get_args() refQuery=pd.read_json("refQuery.json") if refQuery["File_URL"].count() == 1: - print(refQuery["File_URL"].values[0]) + if args.return == "URL": + print(refQuery["File_URL"].values[0]) + elif args.return == "MD5": + print(refQuery["File_MD5"].values[0]) else: raise Exception("Multple references found: \n%s" % refQuery["RID"])