Put getRef in CI stage and add get from datahub

.gitlab-ci.yml

@@ -9,6 +9,7 @@ before_script:
 
 stages:
   - unit
+  - references
   - integration
   - consistency
 
@@ -74,19 +75,6 @@ inferMetadata:
     if [[ ${ended} == "" ]]; then exit 1; fi
   - pytest -m inferMetadata
 
-getRef:
-  stage: unit
-  only:
-    - push
-    - tags
-  except:
-    - merge_requests
-  script:
-  - mkdir -p hu
-  - mkdir -p mo
-  - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./hu/
-  - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./mo/
-
 trimData:
   stage: unit
   only:
@@ -211,6 +199,69 @@ outputBag:
   - pytest -m outputBag
 
 
+humanBioHPC:
+  stage: reference
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
+  script:
+  - mkdir -p hu
+  - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./hu/
+
+mouseBioHPC:
+  stage: reference
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
+  script:
+  - mkdir -p mo
+  - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./mo/
+
+humanDataHub:
+  stage: reference
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
+  script:
+    referenceBase=dev.gudmap.org
+    refName=GRCh
+    refHuVersion=38.p12.v31
+    references=$(echo ${referenceBase}/${refName}${refHuVersion})
+    GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1)
+    GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2)
+    GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
+    query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
+    curl --request GET ${query} > refQuery.json
+    refURL=$(python ./workflow/scripts/extractRefData.py)
+    deriva-hatrac-cli --host ${referenceBase} get ${refURL}
+
+mousenDataHub:
+  stage: reference
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
+  script:
+    referenceBase=dev.gudmap.org
+    refName=GRCm
+    refHuVersion=38.p6.vM22
+    references=$(echo ${referenceBase}/${refName}${refHuVersion})
+    GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1)
+    GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2)
+    GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
+    query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
+    curl --request GET ${query} > refQuery.json
+    refURL=$(python ./workflow/scripts/extractRefData.py)
+    deriva-hatrac-cli --host ${referenceBase} get ${refURL}
+
+
 integration_se:
   stage: integration
   only: [merge_requests]