Move Override stage in CI to integration (and update bag test-data location

.gitlab-ci.yml

@@ -11,7 +11,6 @@ stages:
   - unit
   - integration
   - consistency
-  - override
 
 getBag:
   stage: unit
@@ -24,7 +23,7 @@ getData:
   stage: unit
   script:
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
-  - unzip ./test_data/bagit/Replicate_Q-Y5F6.zip
+  - unzip ./test_data/bag/Replicate_Q-Y5F6.zip
   - singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' bash ./workflow/scripts/bdbagFetch.sh Replicate_Q-Y5F6 Replicate_Q-Y5F6 TEST
   - pytest -m getData
 
@@ -173,33 +172,12 @@ integration_pe:
       - PE_multiqc_data.json
     expire_in: 7 days
 
-consistency:
-  stage: consistency
-  script:
-  - grep -m 1 \"Assigned\":.[0-9] SE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedSE.txt
-  - grep -m 1 \"Assigned\":.[0-9] PE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedPE.txt
-  - echo 7742416 > assignedExpectSE.txt
-  - echo 2599140 > assignedExpectPE.txt
-  - pytest -m consistencySE
-  - pytest -m consistencyPE
-  artifacts:
-    name: "$CI_JOB_NAME"
-    when: always
-    paths:
-      - SE_multiqc_data.json
-      - PE_multiqc_data.json
-      - assignedSE.txt
-      - assignedPE.txt
-      - assignedExpectSE.txt
-      - assignedExpectPE.txt
-    expire_in: 7 days
-
-inputBag:
-  stage: override
+override_inputBag:
+  stage: integration
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --inputBagForce test_data/bagit/Replicate_Q-Y5F6.zip --ci true
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --inputBagForce test_data/bag/Replicate_Q-Y5F6.zip --ci true
   - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_PE_multiqc_data.json \;
   artifacts:
     name: "$CI_JOB_NAME"
@@ -208,8 +186,8 @@ inputBag:
       - inputBagOverride_PE_multiqc_data.json
     expire_in: 7 days
 
-fastq:
-  stage: override
+override_fastq:
+  stage: integration
   script:
   - hostname
     - ulimit -a
@@ -220,4 +198,25 @@ fastq:
     when: always
     paths:
       - fastqOverride_PE_multiqc_data.json
+    expire_in: 7 days
+
+consistency:
+  stage: consistency
+  script:
+  - grep -m 1 \"Assigned\":.[0-9] SE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedSE.txt
+  - grep -m 1 \"Assigned\":.[0-9] PE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedPE.txt
+  - echo 7742416 > assignedExpectSE.txt
+  - echo 2599140 > assignedExpectPE.txt
+  - pytest -m consistencySE
+  - pytest -m consistencyPE
+  artifacts:
+    name: "$CI_JOB_NAME"
+    when: always
+    paths:
+      - SE_multiqc_data.json
+      - PE_multiqc_data.json
+      - assignedSE.txt
+      - assignedPE.txt
+      - assignedExpectSE.txt
+      - assignedExpectPE.txt
     expire_in: 7 days
\ No newline at end of file