Add new py tests for integration and override

.gitlab-ci.yml

@@ -666,6 +666,7 @@ integration_se:
   - ulimit -a
   - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./SE_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \;
+  - pytest -m completionIntegration_se
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -690,6 +691,7 @@ integration_pe:
   - ulimit -a
   - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./PE_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \;
+  - pytest -m completionIntegration_pe
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -791,6 +793,7 @@ override_inputBag:
   - ulimit -a
   - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source dev --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true -with-report ./inputBagOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_PE_multiqc_data.json \;
+  - pytest -m completionOverride_inputBag
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -813,6 +816,7 @@ override_fastq:
   - ulimit -a
   - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6  --source dev --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true -with-report ./fastqOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_PE_multiqc_data.json \;
+  - pytest -m completionOverride_fastq
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -835,6 +839,7 @@ override_species:
   - ulimit -a
   - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --speciesForce 'Homo sapiens' --upload false --dev false --ci true -with-report ./speciesOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_PE_multiqc_data.json \;
+  - pytest -m completionOverride_spike
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -857,6 +862,7 @@ override_stranded:
   - ulimit -a
   - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --strandedForce unstranded --upload false --dev false --ci true -with-report ./strandedOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./strandedOverride_PE_multiqc_data.json \;
+  - pytest -m completionOverride_stranded
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -868,7 +874,7 @@ override_stranded:
     when:
       - always
 
-override_stpike:
+override_spike:
   stage: integration
   only: [merge_requests]
   except:
@@ -879,6 +885,7 @@ override_stpike:
   - ulimit -a
   - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --spikeForce t --upload false --dev false --ci true -with-report ./spikeOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./spikeOverride_PE_multiqc_data.json \;
+  - pytest -m completionOverride_spike
   artifacts:
     name: "$CI_JOB_NAME"
     when: always

CHANGELOG.md

@@ -23,6 +23,7 @@
 * Data-hub column title change from "Has_Strand_Specific_Information" to "Strandedness" (#115)
 * Merge data error pre-inference execution run upload/finalize to 1 process
 * Change uploadOutputBag logic to change reuse hatrac file if alread exists (re-uses Output_Bag entry by reassigning Execution_Run RID) (#112)
+* Add new CI py tests for override and integration
 
 *Known Bugs*
 * Override params (inputBag, fastq, species) aren't checked for integrity

workflow/tests/test_completion.py

+#!/usr/bin/env python3
+
+import pytest
+import pandas as pd
+from io import StringIO
+import os
+import json
+
+test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
+    '/../../'
+
+
+@pytest.mark.consistencySE
+def test_consistencySE():
+    assert os.path.exists(os.path.join(
+        test_output_path, 'SE_multiqc_data.json'))
+
+    with open(os.path.join(
+        test_output_path, 'SE_multiqc_data.json')) as f:
+        assigned_reads_json = json.load(f)
+    assigned_reads = assigned_reads_json['report_general_stats_data'][4]['16-1ZX4_sorted']['Assigned']
+    baseline = 7746121
+    baseline_hi = baseline+(baseline*0.05)
+    baseline_lo = baseline-(baseline*0.05)
+    assert (assigned_reads >= baseline_lo)
+    assert (assigned_reads <=  baseline_hi)
+
+
+@pytest.mark.consistencyPE
+def test_consistencyPE():
+    assert os.path.exists(os.path.join(
+        test_output_path, 'PE_multiqc_data.json'))
+
+    with open(os.path.join(
+        test_output_path, 'PE_multiqc_data.json')) as f:
+        assigned_reads_json = json.load(f)
+    assigned_reads = assigned_reads_json['report_general_stats_data'][4]['Q-Y5JA_sorted']['Assigned']
+    baseline = 2596053
+    baseline_hi = baseline+(baseline*0.05)
+    baseline_lo = baseline-(baseline*0.05)
+    assert (assigned_reads >= baseline_lo)
+    assert (assigned_reads <=  baseline_hi)