Merge branch '60-consistency_tests' into 'develop'

Update consistency checks to be fully in python.

Closes #60

See merge request !46

.gitlab-ci.yml

@@ -397,10 +397,6 @@ consistency:
     variables:
         - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/
   script:
-  - grep -m 1 \"Assigned\":.[0-9] SE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedSE.txt
-  - grep -m 1 \"Assigned\":.[0-9] PE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedPE.txt
-  - echo 7742416 > assignedExpectSE.txt
-  - echo 2599140 > assignedExpectPE.txt
   - pytest -m consistencySE
   - pytest -m consistencyPE
   artifacts:
@@ -409,8 +405,4 @@ consistency:
     paths:
       - SE_multiqc_data.json
       - PE_multiqc_data.json
-      - assignedSE.txt
-      - assignedPE.txt
-      - assignedExpectSE.txt
-      - assignedExpectPE.txt
     expire_in: 7 days

CHANGELOG.md

@@ -7,6 +7,7 @@
 * Remove (comment out) option to pull references from S3
 * Make pull references from BioHPC default (including in biohpc.config)
 * Start using new gudmaprbk dockerhub (images autobuilt)
+* Moved consistency checks to be fully python
 
 *Known Bugs*
 * Datahub reference pull uses dev.gudmap.org as source until referencencs are placed on production

workflow/tests/test_consistency.py

@@ -4,6 +4,7 @@ import pytest
 import pandas as pd
 from io import StringIO
 import os
+import json
 
 test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
     '/../../'
@@ -13,23 +14,21 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
 def test_consistencySE():
     assert os.path.exists(os.path.join(
         test_output_path, 'SE_multiqc_data.json'))
-    assert readAssigned("assignedSE.txt", "assignedExpectSE.txt")
+
+    with open(os.path.join(
+        test_output_path, 'SE_multiqc_data.json')) as f:
+        assigned_reads_json = json.load(f)
+    assigned_reads = assigned_reads_json['report_general_stats_data'][4]['16-1ZX4']['Assigned']
+    assert  assigned_reads == 7742416
 
 
 @pytest.mark.consistencyPE
 def test_consistencyPE():
     assert os.path.exists(os.path.join(
         test_output_path, 'PE_multiqc_data.json'))
-    assert readAssigned("assignedPE.txt", "assignedExpectPE.txt")
-
-
-def readAssigned(fileAssigned, fileExpectAssigned):
-    data = False
-    assigned = open(fileAssigned, "r")
-    expect = open(fileExpectAssigned, "r")
-    lineAssigned = assigned.readline()
-    lineExpect = expect.readline()
-    if int(lineAssigned.strip()) < (int(lineExpect.strip())+(int(lineExpect.strip())*0.00001)) and int(lineAssigned.strip()) > (int(lineExpect.strip())-(int(lineExpect.strip())*0.00001)):
-        data = True
 
-    return data
+    with open(os.path.join(
+        test_output_path, 'PE_multiqc_data.json')) as f:
+        assigned_reads_json = json.load(f)
+    assigned_reads = assigned_reads_json['report_general_stats_data'][4]['Q-Y5JA']['Assigned']
+    assert  assigned_reads == 2599149