diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20659ed35f42727c0f2284be96318dc1d7162220..91c766d055e333005dbc44c0cdc4a7f444e771c9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -397,10 +397,6 @@ consistency: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - grep -m 1 \"Assigned\":.[0-9] SE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedSE.txt - - grep -m 1 \"Assigned\":.[0-9] PE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedPE.txt - - echo 7742416 > assignedExpectSE.txt - - echo 2599140 > assignedExpectPE.txt - pytest -m consistencySE - pytest -m consistencyPE artifacts: @@ -409,8 +405,4 @@ consistency: paths: - SE_multiqc_data.json - PE_multiqc_data.json - - assignedSE.txt - - assignedPE.txt - - assignedExpectSE.txt - - assignedExpectPE.txt expire_in: 7 days diff --git a/CHANGELOG.md b/CHANGELOG.md index d9de614cb1b0a85c3442a9f9f8db16fdd000aaeb..33465e4166f2e1f2bb8ab989f9c4a89e7643fc8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Remove (comment out) option to pull references from S3 * Make pull references from BioHPC default (including in biohpc.config) * Start using new gudmaprbk dockerhub (images autobuilt) +* Moved consistency checks to be fully python *Known Bugs* * Datahub reference pull uses dev.gudmap.org as source until referencencs are placed on production diff --git a/workflow/tests/test_consistency.py b/workflow/tests/test_consistency.py index 49991489ef71eccb2a88a039420661dfc1f0a303..8b7eb86f9fff3e252d5de463a5f3ec188d997d6e 100644 --- a/workflow/tests/test_consistency.py +++ b/workflow/tests/test_consistency.py @@ -4,6 +4,7 @@ import pytest import pandas as pd from io import StringIO import os +import json test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ '/../../' @@ -13,23 +14,21 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ def test_consistencySE(): assert os.path.exists(os.path.join( test_output_path, 'SE_multiqc_data.json')) - assert readAssigned("assignedSE.txt", "assignedExpectSE.txt") + + with open(os.path.join( + test_output_path, 'SE_multiqc_data.json')) as f: + assigned_reads_json = json.load(f) + assigned_reads = assigned_reads_json['report_general_stats_data'][4]['16-1ZX4']['Assigned'] + assert assigned_reads == 7742416 @pytest.mark.consistencyPE def test_consistencyPE(): assert os.path.exists(os.path.join( test_output_path, 'PE_multiqc_data.json')) - assert readAssigned("assignedPE.txt", "assignedExpectPE.txt") - - -def readAssigned(fileAssigned, fileExpectAssigned): - data = False - assigned = open(fileAssigned, "r") - expect = open(fileExpectAssigned, "r") - lineAssigned = assigned.readline() - lineExpect = expect.readline() - if int(lineAssigned.strip()) < (int(lineExpect.strip())+(int(lineExpect.strip())*0.00001)) and int(lineAssigned.strip()) > (int(lineExpect.strip())-(int(lineExpect.strip())*0.00001)): - data = True - return data + with open(os.path.join( + test_output_path, 'PE_multiqc_data.json')) as f: + assigned_reads_json = json.load(f) + assigned_reads = assigned_reads_json['report_general_stats_data'][4]['Q-Y5JA']['Assigned'] + assert assigned_reads == 2599149