diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f71ff415aa60f09156048a4bf26759dee0c628a..b36504f9ea56479147ffeb414ae758b948736fa6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,7 @@ before_script: stages: - unit - integration + - consistency getBag: stage: unit @@ -29,14 +30,16 @@ getData: parseMetadata: stage: unit script: - - singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID - - singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p expRID - - singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p studyRID - - singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p endsMeta - - singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p endsManual - - singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p stranded - - singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p spike - - singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p species + - rep=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID) + - exp=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p expRID) + - study=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p studyRID) + - endsMeta=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p endsMeta) + - endsManual=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p endsManual) + - stranded=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p stranded) + - spike=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p spike) + - species=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p species) + - echo -e "${endsMeta},${endsManual},${stranded},${spike},${species},${exp},${study},${rep}" > design.csv + - pytest -m parseMetadata inferMetadata: stage: unit @@ -129,17 +132,50 @@ integration_se: script: - hostname - ulimit -a - - nextflow -bg run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --ci true + - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 -with-dag dag.png --ci true + - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \; + artifacts: + name: "$CI_JOB_NAME" + when: always + paths: + - output/qc/ + - SE_multiqc_data.json + expire_in: 7 days integration_pe: stage: integration script: - hostname - ulimit -a - - nextflow -bg run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA -with-dag dag.png --ci true + - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA -with-dag dag.png --ci true + - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \; artifacts: name: "$CI_JOB_NAME" when: always paths: - dag.png + - output/qc/ + - PE_multiqc_data.json expire_in: 7 days + +consistency: + stage: consistency + script: + - grep -m 1 \"Assigned\":.[0-9] SE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedSE.txt + - grep -m 1 \"Assigned\":.[0-9] PE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedPE.txt + - echo 7742416 > assignedExpectSE.txt + - echo 2599149 > assignedExpectPE.txt + - pytest -m consistencySE + - pytest -m consistencyPE + artifacts: + name: "$CI_JOB_NAME" + when: always + paths: + - SE_multiqc_data.json + - PE_multiqc_data.json + - assignedSE.txt + - assignedPE.txt + - assignedExpectSE.txt + - assignedExpectPE.txt + expire_in: 7 days + diff --git a/workflow/tests/test_consistency.py b/workflow/tests/test_consistency.py new file mode 100644 index 0000000000000000000000000000000000000000..0aecc02000f301eaf237a7e6ae6270761e01687e --- /dev/null +++ b/workflow/tests/test_consistency.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import pytest +import pandas as pd +from io import StringIO +import os + +test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ + '/../../' + +@pytest.mark.consistencySE +def test_consistencySE(): + assert os.path.exists(os.path.join(test_output_path, 'SE_multiqc_data.json')) + assert readAssigned("assignedSE.txt","assignedExpectSE.txt") + +@pytest.mark.consistencyPE +def test_consistencyPE(): + assert os.path.exists(os.path.join(test_output_path, 'PE_multiqc_data.json')) + assert readAssigned("assignedPE.txt","assignedExpectPE.txt") + +def readAssigned(fileAssigned,fileExpectAssigned): + data = False + assigned = open(fileAssigned, "r") + expect = open(fileExpectAssigned, "r") + lineAssigned = assigned.readline() + lineExpect = expect.readline() + if lineAssigned.strip() == lineExpect.strip(): + data = True + + return data diff --git a/workflow/tests/test_dataQC.py b/workflow/tests/test_dataQC.py index 28e96284e8e4b1f16faf8faae59c33e77861ec60..e77d4680fd8eac61c3a9b9a8fd175136a61244b9 100644 --- a/workflow/tests/test_dataQC.py +++ b/workflow/tests/test_dataQC.py @@ -11,4 +11,13 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ @pytest.mark.dataQC def test_dataQC(): assert os.path.exists(os.path.join(test_output_path, 'Q-Y5F6_1M.se.sorted.deduped.tin.xls')) + assert countLines(os.path.join(test_output_path, 'Q-Y5F6_1M.se.sorted.deduped.tin.xls')) +def countLines(fileName): + data = False + file = open(fileName, "r") + file.readline() + if file.readlines()[6] != "geneID": + data = True + + return data diff --git a/workflow/tests/test_parseMetadata.py b/workflow/tests/test_parseMetadata.py new file mode 100644 index 0000000000000000000000000000000000000000..31a9674be28708b08fa54fb75fde541949b5278a --- /dev/null +++ b/workflow/tests/test_parseMetadata.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import pytest +import pandas as pd +from io import StringIO +import os + +test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ + '/../../' + +@pytest.mark.parseMetadata +def test_parseMetadata(): + assert os.path.exists(os.path.join(test_output_path, 'design.csv')) + assert readLine(os.path.join(test_output_path, 'design.csv')) + +def readLine(fileName): + data = False + file = open(fileName, "r") + line = file.readline() + if line.strip() == "uk,se,unstranded,no,Homo sapiens,Experiment_RID,Study_RID,Replicate_RID": + data = True + + return data