From ecd4e76f3c90af4695cfa7292a22021c7937273a Mon Sep 17 00:00:00 2001
From: "Gervaise H. Henry" <gervaise.henry@utsouthwestern.edu>
Date: Tue, 26 Jan 2021 16:36:46 -0600
Subject: [PATCH] Add new py tests for integration and override
---
.gitlab-ci.yml | 9 ++++++-
CHANGELOG.md | 1 +
workflow/tests/test_completion.py | 42 +++++++++++++++++++++++++++++++
3 files changed, 51 insertions(+), 1 deletion(-)
create mode 100644 workflow/tests/test_completion.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2b0b70a..9c02255 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -666,6 +666,7 @@ integration_se:
- ulimit -a
- nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./SE_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \;
+ - pytest -m completionIntegration_se
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -690,6 +691,7 @@ integration_pe:
- ulimit -a
- nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./PE_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \;
+ - pytest -m completionIntegration_pe
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -791,6 +793,7 @@ override_inputBag:
- ulimit -a
- nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source dev --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true -with-report ./inputBagOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_PE_multiqc_data.json \;
+ - pytest -m completionOverride_inputBag
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -813,6 +816,7 @@ override_fastq:
- ulimit -a
- nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source dev --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true -with-report ./fastqOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_PE_multiqc_data.json \;
+ - pytest -m completionOverride_fastq
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -835,6 +839,7 @@ override_species:
- ulimit -a
- nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --speciesForce 'Homo sapiens' --upload false --dev false --ci true -with-report ./speciesOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_PE_multiqc_data.json \;
+ - pytest -m completionOverride_spike
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -857,6 +862,7 @@ override_stranded:
- ulimit -a
- nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --strandedForce unstranded --upload false --dev false --ci true -with-report ./strandedOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./strandedOverride_PE_multiqc_data.json \;
+ - pytest -m completionOverride_stranded
artifacts:
name: "$CI_JOB_NAME"
when: always
@@ -868,7 +874,7 @@ override_stranded:
when:
- always
-override_stpike:
+override_spike:
stage: integration
only: [merge_requests]
except:
@@ -879,6 +885,7 @@ override_stpike:
- ulimit -a
- nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --spikeForce t --upload false --dev false --ci true -with-report ./spikeOverride_report.html
- find . -type f -name "multiqc_data.json" -exec cp {} ./spikeOverride_PE_multiqc_data.json \;
+ - pytest -m completionOverride_spike
artifacts:
name: "$CI_JOB_NAME"
when: always
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e45a78d..66553e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@
* Data-hub column title change from "Has_Strand_Specific_Information" to "Strandedness" (#115)
* Merge data error pre-inference execution run upload/finalize to 1 process
* Change uploadOutputBag logic to change reuse hatrac file if alread exists (re-uses Output_Bag entry by reassigning Execution_Run RID) (#112)
+* Add new CI py tests for override and integration
*Known Bugs*
* Override params (inputBag, fastq, species) aren't checked for integrity
diff --git a/workflow/tests/test_completion.py b/workflow/tests/test_completion.py
new file mode 100644
index 0000000..aa04f19
--- /dev/null
+++ b/workflow/tests/test_completion.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+import pytest
+import pandas as pd
+from io import StringIO
+import os
+import json
+
+test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
+ '/../../'
+
+
+@pytest.mark.consistencySE
+def test_consistencySE():
+ assert os.path.exists(os.path.join(
+ test_output_path, 'SE_multiqc_data.json'))
+
+ with open(os.path.join(
+ test_output_path, 'SE_multiqc_data.json')) as f:
+ assigned_reads_json = json.load(f)
+ assigned_reads = assigned_reads_json['report_general_stats_data'][4]['16-1ZX4_sorted']['Assigned']
+ baseline = 7746121
+ baseline_hi = baseline+(baseline*0.05)
+ baseline_lo = baseline-(baseline*0.05)
+ assert (assigned_reads >= baseline_lo)
+ assert (assigned_reads <= baseline_hi)
+
+
+@pytest.mark.consistencyPE
+def test_consistencyPE():
+ assert os.path.exists(os.path.join(
+ test_output_path, 'PE_multiqc_data.json'))
+
+ with open(os.path.join(
+ test_output_path, 'PE_multiqc_data.json')) as f:
+ assigned_reads_json = json.load(f)
+ assigned_reads = assigned_reads_json['report_general_stats_data'][4]['Q-Y5JA_sorted']['Assigned']
+ baseline = 2596053
+ baseline_hi = baseline+(baseline*0.05)
+ baseline_lo = baseline-(baseline*0.05)
+ assert (assigned_reads >= baseline_lo)
+ assert (assigned_reads <= baseline_hi)
--
GitLab