Revert SE integration test and server for all integrations

.gitlab-ci.yml

@@ -664,7 +664,7 @@ integration_se:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-3EMJ --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./SE_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --upload true -with-dag dag.png --dev false --ci true -with-report ./SE_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \;
   - pytest -m completionIntegration_se
   artifacts:
@@ -689,7 +689,7 @@ integration_pe:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./PE_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png --dev false --ci true -with-report ./PE_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \;
   - pytest -m completionIntegration_pe
   artifacts:
@@ -716,7 +716,7 @@ failAmbiguousSpecies:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./failAmbiguousSpecies_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source staging --upload true -with-dag dag.png --dev false --ci true -with-report ./failAmbiguousSpecies_report.html
   retry:
     max: 0
     when:
@@ -731,7 +731,7 @@ failTrunkation:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ET --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./failTrunkation_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ET --source staging --upload true -with-dag dag.png --dev false --ci true -with-report ./failTrunkation_report.html
   retry:
     max: 0
     when:
@@ -746,7 +746,7 @@ failMismatchR1R2:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-CWH4 --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./failMismatchR1R2_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-CWH4 --source staging --upload true -with-dag dag.png --dev false --ci true -with-report ./failMismatchR1R2_report.html
   retry:
     max: 0
     when:
@@ -761,7 +761,7 @@ failUnexpectedMeta:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 14-3R4R --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./failUnexpectedMeta_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 14-3R4R --source staging --upload true -with-dag dag.png --dev false --ci true -with-report ./failUnexpectedMeta_report.html
   retry:
     max: 0
     when:
@@ -776,7 +776,7 @@ failFileStructure:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5HT --source dev --upload true -with-dag dag.png --dev false --ci true -with-report ./failFileStructure_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5HT --source staging --upload true -with-dag dag.png --dev false --ci true -with-report ./failFileStructure_report.html
   retry:
     max: 0
     when:
@@ -791,7 +791,7 @@ override_inputBag:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source dev --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true -with-report ./inputBagOverride_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true -with-report ./inputBagOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_PE_multiqc_data.json \;
   - pytest -m completionOverride_inputBag
   artifacts:
@@ -814,7 +814,7 @@ override_fastq:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6  --source dev --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true -with-report ./fastqOverride_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6  --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true -with-report ./fastqOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_PE_multiqc_data.json \;
   - pytest -m completionOverride_fastq
   artifacts:
@@ -837,7 +837,7 @@ override_species:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --speciesForce 'Homo sapiens' --upload false --dev false --ci true -with-report ./speciesOverride_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source staging --speciesForce 'Homo sapiens' --upload false --dev false --ci true -with-report ./speciesOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_PE_multiqc_data.json \;
   - pytest -m completionOverride_spike
   artifacts:
@@ -860,7 +860,7 @@ override_stranded:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --strandedForce unstranded --upload false --dev false --ci true -with-report ./strandedOverride_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source staging --strandedForce unstranded --upload false --dev false --ci true -with-report ./strandedOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./strandedOverride_PE_multiqc_data.json \;
   - pytest -m completionOverride_stranded
   artifacts:
@@ -883,7 +883,7 @@ override_spike:
   script:
   - hostname
   - ulimit -a
-  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source dev --spikeForce t --upload false --dev false --ci true -with-report ./spikeOverride_report.html
+  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source staging --spikeForce t --upload false --dev false --ci true -with-report ./spikeOverride_report.html
   - find . -type f -name "multiqc_data.json" -exec cp {} ./spikeOverride_PE_multiqc_data.json \;
   - pytest -m completionOverride_spike
   artifacts:

workflow/tests/test_completion.py

@@ -10,33 +10,43 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
     '/../../'
 
 
-@pytest.mark.consistencySE
-def test_consistencySE():
+@pytest.mark.completionIntegration_se
+def test_multiqc():
     assert os.path.exists(os.path.join(
         test_output_path, 'SE_multiqc_data.json'))
 
-    with open(os.path.join(
-        test_output_path, 'SE_multiqc_data.json')) as f:
-        assigned_reads_json = json.load(f)
-    assigned_reads = assigned_reads_json['report_general_stats_data'][4]['16-1ZX4_sorted']['Assigned']
-    baseline = 7746121
-    baseline_hi = baseline+(baseline*0.05)
-    baseline_lo = baseline-(baseline*0.05)
-    assert (assigned_reads >= baseline_lo)
-    assert (assigned_reads <=  baseline_hi)
 
-
-@pytest.mark.consistencyPE
-def test_consistencyPE():
+@pytest.mark.completionIntegration_pe
+def test_multiqc():
     assert os.path.exists(os.path.join(
         test_output_path, 'PE_multiqc_data.json'))
 
-    with open(os.path.join(
-        test_output_path, 'PE_multiqc_data.json')) as f:
-        assigned_reads_json = json.load(f)
-    assigned_reads = assigned_reads_json['report_general_stats_data'][4]['Q-Y5JA_sorted']['Assigned']
-    baseline = 2596053
-    baseline_hi = baseline+(baseline*0.05)
-    baseline_lo = baseline-(baseline*0.05)
-    assert (assigned_reads >= baseline_lo)
-    assert (assigned_reads <=  baseline_hi)
+
+@pytest.mark.completionOverride_inputBag
+def test_multiqc():
+    assert os.path.exists(os.path.join(
+        test_output_path, 'inputBagOverride_PE_multiqc_data.json'))
+
+
+@pytest.mark.completionOverride_fastq
+def test_multiqc():
+    assert os.path.exists(os.path.join(
+        test_output_path, 'fastqOverride_PE_multiqc_data.json'))
+
+
+@pytest.mark.completionOverride_species
+def test_multiqc():
+    assert os.path.exists(os.path.join(
+        test_output_path, 'speciesOverride_PE_multiqc_data.json'))
+
+
+@pytest.mark.completionOverride_stranded
+def test_multiqc():
+    assert os.path.exists(os.path.join(
+        test_output_path, 'strandedOverride_PE_multiqc_data.json'))
+
+
+@pytest.mark.completionOverride_spike
+def test_multiqc():
+    assert os.path.exists(os.path.join(
+        test_output_path, 'spikeOverride_PE_multiqc_data.json'))
\ No newline at end of file