Merge branch '28-AddPytestCI' into 'develop'

Resolve "Add pytest to CI"

Closes #28

See merge request !51

.gitlab-ci.yml

 before_script:
   - module load astrocyte
   - module load python/3.6.1-2-anaconda
+  - pip install --user pytest-pythonpath==0.7.1 pytest-cov==2.5.1
   - module load nextflow/0.31.1_Ignite
-  - mkdir test_data/hu.v2s1r500
-  - mkdir -p test_data/mu.v2s2r10k
-  - mkdir -p test_data/hu.v3s2r10k
-  - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v2s1r500/* test_data/hu.v2s1r500/
-  - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/mu.v2s2r10k/* test_data/mu.v2s2r10k/
+  - mkdir test_data/hu.v3s1r500
+  - mkdir test_data/mu.v3s1r500
+  - mkdir test_data/hu.v3s2r10k
+  - mkdir test_data/mu.v3s2r10k
+  - mkdir test_data/hu.v2s2r10k
+  - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s1r500/* test_data/hu.v3s1r500/
+  - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/mu.v3s1r500/* test_data/mu.v3s1r500/
   - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v3s2r10k/* test_data/hu.v3s2r10k/
+  - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/mu.v3s2r10k/* test_data/mu.v3s2r10k/
+  - ln -sfn /project/shared/bicf_workflow_ref/workflow_testdata/cellranger/cellranger_count/hu.v2s2r10k/* test_data/hu.v2s2r10k/
 
 stages:
   - astrocyte
@@ -21,14 +26,21 @@ astrocyte_check:
   artifacts:
     expire_in: 2 days
   retry:
-    max: 2
+    max: 1
     when:
       - always
 
-run_hu.cr3v2ref3.0.0:
+simple_1:
   stage: simple
+  only:
+    - branches
+    - tags
+  except:
+    - develop
+    - master
   script:
-  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v2s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v2s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'two' --version '3.0.2'
+  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.0.2'
+  - pytest -m count302
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -36,16 +48,21 @@ run_hu.cr3v2ref3.0.0:
       - .nextflow.log
     expire_in: 2 days
   retry:
-    max: 2
+    max: 1
     when:
       - always
 
-run_mu.cr2v2ref1.2.0:
-  stage: detailed
+simple_2:
+  stage: simple
+  only:
+    - branches
   except:
+    - develop
+    - master
     - tags
   script:
-  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v2s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v2s2r10k/design.csv" --genome 'mm10-1.2.0' --kitVersion 'auto' --version '2.1.1'
+  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s1r500/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.1'
+  - pytest -m count301
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -53,16 +70,20 @@ run_mu.cr2v2ref1.2.0:
       - .nextflow.log
     expire_in: 2 days
   retry:
-    max: 2
+    max: 1
     when:
       - always
 
-run_hu.cr3v3ref3.0.0:
+detailed_1:
   stage: detailed
+  only:
+    - develop
+    - master
   except:
     - tags
   script:
   - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.0.2'
+  - pytest -m count302
   artifacts:
     name: "$CI_JOB_NAME"
     when: always
@@ -70,6 +91,48 @@ run_hu.cr3v3ref3.0.0:
       - .nextflow.log
     expire_in: 2 days
   retry:
-    max: 2
+    max: 1
     when:
-      - always
\ No newline at end of file
+      - always
+
+detailed_2:
+  stage: detailed
+  only:
+    - develop
+    - master
+  except:
+    - tags
+  script:
+  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/mu.v3s2r10k/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.2'
+  - pytest -m count302
+  artifacts:
+    name: "$CI_JOB_NAME"
+    when: always
+    paths:
+      - .nextflow.log
+    expire_in: 2 days
+  retry:
+    max: 1
+    when:
+      - always
+
+detailed_3:
+  stage: detailed
+  only:
+    - develop
+    - master
+  except:
+    - tags
+  script:
+  - nextflow run workflow/main.nf --fastq "$CI_PROJECT_DIR/test_data/hu.v2s2r10k/*.fastq.gz" --designFile "$CI_PROJECT_DIR/test_data/hu.v2s2r10k/design.csv" --genome 'GRCh38-1.2.0' --kitVersion 'two' --version '2.1.1'
+  - pytest -m count211
+  artifacts:
+    name: "$CI_JOB_NAME"
+    when: always
+    paths:
+      - .nextflow.log
+    expire_in: 2 days
+  retry:
+    max: 1
+    when:
+      - always

workflow/tests/test_check_design.py

+#!/usr/bin/env python3
+
+import pytest
+import pandas as pd
+from io import StringIO
+import os
+
+test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
+		'/../output/misc/checkDesignFile/run/'
+
+@pytest.mark.count211
+def test_count211_design():
+    assert os.path.exists(os.path.join(test_output_path, 'design.checked.csv'))
+
+@pytest.mark.count301
+def test_count301_design():
+    assert os.path.exists(os.path.join(test_output_path, 'design.checked.csv'))
+
+@pytest.mark.count302
+def test_count302_design():
+    assert os.path.exists(os.path.join(test_output_path, 'design.checked.csv'))

workflow/tests/test_count.py

+#!/usr/bin/env python3
+
+import pytest
+import pandas as pd
+from io import StringIO
+import os
+
+test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
+                    '/../output/'
+
+
+@pytest.mark.count211
+def test_count211_count():
+    assert os.path.exists(os.path.join(test_output_path, 'count211', 'sample1_metrics_summary.tsv'))
+    assert os.path.exists(os.path.join(test_output_path, 'count211', 'sample1', 'outs'))
+
+@pytest.mark.count301
+def test_count301_count():
+    assert os.path.exists(os.path.join(test_output_path, 'count301', 'sample1_metrics_summary.tsv'))
+    assert os.path.exists(os.path.join(test_output_path, 'count301', 'sample1', 'outs'))
+
+@pytest.mark.count302
+def test_count302_count():
+    assert os.path.exists(os.path.join(test_output_path, 'count302', 'sample1_metrics_summary.tsv'))
+    assert os.path.exists(os.path.join(test_output_path, 'count302', 'sample1', 'outs'))

workflow/tests/test_multiqc.py

+#!/usr/bin/env python3
+
+import pytest
+import pandas as pd
+from io import StringIO
+import os
+
+test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
+                '/../output/multiqc/run/'
+
+@pytest.mark.count211
+def test_count211_multiqc():
+    assert os.path.exists(os.path.join(test_output_path, 'multiqc_report.html'))
+
+@pytest.mark.count301
+def test_count301_multiqc():
+    assert os.path.exists(os.path.join(test_output_path, 'multiqc_report.html'))
+
+@pytest.mark.count302
+def test_count302_multiqc():
+    assert os.path.exists(os.path.join(test_output_path, 'multiqc_report.html'))

workflow/tests/test_versions.py

+#!/usr/bin/env python3
+
+import pytest
+import pandas as pd
+from io import StringIO
+import os
+
+test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
+                '/../output/misc/versions/run/'
+
+@pytest.mark.count211
+def test_count211_versions():
+    assert os.path.exists(os.path.join(test_output_path, 'versions_mqc.yaml'))
+    assert os.path.exists(os.path.join(test_output_path, 'references_mqc.yaml'))
+
+@pytest.mark.count301
+def test_count301_versions():
+    assert os.path.exists(os.path.join(test_output_path, 'versions_mqc.yaml'))
+    assert os.path.exists(os.path.join(test_output_path, 'references_mqc.yaml'))
+
+@pytest.mark.count302
+def test_count302_versions():
+    assert os.path.exists(os.path.join(test_output_path, 'versions_mqc.yaml'))
+    assert os.path.exists(os.path.join(test_output_path, 'references_mqc.yaml'))