diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc3e9e879c8d95e072d2918d9b04e69c69601852..5b68a723f60ce5a853db3559e0207b47de39dcc6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,22 @@ before_script: - module load nextflow/0.24.1-SNAPSHOT - ln -s /project/shared/bicf_workflow_ref/workflow_testdata/chipseq/*fastq.gz test_data/ -test: +stages: + - unit + - integration + - acceptance + +user_configuration: + stage: unit + script: + - pytest -m unit + +single_end_mouse: + stage: integration + script: + - nextflow run workflow/main.nf + +single_end_acceptance: + stage: unit script: - - nextflow run workflow/main.nf - - pytest + - pytest -m acceptance diff --git a/workflow/tests/test_call_peaks_macs.py b/workflow/tests/test_call_peaks_macs.py index aba35702c689e3114442648763a855d01611c0cb..62ec2e3bd2f931e44d746d917f2396b0d0494575 100644 --- a/workflow/tests/test_call_peaks_macs.py +++ b/workflow/tests/test_call_peaks_macs.py @@ -8,6 +8,7 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ '/../output/callPeaksMACS/' +@pytest.mark.acceptance def test_call_peaks_macs_singleend(): assert os.path.exists(os.path.join(test_output_path, 'ENCLB144FDT.fc_signal.bw')) assert os.path.exists(os.path.join(test_output_path, 'ENCLB144FDT.pvalue_signal.bw')) @@ -15,6 +16,7 @@ def test_call_peaks_macs_singleend(): assert utils.count_lines(peak_file) == 210349 +@pytest.mark.acceptance def test_call_peaks_macs_pairedend(): # Do the same thing for paired end data pass diff --git a/workflow/tests/test_check_design.py b/workflow/tests/test_check_design.py index 23575ec745da1ff651435edfc34d3b37c156a63f..517c53c71cb31edb18c159b29636381aa66972d0 100644 --- a/workflow/tests/test_check_design.py +++ b/workflow/tests/test_check_design.py @@ -70,6 +70,7 @@ def fastq_files_1(fastq_files): return fastq_df +@pytest.mark.unit def test_check_headers_singleend(design_1): paired = False with pytest.raises(Exception) as excinfo: @@ -77,6 +78,7 @@ def test_check_headers_singleend(design_1): assert str(excinfo.value) == "Missing column headers: ['fastq_read1']" +@pytest.mark.unit def test_check_headers_pairedend(design): paired = True with pytest.raises(Exception) as excinfo: @@ -84,12 +86,14 @@ def test_check_headers_pairedend(design): assert str(excinfo.value) == "Missing column headers: ['fastq_read2']" +@pytest.mark.unit def test_check_controls(design_2): with pytest.raises(Exception) as excinfo: check_design.check_controls(design_2) assert str(excinfo.value) == "Missing control experiments: ['B_1']" +@pytest.mark.unit def test_check_files_missing_files(design, fastq_files_1): paired = False with pytest.raises(Exception) as excinfo: @@ -97,18 +101,22 @@ def test_check_files_missing_files(design, fastq_files_1): assert str(excinfo.value) == "Missing files from design file: ['B_2.fastq.gz']" +@pytest.mark.unit def test_check_files_output_singleend(design, fastq_files): paired = False new_design = check_design.check_files(design, fastq_files, paired) assert new_design.loc[0, 'fastq_read1'] == "/path/to/file/A_1.fastq.gz" +@pytest.mark.unit def test_check_files_output_pairedend(design_3, fastq_files): paired = True new_design = check_design.check_files(design_3, fastq_files, paired) assert new_design.loc[0, 'fastq_read2'] == "/path/to/file/A_2.fastq.gz" + +@pytest.mark.unit def test_check_replicates(design_4): paired = False with pytest.raises(Exception) as excinfo: diff --git a/workflow/tests/test_convert_reads.py b/workflow/tests/test_convert_reads.py index 4ee7ee1c5d55318730b0886c24cfb8463f996f06..bae2ff242a340dcb37c3bc5a5eec34470a69b485 100644 --- a/workflow/tests/test_convert_reads.py +++ b/workflow/tests/test_convert_reads.py @@ -7,11 +7,13 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ '/../output/convertReads/' +@pytest.mark.acceptance def test_convert_reads_singleend(): assert os.path.exists(os.path.join(test_output_path, 'ENCFF646LXU.filt.nodup.tagAlign.gz')) assert os.path.exists(os.path.join(test_output_path, 'ENCFF646LXU.filt.nodup.bedse.gz')) +@pytest.mark.integration def test_map_qc_pairedend(): # Do the same thing for paired end data # Also check that bedpe exists diff --git a/workflow/tests/test_experiment_design.py b/workflow/tests/test_experiment_design.py index bfbd3c01148fb2e31e40f0ddcf25a4f208b92caa..7dca74ce10e4d8b3e4c7c3b9dac8f38cd23b8ad4 100644 --- a/workflow/tests/test_experiment_design.py +++ b/workflow/tests/test_experiment_design.py @@ -24,11 +24,13 @@ def design_tag(): return design_df +@pytest.mark.unit def test_check_update_controls_tag(design_tag): new_design = experiment_design.update_controls(design_tag) assert new_design.loc[0, 'control_tag_align'] == "B_1.tagAlign.gz" +@pytest.mark.acceptance def test_experiment_design_single_end(): design_file = os.path.join(test_output_path, 'ENCSR238SGC.tsv') assert os.path.exists(design_file) @@ -36,6 +38,7 @@ def test_experiment_design_single_end(): assert design_df.shape[0] == 2 +@pytest.mark.acceptance def test_experiment_design_paired_end(): # Do the same thing for paired end data pass diff --git a/workflow/tests/test_experiment_qc.py b/workflow/tests/test_experiment_qc.py index 16f6b006e01032335c33d1a8b9dd3588361bbd4f..d9ab346526bf0c268431e01ea0897ae38adeb11f 100644 --- a/workflow/tests/test_experiment_qc.py +++ b/workflow/tests/test_experiment_qc.py @@ -24,11 +24,13 @@ def design_bam(): return design_df +@pytest.mark.unit def test_check_update_controls(design_bam): new_design = experiment_qc.update_controls(design_bam) assert new_design.loc[0, 'control_reads'] == "B_1.bam" +@pytest.mark.acceptance def test_experiment_qc_singleend(): assert os.path.exists(os.path.join(test_output_path, 'sample_mbs.npz')) assert os.path.exists(os.path.join(test_output_path, 'heatmap_SpearmanCorr.png')) @@ -36,7 +38,7 @@ def test_experiment_qc_singleend(): assert os.path.exists(os.path.join(test_output_path, 'ENCLB144FDT_fingerprint.png')) assert os.path.exists(os.path.join(test_output_path, 'ENCLB831RUI_fingerprint.png')) - +@pytest.mark.acceptance def test_experiment_qc_pairedend(): # Do the same thing for paired end data pass diff --git a/workflow/tests/test_map_qc.py b/workflow/tests/test_map_qc.py index ebc2cd5aae2808986fb59a48250cf53eadb162c4..9255e20e2fedefa108f62030363a7e6fb0be3097 100644 --- a/workflow/tests/test_map_qc.py +++ b/workflow/tests/test_map_qc.py @@ -8,6 +8,7 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ '/../output/filterReads/' +@pytest.mark.acceptance def test_map_qc_singleend(): assert os.path.exists(os.path.join(test_output_path, 'ENCFF646LXU.filt.nodup.bam')) assert os.path.exists(os.path.join(test_output_path, 'ENCFF646LXU.filt.nodup.bam.bai')) diff --git a/workflow/tests/test_map_reads.py b/workflow/tests/test_map_reads.py index 04ffd08084999dfeb2eebde88055e73e9832951f..1bcf609c7f72262bd4c030935fcc7aa7936a5fcb 100644 --- a/workflow/tests/test_map_reads.py +++ b/workflow/tests/test_map_reads.py @@ -7,6 +7,7 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ '/../output/alignReads/' +@pytest.mark.acceptance def test_map_reads_singleend(): assert os.path.exists(os.path.join(test_output_path, 'ENCFF646LXU.srt.bam')) aligned_reads_report = test_output_path + 'ENCFF646LXU.srt.bam.flagstat.qc' @@ -15,6 +16,7 @@ def test_map_reads_singleend(): assert '80050072 + 0 mapped (99.08% : N/A)' in samtools_report[4] +@pytest.mark.acceptance def test_map_reads_pairedend(): # Do the same thing for paired end data pass diff --git a/workflow/tests/test_overlap_peaks.py b/workflow/tests/test_overlap_peaks.py index db22fb868d474ed5e0ede4c84ae08d874935caf6..52fa0e5bc6c5b5b1d78aad9eb224f762239ab604 100644 --- a/workflow/tests/test_overlap_peaks.py +++ b/workflow/tests/test_overlap_peaks.py @@ -25,6 +25,7 @@ def design_diff(): return design_df +@pytest.mark.unit def test_check_update_design(design_diff): new_design = overlap_peaks.update_design(design_diff) assert new_design.shape[0] == 2 @@ -32,12 +33,14 @@ def test_check_update_design(design_diff): assert new_design.loc[0, 'peak_caller'] == "bed" +@pytest.mark.acceptance def test_overlap_peaks_singleend(): assert os.path.exists(os.path.join(test_output_path, 'ENCSR238SGC.rejected.narrowPeak')) peak_file = test_output_path + 'ENCSR238SGC.replicated.narrowPeak' - assert utils.count_lines(peak_file) == 150096 + assert utils.count_lines(peak_file) == 150302 +@pytest.mark.acceptance def test_call_peaks_macs_pairedend(): # Do the same thing for paired end data pass diff --git a/workflow/tests/test_pool_and_psuedoreplicate.py b/workflow/tests/test_pool_and_psuedoreplicate.py index b7a351d94adbbf351d2b6da03575395a09514801..de6f94e99830c10d33250b5398966e025bdaab29 100644 --- a/workflow/tests/test_pool_and_psuedoreplicate.py +++ b/workflow/tests/test_pool_and_psuedoreplicate.py @@ -36,26 +36,31 @@ def design_experiment_3(design_experiment): return design_experiment +@pytest.mark.unit def test_check_replicates(design_experiment): no_reps = pool_and_psuedoreplicate.check_replicates(design_experiment) assert no_reps == 2 +@pytest.mark.unit def test_check_replicates_single(design_experiment_2): no_reps = pool_and_psuedoreplicate.check_replicates(design_experiment_2) assert no_reps == 1 +@pytest.mark.unit def test_check_controls(design_experiment): no_controls = pool_and_psuedoreplicate.check_controls(design_experiment) assert no_controls == 2 +@pytest.mark.unit def test_check_controls_single(design_experiment_3): no_controls = pool_and_psuedoreplicate.check_controls(design_experiment_3) assert no_controls == 1 +@pytest.mark.acceptance def test_pool_and_psuedoreplicate_single_end(): design_file = os.path.join(test_output_path, 'ENCSR238SGC_ppr.tsv') assert os.path.exists(design_file) @@ -63,6 +68,7 @@ def test_pool_and_psuedoreplicate_single_end(): assert design_df.shape[0] == 5 +@pytest.mark.acceptance def test_experiment_design_paired_end(): # Do the same thing for paired end data pass diff --git a/workflow/tests/test_trim_reads.py b/workflow/tests/test_trim_reads.py index 7ca03750cd6f585e078de78d550e3f7fbac824bd..ad90935081333c1dc48559414a5ee8f32bd3f121 100644 --- a/workflow/tests/test_trim_reads.py +++ b/workflow/tests/test_trim_reads.py @@ -9,6 +9,8 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ '/../output/trimReads/' + +@pytest.mark.acceptance def test_trim_reads_singleend(): raw_fastq = test_data_path + 'ENCFF833BLU.fastq.gz' trimmed_fastq = test_output_path + 'ENCFF833BLU_trimmed.fq.gz' @@ -19,6 +21,7 @@ def test_trim_reads_singleend(): assert 'Trimming mode: single-end' in open(trimmed_fastq_report).readlines()[4] +@pytest.mark.acceptance def test_trim_reads_pairedend(): # Do the same thing for paired end data pass diff --git a/workflow/tests/test_utils.py b/workflow/tests/test_utils.py index 0abba1be016686a3df4053ba72a763e6488fb99a..66b9c11be2a982b02e39e8255c11174b7e59465d 100644 --- a/workflow/tests/test_utils.py +++ b/workflow/tests/test_utils.py @@ -29,6 +29,7 @@ def steps_2(steps_1): return steps_1 +@pytest.mark.unit def test_run_one_step(steps_1, capsys): check_output = 'ENCLB144FDT\tENCSR238SGC\tlimb\tH3K4me1\tNone\t1\tENCLB304SBJ\tENCFF833BLU.fastq.gz'.encode('UTF-8') out, err = utils.run_pipe(steps_1) @@ -37,6 +38,7 @@ def test_run_one_step(steps_1, capsys): assert check_output in out +@pytest.mark.unit def test_run_two_step(steps_2, capsys): check_output = 'ENCFF833BLU.fastq.gz\nENCFF646LXU.fastq.gz'.encode('UTF-8') out, err = utils.run_pipe(steps_2) @@ -45,6 +47,7 @@ def test_run_two_step(steps_2, capsys): assert check_output in out +@pytest.mark.unit def test_run_last_step_file(steps_2, capsys, tmpdir): check_output = 'ENCFF833BLU.fastq.gz\nENCFF646LXU.fastq.gz' tmp_outfile = tmpdir.join('output.txt') @@ -54,16 +57,19 @@ def test_run_last_step_file(steps_2, capsys, tmpdir): assert check_output in tmp_outfile.read() +@pytest.mark.unit def test_strip_extensions(): filename = utils.strip_extensions('ENCFF833BLU.fastq.gz', STRIP_EXTENSIONS) assert filename == 'ENCFF833BLU' +@pytest.mark.unit def test_strip_extensions_not_valid(): filename = utils.strip_extensions('ENCFF833BLU.not.valid', STRIP_EXTENSIONS) assert filename == 'ENCFF833BLU.not.valid' +@pytest.mark.unit def test_strip_extensions_missing_basename(): filename = utils.strip_extensions('.fastq.gz', STRIP_EXTENSIONS) assert filename == '.fastq' diff --git a/workflow/tests/test_xcor.py b/workflow/tests/test_xcor.py index b0eafa368e34c27ef095ddd0c5997bf1a13fab79..d1ccead9ac2d5ec54c1e30b8e65d33298e01f454 100644 --- a/workflow/tests/test_xcor.py +++ b/workflow/tests/test_xcor.py @@ -8,6 +8,7 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \ '/../output/crossReads/' +@pytest.mark.acceptance def test_convert_reads_singleend(): assert os.path.exists(os.path.join(test_output_path, 'ENCFF833BLU.filt.nodup.tagAlign.15.tagAlign.gz.cc.plot.pdf')) qc_file = os.path.join(test_output_path,"ENCFF833BLU.filt.nodup.tagAlign.15.tagAlign.gz.cc.qc") @@ -17,6 +18,7 @@ def test_convert_reads_singleend(): assert df_xcor[9].iloc[0] == 1.266678 +@pytest.mark.acceptance def test_map_qc_pairedend(): # Do the same thing for paired end data pass