diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7aff1aa2319a86ca01821b06f2092aee9d62f654..feefbbdffa1f7915c6714ecba7ae00082546f624 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,6 +14,11 @@ stages:
getBag:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- ln -sfn `readlink -e ./test_data/auth/credential.json` ~/.deriva/credential.json
- singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' deriva-download-cli dev.gudmap.org --catalog 2 ./workflow/conf/replicate_export_config.json . rid=Q-Y5F6
@@ -21,6 +26,11 @@ getBag:
getData:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
- unzip ./test_data/bag/Replicate_Q-Y5F6.zip
@@ -29,6 +39,11 @@ getData:
parseMetadata:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- rep=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID)
- exp=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p expRID)
@@ -44,6 +59,11 @@ parseMetadata:
inferMetadata:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- >
align=$(echo $(grep "Overall alignment rate" ./test_data/meta/Q-Y5F6_1M.se.alignSummary.txt | cut -f2 -d ':' | cut -f2 -d ' ' | tr -d '%')) &&
@@ -56,6 +76,11 @@ inferMetadata:
getRef:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- mkdir -p hu
- mkdir -p mo
@@ -64,6 +89,11 @@ getRef:
trimData:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- singularity run 'docker://bicf/trimgalore:1.1' trim_galore --gzip -q 25 --length 35 --basename Q-Y5F6_1M.se ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz
- singularity run 'docker://bicf/trimgalore:1.1' trim_galore --gzip -q 25 --length 35 --paired --basename Q-Y5F6_1M.pe ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz ./test_data/fastq/small/Q-Y5F6_1M.R2.fastq.gz
@@ -73,12 +103,22 @@ trimData:
downsampleData:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- singularity run 'docker://bicf/seqtk:2.0.1_indev' seqtk sample -s100 ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz 1000 1> sampled.1.fq
- pytest -m downsampleData
alignData:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- singularity run 'docker://bicf/gudmaprbkaligner:2.0.1_indev' hisat2 -p 20 --add-chrname --un-gz Q-Y5F6_1M.se.unal.gz -S Q-Y5F6_1M.se.sam -x /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2/genome --rna-strandness F -U ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz --summary-file Q-Y5F6_1M.se.alignSummary.txt --new-summary
- singularity run 'docker://bicf/gudmaprbkaligner:2.0.1_indev' samtools view -1 -@ 20 -F 4 -F 8 -F 256 -o Q-Y5F6_1M.se.bam Q-Y5F6_1M.se.sam
@@ -92,6 +132,11 @@ alignData:
dedupData:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- singularity run 'docker://bicf/gudmaprbkdedup:2.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates I=./test_data/bam/small/Q-Y5F6_1M.se.sorted.bam O=Q-Y5F6_1M.se.deduped.bam M=Q-Y5F6_1M.se.deduped.Metrics.txt REMOVE_DUPLICATES=true
- singularity run 'docker://bicf/gudmaprbkdedup:2.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F6_1M.se.sorted.deduped.bam ./test_data/bam/small/Q-Y5F6_1M.se.deduped.bam
@@ -104,6 +149,11 @@ dedupData:
countData:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/geneID.tsv
- ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/Entrez.tsv
@@ -115,18 +165,33 @@ countData:
makeBigWig:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- singularity run 'docker://bicf/deeptools3.3:2.0.1_indev' bamCoverage -p 20 -b ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam -o Q-Y5F6_1M.se.bw
- pytest -m makeBigWig
fastqc:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- singularity run 'docker://bicf/fastqc:2.0.1_indev' fastqc ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -o .
- pytest -m fastqc
dataQC:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- echo -e "geneID\tchrom\ttx_start\ttx_end\tTIN" > Q-Y5F6_1M.se.sorted.deduped.tin.xls
- for i in {"chr8","chr4","chrY"}; do
@@ -135,6 +200,11 @@ dataQC:
outputBag:
stage: unit
+ only:
+ - push
+ - tags
+ except:
+ - merge_requests
script:
- mkdir test
- singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' bdbag test --archiver zip
@@ -143,6 +213,11 @@ outputBag:
integration_se:
stage: integration
+ only:
+ - merge_requests
+ except:
+ refs:
+ - master
script:
- hostname
- ulimit -a
@@ -156,9 +231,18 @@ integration_se:
- output/report/
- SE_multiqc_data.json
expire_in: 7 days
+ retry:
+ max: 1
+ when:
+ - always
integration_pe:
stage: integration
+ only:
+ - merge_requests
+ except:
+ refs:
+ - master
script:
- hostname
- ulimit -a
@@ -173,9 +257,17 @@ integration_pe:
- output/report/
- PE_multiqc_data.json
expire_in: 7 days
+ retry:
+ max: 1
+ when:
+ - always
override_inputBag:
stage: integration
+ only:
+ - merge_requests
+ refs:
+ - develop
script:
- hostname
- ulimit -a
@@ -187,9 +279,17 @@ override_inputBag:
paths:
- inputBagOverride_PE_multiqc_data.json
expire_in: 7 days
+ retry:
+ max: 1
+ when:
+ - always
override_fastq:
stage: integration
+ only:
+ - merge_requests
+ refs:
+ - develop
script:
- hostname
- ulimit -a
@@ -201,9 +301,19 @@ override_fastq:
paths:
- fastqOverride_PE_multiqc_data.json
expire_in: 7 days
+ retry:
+ max: 1
+ when:
+ - always
override_species:
stage: integration
+ only:
+ - merge_requests
+ - tags
+ except:
+ refs:
+ - master
script:
- hostname
- ulimit -a
@@ -215,9 +325,19 @@ override_species:
paths:
- speciesOverride_PE_multiqc_data.json
expire_in: 7 days
+ retry:
+ max: 1
+ when:
+ - always
+
consistency:
stage: consistency
+ only:
+ - merge_requests
+ except:
+ refs:
+ - master
script:
- grep -m 1 \"Assigned\":.[0-9] SE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedSE.txt
- grep -m 1 \"Assigned\":.[0-9] PE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedPE.txt