diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a6fe3b56e0f528075ae7786c157751466ecca8d..376e3cc67b1336b3b54548ce3651a4d59df0e9fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,6 +29,7 @@ build_badges: - tags before_script: - module load singularity/3.5.3 + - export SINGULARITY_CACHEDIR=/project/BICF/BICF_Core/shared/gudmap/singularity_cache/ - chmod +x ./workflow/scripts/get_updated_badge_info.sh script: - echo "Building badges" @@ -62,10 +63,10 @@ getBag: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/credential.json` ~/.deriva/credential.json - - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-download-cli --version > version_deriva.txt - - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-download-cli staging.gudmap.org --catalog 2 ./workflow/conf/Replicate_For_Input_Bag.json . rid=Q-Y5F6 - - pytest -m getBag + - ln -sfn `readlink -e ./test_data/auth/credential.json` ~/.deriva/credential.json + - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-download-cli --version > version_deriva.txt + - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-download-cli staging.gudmap.org --catalog 2 ./workflow/conf/Replicate_For_Input_Bag.json . rid=Q-Y5F6 + - pytest -m getBag artifacts: name: "$CI_JOB_NAME" when: always @@ -82,11 +83,11 @@ getData: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' bdbag --version > version_bdbag.txt - - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - - unzip ./test_data/bag/Q-Y5F6_inputBag_xxxxtest.zip - - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' bash ./workflow/scripts/bdbag_fetch.sh Q-Y5F6_inputBag Q-Y5F6 - - pytest -m getData + - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' bdbag --version > version_bdbag.txt + - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt + - unzip ./test_data/bag/Q-Y5F6_inputBag_xxxxtest.zip + - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' bash ./workflow/scripts/bdbag_fetch.sh Q-Y5F6_inputBag Q-Y5F6 + - pytest -m getData artifacts: name: "$CI_JOB_NAME" when: always @@ -103,19 +104,19 @@ parseMetadata: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/python3:1.0.0' python3 --version > version_python.txt - - rep=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID) - - exp=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p expRID) - - study=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p studyRID) - - endsRaw=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p endsMeta) - - endsMeta="uk" - - endsManual="se" - - stranded=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p stranded) - - spike=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p spike) - - species=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p species) - - readLength=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p readLength) - - echo -e "${endsMeta},${endsRaw},${endsManual},${stranded},${spike},${species},${readLength},${exp},${study},${rep}" > design.csv - - pytest -m parseMetadata + - singularity run 'docker://gudmaprbk/python3:1.0.0' python3 --version > version_python.txt + - rep=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID) + - exp=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p expRID) + - study=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p studyRID) + - endsRaw=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p endsMeta) + - endsMeta="uk" + - endsManual="se" + - stranded=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p stranded) + - spike=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p spike) + - species=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p species) + - readLength=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p readLength) + - echo -e "${endsMeta},${endsRaw},${endsManual},${stranded},${spike},${species},${readLength},${exp},${study},${rep}" > design.csv + - pytest -m parseMetadata artifacts: name: "$CI_JOB_NAME" when: always @@ -132,9 +133,9 @@ fastqc: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc --version > version_fastqc.txt - - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -o . - - pytest -m fastqc + - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc --version > version_fastqc.txt + - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -o . + - pytest -m fastqc artifacts: name: "$CI_JOB_NAME" when: always @@ -151,11 +152,11 @@ seqwho: - merge_requests - schedules script: - - wget -O SeqWho.ix https://cloud.biohpc.swmed.edu/index.php/s/eeNWqZz8jqN5zWY/download - - mkdir -p SeqWho_call_plots/test_data/fastq/small/ - - singularity run 'docker://gudmaprbk/seqwho0.0.1:1.0.0' seqwho.py -h | grep -o Version.* > version_seqwho.txt - - singularity run 'docker://gudmaprbk/seqwho0.0.1:1.0.0' seqwho.py -f test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -x SeqWho.ix - - pytest -m seqwho + - wget -O SeqWho.ix https://cloud.biohpc.swmed.edu/index.php/s/eeNWqZz8jqN5zWY/download + - mkdir -p SeqWho_call_plots/test_data/fastq/small/ + - singularity run 'docker://gudmaprbk/seqwho0.0.1:1.0.0' seqwho.py -h | grep -o Version.* > version_seqwho.txt + - singularity run 'docker://gudmaprbk/seqwho0.0.1:1.0.0' seqwho.py -f test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -x SeqWho.ix + - pytest -m seqwho artifacts: name: "$CI_JOB_NAME" when: always @@ -172,12 +173,12 @@ trimData: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --version > version_trimgalore.txt - - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --gzip -q 25 --length 35 --basename Q-Y5F6_1M.se ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz - - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --gzip -q 25 --length 35 --paired --basename Q-Y5F6_1M.pe ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz ./test_data/fastq/small/Q-Y5F6_1M.R2.fastq.gz - - readLengthSE=$(zcat *_trimmed.fq.gz | awk '{if(NR%4==2) print length($1)}' | sort -n | awk '{a[NR]=$0}END{print(NR%2==1)?a[int(NR/2)+1]:(a[NR/2]+a[NR/2+1])/2}') - - readLengthPE=$(zcat *_1.fq.gz | awk '{if(NR%4==2) print length($1)}' | sort -n | awk '{a[NR]=$0}END{print(NR%2==1)?a[int(NR/2)+1]:(a[NR/2]+a[NR/2+1])/2}') - - pytest -m trimData + - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --version > version_trimgalore.txt + - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --gzip -q 25 --length 35 --basename Q-Y5F6_1M.se ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz + - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --gzip -q 25 --length 35 --paired --basename Q-Y5F6_1M.pe ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz ./test_data/fastq/small/Q-Y5F6_1M.R2.fastq.gz + - readLengthSE=$(zcat *_trimmed.fq.gz | awk '{if(NR%4==2) print length($1)}' | sort -n | awk '{a[NR]=$0}END{print(NR%2==1)?a[int(NR/2)+1]:(a[NR/2]+a[NR/2+1])/2}') + - readLengthPE=$(zcat *_1.fq.gz | awk '{if(NR%4==2) print length($1)}' | sort -n | awk '{a[NR]=$0}END{print(NR%2==1)?a[int(NR/2)+1]:(a[NR/2]+a[NR/2+1])/2}') + - pytest -m trimData artifacts: name: "$CI_JOB_NAME" when: always @@ -194,9 +195,9 @@ downsampleData: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/seqtk1.3:1.0.0' seqtk 2>&1 | grep -o Version.* > version_seqtk.txt& - - singularity run 'docker://gudmaprbk/seqtk1.3:1.0.0' seqtk sample -s100 ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz 1000 1> sampled.1.fq - - pytest -m downsampleData + - singularity run 'docker://gudmaprbk/seqtk1.3:1.0.0' seqtk 2>&1 | grep -o Version.* > version_seqtk.txt& + - singularity run 'docker://gudmaprbk/seqtk1.3:1.0.0' seqtk sample -s100 ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz 1000 1> sampled.1.fq + - pytest -m downsampleData artifacts: name: "$CI_JOB_NAME" when: always @@ -213,15 +214,15 @@ inferMetadata: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' infer_experiment.py --version > version_rseqc.txt - - > - align=$(echo $(grep "Overall alignment rate" ./test_data/meta/Q-Y5F6_1M.se.alignSummary.txt | cut -f2 -d ':' | cut -f2 -d ' ' | tr -d '%')) && - if [[ ${align} == "" ]]; then exit 1; fi - - > - singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' infer_experiment.py -r "/project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/annotation/genome.bed" -i "./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam" 1>> Q-Y5F6_1M.se.inferMetadata.log && - ended=`singularity run 'gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/infer_meta.sh endness Q-Y5F6_1M.se.inferMetadata.log` && - if [[ ${ended} == "" ]]; then exit 1; fi - - pytest -m inferMetadata + - singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' infer_experiment.py --version > version_rseqc.txt + - > + align=$(echo $(grep "Overall alignment rate" ./test_data/meta/Q-Y5F6_1M.se.alignSummary.txt | cut -f2 -d ':' | cut -f2 -d ' ' | tr -d '%')) && + if [[ ${align} == "" ]]; then exit 1; fi + - > + singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' infer_experiment.py -r "/project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/annotation/genome.bed" -i "./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam" 1>> Q-Y5F6_1M.se.inferMetadata.log && + ended=`singularity run 'gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/infer_meta.sh endness Q-Y5F6_1M.se.inferMetadata.log` && + if [[ ${ended} == "" ]]; then exit 1; fi + - pytest -m inferMetadata artifacts: name: "$CI_JOB_NAME" when: always @@ -238,17 +239,17 @@ alignData: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 --version > version_hisat2.txt - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools --version > version_samtools.txt - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 -p 20 --add-chrname --un-gz Q-Y5F6_1M.se.unal.gz -S Q-Y5F6_1M.se.sam -x /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/hisat2/genome --rna-strandness F -U ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz --summary-file Q-Y5F6_1M.se.alignSummary.txt --new-summary - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools view -1 -@ 20 -F 4 -F 8 -F 256 -o Q-Y5F6_1M.se.bam Q-Y5F6_1M.se.sam - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F6_1M.se.sorted.bam Q-Y5F6_1M.se.bam - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools index -@ 20 -b Q-Y5F6_1M.se.sorted.bam Q-Y5F6_1M.se.sorted.bam.bai - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 -p 20 --add-chrname --un-gz Q-Y5F6_1M.pe.unal.gz -S Q-Y5F6_1M.pe.sam -x /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/hisat2/genome --rna-strandness FR --no-mixed --no-discordant -1 ./test_data/fastq/small/Q-Y5F6_1M.pe_val_1.fq.gz -2 ./test_data/fastq/small/Q-Y5F6_1M.pe_val_2.fq.gz --summary-file Q-Y5F6_1M.pe.alignSummary.txt --new-summary - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools view -1 -@ 20 -F 4 -F 8 -F 256 -o Q-Y5F6_1M.pe.bam Q-Y5F6_1M.pe.sam - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F6_1M.pe.sorted.bam Q-Y5F6_1M.pe.bam - - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools index -@ 20 -b Q-Y5F6_1M.pe.sorted.bam Q-Y5F6_1M.pe.sorted.bam.bai - - pytest -m alignData + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 --version > version_hisat2.txt + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools --version > version_samtools.txt + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 -p 20 --add-chrname --un-gz Q-Y5F6_1M.se.unal.gz -S Q-Y5F6_1M.se.sam -x /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/hisat2/genome --rna-strandness F -U ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz --summary-file Q-Y5F6_1M.se.alignSummary.txt --new-summary + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools view -1 -@ 20 -F 4 -F 8 -F 256 -o Q-Y5F6_1M.se.bam Q-Y5F6_1M.se.sam + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F6_1M.se.sorted.bam Q-Y5F6_1M.se.bam + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools index -@ 20 -b Q-Y5F6_1M.se.sorted.bam Q-Y5F6_1M.se.sorted.bam.bai + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 -p 20 --add-chrname --un-gz Q-Y5F6_1M.pe.unal.gz -S Q-Y5F6_1M.pe.sam -x /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/hisat2/genome --rna-strandness FR --no-mixed --no-discordant -1 ./test_data/fastq/small/Q-Y5F6_1M.pe_val_1.fq.gz -2 ./test_data/fastq/small/Q-Y5F6_1M.pe_val_2.fq.gz --summary-file Q-Y5F6_1M.pe.alignSummary.txt --new-summary + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools view -1 -@ 20 -F 4 -F 8 -F 256 -o Q-Y5F6_1M.pe.bam Q-Y5F6_1M.pe.sam + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F6_1M.pe.sorted.bam Q-Y5F6_1M.pe.bam + - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools index -@ 20 -b Q-Y5F6_1M.pe.sorted.bam Q-Y5F6_1M.pe.sorted.bam.bai + - pytest -m alignData artifacts: name: "$CI_JOB_NAME" when: always @@ -266,16 +267,16 @@ dedupData: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools --version > version_samtools.txt - - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates --version 2> version_markdups.txt& - - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates I=./test_data/bam/small/Q-Y5F6_1M.se.sorted.bam O=Q-Y5F6_1M.se.deduped.bam M=Q-Y5F6_1M.se.deduped.Metrics.txt REMOVE_DUPLICATES=true - - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F6_1M.se.sorted.deduped.bam ./test_data/bam/small/Q-Y5F6_1M.se.deduped.bam - - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools index -@ 20 -b ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam Q-Y5F6_1M.se.sorted.deduped.bam.bai - - > - for i in {"chr8","chr4","chrY"}; do - echo "samtools view -b Q-Y5F6_1M.se.sorted.deduped.bam ${i} > Q-Y5F6_1M.se.sorted.deduped.${i}.bam; samtools index -@ 20 -b Q-Y5F6_1M.se.sorted.deduped.${i}.bam Q-Y5F6_1M.se.sorted.deduped.${i}.bam.bai;"; - done | singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' parallel -j 20 -k - - pytest -m dedupData + - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools --version > version_samtools.txt + - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates --version 2> version_markdups.txt& + - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates I=./test_data/bam/small/Q-Y5F6_1M.se.sorted.bam O=Q-Y5F6_1M.se.deduped.bam M=Q-Y5F6_1M.se.deduped.Metrics.txt REMOVE_DUPLICATES=true + - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F6_1M.se.sorted.deduped.bam ./test_data/bam/small/Q-Y5F6_1M.se.deduped.bam + - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools index -@ 20 -b ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam Q-Y5F6_1M.se.sorted.deduped.bam.bai + - > + for i in {"chr8","chr4","chrY"}; do + echo "samtools view -b Q-Y5F6_1M.se.sorted.deduped.bam ${i} > Q-Y5F6_1M.se.sorted.deduped.${i}.bam; samtools index -@ 20 -b Q-Y5F6_1M.se.sorted.deduped.${i}.bam Q-Y5F6_1M.se.sorted.deduped.${i}.bam.bai;"; + done | singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' parallel -j 20 -k + - pytest -m dedupData artifacts: name: "$CI_JOB_NAME" when: always @@ -293,15 +294,15 @@ countData: - merge_requests - schedules script: - - ln -s /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/metadata/geneID.tsv - - ln -s /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/metadata/Entrez.tsv - - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' featureCounts -T 20 -a /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/annotation/genome.gtf -G /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/sequence/genome.fna -g 'gene_name' --extraAttributes 'gene_id' -o Q-Y5F6_1M.se_countData -s 1 -R SAM --primary --ignoreDup ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam - - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' Rscript ./workflow/scripts/calculateTPM.R --count ./test_data/counts/small/Q-Y5F6_1M.se_countData - - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' Rscript ./workflow/scripts/convertGeneSymbols.R --repRID Q-Y5F6_1M.se - - assignedReads=$(grep -m 1 'Assigned' *.summary | grep -oe '\([0-9.]*\)') - - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' featureCounts -v &> version_featurecounts.txt - - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' R --version > version_r.txt - - pytest -m makeFeatureCounts + - ln -s /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/metadata/geneID.tsv + - ln -s /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/metadata/Entrez.tsv + - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' featureCounts -T 20 -a /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/annotation/genome.gtf -G /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/sequence/genome.fna -g 'gene_name' --extraAttributes 'gene_id' -o Q-Y5F6_1M.se_countData -s 1 -R SAM --primary --ignoreDup ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam + - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' Rscript ./workflow/scripts/calculateTPM.R --count ./test_data/counts/small/Q-Y5F6_1M.se_countData + - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' Rscript ./workflow/scripts/convertGeneSymbols.R --repRID Q-Y5F6_1M.se + - assignedReads=$(grep -m 1 'Assigned' *.summary | grep -oe '\([0-9.]*\)') + - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' featureCounts -v &> version_featurecounts.txt + - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' R --version > version_r.txt + - pytest -m makeFeatureCounts artifacts: name: "$CI_JOB_NAME" when: always @@ -319,9 +320,9 @@ makeBigWig: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' deeptools --version > version_deeptools.txt - - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' bamCoverage -p 20 -b ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam -o Q-Y5F6_1M.se.bw - - pytest -m makeBigWig + - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' deeptools --version > version_deeptools.txt + - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' bamCoverage -p 20 -b ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam -o Q-Y5F6_1M.se.bw + - pytest -m makeBigWig artifacts: name: "$CI_JOB_NAME" when: always @@ -337,12 +338,12 @@ dataQC: except: - merge_requests script: - - echo -e "geneID\tchrom\ttx_start\ttx_end\tTIN" > Q-Y5F6_1M.se.sorted.deduped.tin.xls - - > - for i in {"chr8","chr4","chrY"}; do - echo "tin.py -i ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.${i}.bam -r /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/annotation/genome.bed; cat Q-Y5F6_1M.se.sorted.deduped.${i}.tin.xls | tr -s \"\\w\" \"\\t\" | grep -P \"\\t${i}\\t\";" - done | singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' parallel -j 20 -k >> Q-Y5F6_1M.se.sorted.deduped.tin.xls - - pytest -m dataQC + - echo -e "geneID\tchrom\ttx_start\ttx_end\tTIN" > Q-Y5F6_1M.se.sorted.deduped.tin.xls + - > + for i in {"chr8","chr4","chrY"}; do + echo "tin.py -i ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.${i}.bam -r /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/annotation/genome.bed; cat Q-Y5F6_1M.se.sorted.deduped.${i}.tin.xls | tr -s \"\\w\" \"\\t\" | grep -P \"\\t${i}\\t\";" + done | singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' parallel -j 20 -k >> Q-Y5F6_1M.se.sorted.deduped.tin.xls + - pytest -m dataQC uploadInputBag: stage: unit @@ -353,23 +354,23 @@ uploadInputBag: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - - echo THIS IS A TEST FILE > test.txt - - > - md5=$(md5sum ./test.txt | awk '{ print $1 }') && - size=$(wc -c < ./test.txt) && - exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Input_Bag/File_MD5=${md5}) && - if [ "${exist}" == "[]" ]; then - cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && - cookie=${cookie:11:-1} && - loc=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host staging.gudmap.org put ./test.txt /hatrac/resources/rnaseq/pipeline/input_bag/TEST/test.txt --parents) && - rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_input_bag.py -f test.txt -l ${loc} -s ${md5} -b ${size} -n 'This is a test input bag' -o staging.gudmap.org -c ${cookie}) && - echo ${rid} test input bag created - else - rid=$(echo ${exist} | grep -o '\"RID\":\".*\",\"RCT') && - rid=${rid:8:-6} && - echo ${rid} test input bag already exists - fi + - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json + - echo THIS IS A TEST FILE > test.txt + - > + md5=$(md5sum ./test.txt | awk '{ print $1 }') && + size=$(wc -c < ./test.txt) && + exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Input_Bag/File_MD5=${md5}) && + if [ "${exist}" == "[]" ]; then + cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && + cookie=${cookie:11:-1} && + loc=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host staging.gudmap.org put ./test.txt /hatrac/resources/rnaseq/pipeline/input_bag/TEST/test.txt --parents) && + rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_input_bag.py -f test.txt -l ${loc} -s ${md5} -b ${size} -n 'This is a test input bag' -o staging.gudmap.org -c ${cookie}) && + echo ${rid} test input bag created + else + rid=$(echo ${exist} | grep -o '\"RID\":\".*\",\"RCT') && + rid=${rid:8:-6} && + echo ${rid} test input bag already exists + fi uploadExecutionRun: stage: unit @@ -380,20 +381,20 @@ uploadExecutionRun: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - - > - exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Replicate=17-BTFJ) && - cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && - cookie=${cookie:11:-1} && - if [ "${exist}" == "[]" ]; then - rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_execution_run.py -r 17-BTFJ -w 17-BV2Y -g 17-BV90 -i 17-BTFT -s Success -d 'This is a test execution run' -o staging.gudmap.org -c ${cookie} -u F) && - echo ${rid} test execution run created - else - rid=$(echo ${exist} | grep -o '\"RID\":\".*\",\"RCT') && - rid=${rid:7:-6} && - rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_execution_run.py -r 17-BTFJ -w 17-BV2Y -g 17-BV90 -i 17-BTFT -s Success -d 'This is a test execution run' -o staging.gudmap.org -c ${cookie} -u ${rid}) && - echo ${rid} test execution run already exists - fi + - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json + - > + exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Replicate=17-BTFJ) && + cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && + cookie=${cookie:11:-1} && + if [ "${exist}" == "[]" ]; then + rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_execution_run.py -r 17-BTFJ -w 17-BV2Y -g 17-BV90 -i 17-BTFT -s Success -d 'This is a test execution run' -o staging.gudmap.org -c ${cookie} -u F) && + echo ${rid} test execution run created + else + rid=$(echo ${exist} | grep -o '\"RID\":\".*\",\"RCT') && + rid=${rid:7:-6} && + rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_execution_run.py -r 17-BTFJ -w 17-BV2Y -g 17-BV90 -i 17-BTFT -s Success -d 'This is a test execution run' -o staging.gudmap.org -c ${cookie} -u ${rid}) && + echo ${rid} test execution run already exists + fi uploadQC: stage: unit @@ -404,20 +405,20 @@ uploadQC: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - - > - exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:mRNA_QC/Replicate=17-BTFJ) && - cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && - cookie=${cookie:11:-1} && - if [ "${exist}" != "[]" ]; then - rids=$(echo ${exist} | grep -o '\"RID\":\".\{7\}' | sed 's/^.\{7\}//') && - for rid in ${rids}; do - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/delete_entry.py -r ${rid} -t mRNA_QC -o staging.gudmap.org -c ${cookie} - done - echo all old mRNA QC RIDs deleted - fi - rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_qc.py -r 17-BTFJ -e 17-BVDJ -p "Single End" -s forward -l 35 -w 5 -f 1 -t 1 -n "This is a test mRNA QC" -o staging.gudmap.org -c ${cookie} -u F) - echo ${rid} test mRNA QC created + - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json + - > + exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:mRNA_QC/Replicate=17-BTFJ) && + cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && + cookie=${cookie:11:-1} && + if [ "${exist}" != "[]" ]; then + rids=$(echo ${exist} | grep -o '\"RID\":\".\{7\}' | sed 's/^.\{7\}//') && + for rid in ${rids}; do + singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/delete_entry.py -r ${rid} -t mRNA_QC -o staging.gudmap.org -c ${cookie} + done + echo all old mRNA QC RIDs deleted + fi + rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_qc.py -r 17-BTFJ -e 17-BVDJ -p "Single End" -s forward -l 35 -w 5 -f 1 -t 1 -n "This is a test mRNA QC" -o staging.gudmap.org -c ${cookie} -u F) + echo ${rid} test mRNA QC created uploadProcessedFile: stage: unit @@ -428,27 +429,27 @@ uploadProcessedFile: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - - echo THIS IS A TEST FILE > 17-BTFJ_test.csv - - mkdir -p ./deriva/Seq/pipeline/17-BTFE/17-BVDJ/ - - mv 17-BTFJ_test.csv ./deriva/Seq/pipeline/17-BTFE/17-BVDJ/17-BTFJ_test.csv - - > - exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Processed_File/Replicate=17-BTFJ) && - cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && - cookie=${cookie:11:-1} && - if [ "${exist}" != "[]" ]; then - rids=$(echo ${exist} | grep -o '\"RID\":\".\{7\}' | sed 's/^.\{7\}//') && - for rid in ${rids}; do - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/delete_entry.py -r ${rid} -t Processed_File -o staging.gudmap.org -c ${cookie} - done - echo all old processed file RIDs deleted - fi + - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json + - echo THIS IS A TEST FILE > 17-BTFJ_test.csv + - mkdir -p ./deriva/Seq/pipeline/17-BTFE/17-BVDJ/ + - mv 17-BTFJ_test.csv ./deriva/Seq/pipeline/17-BTFE/17-BVDJ/17-BTFJ_test.csv + - > + exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Processed_File/Replicate=17-BTFJ) && + cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && + cookie=${cookie:11:-1} && + if [ "${exist}" != "[]" ]; then + rids=$(echo ${exist} | grep -o '\"RID\":\".\{7\}' | sed 's/^.\{7\}//') && + for rid in ${rids}; do + singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/delete_entry.py -r ${rid} -t Processed_File -o staging.gudmap.org -c ${cookie} + done + echo all old processed file RIDs deleted + fi singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-upload-cli --catalog 2 --token ${cookie:9} staging.gudmap.org ./deriva echo test processed file uploaded - - mkdir test - - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' bdbag test --archiver zip - - echo test output bag created - - pytest -m outputBag + - mkdir test + - singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' bdbag test --archiver zip + - echo test output bag created + - pytest -m outputBag uploadOutputBag: stage: unit @@ -459,23 +460,23 @@ uploadOutputBag: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - - echo THIS IS A TEST FILE > test.txt - - > - md5=$(md5sum ./test.txt | awk '{ print $1 }') && - size=$(wc -c < ./test.txt) && - exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Output_Bag/File_MD5=${md5}) && - if [ "${exist}" == "[]" ]; then - cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && - cookie=${cookie:11:-1} && - loc=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host staging.gudmap.org put ./test.txt /hatrac/resources/rnaseq/pipeline/output_bag/TEST/test.txt --parents) && - rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_output_bag.py -e 17-BVDJ -f test.txt -l ${loc} -s ${md5} -b ${size} -n 'This is a test output bag' -o staging.gudmap.org -c ${cookie}) && - echo ${rid} test output bag created - else - rid=$(echo ${exist} | grep -o '\"RID\":\".*\",\"RCT') && - rid=${rid:8:-6} && - echo ${rid} test output bag already exists - fi + - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json + - echo THIS IS A TEST FILE > test.txt + - > + md5=$(md5sum ./test.txt | awk '{ print $1 }') && + size=$(wc -c < ./test.txt) && + exist=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Output_Bag/File_MD5=${md5}) && + if [ "${exist}" == "[]" ]; then + cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') && + cookie=${cookie:11:-1} && + loc=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host staging.gudmap.org put ./test.txt /hatrac/resources/rnaseq/pipeline/output_bag/TEST/test.txt --parents) && + rid=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' python3 ./workflow/scripts/upload_output_bag.py -e 17-BVDJ -f test.txt -l ${loc} -s ${md5} -b ${size} -n 'This is a test output bag' -o staging.gudmap.org -c ${cookie}) && + echo ${rid} test output bag created + else + rid=$(echo ${exist} | grep -o '\"RID\":\".*\",\"RCT') && + rid=${rid:8:-6} && + echo ${rid} test output bag already exists + fi generateVersions: @@ -487,9 +488,9 @@ generateVersions: - merge_requests - schedules script: - - singularity run 'docker://gudmaprbk/multiqc1.9:1.0.0' multiqc --version > version_multiqc.txt - - python ./workflow/scripts/generate_versions.py -o software_versions - - python ./workflow/scripts/generate_references.py -r ./docs/references.md -o software_references + - singularity run 'docker://gudmaprbk/multiqc1.9:1.0.0' multiqc --version > version_multiqc.txt + - python ./workflow/scripts/generate_versions.py -o software_versions + - python ./workflow/scripts/generate_references.py -r ./docs/references.md -o software_references artifacts: name: "$CI_JOB_NAME" when: always @@ -508,8 +509,8 @@ human_BioHPC: - merge_requests - schedules script: - - mkdir -p hu - - cp -R /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/hisat2 ./hu/ + - mkdir -p hu + - cp -R /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/hisat2 ./hu/ mouse_BioHPC: stage: reference @@ -520,8 +521,8 @@ mouse_BioHPC: - merge_requests - schedules script: - - mkdir -p mo - - cp -R /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/hisat2 ./mo/ + - mkdir -p mo + - cp -R /project/BICF/BICF_Core/shared/gudmap/references/new/GRCh38.p13.v36/data/hisat2 ./mo/ human_dev: stage: reference @@ -532,22 +533,22 @@ human_dev: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - - referenceBase=dev.gudmap.org - - refName=GRCh - - references=$(echo ${referenceBase}/${refName}${refHuVersion}) - - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) - - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) - - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) - - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) - - curl --request GET ${query} > refQuery.json - - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) - - loc=$(dirname ${refURL}) - - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi - - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') - - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) - - test=$(echo ${test} | grep -o ${filename}) - - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi + - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt + - referenceBase=dev.gudmap.org + - refName=GRCh + - references=$(echo ${referenceBase}/${refName}${refHuVersion}) + - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) + - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) + - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) + - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) + - curl --request GET ${query} > refQuery.json + - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) + - loc=$(dirname ${refURL}) + - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi + - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') + - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) + - test=$(echo ${test} | grep -o ${filename}) + - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi mouse_dev: stage: reference @@ -558,22 +559,22 @@ mouse_dev: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - - referenceBase=dev.gudmap.org - - refName=GRCm - - references=$(echo ${referenceBase}/${refName}${refMoVersion}) - - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) - - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) - - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) - - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) - - curl --request GET ${query} > refQuery.json - - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) - - loc=$(dirname ${refURL}) - - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi - - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') - - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) - - test=$(echo ${test} | grep -o ${filename}) - - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi + - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt + - referenceBase=dev.gudmap.org + - refName=GRCm + - references=$(echo ${referenceBase}/${refName}${refMoVersion}) + - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) + - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) + - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) + - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) + - curl --request GET ${query} > refQuery.json + - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) + - loc=$(dirname ${refURL}) + - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi + - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') + - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) + - test=$(echo ${test} | grep -o ${filename}) + - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi human_staging: stage: reference @@ -584,22 +585,22 @@ human_staging: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - - referenceBase=staging.gudmap.org - - refName=GRCh - - references=$(echo ${referenceBase}/${refName}${refHuVersion}) - - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) - - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) - - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) - - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) - - curl --request GET ${query} > refQuery.json - - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) - - loc=$(dirname ${refURL}) - - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi - - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') - - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) - - test=$(echo ${test} | grep -o ${filename}) - - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi + - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt + - referenceBase=staging.gudmap.org + - refName=GRCh + - references=$(echo ${referenceBase}/${refName}${refHuVersion}) + - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) + - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) + - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) + - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) + - curl --request GET ${query} > refQuery.json + - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) + - loc=$(dirname ${refURL}) + - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi + - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') + - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) + - test=$(echo ${test} | grep -o ${filename}) + - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi mouse_staging: stage: reference @@ -610,23 +611,23 @@ mouse_staging: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - - referenceBase=staging.gudmap.org - - refName=GRCm - - refHuVersion=38.p6.vM22 - - references=$(echo ${referenceBase}/${refName}${refMoVersion}) - - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) - - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) - - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) - - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) - - curl --request GET ${query} > refQuery.json - - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) - - loc=$(dirname ${refURL}) - - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi - - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') - - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) - - test=$(echo ${test} | grep -o ${filename}) - - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi + - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt + - referenceBase=staging.gudmap.org + - refName=GRCm + - refHuVersion=38.p6.vM22 + - references=$(echo ${referenceBase}/${refName}${refMoVersion}) + - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) + - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) + - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) + - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) + - curl --request GET ${query} > refQuery.json + - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) + - loc=$(dirname ${refURL}) + - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi + - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') + - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) + - test=$(echo ${test} | grep -o ${filename}) + - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi human_prod: stage: reference @@ -637,22 +638,22 @@ human_prod: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - - referenceBase=www.gudmap.org - - refName=GRCh - - references=$(echo ${referenceBase}/${refName}${refHuVersion}) - - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) - - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) - - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) - - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) - - curl --request GET ${query} > refQuery.json - - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) - - loc=$(dirname ${refURL}) - - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi - - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') - - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) - - test=$(echo ${test} | grep -o ${filename}) - - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi + - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt + - referenceBase=www.gudmap.org + - refName=GRCh + - references=$(echo ${referenceBase}/${refName}${refHuVersion}) + - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) + - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) + - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) + - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) + - curl --request GET ${query} > refQuery.json + - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) + - loc=$(dirname ${refURL}) + - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi + - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') + - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) + - test=$(echo ${test} | grep -o ${filename}) + - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi mouse_prod: stage: reference @@ -663,23 +664,23 @@ mouse_prod: - merge_requests - schedules script: - - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - - referenceBase=www.gudmap.org - - refName=GRCm - - refHuVersion=38.p6.vM22 - - references=$(echo ${referenceBase}/${refName}${refMoVersion}) - - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) - - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) - - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) - - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) - - curl --request GET ${query} > refQuery.json - - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) - - loc=$(dirname ${refURL}) - - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi - - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') - - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) - - test=$(echo ${test} | grep -o ${filename}) - - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi + - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt + - referenceBase=www.gudmap.org + - refName=GRCm + - refHuVersion=38.p6.vM22 + - references=$(echo ${referenceBase}/${refName}${refMoVersion}) + - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1) + - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2) + - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3) + - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE}/Used_Spike_Ins=false) + - curl --request GET ${query} > refQuery.json + - refURL=$(python ./workflow/scripts/extract_ref_data.py --returnParam URL) + - loc=$(dirname ${refURL}) + - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi + - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)') + - test=$(singularity run 'docker://gudmaprbk/deriva1.4:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/) + - test=$(echo ${test} | grep -o ${filename}) + - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi integration_se: @@ -689,11 +690,11 @@ integration_se: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./SE_report.html - - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \; - - pytest -m completionMultiqc --filename SE_multiqc_data.json + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging --refSource datahub --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./SE_report.html + - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \; + - pytest -m completionMultiqc --filename SE_multiqc_data.json artifacts: name: "$CI_JOB_NAME" when: always @@ -714,11 +715,11 @@ integration_pe: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./PE_report.html - - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \; - - pytest -m completionMultiqc --filename PE_multiqc_data.json + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./PE_report.html + - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \; + - pytest -m completionMultiqc --filename PE_multiqc_data.json artifacts: name: "$CI_JOB_NAME" when: always @@ -741,9 +742,9 @@ failAmbiguousSpecies: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failAmbiguousSpecies_report.html + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failAmbiguousSpecies_report.html retry: max: 0 when: @@ -756,9 +757,9 @@ failTrunkation: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ET --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failTrunkation_report.html + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ET --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failTrunkation_report.html retry: max: 0 when: @@ -771,9 +772,9 @@ failMismatchR1R2: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-CWH4 --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failMismatchR1R2_report.html + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-CWH4 --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failMismatchR1R2_report.html retry: max: 0 when: @@ -786,9 +787,9 @@ failUnexpectedMeta: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 14-3R4R --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failUnexpectedMeta_report.html + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 14-3R4R --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failUnexpectedMeta_report.html retry: max: 0 when: @@ -801,9 +802,9 @@ failFileStructure: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5HT --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failFileStructure_report.html + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5HT --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failFileStructure_report.html retry: max: 0 when: @@ -816,9 +817,9 @@ failSeqType: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-DNDJ --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failSeqType_report.html + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-DNDJ --source staging --upload true -with-dag dag.png --dev false --ci true --track true -with-report ./failSeqType_report.html retry: max: 0 when: @@ -831,12 +832,12 @@ override_inputBag: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true --track false -with-report ./inputBagOverride_report.html - - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_multiqc_data.json \; - - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./inputBagOverride_multiqc.html \; - - pytest -m completionMultiqc --filename inputBagOverride_multiqc_data.json + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true --track false -with-report ./inputBagOverride_report.html + - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_multiqc_data.json \; + - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./inputBagOverride_multiqc.html \; + - pytest -m completionMultiqc --filename inputBagOverride_multiqc_data.json artifacts: name: "$CI_JOB_NAME" when: always @@ -856,12 +857,12 @@ override_fastq: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true --track false -with-report ./fastqOverride_report.html - - find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_multiqc_data.json \; - - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./fastqOverride_multiqc.html \; - - pytest -m completionMultiqc --filename fastqOverride_multiqc_data.json + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true --track false -with-report ./fastqOverride_report.html + - find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_multiqc_data.json \; + - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./fastqOverride_multiqc.html \; + - pytest -m completionMultiqc --filename fastqOverride_multiqc_data.json artifacts: name: "$CI_JOB_NAME" when: always @@ -881,12 +882,12 @@ override_species: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true --dev false --ci true --track false -with-report ./speciesOverride_report.html - - find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_multiqc_data.json \; - - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./speciesOverride_multiqc.html \; - - pytest -m completionMultiqc --filename speciesOverride_multiqc_data.json + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EW --source staging --speciesForce 'Homo sapiens' --upload true --dev false --ci true --track false -with-report ./speciesOverride_report.html + - find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_multiqc_data.json \; + - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./speciesOverride_multiqc.html \; + - pytest -m completionMultiqc --filename speciesOverride_multiqc_data.json artifacts: name: "$CI_JOB_NAME" when: always @@ -906,12 +907,12 @@ override_stranded: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true --dev false --ci true --track false -with-report ./strandedOverride_report.html - - find . -type f -name "multiqc_data.json" -exec cp {} ./strandedOverride_multiqc_data.json \; - - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./strandedOverride_multiqc.html \; - - pytest -m completionMultiqc --filename strandedOverride_multiqc_data.json + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5EY --source staging --strandedForce unstranded --upload true --dev false --ci true --track false -with-report ./strandedOverride_report.html + - find . -type f -name "multiqc_data.json" -exec cp {} ./strandedOverride_multiqc_data.json \; + - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./strandedOverride_multiqc.html \; + - pytest -m completionMultiqc --filename strandedOverride_multiqc_data.json artifacts: name: "$CI_JOB_NAME" when: always @@ -931,12 +932,12 @@ override_spike: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - hostname - - ulimit -a - - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true --dev false --ci true --track false -with-report ./spikeOverride_report.html - - find . -type f -name "multiqc_data.json" -exec cp {} ./spikeOverride_multiqc_data.json \; - - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./spikeOverride_multiqc.html \; - - pytest -m completionMultiqc --filename spikeOverride_multiqc_data.json + - hostname + - ulimit -a + - nextflow -q run ./rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F0 --source staging --spikeForce true --upload true --dev false --ci true --track false -with-report ./spikeOverride_report.html + - find . -type f -name "multiqc_data.json" -exec cp {} ./spikeOverride_multiqc_data.json \; + - find ./**/report/ -type f -name "*multiqc.html" -exec cp {} ./spikeOverride_multiqc.html \; + - pytest -m completionMultiqc --filename spikeOverride_multiqc_data.json artifacts: name: "$CI_JOB_NAME" when: always @@ -957,8 +958,8 @@ consistency: variables: - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/ script: - - pytest -m consistencySE - - pytest -m consistencyPE + - pytest -m consistencySE + - pytest -m consistencyPE artifacts: name: "$CI_JOB_NAME" when: always