diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 213da881fe1e3cf9419aad1db990d091d0202e43..d2b97b78af84f2e21db86ca9de15c16b3ce5ce2e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,7 @@ before_script: - mkdir -p ~/.deriva - mkdir -p ~/.bdbag + stages: - unit - aggregation @@ -14,6 +15,7 @@ stages: - integration - consistency + getBag: stage: unit only: @@ -136,7 +138,6 @@ downsampleData: - singularity run 'docker://bicf/seqtk:2.0.1_indev' seqtk sample -s100 ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz 1000 1> sampled.1.fq - pytest -m downsampleData - alignData: stage: unit only: @@ -164,7 +165,6 @@ alignData: - version_samtools.txt expire_in: 7 days - dedupData: stage: unit only: @@ -262,22 +262,48 @@ dataQC: - merge_requests script: - echo -e "geneID\tchrom\ttx_start\ttx_end\tTIN" > Q-Y5F6_1M.se.sorted.deduped.tin.xls - - for i in {"chr8","chr4","chrY"}; do - echo "tin.py -i ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.${i}.bam -r /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/bed/genome.bed; cat Q-Y5F6_1M.se.sorted.deduped.${i}.tin.xls | tr -s \"\\w\" \"\\t\" | grep -P \"\\t${i}\\t\";"; done | singularity run 'docker://bicf/rseqc3.0:2.0.1_indev' parallel -j 20 -k >> Q-Y5F6_1M.se.sorted.deduped.tin.xls + - > + for i in {"chr8","chr4","chrY"}; do + echo "tin.py -i ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.${i}.bam -r /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/bed/genome.bed; cat Q-Y5F6_1M.se.sorted.deduped.${i}.tin.xls | tr -s \"\\w\" \"\\t\" | grep -P \"\\t${i}\\t\";" + done | singularity run 'docker://bicf/rseqc3.0:2.0.1_indev' parallel -j 20 -k >> Q-Y5F6_1M.se.sorted.deduped.tin.xls - pytest -m dataQC outputBag: stage: unit only: - push - - tags - except: - merge_requests script: - mkdir test - singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' bdbag test --archiver zip - pytest -m outputBag +outputBag: + stage: unit + only: + - push + - tags + except: + - merge_requests + script: + - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json + - hatrac=$(singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' deriva-hatrac-cli --host dev.gudmap.org ls /hatrac/resources/rnaseq/pipeline/input_bag/ | grep -o TEST) + - > + if [ -z "${hatrac}" ]; then + singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' deriva-hatrac-cli --host ${source} mkdir /hatrac/resources/rnaseq/pipeline/input_bag/TEST + fi + - echo "THIS IS A TEST FILE" > test.txt + - md5=$(md5sum ./test.txt | awk '{ print $1 }') + - size=$(wc -c < ./test.txt) + - exist=$(singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' curl -s https://${source}/ermrest/catalog/2/entity/RNASeq:Input_Bag/File_MD5=${md5}) + - > + if [ "\${exist}" == "[]" ]; then + cookie=$(cat credential.json | grep -A 1 '\"dev.gudmap.org": {' | grep -o '\"cookie\": \".*\"') + cookie=${cookie:11:-1} + loc=$(singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' deriva-hatrac-cli --host dev.gudmap.org put ./test.txt /hatrac/resources/rnaseq/pipeline/input_bag/TEST/test.txt) + inputBag_rid=$(singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' python3 uploadInputBag.py -f test.txt -l ${loc} -s ${md5} -b ${size} -o dev.gudmap.org -c ${cookie}) + fi + - #pytest -m uploadInputBag generateVersions: stage: aggregation