.gitlab-ci.yml

before_script:
  - module load python/3.6.4-anaconda
  - pip install --user  attrs==19.1.0 pytest-pythonpath==0.7.1 pytest-cov==2.5.1 deriva==1.3.0
  - module load singularity/3.5.3
  - module load nextflow/20.01.0
  - ln -sfn /project/BICF/BICF_Core/shared/gudmap/test_data/* ./test_data/
  - mkdir -p ~/.deriva
  - mkdir -p ~/.bdbag

variables:
  refMoVersion: "38.p6.vM22"
  refHuVersion: "38.p12.v31"
  refERCCVersion: "92"

stages:
  - unit
  - aggregation
  - reference
  - integration
  - consistency


getBag:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/credential.json` ~/.deriva/credential.json
  - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-download-cli --version > version_deriva.txt
  - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-download-cli dev.gudmap.org --catalog 2 ./workflow/conf/Replicate_For_Input_Bag.json . rid=Q-Y5F4
  - pytest -m getBag
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_deriva.txt
    expire_in: 7 days

getData:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' bdbag --version > version_bdbag.txt
  - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
  - unzip ./test_data/bag/Q-Y5F4_inputBag.zip
  - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' bash ./workflow/scripts/bdbagFetch.sh Q-Y5F4_inputBag Q-Y5F4 TEST
  - pytest -m getData
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_bdbag.txt
    expire_in: 7 days

parseMetadata:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/python3:1.0.0' python3 --version > version_python.txt
  - rep=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID)
  - exp=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p expRID)
  - study=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p studyRID)
  - endsMeta=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p endsMeta)
  - endsManual=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p endsManual)
  - stranded=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p stranded)
  - spike=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p spike)
  - species=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p species)
  - readLength=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.stageNew.csv" -p readLength)
  - echo -e "${endsMeta},${endsManual},${stranded},${spike},${species},${readLength},${exp},${study},${rep}" > design.csv
  - pytest -m parseMetadata
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_python.txt
    expire_in: 7 days

inferMetadata:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' infer_experiment.py --version > version_rseqc.txt
  - >
    align=$(echo $(grep "Overall alignment rate" ./test_data/meta/Q-Y5F4_1M.se.alignSummary.txt | cut -f2 -d ':' | cut -f2 -d ' ' | tr -d '%')) &&
    if [[ ${align} == "" ]]; then exit 1; fi
  - >
    singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' infer_experiment.py -r "/project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/bed/genome.bed" -i "./test_data/bam/small/Q-Y5F4_1M.se.sorted.deduped.bam" 1>> Q-Y5F4_1M.se.inferMetadata.log &&
    ended=`singularity run 'gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/inferMeta.sh endness Q-Y5F4_1M.se.inferMetadata.log` &&
    if [[ ${ended} == "" ]]; then exit 1; fi
  - pytest -m inferMetadata
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_rseqc.txt
    expire_in: 7 days

trimData:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --version > version_trimgalore.txt
  - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --gzip -q 25 --length 35 --basename Q-Y5F4_1M.se ./test_data/fastq/small/Q-Y5F4_1M.R1.fastq.gz
  - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --gzip -q 25 --length 35 --paired --basename Q-Y5F4_1M.pe ./test_data/fastq/small/Q-Y5F4_1M.R1.fastq.gz ./test_data/fastq/small/Q-Y5F4_1M.R2.fastq.gz
  - readLengthSE=$(zcat *_trimmed.fq.gz | awk '{if(NR%4==2) print length($1)}' | sort -n | awk '{a[NR]=$0}END{print(NR%2==1)?a[int(NR/2)+1]:(a[NR/2]+a[NR/2+1])/2}')
  - readLengthPE=$(zcat *_1.fq.gz | awk '{if(NR%4==2) print length($1)}' | sort -n | awk '{a[NR]=$0}END{print(NR%2==1)?a[int(NR/2)+1]:(a[NR/2]+a[NR/2+1])/2}')
  - pytest -m trimData
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_trimgalore.txt
    expire_in: 7 days

downsampleData:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/seqtk1.3:1.0.0' seqtk sample -s100 ./test_data/fastq/small/Q-Y5F4_1M.se_trimmed.fq.gz 1000 1> sampled.1.fq
  - pytest -m downsampleData

alignData:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 --version > version_hisat2.txt
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools --version > version_samtools.txt
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 -p 20 --add-chrname --un-gz Q-Y5F4_1M.se.unal.gz -S Q-Y5F4_1M.se.sam -x /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2/genome --rna-strandness F -U ./test_data/fastq/small/Q-Y5F4_1M.se_trimmed.fq.gz --summary-file Q-Y5F4_1M.se.alignSummary.txt --new-summary
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools view -1 -@ 20 -F 4 -F 8 -F 256 -o Q-Y5F4_1M.se.bam Q-Y5F4_1M.se.sam
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F4_1M.se.sorted.bam Q-Y5F4_1M.se.bam
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools index -@ 20 -b Q-Y5F4_1M.se.sorted.bam Q-Y5F4_1M.se.sorted.bam.bai
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 -p 20 --add-chrname --un-gz Q-Y5F4_1M.pe.unal.gz -S Q-Y5F4_1M.pe.sam -x /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2/genome --rna-strandness FR --no-mixed --no-discordant -1 ./test_data/fastq/small/Q-Y5F4_1M.pe_val_1.fq.gz -2 ./test_data/fastq/small/Q-Y5F4_1M.pe_val_2.fq.gz --summary-file Q-Y5F4_1M.pe.alignSummary.txt --new-summary
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools view -1 -@ 20 -F 4 -F 8 -F 256 -o Q-Y5F4_1M.pe.bam Q-Y5F4_1M.pe.sam
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F4_1M.pe.sorted.bam Q-Y5F4_1M.pe.bam
  - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools index -@ 20 -b Q-Y5F4_1M.pe.sorted.bam Q-Y5F4_1M.pe.sorted.bam.bai
  - pytest -m alignData
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_hisat2.txt
      - version_samtools.txt
    expire_in: 7 days

dedupData:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools --version > version_samtools.txt
  - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates --version 2> version_markdups.txt&
  - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates I=./test_data/bam/small/Q-Y5F4_1M.se.sorted.bam O=Q-Y5F4_1M.se.deduped.bam M=Q-Y5F4_1M.se.deduped.Metrics.txt REMOVE_DUPLICATES=true
  - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F4_1M.se.sorted.deduped.bam ./test_data/bam/small/Q-Y5F4_1M.se.deduped.bam
  - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools index -@ 20 -b ./test_data/bam/small/Q-Y5F4_1M.se.sorted.deduped.bam Q-Y5F4_1M.se.sorted.deduped.bam.bai
  - >
    for i in {"chr8","chr4","chrY"}; do
      echo "samtools view -b Q-Y5F4_1M.se.sorted.deduped.bam ${i} > Q-Y5F4_1M.se.sorted.deduped.${i}.bam; samtools index -@ 20 -b Q-Y5F4_1M.se.sorted.deduped.${i}.bam Q-Y5F4_1M.se.sorted.deduped.${i}.bam.bai;";
    done | singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' parallel -j 20 -k
  - pytest -m dedupData
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_markdups.txt
      - version_samtools.txt
    expire_in: 7 days

countData:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/geneID.tsv
  - ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/Entrez.tsv
  - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' featureCounts -T 20 -a /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/genome.gtf -G /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/genome.fna -g 'gene_name' --extraAttributes 'gene_id' -o Q-Y5F4_1M.se.countData -s 1 -R SAM --primary --ignoreDup ./test_data/bam/small/Q-Y5F4_1M.se.sorted.deduped.bam
  - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' Rscript ./workflow/scripts/calculateTPM.R --count ./test_data/counts/small/Q-Y5F4_1M.se.countData
  - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' Rscript ./workflow/scripts/convertGeneSymbols.R --repRID Q-Y5F4_1M.se
  - assignedReads=$(grep -m 1 'Assigned' *.summary | grep -oe '\([0-9.]*\)')
  - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' featureCounts -v &> version_featurecounts.txt
  - singularity run 'docker://gudmaprbk/subread2.0.1:1.0.0' R --version > version_r.txt
  - pytest -m makeFeatureCounts
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_featurecounts.txt
      - version_r.txt
    expire_in: 7 days

makeBigWig:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' deeptools --version > version_deeptools.txt
  - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' bamCoverage -p 20 -b ./test_data/bam/small/Q-Y5F4_1M.se.sorted.deduped.bam -o Q-Y5F4_1M.se.bw
  - pytest -m makeBigWig
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_deeptools.txt
    expire_in: 7 days

fastqc:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc --version > version_fastqc.txt
  - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc ./test_data/fastq/small/Q-Y5F4_1M.R1.fastq.gz -o .
  - pytest -m fastqc
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - version_fastqc.txt
    expire_in: 7 days


dataQC:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - echo -e  "geneID\tchrom\ttx_start\ttx_end\tTIN" > Q-Y5F4_1M.se.sorted.deduped.tin.xls
  - >
    for i in {"chr8","chr4","chrY"}; do
      echo "tin.py -i ./test_data/bam/small/Q-Y5F4_1M.se.sorted.deduped.${i}.bam -r /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/bed/genome.bed; cat Q-Y5F4_1M.se.sorted.deduped.${i}.tin.xls | tr -s \"\\w\" \"\\t\" | grep -P \"\\t${i}\\t\";"
    done | singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' parallel -j 20 -k >> Q-Y5F4_1M.se.sorted.deduped.tin.xls
  - pytest -m dataQC

uploadInputBag:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
  - echo THIS IS A TEST FILE > test.txt
  - >
    md5=$(md5sum ./test.txt | awk '{ print $1 }') &&
    size=$(wc -c < ./test.txt) &&
    exist=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Input_Bag/File_MD5=${md5}) &&
    if [ "${exist}" == "[]" ]; then
      cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') &&
      cookie=${cookie:11:-1} &&
      loc=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host staging.gudmap.org put ./test.txt /hatrac/resources/rnaseq/pipeline/input_bag/TEST/test.txt --parents) &&
      rid=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' python3 ./workflow/scripts/uploadInputBag.py -f test.txt -l ${loc} -s ${md5} -b ${size} -n 'This is a test input bag' -o staging.gudmap.org -c ${cookie}) &&
      echo ${rid} test input bag created
    else
      rid=$(echo ${exist} | grep -o '\"RID\":\".*\",\"RCT') &&
      rid=${rid:8:-6} &&
      echo ${rid} test input bag already exists
    fi

uploadExecutionRun:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
  - >
    exist=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Replicate=17-BTFJ) &&
    cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') &&
    cookie=${cookie:11:-1} &&
    if [ "${exist}" == "[]" ]; then
      rid=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' python3 ./workflow/scripts/uploadExecutionRun.py -r 17-BTFJ -w 17-BTFM -g 17-BT50 -i 17-BTFT -s Success -d 'This is a test execution run' -o staging.gudmap.org -c ${cookie} -u F) &&
      echo ${rid} test execution run created
    else
      rid=$(echo ${exist} | grep -o '\"RID\":\".*\",\"RCT') &&
      rid=${rid:7:-6} &&
      rid=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' python3 ./workflow/scripts/uploadExecutionRun.py -r 17-BTFJ -w 17-BTFM -g 17-BT50 -i 17-BTFT -s Success -d 'This is a test execution run' -o staging.gudmap.org -c ${cookie} -u ${rid}) &&
      echo ${rid} test execution run already exists
    fi

uploadQC:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
  - >
    exist=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:mRNA_QC/Replicate=17-BTFJ) &&
    cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') &&
    cookie=${cookie:11:-1} &&
    if [ "${exist}" != "[]" ]; then
      rids=$(echo ${exist} | grep -o '\"RID\":\".\{7\}' | sed 's/^.\{7\}//') &&
      for rid in ${rids}; do
        singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' python3 ./workflow/scripts/deleteEntry.py -r ${rid} -t mRNA_QC -o staging.gudmap.org -c ${cookie}
      done
      echo all old mRNA QC RIDs deleted
    fi
      rid=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' python3 ./workflow/scripts/uploadQC.py -r 17-BTFJ -e 17-BTG4 -p "Single Read" -s forward -l 35 -w 5 -f 1 -n "This is a test mRNA QC" -o staging.gudmap.org -c ${cookie} -u F)
      echo ${rid} test mRNA QC created

uploadProcessedFile:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
  - echo THIS IS A TEST FILE > 17-BTFJ_test.csv
  - mkdir -p ./deriva/Seq/pipeline/17-BTFE/17-BTG4/
  - mv 17-BTFJ_test.csv ./deriva/Seq/pipeline/17-BTFE/17-BTG4/17-BTFJ_test.csv
  - >
    exist=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Processed_File/Replicate=17-BTFJ) &&
    cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') &&
    cookie=${cookie:11:-1} &&
    if [ "${exist}" != "[]" ]; then
      rids=$(echo ${exist} | grep -o '\"RID\":\".\{7\}' | sed 's/^.\{7\}//') &&
      for rid in ${rids}; do
        singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' python3 ./workflow/scripts/deleteEntry.py -r ${rid} -t Processed_File -o staging.gudmap.org -c ${cookie}
      done
      echo all old processed file RIDs deleted
    fi
      singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-upload-cli --catalog 2 --token ${cookie:9} staging.gudmap.org ./deriva
      echo test processed file uploaded
  - mkdir test
  - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' bdbag test --archiver zip
  - echo test output bag created
  - pytest -m outputBag

uploadOutputBag:
  stage: unit
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
  - echo THIS IS A TEST FILE > test.txt
  - >
    md5=$(md5sum ./test.txt | awk '{ print $1 }') &&
    size=$(wc -c < ./test.txt) &&
    exist=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Output_Bag/File_MD5=${md5}) &&
    if [ "${exist}" == "[]" ]; then
      cookie=$(cat credential.json | grep -A 1 '\"staging.gudmap.org\": {' | grep -o '\"cookie\": \".*\"') &&
      cookie=${cookie:11:-1} &&
      loc=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host staging.gudmap.org put ./test.txt /hatrac/resources/rnaseq/pipeline/output_bag/TEST/test.txt --parents) &&
      rid=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' python3 ./workflow/scripts/uploadOutputBag.py -e 17-BTG4 -f test.txt -l ${loc} -s ${md5} -b ${size} -n 'This is a test output bag' -o staging.gudmap.org -c ${cookie}) &&
      echo ${rid} test output bag created
    else
      rid=$(echo ${exist} | grep -o '\"RID\":\".*\",\"RCT') &&
      rid=${rid:8:-6} &&
      echo ${rid} test output bag already exists
    fi


generateVersions:
  stage: aggregation
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - singularity run 'docker://gudmaprbk/multiqc1.9:1.0.0' multiqc --version > version_multiqc.txt
  - python ./workflow/scripts/generate_versions.py -o software_versions
  - python ./workflow/scripts/generate_references.py -r ./docs/references.md -o software_references
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - software_references_mqc.yaml
      - software_versions_mqc.yaml
    expire_in: 7 days


human_BioHPC:
  stage: reference
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - mkdir -p hu
  - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./hu/

mouse_BioHPC:
  stage: reference
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - mkdir -p mo
  - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./mo/

human_dev:
  stage: reference
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
  - referenceBase=dev.gudmap.org
  - refName=GRCh
  - references=$(echo ${referenceBase}/${refName}${refHuVersion})
  - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1)
  - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2)
  - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
  - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
  - curl --request GET ${query} > refQuery.json
  - refURL=$(python ./workflow/scripts/extractRefData.py --returnParam URL)
  - loc=$(dirname ${refURL})
  - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi
  - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)')
  - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/)
  - test=$(echo ${test} | grep -o ${filename})
  - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi

mouse_dev:
  stage: reference
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
  - referenceBase=dev.gudmap.org
  - refName=GRCm
  - references=$(echo ${referenceBase}/${refName}${refHuVersion})
  - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1)
  - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2)
  - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
  - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
  - curl --request GET ${query} > refQuery.json
  - refURL=$(python ./workflow/scripts/extractRefData.py --returnParam URL)
  - loc=$(dirname ${refURL})
  - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi
  - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)')
  - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/)
  - test=$(echo ${test} | grep -o ${filename})
  - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi

human_staging:
  stage: reference
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
  - referenceBase=staging.gudmap.org
  - refName=GRCh
  - references=$(echo ${referenceBase}/${refName}${refHuVersion})
  - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1)
  - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2)
  - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
  - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
  - curl --request GET ${query} > refQuery.json
  - refURL=$(python ./workflow/scripts/extractRefData.py --returnParam URL)
  - loc=$(dirname ${refURL})
  - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi
  - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)')
  - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/)
  - test=$(echo ${test} | grep -o ${filename})
  - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi

mouse_staging:
  stage: reference
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
  - referenceBase=staging.gudmap.org
  - refName=GRCm
  - refHuVersion=38.p6.vM22
  - references=$(echo ${referenceBase}/${refName}${refHuVersion})
  - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1)
  - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2)
  - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
  - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
  - curl --request GET ${query} > refQuery.json
  - refURL=$(python ./workflow/scripts/extractRefData.py --returnParam URL)
  - loc=$(dirname ${refURL})
  - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi
  - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)')
  - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/)
  - test=$(echo ${test} | grep -o ${filename})
  - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi

human_prod:
  stage: reference
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
  - referenceBase=www.gudmap.org
  - refName=GRCh
  - references=$(echo ${referenceBase}/${refName}${refHuVersion})
  - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1)
  - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2)
  - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
  - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
  - curl --request GET ${query} > refQuery.json
  - refURL=$(python ./workflow/scripts/extractRefData.py --returnParam URL)
  - loc=$(dirname ${refURL})
  - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi
  - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)')
  - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/)
  - test=$(echo ${test} | grep -o ${filename})
  - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi

mouse_prod:
  stage: reference
  only:
    - push
    - tags
  except:
    - merge_requests
  script:
  - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
  - referenceBase=www.gudmap.org
  - refName=GRCm
  - refHuVersion=38.p6.vM22
  - references=$(echo ${referenceBase}/${refName}${refHuVersion})
  - GRCv=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f1)
  - GRCp=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f2)
  - GENCODE=$(echo ${references} | grep -o ${refName}.* | cut -d '.' -f3)
  - query=$(echo 'https://'${referenceBase}'/ermrest/catalog/2/entity/RNASeq:Reference_Genome/Reference_Version='${GRCv}'.'${GRCp}'/Annotation_Version=GENCODE%20'${GENCODE})
  - curl --request GET ${query} > refQuery.json
  - refURL=$(python ./workflow/scripts/extractRefData.py --returnParam URL)
  - loc=$(dirname ${refURL})
  - if [ "${loc}" = "/hatrac/*" ]; then echo "reference not present in hatrac"; exit 1; fi
  - filename=$(echo $(basename ${refURL}) | grep -oP '.*(?=:)')
  - test=$(singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-hatrac-cli --host ${referenceBase} ls ${loc}/)
  - test=$(echo ${test} | grep -o ${filename})
  - if [ "${test}" == "" ]; then echo "reference file not present"; exit 1; fi


integration_se:
  stage: integration
  only: [merge_requests]
  except:
    variables:
        - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/
  script:
  - hostname
  - ulimit -a
  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 -with-dag dag.png --ci true --email 'venkat.malladi@utsouthwestern.edu,Gervaise.Henry@UTSouthwestern.edu'
  - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \;
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - output/qc/
      - output/report/
      - SE_multiqc_data.json
    expire_in: 7 days
  retry:
    max: 0
    when:
      - always

integration_pe:
  stage: integration
  only: [merge_requests]
  except:
    variables:
        - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/
  script:
  - hostname
  - ulimit -a
  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA -with-dag dag.png --ci true
  - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \;
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - dag.png
      - output/qc/
      - output/report/
      - PE_multiqc_data.json
    expire_in: 7 days
  retry:
    max: 0
    when:
      - always


override_inputBag:
  stage: integration
  only: [merge_requests]
  except:
    variables:
        - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/
  script:
  - hostname
  - ulimit -a
  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F4 --inputBagForce ./test_data/bag/Q-Y5F4_inputBag_20201228.zip --upload false --ci true
  - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_PE_multiqc_data.json \;
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - inputBagOverride_PE_multiqc_data.json
    expire_in: 7 days
  retry:
    max: 0
    when:
      - always

override_fastq:
  stage: integration
  only: [merge_requests]
  except:
    variables:
        - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/
  script:
  - hostname
  - ulimit -a
  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F4 --fastqsForce './test_data/fastq/small/Q-Y5F4_1M.R{1,2}.fastq.gz' --upload false --ci true
  - find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_PE_multiqc_data.json \;
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - fastqOverride_PE_multiqc_data.json
    expire_in: 7 days
  retry:
    max: 0
    when:
      - always

override_species:
  stage: integration
  only: [merge_requests]
  except:
    variables:
        - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/
  script:
  - hostname
  - ulimit -a
  - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --speciesForce 'Homo sapiens' --upload false --ci true
  - find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_PE_multiqc_data.json \;
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - speciesOverride_PE_multiqc_data.json
    expire_in: 7 days
  retry:
    max: 0
    when:
      - always


consistency:
  stage: consistency
  only: [merge_requests]
  except:
    variables:
        - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /master/
  script:
  - pytest -m consistencySE
  - pytest -m consistencyPE
  artifacts:
    name: "$CI_JOB_NAME"
    when: always
    paths:
      - SE_multiqc_data.json
      - PE_multiqc_data.json
    expire_in: 7 days