diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5dc43603a40dd59ce926bf9a69f2655b7069f99e..a09f1786745c1678022cc9c8516b6bb7c61d1cea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1054,3 +1054,72 @@ dnanexus: paths: - badges/ allow_failure: true + + aws: + stage: environment + only: + variables: + - $awsEnv == "true" + except: + - push + - tags + - merge_requests + script: + - hostname + - ulimit -a + - mkdir -p ./badges/env + - curl --request GET https://img.shields.io/badge/Envronment%3A%20AWS-inactive-critical?style=flat > ./badges/env/aws.svg + - module load awscli/1.19.17 + - > + export AWS_ACCESS_KEY_ID=${aws_accesskeyid} + export AWS_SECRET_ACCESS_KEY=${aws_secretaccesskey} + aws configure set region ${aws_region} + - aws s3 cp ./test_data/auth/ s3://bicf-output/ci-env/auth/ --exclude "*" --include "c*" --recursive + - aws s3 cp ./test_data/fastq/xsmall/ s3://bicf-output/ci-env/input/ --exclude "*" --include "Q-Y5F6_10K.R*.fastq.gz" --recursive + - > + id=$(aws batch submit-job\ + --job-name nf-GUDMAP_RBK_ci-env\ + --job-queue default-bicf\ + --job-definition nextflow-bicf-nextflow\ + --container-overrides command=\ + "utsw-bicf/gudmap_rbk.rna-seq","-r","env.ci",\ + "-profile","aws",\ + "--deriva","s3://bicf-output/ci-env/auth/credential.json",\ + "--bdbag","s3://bicf-output/ci-env/auth/cookies.txt",\ + "--repRID","Q-Y5F6",\ + "--source","staging",\ + "--upload","false",\ + "--dev","false",\ + "--ci","true",\ + "--track","false",\ + "-with-report","s3://bicf-output/ci-env/output/Q-Y5F6_fastqoverride_report.html",\ + "--refSource","datahub",\ + "--outDir","s3://bicf-output/ci-env/output/Q-Y5F6_fastqoverride",\ + "--fastqsForce","s3://bicf-output/ci-env/input/*.fastq.gz") + id=$(echo ${id}| grep -oP "jobId\K.*" | tr -d '"' | tr -d ":" | tr -d " " | tr -d "}") + - > + status=$(aws batch describe-jobs --jobs ${id} | grep -oP "status\": \K.*" | tr -d '"' | tr -d ',') && + until [[ "${status}" == "SUCCEEDED" || "${status}" == "FAILED" ]]; do + status=$(aws batch describe-jobs --jobs ${id} | grep -oP "status\": \K.*" | tr -d '"' | tr -d ',') && + echo ${status} && + sleep 30 + done + - > + if [ "${status}" == "SUCCEEDED" ]; then + curl --request GET https://img.shields.io/badge/Envronment%3A%20AWS-run%20succesful-success?style=flat > ./badges/env/dnanexus.svg + else + curl --request GET https://img.shields.io/badge/Envronment%3A%20AWS-run%20failed-critical?style=flat > ./badges/env/dnanexus.svg + fi + after_script: + - module load awscli/1.19.17 + - > + export AWS_ACCESS_KEY_ID=${aws_accesskeyid} + export AWS_SECRET_ACCESS_KEY=${aws_secretaccesskey} + aws configure set region ${aws_region} + - aws s3 rm s3://bicf-output/ci-env/auth/ --recursive + - aws s3 rm s3://bicf-output/ci-env/input/ --recursive + artifacts: + when: always + paths: + - badges/ + allow_failure: true \ No newline at end of file