diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a09f1786745c1678022cc9c8516b6bb7c61d1cea..7d935f564b7bfb20f9a71f2b22527b031728655b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1055,71 +1055,71 @@ dnanexus: - badges/ allow_failure: true - aws: - stage: environment - only: - variables: - - $awsEnv == "true" - except: - - push - - tags - - merge_requests - script: - - hostname - - ulimit -a - - mkdir -p ./badges/env - - curl --request GET https://img.shields.io/badge/Envronment%3A%20AWS-inactive-critical?style=flat > ./badges/env/aws.svg - - module load awscli/1.19.17 - - > - export AWS_ACCESS_KEY_ID=${aws_accesskeyid} - export AWS_SECRET_ACCESS_KEY=${aws_secretaccesskey} - aws configure set region ${aws_region} - - aws s3 cp ./test_data/auth/ s3://bicf-output/ci-env/auth/ --exclude "*" --include "c*" --recursive - - aws s3 cp ./test_data/fastq/xsmall/ s3://bicf-output/ci-env/input/ --exclude "*" --include "Q-Y5F6_10K.R*.fastq.gz" --recursive - - > - id=$(aws batch submit-job\ - --job-name nf-GUDMAP_RBK_ci-env\ - --job-queue default-bicf\ - --job-definition nextflow-bicf-nextflow\ - --container-overrides command=\ - "utsw-bicf/gudmap_rbk.rna-seq","-r","env.ci",\ - "-profile","aws",\ - "--deriva","s3://bicf-output/ci-env/auth/credential.json",\ - "--bdbag","s3://bicf-output/ci-env/auth/cookies.txt",\ - "--repRID","Q-Y5F6",\ - "--source","staging",\ - "--upload","false",\ - "--dev","false",\ - "--ci","true",\ - "--track","false",\ - "-with-report","s3://bicf-output/ci-env/output/Q-Y5F6_fastqoverride_report.html",\ - "--refSource","datahub",\ - "--outDir","s3://bicf-output/ci-env/output/Q-Y5F6_fastqoverride",\ - "--fastqsForce","s3://bicf-output/ci-env/input/*.fastq.gz") - id=$(echo ${id}| grep -oP "jobId\K.*" | tr -d '"' | tr -d ":" | tr -d " " | tr -d "}") - - > +aws: + stage: environment + only: + variables: + - $awsEnv == "true" + except: + - push + - tags + - merge_requests + script: + - hostname + - ulimit -a + - mkdir -p ./badges/env + - curl --request GET https://img.shields.io/badge/Envronment%3A%20AWS-inactive-critical?style=flat > ./badges/env/aws.svg + - module load awscli/1.19.17 + - > + export AWS_ACCESS_KEY_ID=${aws_accesskeyid} + export AWS_SECRET_ACCESS_KEY=${aws_secretaccesskey} + aws configure set region ${aws_region} + - aws s3 cp ./test_data/auth/ s3://bicf-output/ci-env/auth/ --exclude "*" --include "c*" --recursive + - aws s3 cp ./test_data/fastq/xsmall/ s3://bicf-output/ci-env/input/ --exclude "*" --include "Q-Y5F6_10K.R*.fastq.gz" --recursive + - > + id=$(aws batch submit-job\ + --job-name nf-GUDMAP_RBK_ci-env\ + --job-queue default-bicf\ + --job-definition nextflow-bicf-nextflow\ + --container-overrides command=\ + "utsw-bicf/gudmap_rbk.rna-seq","-r","env.ci",\ + "-profile","aws",\ + "--deriva","s3://bicf-output/ci-env/auth/credential.json",\ + "--bdbag","s3://bicf-output/ci-env/auth/cookies.txt",\ + "--repRID","Q-Y5F6",\ + "--source","staging",\ + "--upload","false",\ + "--dev","false",\ + "--ci","true",\ + "--track","false",\ + "-with-report","s3://bicf-output/ci-env/output/Q-Y5F6_fastqoverride_report.html",\ + "--refSource","datahub",\ + "--outDir","s3://bicf-output/ci-env/output/Q-Y5F6_fastqoverride",\ + "--fastqsForce","s3://bicf-output/ci-env/input/*.fastq.gz") + id=$(echo ${id}| grep -oP "jobId\K.*" | tr -d '"' | tr -d ":" | tr -d " " | tr -d "}") + - > + status=$(aws batch describe-jobs --jobs ${id} | grep -oP "status\": \K.*" | tr -d '"' | tr -d ',') && + until [[ "${status}" == "SUCCEEDED" || "${status}" == "FAILED" ]]; do status=$(aws batch describe-jobs --jobs ${id} | grep -oP "status\": \K.*" | tr -d '"' | tr -d ',') && - until [[ "${status}" == "SUCCEEDED" || "${status}" == "FAILED" ]]; do - status=$(aws batch describe-jobs --jobs ${id} | grep -oP "status\": \K.*" | tr -d '"' | tr -d ',') && - echo ${status} && - sleep 30 - done - - > - if [ "${status}" == "SUCCEEDED" ]; then - curl --request GET https://img.shields.io/badge/Envronment%3A%20AWS-run%20succesful-success?style=flat > ./badges/env/dnanexus.svg - else - curl --request GET https://img.shields.io/badge/Envronment%3A%20AWS-run%20failed-critical?style=flat > ./badges/env/dnanexus.svg - fi - after_script: - - module load awscli/1.19.17 - - > - export AWS_ACCESS_KEY_ID=${aws_accesskeyid} - export AWS_SECRET_ACCESS_KEY=${aws_secretaccesskey} - aws configure set region ${aws_region} - - aws s3 rm s3://bicf-output/ci-env/auth/ --recursive - - aws s3 rm s3://bicf-output/ci-env/input/ --recursive - artifacts: - when: always - paths: - - badges/ - allow_failure: true \ No newline at end of file + echo ${status} && + sleep 30 + done + - > + if [ "${status}" == "SUCCEEDED" ]; then + curl --request GET https://img.shields.io/badge/Envronment%3A%20AWS-run%20succesful-success?style=flat > ./badges/env/dnanexus.svg + else + curl --request GET https://img.shields.io/badge/Envronment%3A%20AWS-run%20failed-critical?style=flat > ./badges/env/dnanexus.svg + fi + after_script: + - module load awscli/1.19.17 + - > + export AWS_ACCESS_KEY_ID=${aws_accesskeyid} + export AWS_SECRET_ACCESS_KEY=${aws_secretaccesskey} + aws configure set region ${aws_region} + - aws s3 rm s3://bicf-output/ci-env/auth/ --recursive + - aws s3 rm s3://bicf-output/ci-env/input/ --recursive + artifacts: + when: always + paths: + - badges/ + allow_failure: true \ No newline at end of file