diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 099f96b016e91f3be2c47c4de888512bc38bacec..337b001fd3bda2ab85b9f17fb050a07d2354f42c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -21,8 +21,9 @@ stages:
- integration
- consistency
-build_badges:
+build_badges_versions:
stage: badges
+ except: schedules
before_script:
- module load singularity/3.5.3
- chmod +x ./workflow/scripts/get_updated_badge_info.sh
@@ -33,6 +34,20 @@ build_badges:
paths:
- badges/
+build_badges_counts:
+ stage: badges
+ only: schedules
+ before_script:
+ - module load singularity/3.5.3
+ - chmod +x ./workflow/scripts/get_updated_badge_info.sh
+ script:
+ - echo "Building badges"
+ - singularity run 'docker://gudmaprbk/gudmap-rbk_base:1.0.0' bash ./workflow/scripts/get_updated_badge_info.sh
+ - singularity run 'docker://gudmaprbk/gudmap-rbk_base:1.0.0' bash ./workflow/scripts/get_updated_rep_count.sh
+ artifacts:
+ paths:
+ - badges/
+
pages:
stage: deploy
dependencies:
@@ -51,6 +66,7 @@ getBag:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/credential.json` ~/.deriva/credential.json
- singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-download-cli --version > version_deriva.txt
@@ -70,6 +86,7 @@ getData:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' bdbag --version > version_bdbag.txt
- ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
@@ -90,6 +107,7 @@ parseMetadata:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/python3:1.0.0' python3 --version > version_python.txt
- rep=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID)
@@ -117,6 +135,7 @@ inferMetadata:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' infer_experiment.py --version > version_rseqc.txt
- >
@@ -141,6 +160,7 @@ trimData:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --version > version_trimgalore.txt
- singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --gzip -q 25 --length 35 --basename Q-Y5F6_1M.se ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz
@@ -162,6 +182,7 @@ downsampleData:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/seqtk1.3:1.0.0' seqtk sample -s100 ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz 1000 1> sampled.1.fq
- pytest -m downsampleData
@@ -173,6 +194,7 @@ alignData:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 --version > version_hisat2.txt
- singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools --version > version_samtools.txt
@@ -200,6 +222,7 @@ dedupData:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools --version > version_samtools.txt
- singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates --version 2> version_markdups.txt&
@@ -226,6 +249,7 @@ countData:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/geneID.tsv
- ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/Entrez.tsv
@@ -251,6 +275,7 @@ makeBigWig:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' deeptools --version > version_deeptools.txt
- singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' bamCoverage -p 20 -b ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam -o Q-Y5F6_1M.se.bw
@@ -269,6 +294,7 @@ fastqc:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc --version > version_fastqc.txt
- singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -o .
@@ -303,6 +329,7 @@ uploadInputBag:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
- echo THIS IS A TEST FILE > test.txt
@@ -329,6 +356,7 @@ uploadExecutionRun:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
- >
@@ -352,6 +380,7 @@ uploadQC:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
- >
@@ -375,6 +404,7 @@ uploadProcessedFile:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
- echo THIS IS A TEST FILE > 17-BTFJ_test.csv
@@ -405,6 +435,7 @@ uploadOutputBag:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
- echo THIS IS A TEST FILE > test.txt
@@ -432,6 +463,7 @@ generateVersions:
- tags
except:
- merge_requests
+ - schedules
script:
- singularity run 'docker://gudmaprbk/multiqc1.9:1.0.0' multiqc --version > version_multiqc.txt
- python ./workflow/scripts/generate_versions.py -o software_versions
@@ -452,6 +484,7 @@ human_BioHPC:
- tags
except:
- merge_requests
+ - schedules
script:
- mkdir -p hu
- cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./hu/
@@ -463,6 +496,7 @@ mouse_BioHPC:
- tags
except:
- merge_requests
+ - schedules
script:
- mkdir -p mo
- cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./mo/
@@ -474,6 +508,7 @@ human_dev:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
- referenceBase=dev.gudmap.org
@@ -499,6 +534,7 @@ mouse_dev:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
- referenceBase=dev.gudmap.org
@@ -524,6 +560,7 @@ human_staging:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
- referenceBase=staging.gudmap.org
@@ -549,6 +586,7 @@ mouse_staging:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
- referenceBase=staging.gudmap.org
@@ -575,6 +613,7 @@ human_prod:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
- referenceBase=www.gudmap.org
@@ -600,6 +639,7 @@ mouse_prod:
- tags
except:
- merge_requests
+ - schedules
script:
- ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
- referenceBase=www.gudmap.org
diff --git a/workflow/scripts/get_updated_rep_count.sh b/workflow/scripts/get_updated_rep_count.sh
new file mode 100644
index 0000000000000000000000000000000000000000..2e6e1a50faceba87a76206488ee3c35cd7f91331
--- /dev/null
+++ b/workflow/scripts/get_updated_rep_count.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+echo "collecting stats for badges"
+latest_release_tag=$(git tag --sort=-committerdate -l *.*.* | head -1)
+current_pipeline_version=$(git show ${latest_release_tag}:workflow/nextflow.config | grep -o version.* | grep -oP "(?<=').*(?=')")
+
+echo "collecting workflow RIDs from servers"
+dev_workflow_RID=$(curl -s https://dev.gudmap.org/ermrest/catalog/2/entity/RNASeq:Workflow/Version=${current_pipeline_version} | grep -o '\"RID\":\".*\",\"RCT')
+dev_workflow_RID=${dev_workflow_RID:7:-6}
+staging_workflow_RID=$(curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Workflow/Version=${current_pipeline_version} | grep -o '\"RID\":\".*\",\"RCT')
+staging_workflow_RID=${staging_workflow_RID:7:-6}
+prod_workflow_RID=$(curl -s https://www.gudmap.org/ermrest/catalog/2/entity/RNASeq:Workflow/Version=${current_pipeline_version} | grep -o '\"RID\":\".*\",\"RCT')
+prod_workflow_RID=${prod_workflow_RID:7:-6}
+
+echo "collecting unique replicates with successful execution runs"
+dev_count=$(curl -s https://dev.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Execution_Status=Success/Workflow=${dev_workflow_RID} | grep -o \"Replicate\".*,\"Workflow | grep -oP "(?<=\"Replicate\":\").*(?=\",\"Workflow)" | sort | uniq | wc -l)
+staging_count=$(curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Execution_Status=Success/Workflow=${staging_workflow_RID} | grep -o \"Replicate\".*,\"Workflow | grep -oP "(?<=\"Replicate\":\").*(?=\",\"Workflow)" | sort | uniq | wc -l)
+prod_count=$(curl -s https://prod.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Execution_Status=Success/Workflow=${prod_workflow_RID} | grep -o \"Replicate\".*,\"Workflow | grep -oP "(?<=\"Replicate\":\").*(?=\",\"Workflow)" | sort | uniq | wc -l)
+
+echo "collecting badges"
+mkdir -p ./badges/counts
+curl --request GET https://img.shields.io/badge/Development%20Replicate%20Count-${dev_count}-informational?style=flat > ./badges/counts/dev_counts.svg
+curl --request GET https://img.shields.io/badge/Staging%20Replicate%20Count-${staging_count}-informational?style=flat > ./badges/counts/staging_counts.svg
+curl --request GET https://img.shields.io/badge/Production%20Replicate%20Count-${prod_count}-informational?style=flat > ./badges/counts/prod_counts.svg