Add badges for replicate counts

.gitlab-ci.yml

@@ -21,8 +21,9 @@ stages:
   - integration
   - consistency
 
-build_badges:
+build_badges_versions:
   stage: badges
+  except: schedules
   before_script:
     - module load singularity/3.5.3
     - chmod +x ./workflow/scripts/get_updated_badge_info.sh
@@ -33,6 +34,20 @@ build_badges:
     paths:
       - badges/
 
+build_badges_counts:
+  stage: badges
+  only: schedules
+  before_script:
+    - module load singularity/3.5.3
+    - chmod +x ./workflow/scripts/get_updated_badge_info.sh
+  script:
+    - echo "Building badges"
+    - singularity run 'docker://gudmaprbk/gudmap-rbk_base:1.0.0' bash ./workflow/scripts/get_updated_badge_info.sh
+    - singularity run 'docker://gudmaprbk/gudmap-rbk_base:1.0.0' bash ./workflow/scripts/get_updated_rep_count.sh
+  artifacts:
+    paths:
+      - badges/
+
 pages:
   stage: deploy
   dependencies:
@@ -51,6 +66,7 @@ getBag:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/credential.json` ~/.deriva/credential.json
   - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-download-cli --version > version_deriva.txt
@@ -70,6 +86,7 @@ getData:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' bdbag --version > version_bdbag.txt
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
@@ -90,6 +107,7 @@ parseMetadata:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/python3:1.0.0' python3 --version > version_python.txt
   - rep=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID)
@@ -117,6 +135,7 @@ inferMetadata:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' infer_experiment.py --version > version_rseqc.txt
   - >
@@ -141,6 +160,7 @@ trimData:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --version > version_trimgalore.txt
   - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --gzip -q 25 --length 35 --basename Q-Y5F6_1M.se ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz
@@ -162,6 +182,7 @@ downsampleData:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/seqtk1.3:1.0.0' seqtk sample -s100 ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz 1000 1> sampled.1.fq
   - pytest -m downsampleData
@@ -173,6 +194,7 @@ alignData:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 --version > version_hisat2.txt
   - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools --version > version_samtools.txt
@@ -200,6 +222,7 @@ dedupData:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools --version > version_samtools.txt
   - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates --version 2> version_markdups.txt&
@@ -226,6 +249,7 @@ countData:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/geneID.tsv
   - ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/Entrez.tsv
@@ -251,6 +275,7 @@ makeBigWig:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' deeptools --version > version_deeptools.txt
   - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' bamCoverage -p 20 -b ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam -o Q-Y5F6_1M.se.bw
@@ -269,6 +294,7 @@ fastqc:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc --version > version_fastqc.txt
   - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -o .
@@ -303,6 +329,7 @@ uploadInputBag:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
   - echo THIS IS A TEST FILE > test.txt
@@ -329,6 +356,7 @@ uploadExecutionRun:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
   - >
@@ -352,6 +380,7 @@ uploadQC:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
   - >
@@ -375,6 +404,7 @@ uploadProcessedFile:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
   - echo THIS IS A TEST FILE > 17-BTFJ_test.csv
@@ -405,6 +435,7 @@ uploadOutputBag:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json
   - echo THIS IS A TEST FILE > test.txt
@@ -432,6 +463,7 @@ generateVersions:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - singularity run 'docker://gudmaprbk/multiqc1.9:1.0.0' multiqc --version > version_multiqc.txt
   - python ./workflow/scripts/generate_versions.py -o software_versions
@@ -452,6 +484,7 @@ human_BioHPC:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - mkdir -p hu
   - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./hu/
@@ -463,6 +496,7 @@ mouse_BioHPC:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - mkdir -p mo
   - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./mo/
@@ -474,6 +508,7 @@ human_dev:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
   - referenceBase=dev.gudmap.org
@@ -499,6 +534,7 @@ mouse_dev:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
   - referenceBase=dev.gudmap.org
@@ -524,6 +560,7 @@ human_staging:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
   - referenceBase=staging.gudmap.org
@@ -549,6 +586,7 @@ mouse_staging:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
   - referenceBase=staging.gudmap.org
@@ -575,6 +613,7 @@ human_prod:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
   - referenceBase=www.gudmap.org
@@ -600,6 +639,7 @@ mouse_prod:
     - tags
   except:
     - merge_requests
+    - schedules
   script:
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
   - referenceBase=www.gudmap.org

workflow/scripts/get_updated_rep_count.sh

+#!/bin/bash
+
+echo "collecting stats for badges"
+latest_release_tag=$(git tag --sort=-committerdate -l *.*.* | head -1)
+current_pipeline_version=$(git show ${latest_release_tag}:workflow/nextflow.config | grep -o version.* | grep -oP "(?<=').*(?=')")
+
+echo "collecting workflow RIDs from servers"
+dev_workflow_RID=$(curl -s https://dev.gudmap.org/ermrest/catalog/2/entity/RNASeq:Workflow/Version=${current_pipeline_version} | grep -o '\"RID\":\".*\",\"RCT')
+dev_workflow_RID=${dev_workflow_RID:7:-6}
+staging_workflow_RID=$(curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Workflow/Version=${current_pipeline_version} | grep -o '\"RID\":\".*\",\"RCT')
+staging_workflow_RID=${staging_workflow_RID:7:-6}
+prod_workflow_RID=$(curl -s https://www.gudmap.org/ermrest/catalog/2/entity/RNASeq:Workflow/Version=${current_pipeline_version} | grep -o '\"RID\":\".*\",\"RCT')
+prod_workflow_RID=${prod_workflow_RID:7:-6}
+
+echo "collecting unique replicates with successful execution runs"
+dev_count=$(curl -s https://dev.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Execution_Status=Success/Workflow=${dev_workflow_RID} | grep -o \"Replicate\".*,\"Workflow | grep -oP "(?<=\"Replicate\":\").*(?=\",\"Workflow)" | sort | uniq | wc -l)
+staging_count=$(curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Execution_Status=Success/Workflow=${staging_workflow_RID} | grep -o \"Replicate\".*,\"Workflow | grep -oP "(?<=\"Replicate\":\").*(?=\",\"Workflow)" | sort | uniq | wc -l)
+prod_count=$(curl -s https://prod.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Execution_Status=Success/Workflow=${prod_workflow_RID} | grep -o \"Replicate\".*,\"Workflow | grep -oP "(?<=\"Replicate\":\").*(?=\",\"Workflow)" | sort | uniq | wc -l)
+
+echo "collecting badges"
+mkdir -p ./badges/counts
+curl --request GET https://img.shields.io/badge/Development%20Replicate%20Count-${dev_count}-informational?style=flat > ./badges/counts/dev_counts.svg
+curl --request GET https://img.shields.io/badge/Staging%20Replicate%20Count-${staging_count}-informational?style=flat > ./badges/counts/staging_counts.svg
+curl --request GET https://img.shields.io/badge/Production%20Replicate%20Count-${prod_count}-informational?style=flat > ./badges/counts/prod_counts.svg