diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b924a7431d02ddd74e88341077a90a639bf793f4..b0131c6ff424bf4002075c28dcf6e6cbb207536c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,7 @@ build_badges: script: - echo "Building badges" - singularity run 'docker://gudmaprbk/gudmap-rbk_base:1.0.0' bash ./workflow/scripts/get_updated_badge_info.sh + - singularity run 'docker://gudmaprbk/gudmap-rbk_base:1.0.0' bash ./workflow/scripts/get_updated_rep_count.sh artifacts: paths: - badges/ @@ -46,7 +47,8 @@ pages: dependencies: - build_badges script: - - mv badges/ public/ + - mkdir -p public + - mv badges/ public/badges/ artifacts: paths: - public @@ -58,6 +60,7 @@ getBag: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/credential.json` ~/.deriva/credential.json - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' deriva-download-cli --version > version_deriva.txt @@ -77,6 +80,7 @@ getData: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' bdbag --version > version_bdbag.txt - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt @@ -97,6 +101,7 @@ parseMetadata: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/python3:1.0.0' python3 --version > version_python.txt - rep=$(singularity run 'docker://gudmaprbk/python3:1.0.0' python3 ./workflow/scripts/parse_meta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID) @@ -124,6 +129,7 @@ inferMetadata: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/rseqc4.0.0:1.0.0' infer_experiment.py --version > version_rseqc.txt - > @@ -148,6 +154,7 @@ trimData: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --version > version_trimgalore.txt - singularity run 'docker://gudmaprbk/trimgalore0.6.5:1.0.0' trim_galore --gzip -q 25 --length 35 --basename Q-Y5F6_1M.se ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz @@ -169,6 +176,7 @@ downsampleData: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/seqtk1.3:1.0.0' seqtk sample -s100 ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz 1000 1> sampled.1.fq - pytest -m downsampleData @@ -180,6 +188,7 @@ alignData: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' hisat2 --version > version_hisat2.txt - singularity run 'docker://gudmaprbk/hisat2.2.1:1.0.0' samtools --version > version_samtools.txt @@ -207,6 +216,7 @@ dedupData: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' samtools --version > version_samtools.txt - singularity run 'docker://gudmaprbk/picard2.23.9:1.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates --version 2> version_markdups.txt& @@ -233,6 +243,7 @@ countData: - tags except: - merge_requests + - schedules script: - ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/geneID.tsv - ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/Entrez.tsv @@ -258,6 +269,7 @@ makeBigWig: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' deeptools --version > version_deeptools.txt - singularity run 'docker://gudmaprbk/deeptools3.5.0:1.0.0' bamCoverage -p 20 -b ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam -o Q-Y5F6_1M.se.bw @@ -276,6 +288,7 @@ fastqc: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc --version > version_fastqc.txt - singularity run 'docker://gudmaprbk/fastqc0.11.9:1.0.0' fastqc ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -o . @@ -310,6 +323,7 @@ uploadInputBag: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - echo THIS IS A TEST FILE > test.txt @@ -336,6 +350,7 @@ uploadExecutionRun: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - > @@ -359,6 +374,7 @@ uploadQC: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - > @@ -382,6 +398,7 @@ uploadProcessedFile: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - echo THIS IS A TEST FILE > 17-BTFJ_test.csv @@ -412,6 +429,7 @@ uploadOutputBag: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/credential.json` ./credential.json - echo THIS IS A TEST FILE > test.txt @@ -439,6 +457,7 @@ generateVersions: - tags except: - merge_requests + - schedules script: - singularity run 'docker://gudmaprbk/multiqc1.9:1.0.0' multiqc --version > version_multiqc.txt - python ./workflow/scripts/generate_versions.py -o software_versions @@ -459,6 +478,7 @@ human_BioHPC: - tags except: - merge_requests + - schedules script: - mkdir -p hu - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./hu/ @@ -470,6 +490,7 @@ mouse_BioHPC: - tags except: - merge_requests + - schedules script: - mkdir -p mo - cp -R /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2 ./mo/ @@ -481,6 +502,7 @@ human_dev: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - referenceBase=dev.gudmap.org @@ -506,6 +528,7 @@ mouse_dev: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - referenceBase=dev.gudmap.org @@ -531,6 +554,7 @@ human_staging: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - referenceBase=staging.gudmap.org @@ -556,6 +580,7 @@ mouse_staging: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - referenceBase=staging.gudmap.org @@ -582,6 +607,7 @@ human_prod: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - referenceBase=www.gudmap.org @@ -607,6 +633,7 @@ mouse_prod: - tags except: - merge_requests + - schedules script: - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt - referenceBase=www.gudmap.org @@ -636,7 +663,7 @@ integration_se: script: - hostname - ulimit -a - - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 -with-dag dag.png --ci true --email 'venkat.malladi@utsouthwestern.edu,Gervaise.Henry@UTSouthwestern.edu' + - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID 16-1ZX4 --source staging -with-dag dag.png --dev false --ci true --email 'venkat.malladi@utsouthwestern.edu,Gervaise.Henry@UTSouthwestern.edu' - find . -type f -name "multiqc_data.json" -exec cp {} ./SE_multiqc_data.json \; artifacts: name: "$CI_JOB_NAME" @@ -660,7 +687,7 @@ integration_pe: script: - hostname - ulimit -a - - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA -with-dag dag.png --ci true + - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5JA --source staging -with-dag dag.png --dev false --ci true - find . -type f -name "multiqc_data.json" -exec cp {} ./PE_multiqc_data.json \; artifacts: name: "$CI_JOB_NAME" @@ -686,7 +713,7 @@ override_inputBag: script: - hostname - ulimit -a - - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --ci true + - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --inputBagForce ./test_data/bag/Q-Y5F6_inputBag_xxxxxxxx.zip --upload false --dev false --ci true - find . -type f -name "multiqc_data.json" -exec cp {} ./inputBagOverride_PE_multiqc_data.json \; artifacts: name: "$CI_JOB_NAME" @@ -708,7 +735,7 @@ override_fastq: script: - hostname - ulimit -a - - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --ci true + - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5F6 --source staging --fastqsForce './test_data/fastq/small/Q-Y5F6_1M.R{1,2}.fastq.gz' --upload false --dev false --ci true - find . -type f -name "multiqc_data.json" -exec cp {} ./fastqOverride_PE_multiqc_data.json \; artifacts: name: "$CI_JOB_NAME" @@ -730,7 +757,7 @@ override_species: script: - hostname - ulimit -a - - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --speciesForce 'Homo sapiens' --upload false --ci true + - nextflow -q run ./workflow/rna-seq.nf --deriva ./test_data/auth/credential.json --bdbag ./test_data/auth/cookies.txt --repRID Q-Y5ER --source staging --speciesForce 'Homo sapiens' --upload false --dev false --ci true - find . -type f -name "multiqc_data.json" -exec cp {} ./speciesOverride_PE_multiqc_data.json \; artifacts: name: "$CI_JOB_NAME" diff --git a/README.md b/README.md index 2b558d72173309c71fd6c7c83d1a52f5470825d8..185f4453163575fc4b76e8ccb00aac53c74b25ab 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ |master|develop| |:-:|:-:| |[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/commits/master)|[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/commits/develop)| -|[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/-/tree/master)|[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/-/tree/develop)| -|[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/-/tree/master)|[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/-/tree/develop)| +|[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/-/tree/master)|[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/-/tree/develop)| +|[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/-/tree/master)|[](https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq/-/tree/develop)| <!-- [![DOI]()]() diff --git a/workflow/rna-seq.nf b/workflow/rna-seq.nf index bd1aee4ea8fda6acce19febbec38e0118ed0079b..3a97735fe88059a99de264e03d798dc5e2410032 100644 --- a/workflow/rna-seq.nf +++ b/workflow/rna-seq.nf @@ -18,7 +18,7 @@ params.refMoVersion = "38.p6.vM22" params.refHuVersion = "38.p12.v31" params.refERCCVersion = "92" params.outDir = "${baseDir}/../output" -params.upload = true +params.upload = false params.email = "" @@ -28,6 +28,10 @@ params.inputBagForce = "" params.fastqsForce = "" params.speciesForce = "" +// Define tracking input variables +params.ci = false +params.dev = true + // Parse input variables deriva = Channel @@ -100,8 +104,6 @@ script_deleteEntry_uploadProcessedFile = Channel.fromPath("${baseDir}/scripts/de /* * trackStart: track start of pipeline */ -params.ci = false -params.dev = false process trackStart { container 'docker://bicf/bicfbase:2.1.0' script: diff --git a/workflow/scripts/get_updated_badge_info.sh b/workflow/scripts/get_updated_badge_info.sh index 05a08dc7a6056498f73d26f4dc8cdedd3db58f84..5081fe2716af90f5ec4987a8c6f317a4ca08f7aa 100644 --- a/workflow/scripts/get_updated_badge_info.sh +++ b/workflow/scripts/get_updated_badge_info.sh @@ -2,22 +2,30 @@ echo "collecting stats for badges" latest_release_tag=$(git tag --sort=-committerdate -l *.*.* | head -1) -echo "latest_release_tag =" ${latest_release_tag} current_pipeline_version=$(git show ${latest_release_tag}:workflow/nextflow.config | grep -o version.* | grep -oP "(?<=').*(?=')") -echo "current_pipeline_version =" ${current_pipeline_version} current_nextflow_version=$(git show ${latest_release_tag}:workflow/nextflow.config | grep -o nextflowVersion.* | grep -oP "(?<=').*(?=')") -echo "current_nextflow_version =" ${current_nextflow_version} master_pipeline_version=$(git show origin/master:workflow/nextflow.config | grep -o version.* | grep -oP "(?<=').*(?=')") -echo "master_pipeline_version =" ${master_pipeline_version} master_nextflow_version=$(git show origin/master:workflow/nextflow.config | grep -o nextflowVersion.* | grep -oP "(?<=').*(?=')") -echo "master_nextflow_version =" ${master_nextflow_version} develop_pipeline_version=$(git show origin/develop:workflow/nextflow.config | grep -o version.* | grep -oP "(?<=').*(?=')") -echo "develop_pipeline_version =" ${develop_pipeline_version} develop_nextflow_version=$(git show origin/develop:workflow/nextflow.config | grep -o nextflowVersion.* | grep -oP "(?<=').*(?=')") -echo "develop_nextflow_version =" ${develop_nextflow_version} + +echo "collecting tool version for badges" +python_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o Python.* | grep -oP "(?<=d>).*(?=\<)") +deriva_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o DERIVA.* | grep -oP "(?<=d>).*(?=\<)") +bdbag_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o BDBag.* | grep -oP "(?<=d>).*(?=\<)") +rseqc_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o RSeQC.* | grep -oP "(?<=d>).*(?=\<)") +trimgalore_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o 'Trim Galore!'.* | grep -oP "(?<=d>).*(?=\<)") +hisat2_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o HISAT2.* | grep -oP "(?<=d>).*(?=\<)") +samtools_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o Samtools.* | grep -oP "(?<=d>).*(?=\<)") +picard_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o 'picard (MarkDuplicates)'.* | grep -oP "(?<=d>).*(?=\<)") +featurecounts_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o featureCounts.* | grep -oP "(?<=d>).*(?=\<)") +r_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o '>R<'.* | grep -oP "(?<=d>).*(?=\<)") +deeptools_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o deepTools.* | grep -oP "(?<=d>).*(?=\<)") +fastqc_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o FastQC.* | grep -oP "(?<=d>).*(?=\<)") +multiqc_version=$(git show origin/develop:docs/software_versions_mqc.yaml | grep -o MultiQC.* | grep -oP "(?<=d>).*(?=\<)") echo "collecting badges" -mkdir badges +mkdir -p ./badges/tools curl --request GET https://img.shields.io/badge/Latest%20Release-${latest_release_tag}-informational?style=flat > ./badges/release.svg curl --request GET https://img.shields.io/badge/Pipeline%20Version-${current_pipeline_version}-informational?style=flat > ./badges/releasePipeline.svg curl --request GET https://img.shields.io/badge/Nextflow%20Version-${current_nextflow_version}-informational?style=flat > ./badges/releaseNextflow.svg @@ -25,3 +33,17 @@ curl --request GET https://img.shields.io/badge/Pipeline%20Version-${master_pipe curl --request GET https://img.shields.io/badge/Nextflow%20Version-${master_nextflow_version}-informational?style=flat > ./badges/masterNextflow.svg curl --request GET https://img.shields.io/badge/Pipeline%20Version-${develop_pipeline_version}-informational?style=flat > ./badges/developPipeline.svg curl --request GET https://img.shields.io/badge/Nextflow%20Version-${develop_nextflow_version}-informational?style=flat > ./badges/developNextflow.svg + +curl --request GET https://img.shields.io/badge/Python%20Version-${python_version}-blueviolet?style=flat > ./badges/tools/python.svg +curl --request GET https://img.shields.io/badge/DERIVA%20Version-${deriva_version}-blueviolet?style=flat > ./badges/tools/deriva.svg +curl --request GET https://img.shields.io/badge/BDBag%20Version-${bdbag_version}-blueviolet?style=flat > ./badges/tools/bdbag.svg +curl --request GET https://img.shields.io/badge/RSeQC%20Version-${rseqc_version}-blueviolet?style=flat > ./badges/tools/rseqc.svg +curl --request GET https://img.shields.io/badge/Trim%20Galore%20Version-${trimgalore_version}-blueviolet?style=flat > ./badges/tools/trimgalore.svg +curl --request GET https://img.shields.io/badge/HISAT2%20Version-${hisat2_version}-blueviolet?style=flat > ./badges/tools/hisat2.svg +curl --request GET https://img.shields.io/badge/Samtools%20Version-${samtools_version}-blueviolet?style=flat > ./badges/tools/samtools.svg +curl --request GET https://img.shields.io/badge/picard%20Version-${picard_version}-blueviolet?style=flat > ./badges/tools/picard.svg +curl --request GET https://img.shields.io/badge/featureCounts%20Version-${featurecounts_version}-blueviolet?style=flat > ./badges/tools/featurecounts.svg +curl --request GET https://img.shields.io/badge/R%20Version-${r_version}-blueviolet?style=flat > ./badges/tools/r.svg +curl --request GET https://img.shields.io/badge/deepTools%20Version-${deeptools_version}-blueviolet?style=flat > ./badges/tools/deeptools.svg +curl --request GET https://img.shields.io/badge/FastQC%20Version-${fastqc_version}-blueviolet?style=flat > ./badges/tools/fastqc.svg +curl --request GET https://img.shields.io/badge/MultiQC%20Version-${multiqc_version}-blueviolet?style=flat > ./badges/tools/multiqc.svg \ No newline at end of file diff --git a/workflow/scripts/get_updated_rep_count.sh b/workflow/scripts/get_updated_rep_count.sh new file mode 100644 index 0000000000000000000000000000000000000000..267b4e1bdb5d84a045156abc779e30d739a3616e --- /dev/null +++ b/workflow/scripts/get_updated_rep_count.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +echo "collecting stats for badges" +latest_release_tag=$(git tag --sort=-committerdate -l *.*.* | head -1) +current_pipeline_version=$(git show ${latest_release_tag}:workflow/nextflow.config | grep -o version.* | grep -oP "(?<=').*(?=')") + +echo "collecting workflow RIDs from servers" +dev_workflow_RID=$(curl -s https://dev.gudmap.org/ermrest/catalog/2/entity/RNASeq:Workflow/Version=${current_pipeline_version} | grep -o '\"RID\":\".*\",\"RCT') +dev_workflow_RID=${dev_workflow_RID:7:-6} +staging_workflow_RID=$(curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Workflow/Version=${current_pipeline_version} | grep -o '\"RID\":\".*\",\"RCT') +staging_workflow_RID=${staging_workflow_RID:7:-6} +prod_workflow_RID=$(curl -s https://www.gudmap.org/ermrest/catalog/2/entity/RNASeq:Workflow/Version=${current_pipeline_version} | grep -o '\"RID\":\".*\",\"RCT') +prod_workflow_RID=${prod_workflow_RID:7:-6} + +echo "collecting unique replicates with successful execution runs" +dev_count=$(curl -s https://dev.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Execution_Status=Success/Workflow=${dev_workflow_RID} | grep -o \"Replicate\".*,\"Workflow | grep -oP "(?<=\"Replicate\":\").*(?=\",\"Workflow)" | sort | uniq | wc -l) +staging_count=$(curl -s https://staging.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Execution_Status=Success/Workflow=${staging_workflow_RID} | grep -o \"Replicate\".*,\"Workflow | grep -oP "(?<=\"Replicate\":\").*(?=\",\"Workflow)" | sort | uniq | wc -l) +prod_count=$(curl -s https://prod.gudmap.org/ermrest/catalog/2/entity/RNASeq:Execution_Run/Execution_Status=Success/Workflow=${prod_workflow_RID} | grep -o \"Replicate\".*,\"Workflow | grep -oP "(?<=\"Replicate\":\").*(?=\",\"Workflow)" | sort | uniq | wc -l) + +echo "collecting badges" +mkdir -p ./badges/counts +curl --request GET https://img.shields.io/badge/Development%20Replicate%20Count-${dev_count}-lightgrey?style=flat > ./badges/counts/dev_counts.svg +curl --request GET https://img.shields.io/badge/Staging%20Replicate%20Count-${staging_count}-lightgrey?style=flat > ./badges/counts/staging_counts.svg +curl --request GET https://img.shields.io/badge/Production%20Replicate%20Count-${prod_count}-lightgrey?style=flat > ./badges/counts/prod_counts.svg