Skip to content
Snippets Groups Projects
Commit 24b13f74 authored by Peng Lian's avatar Peng Lian
Browse files

Simplify Gitlab CI/CD

parent 01a16971
Branches
Tags
No related merge requests found
......@@ -9,3 +9,5 @@ workflow/*.csv
*.wordcount
.nextflow/
.rlibrary/
dag.dot*
report.html*
......@@ -7,16 +7,10 @@
stages:
- check
- test
- test_docker
- test_singularity
- cache_docker
- test_gitlab
variables:
GIT_STRATEGY: clone
# Execute this script for everybody. YAML anchor.
.all-script-before: &all-script-before
- pwd
......@@ -31,15 +25,9 @@ variables:
.astrocyte:
before_script:
- *all-script-before
- module load astrocyte/0.4.1
- module load astrocyte/2.0.1
tags:
- cluster_node
.docker:
before_script:
- *all-script-before
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- astrocyte_workflow
### Setup scripts
before_script:
......@@ -54,37 +42,6 @@ astrocyte_check:
script:
- astrocyte_cli check "$CI_PROJECT_DIR"
# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the docker config.
# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
nextflow_test_docker:
stage: test
script:
- module load nextflow/21.04.1
- module load singularity/3.5.3
- cd $CI_PROJECT_DIR/workflow
- nextflow -C $CI_PROJECT_DIR/workflow/configs/biohpc_docker.config run main.nf -with-trace -with-timeline -resume --verbose -bg
artifacts:
paths:
- $CI_PROJECT_DIR/workflow/.nextflow.log
tags:
- cluster_node
# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the singularity config.
# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
nextflow_test_singularity:
stage: test
script:
- module load nextflow/21.04.1
- module load singularity/3.5.3
- cd $CI_PROJECT_DIR/workflow
- nextflow -C $CI_PROJECT_DIR/workflow/configs/biohpc_singularity.config run main.nf -with-trace -with-timeline -resume --verbose -bg
artifacts:
paths:
- $CI_PROJECT_DIR/workflow/.nextflow.log
tags:
- cluster_node
# This runs the workflow with test data and does a simple check for the expected output file.
# This assumes that the 'default' config uses local processing, not containers.
......@@ -93,89 +50,21 @@ astrocyte_test:
stage: test
script:
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.cleanlines.wordcount"
artifacts:
paths:
- $CI_PROJECT_DIR/workflow/.nextflow.log
# This runs the workflow, via Astrocyte, opting to use Docker containers.
docker_astrocyte_test:
stage: test_docker
variables:
config_path : "workflow/configs/"
config_filename : "biohpc_docker.config"
script:
- sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
- nextflow -C workflow/configs/biohpc_docker.config config -flat | grep -oP "process\.'[\w:]+'\.container = '\K([\w:]+)(?=')" | uniq >> containers.txt
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags:
- cluster_node
artifacts:
paths:
- ./containers.txt
# This logs into the GitLab container registry and uses the list of containers parsed in the docker_astrocyte_test job to re-tag
# and push the containers used with the same tag and the same name to the registry.
# NOTE: This is somewhat redundant for true 'caching', but is useful for archiving 'good' containers.
cache_docker_image:
stage: cache_docker
script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- xargs -a containers.txt -I % docker tag "%" "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
- xargs -a containers.txt -I % docker push "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
needs:
- docker_astrocyte_test
# This tests the use of publicly-hosted Docker images (docker.io) in Astrocyte.
singularity_astrocyte_test:
stage: test_singularity
script:
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
# This logs the user into the GitLab Container Registry via the docker daemon and uses docker to run the nextflow pipeline.
# NOTE: If this runs on the same host (with the same docker daemon) login auth may persist from previous steps.
docker_gitlab_test:
stage: test_gitlab
variables:
config_path : "workflow/configs/"
config_filename : "biohpc_docker_gitlab.config"
script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags:
- cluster_node
# This logs the user into the GitLab Container Registry through the SINGULARITY_DOCKER* environment variables.
# The container paths are taken from the config files that are located in ./workflow/configs
singularity_gitlab_test:
stage: test_gitlab
variables:
config_path : "workflow/configs/"
config_filename : "biohpc_singularity_gitlab.config"
script:
- export SINGULARITY_DOCKER_USERNAME=${CI_REGISTRY_USER}
- export SINGULARITY_DOCKER_PASSWORD=${CI_REGISTRY_PASSWORD}
- sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags:
- cluster_node
# This verifies we can install the shiny vizapp dependencies
# and start the vizapp running.
astrocyte_shiny:
astrocyte_viz:
extends: .astrocyte
stage: test
script:
- astrocyte_cli shiny-prepare "$CI_PROJECT_DIR"
- 'timeout 10 astrocyte_cli shiny "$CI_PROJECT_DIR" &'
- 'SHINY_PID=$!'
- astrocyte_cli viz-prepare "$CI_PROJECT_DIR"
- 'timeout 10 astrocyte_cli viz "$CI_PROJECT_DIR" &'
- 'VIZ_PID=$!'
- sleep 5
- unset http_proxy
- unset all_proxy
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment