Commit 0fcff023 authored by Devin OKelly's avatar Devin OKelly
Browse files

Comment up .gitlab-ci.yml

parent 706e728c
# This is the .gitlab-ci.yml file for the astrocyte_example_wordcount project, and demonstrates
# an example CI pipeline for testing and managing Astrocyte/Nextflow workflows that depend on
# containers. Most of the configuration is done by defining containers and executor settings
# in the config files stored in ./workflow/configs/.
stages:
- check
- test
- test_docker
- cache_docker
- test_singularity
- cache_singularity
- cache_docker
- test_gitlab
variables:
GIT_STRATEGY: clone
......@@ -20,8 +26,8 @@ variables:
### All base jobs.
# Astrocyte jobs, assumed to need to run on the cluster where they have access to the module lists.
# Astrocyte jobs, assumed to need to run on the cluster where they have access to the module lists, or on a host that has a suitable 'fake'
# installation of module and modulecmd.
.astrocyte:
before_script:
- *all-script-before
......@@ -34,11 +40,13 @@ variables:
- *all-script-before
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
### Setup scripts
### Setup scripts
before_script:
- *all-script-before
### Tests
# This performs validation of the astrocyte_pkg.yml file
astrocyte_check:
extends: .astrocyte
......@@ -46,6 +54,8 @@ astrocyte_check:
script:
- astrocyte_cli check "$CI_PROJECT_DIR"
# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the docker config.
# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
nextflow_test_docker:
stage: test
script:
......@@ -59,6 +69,9 @@ nextflow_test_docker:
tags:
- cluster_node
# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the singularity config.
# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
nextflow_test_singularity:
stage: test
script:
......@@ -73,8 +86,8 @@ nextflow_test_singularity:
- cluster_node
# This runs the workflow with test data and does a simple
# check for the expected output file
# This runs the workflow with test data and does a simple check for the expected output file.
# This assumes that the 'default' config uses local processing, not containers.
astrocyte_test:
extends: .astrocyte
stage: test
......@@ -86,7 +99,7 @@ astrocyte_test:
- $CI_PROJECT_DIR/workflow/.nextflow.log
# This runs the workflow, opting to use Docker containers.
# This runs the workflow, via Astrocyte, opting to use Docker containers.
docker_astrocyte_test:
stage: test_docker
variables:
......@@ -94,9 +107,7 @@ docker_astrocyte_test:
config_filename : "biohpc_docker.config"
script:
- sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
- echo "Run astrocyte using dockerhub/docker.io registry pulls"
- nextflow -C workflow/configs/biohpc_docker.config config -flat | grep -oP "process\.'[\w:]+'\.container = '\K([\w:]+)(?=')" | uniq >> containers.txt
- echo "BUILD_VERSION=hello" >> build.env
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags:
......@@ -104,43 +115,46 @@ docker_astrocyte_test:
artifacts:
paths:
- ./containers.txt
reports:
dotenv: build.env
# This logs into the GitLab container registry and uses the list of containers parsed in the docker_astrocyte_test job to re-tag
# and push the containers used with the same tag and the same name to the registry.
# NOTE: This is somewhat redundant for true 'caching', but is useful for archiving 'good' containers.
cache_docker_image:
stage: cache_docker
script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- echo "Pull Docker image from the docker.io registry, re-tag, and push to our GitLab"
#xargs -a containers.txt -I % docker pull "%"
- xargs -a containers.txt -I % docker tag "%" "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
- xargs -a containers.txt -I % docker push "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
needs:
- docker_astrocyte_test
# This tests the use of publicly-hosted Docker images (docker.io) in Astrocyte.
singularity_astrocyte_test:
stage: test_singularity
script:
# Note the biohpc_singularity.config replacement.
- sed -i 's|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27biohpc_singularity.config\x27|' astrocyte_pkg.yml
- echo "Run astrocyte using singularity with dockerhub/docker.io registry pulls"
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
# This logs the user into the GitLab Container Registry via the docker daemon and uses docker to run the nextflow pipeline.
# NOTE: If this runs on the same host (with the same docker daemon) login auth may persist from previous steps.
docker_gitlab_test:
stage: test_gitlab
variables:
config_path : "workflow/configs/"
config_filename : "biohpc_docker_gitlab.config"
script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags:
- cluster_node
# This logs the user into the GitLab Container Registry through the SINGULARITY_DOCKER* environment variables.
# The container paths are taken from the config files that are located in ./workflow/configs
singularity_gitlab_test:
stage: test_gitlab
variables:
......@@ -156,11 +170,6 @@ singularity_gitlab_test:
- cluster_node
cache_singularity_image:
stage: cache_singularity
script:
- echo "Pull from our gitlab OCI registry using singularity and save as SIF on the cluster"
# This verifies we can install the shiny vizapp dependencies
# and start the vizapp running.
astrocyte_shiny:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment