Commit 96d532f2 authored by Peng Lian's avatar Peng Lian
Browse files

Merge branch 'LocalDockerRegistry' into 'master'

Merge new CI pipeline with the changes from Peng.

See merge request !3
parents ab1b655e 9de147c8
Pipeline #10287 passed with stages
# Run our tests against a specific version of astrocyte
before_script:
- module load astrocyte/0.4.0
# This is the .gitlab-ci.yml file for the astrocyte_example_wordcount project, and demonstrates
# an example CI pipeline for testing and managing Astrocyte/Nextflow workflows that depend on
# containers. Most of the configuration is done by defining containers and executor settings
# in the config files stored in ./workflow/configs/.
stages:
- check
- test
- test_docker
- test_singularity
- cache_docker
- test_gitlab
variables:
GIT_STRATEGY: clone
# Execute this script for everybody. YAML anchor.
.all-script-before: &all-script-before
- pwd
- env
- ls -laht
- whoami
### All base jobs.
# Astrocyte jobs, assumed to need to run on the cluster where they have access to the module lists, or on a host that has a suitable 'fake'
# installation of module and modulecmd.
.astrocyte:
before_script:
- *all-script-before
- module load astrocyte/0.4.0
tags:
- cluster_node
.docker:
before_script:
- *all-script-before
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
### Setup scripts
before_script:
- *all-script-before
### Tests
# This performs validation of the astrocyte_pkg.yml file
astrocyte_check:
extends: .astrocyte
stage: check
script:
- astrocyte_cli check "$CI_PROJECT_DIR"
# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the docker config.
# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
nextflow_test_docker:
stage: test
script:
- module load nextflow/20.01.0
- module load singularity/3.5.3
- cd $CI_PROJECT_DIR/workflow
- nextflow -C $CI_PROJECT_DIR/workflow/configs/biohpc_docker.config run main.nf -with-trace -with-timeline -resume --verbose -bg
artifacts:
paths:
- $CI_PROJECT_DIR/workflow/.nextflow.log
tags:
- cluster_node
# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the singularity config.
# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
nextflow_test_singularity:
stage: test
script:
- module load nextflow/20.01.0
- module load singularity/3.5.3
- cd $CI_PROJECT_DIR/workflow
- nextflow -C $CI_PROJECT_DIR/workflow/configs/biohpc_singularity.config run main.nf -with-trace -with-timeline -resume --verbose -bg
artifacts:
paths:
- $CI_PROJECT_DIR/workflow/.nextflow.log
tags:
- cluster_node
# This runs the workflow with test data and does a simple
# check for the expected output file
# This runs the workflow with test data and does a simple check for the expected output file.
# This assumes that the 'default' config uses local processing, not containers.
astrocyte_test:
extends: .astrocyte
stage: test
script:
- astrocyte_cli test "$CI_PROJECT_DIR"
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
artifacts:
paths:
- $CI_PROJECT_DIR/workflow/.nextflow.log
# This runs the workflow, via Astrocyte, opting to use Docker containers.
docker_astrocyte_test:
stage: test_docker
variables:
config_path : "workflow/configs/"
config_filename : "biohpc_docker.config"
script:
- sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
- nextflow -C workflow/configs/biohpc_docker.config config -flat | grep -oP "process\.'[\w:]+'\.container = '\K([\w:]+)(?=')" | uniq >> containers.txt
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags:
- cluster_node
artifacts:
paths:
- ./containers.txt
# This logs into the GitLab container registry and uses the list of containers parsed in the docker_astrocyte_test job to re-tag
# and push the containers used with the same tag and the same name to the registry.
# NOTE: This is somewhat redundant for true 'caching', but is useful for archiving 'good' containers.
cache_docker_image:
stage: cache_docker
script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- xargs -a containers.txt -I % docker tag "%" "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
- xargs -a containers.txt -I % docker push "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
needs:
- docker_astrocyte_test
# This tests the use of publicly-hosted Docker images (docker.io) in Astrocyte.
singularity_astrocyte_test:
stage: test_singularity
script:
# Note the biohpc_singularity.config replacement.
- sed -i 's|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27biohpc_singularity.config\x27|' astrocyte_pkg.yml
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
# This logs the user into the GitLab Container Registry via the docker daemon and uses docker to run the nextflow pipeline.
# NOTE: If this runs on the same host (with the same docker daemon) login auth may persist from previous steps.
docker_gitlab_test:
stage: test_gitlab
variables:
config_path : "workflow/configs/"
config_filename : "biohpc_docker_gitlab.config"
script:
- echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags:
- cluster_node
# This logs the user into the GitLab Container Registry through the SINGULARITY_DOCKER* environment variables.
# The container paths are taken from the config files that are located in ./workflow/configs
singularity_gitlab_test:
stage: test_gitlab
variables:
config_path : "workflow/configs/"
config_filename : "biohpc_singularity_gitlab.config"
script:
- export SINGULARITY_DOCKER_USERNAME=${CI_REGISTRY_USER}
- export SINGULARITY_DOCKER_PASSWORD=${CI_REGISTRY_PASSWORD}
- sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
- astrocyte_cli run "$CI_PROJECT_DIR" --option=q
- test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags:
- cluster_node
# This verifies we can install the shiny vizapp dependencies
# and start the vizapp running.
......
......@@ -9,7 +9,7 @@
# A unique identifier for the workflow package, text/underscores only
name: 'example_wordcount'
# Who wrote this?
author: 'David Trudgian, Peng Lian'
author: 'David Trudgian, Peng Lian, Devin OKelly'
# A contact email address for questions
email: 'biohpc-help@utsouthwestern.edu'
# A more informative title for the workflow package
......@@ -19,7 +19,7 @@ description: |
This is a minimal test workflow package that counts the occurences of words
in a test file. It can be used as a template to develop workflows, and as to
test the astrocyte platform. Please note that start from publish version
0.0.8 and test version 0.0.2, this workflow will only support Astrocyte 0.4.0
0.0.8 and test version 0.0.2, this workflow will only support Astrocyte 0.4.1
and above.
#### New Features in Astrocyte 0.4.0 and above ####
......@@ -31,7 +31,7 @@ citation: |
# The minimum Astrocyte version that requires to run this workflow. For old pipelines, which do not have this label
# a default value of 0.3.1 will be assigned automatically. A request of minimum version less than 0.4.0 will be ignored.
minimum_astrocyte_version: '0.4.0'
minimum_astrocyte_version: '0.4.1'
# The Nextflow version that requires to run this workflow. For old pipelines, which do not have this label
# a default value of 0.31.0 will be assigned automatically. Please make sure the requested nextflow version is available
# in the module list.
......@@ -66,6 +66,12 @@ documentation_files:
# Specify versioned module names to ensure reproducability.
workflow_modules:
- Test
# A list of container images requires to run this workflow.
# Specify full path and version names to ensure reproducability.
# This keyword is required when 'container' is specified in Astrocyte 0.4.1 and above
workflow_containers:
- docker://ubuntu:latest
- docker://centos:centos8
# A list of parameters used by the workflow, defining how to present them,
# options etc in the web interface. For each parameter:
......
process {
executor = 'local'
singularity {
enabled = true
runOptions = '--bind /vagrant:/vagrant'
cacheDir = "$baseDir/images/singularity"
}
executor {
name = 'local'
cpus = 1
process {
executor = 'local'
}
docker {
enabled = true
fixOwnership = true
}
process {
executor = 'local'
withName:uppercase {
container = 'ubuntu:latest'
}
withName:tolines {
container = 'ubuntu:latest'
}
withName:wordcounts {
container = 'centos:centos8'
}
}
docker {
enabled = true
fixOwnership = true
}
process {
executor = 'local'
withName:uppercase {
container = 'git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
}
withName:tolines {
container = 'git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
}
withName:wordcounts {
container = 'git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/centos:centos8'
}
}
singularity {
enabled = true
runOptions = '--bind /vagrant:/vagrant'
cacheDir = "$baseDir/images/singularity"
}
process {
executor = 'local'
withName:uppercase {
container = 'ubuntu:latest'
}
withName:tolines {
container = 'ubuntu:latest'
}
withName:wordcounts {
container = 'centos:centos8'
}
}
singularity {
enabled = true
runOptions = '--bind /vagrant:/vagrant'
cacheDir = "$baseDir/images/singularity"
}
process {
executor = 'local'
withName:parameters {
container = 'git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
}
withName:uppercase {
container = 'docker://git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
}
withName:tolines {
container = 'docker://git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
}
withName:wordcounts {
container = 'docker://git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/centos:centos8'
}
}
......@@ -27,6 +27,7 @@ stories = Channel.fromPath( params.story )
process parameters {
cpus 1
//container 'ubuntu:latest'
"""
echo "Test Parameters Provided..."
......@@ -36,7 +37,9 @@ process parameters {
echo "test_string: ${params.test_string}"
echo "test_select: ${params.test_select}"
echo "test_multiselect: ${params.test_multiselect}"
pwd
whoami
env
"""
}
......@@ -74,6 +77,7 @@ process tolines {
process wordcounts {
// Publish the outputs we create here into the workflow output directory
publishDir "$baseDir/output", mode: 'copy'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment