Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
C
cellranger-count
Manage
Activity
Members
Labels
Plan
Issues
3
Issue boards
Milestones
Iterations
Wiki
Requirements
Code
Merge requests
0
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Package Registry
Container Registry
Operate
Environments
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Astrocyte
Workflows
Strand Lab
cellranger-count
Commits
24b13f74
Commit
24b13f74
authored
1 year ago
by
Peng Lian
Browse files
Options
Downloads
Patches
Plain Diff
Simplify Gitlab CI/CD
parent
01a16971
Branches
Branches containing commit
Tags
Tags containing commit
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
.gitignore
+2
-0
2 additions, 0 deletions
.gitignore
.gitlab-ci.yml
+8
-119
8 additions, 119 deletions
.gitlab-ci.yml
with
10 additions
and
119 deletions
.gitignore
+
2
−
0
View file @
24b13f74
...
...
@@ -9,3 +9,5 @@ workflow/*.csv
*.wordcount
.nextflow/
.rlibrary/
dag.dot*
report.html*
This diff is collapsed.
Click to expand it.
.gitlab-ci.yml
+
8
−
119
View file @
24b13f74
...
...
@@ -7,16 +7,10 @@
stages
:
-
check
-
test
-
test_docker
-
test_singularity
-
cache_docker
-
test_gitlab
variables
:
GIT_STRATEGY
:
clone
# Execute this script for everybody. YAML anchor.
.all-script-before
:
&all-script-before
-
pwd
...
...
@@ -31,15 +25,9 @@ variables:
.astrocyte
:
before_script
:
-
*all-script-before
-
module load astrocyte/
0.4
.1
-
module load astrocyte/
2.0
.1
tags
:
-
cluster_node
.docker
:
before_script
:
-
*all-script-before
-
docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-
astrocyte_workflow
### Setup scripts
before_script
:
...
...
@@ -54,37 +42,6 @@ astrocyte_check:
script
:
-
astrocyte_cli check "$CI_PROJECT_DIR"
# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the docker config.
# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
nextflow_test_docker
:
stage
:
test
script
:
-
module load nextflow/21.04.1
-
module load singularity/3.5.3
-
cd $CI_PROJECT_DIR/workflow
-
nextflow -C $CI_PROJECT_DIR/workflow/configs/biohpc_docker.config run main.nf -with-trace -with-timeline -resume --verbose -bg
artifacts
:
paths
:
-
$CI_PROJECT_DIR/workflow/.nextflow.log
tags
:
-
cluster_node
# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the singularity config.
# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
nextflow_test_singularity
:
stage
:
test
script
:
-
module load nextflow/21.04.1
-
module load singularity/3.5.3
-
cd $CI_PROJECT_DIR/workflow
-
nextflow -C $CI_PROJECT_DIR/workflow/configs/biohpc_singularity.config run main.nf -with-trace -with-timeline -resume --verbose -bg
artifacts
:
paths
:
-
$CI_PROJECT_DIR/workflow/.nextflow.log
tags
:
-
cluster_node
# This runs the workflow with test data and does a simple check for the expected output file.
# This assumes that the 'default' config uses local processing, not containers.
...
...
@@ -93,89 +50,21 @@ astrocyte_test:
stage
:
test
script
:
-
astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-
test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
-
test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.
cleanlines.
wordcount"
artifacts
:
paths
:
-
$CI_PROJECT_DIR/workflow/.nextflow.log
# This runs the workflow, via Astrocyte, opting to use Docker containers.
docker_astrocyte_test
:
stage
:
test_docker
variables
:
config_path
:
"
workflow/configs/"
config_filename
:
"
biohpc_docker.config"
script
:
-
sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
-
nextflow -C workflow/configs/biohpc_docker.config config -flat | grep -oP "process\.'[\w:]+'\.container = '\K([\w:]+)(?=')" | uniq >> containers.txt
-
astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-
test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags
:
-
cluster_node
artifacts
:
paths
:
-
./containers.txt
# This logs into the GitLab container registry and uses the list of containers parsed in the docker_astrocyte_test job to re-tag
# and push the containers used with the same tag and the same name to the registry.
# NOTE: This is somewhat redundant for true 'caching', but is useful for archiving 'good' containers.
cache_docker_image
:
stage
:
cache_docker
script
:
-
echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
-
xargs -a containers.txt -I % docker tag "%" "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
-
xargs -a containers.txt -I % docker push "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
needs
:
-
docker_astrocyte_test
# This tests the use of publicly-hosted Docker images (docker.io) in Astrocyte.
singularity_astrocyte_test
:
stage
:
test_singularity
script
:
-
astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-
test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
# This logs the user into the GitLab Container Registry via the docker daemon and uses docker to run the nextflow pipeline.
# NOTE: If this runs on the same host (with the same docker daemon) login auth may persist from previous steps.
docker_gitlab_test
:
stage
:
test_gitlab
variables
:
config_path
:
"
workflow/configs/"
config_filename
:
"
biohpc_docker_gitlab.config"
script
:
-
echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
-
sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
-
astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-
test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags
:
-
cluster_node
# This logs the user into the GitLab Container Registry through the SINGULARITY_DOCKER* environment variables.
# The container paths are taken from the config files that are located in ./workflow/configs
singularity_gitlab_test
:
stage
:
test_gitlab
variables
:
config_path
:
"
workflow/configs/"
config_filename
:
"
biohpc_singularity_gitlab.config"
script
:
-
export SINGULARITY_DOCKER_USERNAME=${CI_REGISTRY_USER}
-
export SINGULARITY_DOCKER_PASSWORD=${CI_REGISTRY_PASSWORD}
-
sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
-
astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-
test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
tags
:
-
cluster_node
# This verifies we can install the shiny vizapp dependencies
# and start the vizapp running.
astrocyte_shiny
:
astrocyte_viz
:
extends
:
.astrocyte
stage
:
test
script
:
-
astrocyte_cli
shiny
-prepare "$CI_PROJECT_DIR"
-
'
timeout
10
astrocyte_cli
shiny
"$CI_PROJECT_DIR"
&'
-
'
SHINY
_PID=$!'
-
astrocyte_cli
viz
-prepare "$CI_PROJECT_DIR"
-
'
timeout
10
astrocyte_cli
viz
"$CI_PROJECT_DIR"
&'
-
'
VIZ
_PID=$!'
-
sleep
5
-
unset http_proxy
-
unset all_proxy
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment