.gitlab-ci.yml 2.95 KB
Newer Older
David Trudgian's avatar
David Trudgian committed
1
2
3
stages:
    - check
    - test
Devin OKelly's avatar
Devin OKelly committed
4
5
6
7
8
    - test_docker
    - cache_docker
    - test_singularity
    - cache_singularity

9
10
variables:
    GIT_STRATEGY: clone
Devin OKelly's avatar
Devin OKelly committed
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28


# Execute this script for everybody. YAML anchor.
.all-script-before: &all-script-before
  - pwd
  - env
  - ls -laht
  - whoami



### All base jobs.

# Astrocyte jobs, assumed to need to run on the cluster where they have access to the module lists.
.astrocyte:
  before_script:
    - *all-script-before
    - module load astrocyte/0.4.0
29
30
    - NXF_DEBUG=3
    - export NXF_DEBUG
Devin OKelly's avatar
Devin OKelly committed
31
32
33
34
35
36
37
38
39
40
41
42
  tags:
    - cluster_node

.docker:
  before_script:
    - *all-script-before
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

### Setup scripts

before_script:
  - *all-script-before
David Trudgian's avatar
David Trudgian committed
43

David Trudgian's avatar
David Trudgian committed
44
# This performs validation of the astrocyte_pkg.yml file
David Trudgian's avatar
David Trudgian committed
45
astrocyte_check:
Devin OKelly's avatar
Devin OKelly committed
46
  extends: .astrocyte
David Trudgian's avatar
David Trudgian committed
47
  stage: check
David Trudgian's avatar
David Trudgian committed
48
49
  script:
  - astrocyte_cli check "$CI_PROJECT_DIR"
Devin OKelly's avatar
Devin OKelly committed
50
  
51
nextflow_test_docker:
Devin OKelly's avatar
Devin OKelly committed
52
53
54
55
  stage: test
  script:
  - module load nextflow/20.01.0 
  - module load singularity/3.5.3 
Devin OKelly's avatar
Devin OKelly committed
56
  - cd $CI_PROJECT_DIR/workflow
57
  - nextflow  -C $CI_PROJECT_DIR/workflow/configs/biohpc_docker.config  run main.nf -with-trace -with-timeline -resume --verbose -bq
58
59
60
  artifacts:
    paths:
    - $CI_PROJECT_DIR/workflow/.nextflow.log
61
62
63
64
65
66
67
68


nextflow_test_singularity:
  stage: test
  script:
  - module load nextflow/20.01.0 
  - module load singularity/3.5.3 
  - cd $CI_PROJECT_DIR/workflow
69
  - nextflow  -C $CI_PROJECT_DIR/workflow/configs/biohpc_singularity.config  run main.nf -with-trace -with-timeline -resume --verbose -bq
70
71
72
73
  artifacts:
    paths:
    - $CI_PROJECT_DIR/workflow/.nextflow.log

74

Devin OKelly's avatar
Devin OKelly committed
75

David Trudgian's avatar
David Trudgian committed
76
77
# This runs the workflow with test data and does a simple
# check for the expected output file
David Trudgian's avatar
David Trudgian committed
78
astrocyte_test:
Devin OKelly's avatar
Devin OKelly committed
79
  extends: .astrocyte
David Trudgian's avatar
David Trudgian committed
80
  stage: test
David Trudgian's avatar
David Trudgian committed
81
  script:
82
  - astrocyte_cli test "$CI_PROJECT_DIR" --verbose 
David Trudgian's avatar
David Trudgian committed
83
  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
84
85
86
  artifacts:
    paths:
    - $CI_PROJECT_DIR/workflow/.nextflow.log  
Devin OKelly's avatar
Devin OKelly committed
87
88
89
90
# This runs the workflow, opting to use Docker containers.
docker_astrocyte_test:
  stage: test_docker
  script:
91
  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
Devin OKelly's avatar
Devin OKelly committed
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
  - echo "Run astrocyte using dockerhub/docker.io registry pulls"
 
cache_docker_image:
  stage: cache_docker
  script:
  - echo "Pull Docker image from the docker.io registry, re-tag, and push to our GitLab"
 
singularity_astrocyte_test:
  stage: test_singularity
  script:
  - echo "Run astrocyte using singularity registry pulls"

cache_singularity_image:
  stage: cache_singularity
  script:
  - echo "Pull from our gitlab OCI registry using singularity and save as SIF on the cluster"
David Trudgian's avatar
David Trudgian committed
108
109
110

# This verifies we can install the shiny vizapp dependencies
# and start the vizapp running.
David Trudgian's avatar
David Trudgian committed
111
112
astrocyte_shiny:
  stage: test
David Trudgian's avatar
David Trudgian committed
113
114
115
116
117
118
119
120
121
  script:
  - astrocyte_cli shiny-prepare "$CI_PROJECT_DIR"
  - 'timeout 10 astrocyte_cli shiny "$CI_PROJECT_DIR" &'
  - 'SHINY_PID=$!'
  - sleep 5
  - unset http_proxy
  - unset all_proxy
  - wget "http://127.0.0.1:8123"