Simplify Gitlab CI/CD

.gitignore

@@ -9,3 +9,5 @@ workflow/*.csv
 *.wordcount
 .nextflow/
 .rlibrary/
+dag.dot*
+report.html*

.gitlab-ci.yml

@@ -7,16 +7,10 @@
 stages:
     - check
     - test
-    - test_docker
-    - test_singularity
-    - cache_docker
-    - test_gitlab
-
 
 variables:
     GIT_STRATEGY: clone
 
-
 # Execute this script for everybody. YAML anchor.
 .all-script-before: &all-script-before
   - pwd
@@ -31,15 +25,9 @@ variables:
 .astrocyte:
   before_script:
     - *all-script-before
-    - module load astrocyte/0.4.1
+    - module load astrocyte/2.0.1
   tags:
-    - cluster_node
-
-.docker:
-  before_script:
-    - *all-script-before
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-
+    - astrocyte_workflow
 
 ### Setup scripts
 before_script:
@@ -54,37 +42,6 @@ astrocyte_check:
   script:
   - astrocyte_cli check "$CI_PROJECT_DIR"
   
-# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the docker config.
-# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
-nextflow_test_docker:
-  stage: test
-  script:
-  - module load nextflow/21.04.1
-  - module load singularity/3.5.3 
-  - cd $CI_PROJECT_DIR/workflow
-  - nextflow  -C $CI_PROJECT_DIR/workflow/configs/biohpc_docker.config  run main.nf -with-trace -with-timeline -resume --verbose -bg
-  artifacts:
-    paths:
-    - $CI_PROJECT_DIR/workflow/.nextflow.log
-  tags:
-    - cluster_node
-
-
-# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the singularity config.
-# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
-nextflow_test_singularity:
-  stage: test
-  script:
-  - module load nextflow/21.04.1
-  - module load singularity/3.5.3 
-  - cd $CI_PROJECT_DIR/workflow
-  - nextflow  -C $CI_PROJECT_DIR/workflow/configs/biohpc_singularity.config  run main.nf -with-trace -with-timeline -resume --verbose -bg
-  artifacts:
-    paths:
-    - $CI_PROJECT_DIR/workflow/.nextflow.log
-  tags:
-    - cluster_node
-
 
 # This runs the workflow with test data and does a simple check for the expected output file.
 #  This assumes that the 'default' config uses local processing, not containers.
@@ -93,89 +50,21 @@ astrocyte_test:
   stage: test
   script:
   - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
+  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.cleanlines.wordcount"
   artifacts:
     paths:
     - $CI_PROJECT_DIR/workflow/.nextflow.log  
 
 
-# This runs the workflow, via Astrocyte, opting to use Docker containers.
-docker_astrocyte_test:
-  stage: test_docker
-  variables: 
-    config_path : "workflow/configs/"
-    config_filename : "biohpc_docker.config"
-  script:
-  - sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
-  - nextflow -C workflow/configs/biohpc_docker.config config -flat | grep -oP "process\.'[\w:]+'\.container = '\K([\w:]+)(?=')" | uniq >> containers.txt
-  - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
-  tags:
-    - cluster_node
-  artifacts:
-    paths:
-    - ./containers.txt
-
-
-# This logs into the GitLab container registry and uses the list of containers parsed in the docker_astrocyte_test job to re-tag 
-#  and push the containers used with the same tag and the same name to the registry.
-# NOTE: This is somewhat redundant for true 'caching', but is useful for archiving 'good' containers.
-cache_docker_image:
-  stage: cache_docker
-  script:
-  - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
-  - xargs -a containers.txt -I % docker tag "%" "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
-  - xargs -a containers.txt -I % docker push "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
-  needs:
-    - docker_astrocyte_test
-
-# This tests the use of publicly-hosted Docker images (docker.io) in Astrocyte.
-singularity_astrocyte_test:
-  stage: test_singularity
-  script:
-  - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
-
-# This logs the user into the GitLab Container Registry via the docker daemon and uses docker to run the nextflow pipeline.
-# NOTE: If this runs on the same host (with the same docker daemon) login auth may persist from previous steps.
-docker_gitlab_test:
-  stage: test_gitlab
-  variables: 
-    config_path : "workflow/configs/"
-    config_filename : "biohpc_docker_gitlab.config"
-  script:
-  - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
-  - sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
-  - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
-  tags:
-    - cluster_node
-
-# This logs the user into the GitLab Container Registry through the SINGULARITY_DOCKER* environment variables.
-# The container paths are taken from the config files that are located in ./workflow/configs
-singularity_gitlab_test:
-  stage: test_gitlab
-  variables: 
-    config_path : "workflow/configs/"
-    config_filename : "biohpc_singularity_gitlab.config"
-  script:
-  - export SINGULARITY_DOCKER_USERNAME=${CI_REGISTRY_USER}
-  - export SINGULARITY_DOCKER_PASSWORD=${CI_REGISTRY_PASSWORD}
-  - sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
-  - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
-  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
-  tags:
-    - cluster_node
-
-
 # This verifies we can install the shiny vizapp dependencies
 # and start the vizapp running.
-astrocyte_shiny:
+astrocyte_viz:
+  extends: .astrocyte
   stage: test
   script:
-  - astrocyte_cli shiny-prepare "$CI_PROJECT_DIR"
-  - 'timeout 10 astrocyte_cli shiny "$CI_PROJECT_DIR" &'
-  - 'SHINY_PID=$!'
+  - astrocyte_cli viz-prepare "$CI_PROJECT_DIR"
+  - 'timeout 10 astrocyte_cli viz "$CI_PROJECT_DIR" &'
+  - 'VIZ_PID=$!'
   - sleep 5
   - unset http_proxy
   - unset all_proxy