Comment up .gitlab-ci.yml

.gitlab-ci.yml

+# This is the .gitlab-ci.yml file for the astrocyte_example_wordcount project, and demonstrates
+# an example CI pipeline for testing and managing Astrocyte/Nextflow workflows that depend on
+# containers. Most of the configuration is done by defining containers and executor settings
+# in the config files stored in ./workflow/configs/. 
+
+
 stages:
     - check
     - test
     - test_docker
-    - cache_docker
     - test_singularity
-    - cache_singularity
+    - cache_docker
     - test_gitlab
 
+
 variables:
     GIT_STRATEGY: clone
 
@@ -20,8 +26,8 @@ variables:
 
 
 ### All base jobs.
-
-# Astrocyte jobs, assumed to need to run on the cluster where they have access to the module lists.
+# Astrocyte jobs, assumed to need to run on the cluster where they have access to the module lists, or on a host that has a suitable 'fake' 
+#  installation of module and modulecmd.
 .astrocyte:
   before_script:
     - *all-script-before
@@ -34,11 +40,13 @@ variables:
     - *all-script-before
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
-### Setup scripts
 
+### Setup scripts
 before_script:
   - *all-script-before
 
+
+### Tests 
 # This performs validation of the astrocyte_pkg.yml file
 astrocyte_check:
   extends: .astrocyte
@@ -46,6 +54,8 @@ astrocyte_check:
   script:
   - astrocyte_cli check "$CI_PROJECT_DIR"
   
+# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the docker config.
+# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
 nextflow_test_docker:
   stage: test
   script:
@@ -59,6 +69,9 @@ nextflow_test_docker:
   tags:
     - cluster_node
 
+
+# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the singularity config.
+# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
 nextflow_test_singularity:
   stage: test
   script:
@@ -73,8 +86,8 @@ nextflow_test_singularity:
     - cluster_node
 
 
-# This runs the workflow with test data and does a simple
-# check for the expected output file
+# This runs the workflow with test data and does a simple check for the expected output file.
+#  This assumes that the 'default' config uses local processing, not containers.
 astrocyte_test:
   extends: .astrocyte
   stage: test
@@ -86,7 +99,7 @@ astrocyte_test:
     - $CI_PROJECT_DIR/workflow/.nextflow.log  
 
 
-# This runs the workflow, opting to use Docker containers.
+# This runs the workflow, via Astrocyte, opting to use Docker containers.
 docker_astrocyte_test:
   stage: test_docker
   variables: 
@@ -94,9 +107,7 @@ docker_astrocyte_test:
     config_filename : "biohpc_docker.config"
   script:
   - sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
-  - echo "Run astrocyte using dockerhub/docker.io registry pulls"
   - nextflow -C workflow/configs/biohpc_docker.config config -flat | grep -oP "process\.'[\w:]+'\.container = '\K([\w:]+)(?=')" | uniq >> containers.txt
-  - echo "BUILD_VERSION=hello" >> build.env
   - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
   - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
   tags:
@@ -104,43 +115,46 @@ docker_astrocyte_test:
   artifacts:
     paths:
     - ./containers.txt
-    reports:
-     dotenv: build.env
 
+
+# This logs into the GitLab container registry and uses the list of containers parsed in the docker_astrocyte_test job to re-tag 
+#  and push the containers used with the same tag and the same name to the registry.
+# NOTE: This is somewhat redundant for true 'caching', but is useful for archiving 'good' containers.
 cache_docker_image:
   stage: cache_docker
   script:
   - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
-  - echo "Pull Docker image from the docker.io registry, re-tag, and push to our GitLab"
-   #xargs -a containers.txt -I %  docker pull "%" 
   - xargs -a containers.txt -I % docker tag "%" "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
   - xargs -a containers.txt -I % docker push "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
   needs:
     - docker_astrocyte_test
 
+# This tests the use of publicly-hosted Docker images (docker.io) in Astrocyte.
 singularity_astrocyte_test:
   stage: test_singularity
   script:
   # Note the biohpc_singularity.config replacement.
   - sed -i 's|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27biohpc_singularity.config\x27|' astrocyte_pkg.yml
-  - echo "Run astrocyte using singularity with dockerhub/docker.io registry pulls"
   - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
   - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
 
-
+# This logs the user into the GitLab Container Registry via the docker daemon and uses docker to run the nextflow pipeline.
+# NOTE: If this runs on the same host (with the same docker daemon) login auth may persist from previous steps.
 docker_gitlab_test:
   stage: test_gitlab
   variables: 
     config_path : "workflow/configs/"
     config_filename : "biohpc_docker_gitlab.config"
   script:
+  - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
   - sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
   - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
   - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
   tags:
     - cluster_node
 
-
+# This logs the user into the GitLab Container Registry through the SINGULARITY_DOCKER* environment variables.
+# The container paths are taken from the config files that are located in ./workflow/configs
 singularity_gitlab_test:
   stage: test_gitlab
   variables: 
@@ -156,11 +170,6 @@ singularity_gitlab_test:
     - cluster_node
 
 
-cache_singularity_image:
-  stage: cache_singularity
-  script:
-  - echo "Pull from our gitlab OCI registry using singularity and save as SIF on the cluster"
-
 # This verifies we can install the shiny vizapp dependencies
 # and start the vizapp running.
 astrocyte_shiny: