Merge branch 'LocalDockerRegistry' into 'master'

Merge new CI pipeline with the changes from Peng.

See merge request biohpc/astrocyte_example_wordcount!3

.gitlab-ci.yml

-# Run our tests against a specific version of astrocyte
-before_script:
-  - module load astrocyte/0.4.0
+# This is the .gitlab-ci.yml file for the astrocyte_example_wordcount project, and demonstrates
+# an example CI pipeline for testing and managing Astrocyte/Nextflow workflows that depend on
+# containers. Most of the configuration is done by defining containers and executor settings
+# in the config files stored in ./workflow/configs/. 
 
 
 stages:
     - check
     - test
+    - test_docker
+    - test_singularity
+    - cache_docker
+    - test_gitlab
+
+
+variables:
+    GIT_STRATEGY: clone
+
+
+# Execute this script for everybody. YAML anchor.
+.all-script-before: &all-script-before
+  - pwd
+  - env
+  - ls -laht
+  - whoami
+
+
+### All base jobs.
+# Astrocyte jobs, assumed to need to run on the cluster where they have access to the module lists, or on a host that has a suitable 'fake' 
+#  installation of module and modulecmd.
+.astrocyte:
+  before_script:
+    - *all-script-before
+    - module load astrocyte/0.4.0
+  tags:
+    - cluster_node
+
+.docker:
+  before_script:
+    - *all-script-before
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+
 
+### Setup scripts
+before_script:
+  - *all-script-before
+
+
+### Tests 
 # This performs validation of the astrocyte_pkg.yml file
 astrocyte_check:
+  extends: .astrocyte
   stage: check
   script:
   - astrocyte_cli check "$CI_PROJECT_DIR"
+  
+# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the docker config.
+# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
+nextflow_test_docker:
+  stage: test
+  script:
+  - module load nextflow/20.01.0 
+  - module load singularity/3.5.3 
+  - cd $CI_PROJECT_DIR/workflow
+  - nextflow  -C $CI_PROJECT_DIR/workflow/configs/biohpc_docker.config  run main.nf -with-trace -with-timeline -resume --verbose -bg
+  artifacts:
+    paths:
+    - $CI_PROJECT_DIR/workflow/.nextflow.log
+  tags:
+    - cluster_node
+
+
+# This runs the workflow at the level of Nextflow, using the 'fake' module and modulecmd, with the singularity config.
+# NOTE: The option -bg here only works if there's no further tasks (e.g. testing the existence of a file). Use -q for that.
+nextflow_test_singularity:
+  stage: test
+  script:
+  - module load nextflow/20.01.0 
+  - module load singularity/3.5.3 
+  - cd $CI_PROJECT_DIR/workflow
+  - nextflow  -C $CI_PROJECT_DIR/workflow/configs/biohpc_singularity.config  run main.nf -with-trace -with-timeline -resume --verbose -bg
+  artifacts:
+    paths:
+    - $CI_PROJECT_DIR/workflow/.nextflow.log
+  tags:
+    - cluster_node
 
-# This runs the workflow with test data and does a simple
-# check for the expected output file
+
+# This runs the workflow with test data and does a simple check for the expected output file.
+#  This assumes that the 'default' config uses local processing, not containers.
 astrocyte_test:
+  extends: .astrocyte
   stage: test
   script:
-  - astrocyte_cli test "$CI_PROJECT_DIR"
+  - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
+  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
+  artifacts:
+    paths:
+    - $CI_PROJECT_DIR/workflow/.nextflow.log  
+
+
+# This runs the workflow, via Astrocyte, opting to use Docker containers.
+docker_astrocyte_test:
+  stage: test_docker
+  variables: 
+    config_path : "workflow/configs/"
+    config_filename : "biohpc_docker.config"
+  script:
+  - sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
+  - nextflow -C workflow/configs/biohpc_docker.config config -flat | grep -oP "process\.'[\w:]+'\.container = '\K([\w:]+)(?=')" | uniq >> containers.txt
+  - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
+  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
+  tags:
+    - cluster_node
+  artifacts:
+    paths:
+    - ./containers.txt
+
+
+# This logs into the GitLab container registry and uses the list of containers parsed in the docker_astrocyte_test job to re-tag 
+#  and push the containers used with the same tag and the same name to the registry.
+# NOTE: This is somewhat redundant for true 'caching', but is useful for archiving 'good' containers.
+cache_docker_image:
+  stage: cache_docker
+  script:
+  - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+  - xargs -a containers.txt -I % docker tag "%" "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
+  - xargs -a containers.txt -I % docker push "$CI_REGISTRY/$CI_PROJECT_ROOT_NAMESPACE/$CI_PROJECT_NAME/%"
+  needs:
+    - docker_astrocyte_test
+
+# This tests the use of publicly-hosted Docker images (docker.io) in Astrocyte.
+singularity_astrocyte_test:
+  stage: test_singularity
+  script:
+  # Note the biohpc_singularity.config replacement.
+  - sed -i 's|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27biohpc_singularity.config\x27|' astrocyte_pkg.yml
+  - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
+  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
+
+# This logs the user into the GitLab Container Registry via the docker daemon and uses docker to run the nextflow pipeline.
+# NOTE: If this runs on the same host (with the same docker daemon) login auth may persist from previous steps.
+docker_gitlab_test:
+  stage: test_gitlab
+  variables: 
+    config_path : "workflow/configs/"
+    config_filename : "biohpc_docker_gitlab.config"
+  script:
+  - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+  - sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
+  - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
   - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
+  tags:
+    - cluster_node
+
+# This logs the user into the GitLab Container Registry through the SINGULARITY_DOCKER* environment variables.
+# The container paths are taken from the config files that are located in ./workflow/configs
+singularity_gitlab_test:
+  stage: test_gitlab
+  variables: 
+    config_path : "workflow/configs/"
+    config_filename : "biohpc_singularity_gitlab.config"
+  script:
+  - export SINGULARITY_DOCKER_USERNAME=${CI_REGISTRY_USER}
+  - export SINGULARITY_DOCKER_PASSWORD=${CI_REGISTRY_PASSWORD}
+  - sed -i "s|nextflow_config\x3A \x27biohpc.config\x27|nextflow_config\x3A \x27${config_filename}\x27|" astrocyte_pkg.yml
+  - astrocyte_cli run "$CI_PROJECT_DIR" --option=q
+  - test -s "$CI_PROJECT_DIR/workflow/output/mobydick.txt.uppercase.tolines.wordcount"
+  tags:
+    - cluster_node
+
 
 # This verifies we can install the shiny vizapp dependencies
 # and start the vizapp running.

astrocyte_pkg.yml

@@ -9,7 +9,7 @@
 # A unique identifier for the workflow package, text/underscores only
 name: 'example_wordcount'
 # Who wrote this?
-author: 'David Trudgian, Peng Lian'
+author: 'David Trudgian, Peng Lian, Devin OKelly'
 # A contact email address for questions
 email: 'biohpc-help@utsouthwestern.edu'
 # A more informative title for the workflow package
@@ -19,7 +19,7 @@ description: |
   This is a minimal test workflow package that counts the occurences of words
   in a test file. It can be used as a template to develop workflows, and as to
   test the astrocyte platform. Please note that start from publish version 
-  0.0.8 and test version 0.0.2, this workflow will only support Astrocyte 0.4.0
+  0.0.8 and test version 0.0.2, this workflow will only support Astrocyte 0.4.1
   and above.
 
 #### New Features in Astrocyte 0.4.0 and above ####
@@ -31,7 +31,7 @@ citation: |
   
 # The minimum Astrocyte version that requires to run this workflow. For old pipelines, which do not have this label 
 # a default value of 0.3.1 will be assigned automatically. A request of minimum version less than 0.4.0 will be ignored.
-minimum_astrocyte_version: '0.4.0'
+minimum_astrocyte_version: '0.4.1'
 # The Nextflow version that requires to run this workflow.  For old pipelines, which do not have this label 
 # a default value of 0.31.0 will be assigned automatically. Please make sure the requested nextflow version is available
 # in the module list.
@@ -66,6 +66,12 @@ documentation_files:
 # Specify versioned module names to ensure reproducability.
 workflow_modules:
   - Test
+# A list of container images requires to run this workflow.
+# Specify full path and version names to ensure reproducability.
+# This keyword is required when 'container' is specified in Astrocyte 0.4.1 and above
+workflow_containers:
+  - docker://ubuntu:latest
+  - docker://centos:centos8
 
 # A list of parameters used by the workflow, defining how to present them,
 # options etc in the web interface. For each parameter:

workflow/configs/biohpc.config

-process {
-    executor = 'local'
+singularity {
+  enabled = true
+  runOptions = '--bind /vagrant:/vagrant'
+  cacheDir = "$baseDir/images/singularity"
 }
 
-executor {
-    name = 'local'
-    cpus = 1
+process {
+    executor = 'local'
 }

workflow/configs/biohpc_docker.config

+docker {
+  enabled = true
+  fixOwnership = true
+}
+
+
+process {
+    executor = 'local'
+    withName:uppercase {
+        container = 'ubuntu:latest'
+    }
+    withName:tolines {
+        container = 'ubuntu:latest'
+    }
+    withName:wordcounts {
+        container = 'centos:centos8'
+    }
+}

workflow/configs/biohpc_docker_gitlab.config

+docker {
+  enabled = true
+  fixOwnership = true
+}
+
+
+process {
+    executor = 'local'
+    withName:uppercase {
+        container = 'git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
+    }
+    withName:tolines {
+        container = 'git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
+    }
+    withName:wordcounts {
+        container = 'git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/centos:centos8'
+    }
+}

workflow/configs/biohpc_singularity.config

+singularity {
+  enabled = true
+  runOptions = '--bind /vagrant:/vagrant'
+  cacheDir = "$baseDir/images/singularity"
+}
+
+process {
+    executor = 'local'
+    withName:uppercase {
+        container = 'ubuntu:latest'
+    }
+    withName:tolines {
+        container = 'ubuntu:latest'
+    }
+    withName:wordcounts {
+        container = 'centos:centos8'
+    }
+}

workflow/configs/biohpc_singularity_gitlab.config

+singularity {
+  enabled = true
+  runOptions = '--bind /vagrant:/vagrant'
+  cacheDir = "$baseDir/images/singularity"
+}
+
+
+process {
+    executor = 'local'
+    withName:parameters {
+        container = 'git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
+    }
+    withName:uppercase {
+        container = 'docker://git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
+    }
+    withName:tolines {
+        container = 'docker://git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/ubuntu:latest'
+    }
+    withName:wordcounts {
+        container = 'docker://git.biohpc.swmed.edu:5050/s190450/astrocyte_example_wordcount/centos:centos8'
+    }
+}

workflow/main.nf

@@ -27,6 +27,7 @@ stories = Channel.fromPath( params.story )
 process parameters {
 
     cpus 1
+    //container 'ubuntu:latest'
 
     """
     echo "Test Parameters Provided..."
@@ -36,7 +37,9 @@ process parameters {
     echo "test_string: ${params.test_string}"
     echo "test_select: ${params.test_select}"
     echo "test_multiselect: ${params.test_multiselect}"
-
+    pwd
+    whoami
+    env 
     """
 }
 
@@ -74,6 +77,7 @@ process tolines {
 
 process wordcounts {
 
+
     // Publish the outputs we create here into the workflow output directory
     publishDir "$baseDir/output", mode: 'copy'