.gitlab-ci.yml

@@ -2,25 +2,34 @@
 # Brandi L. Cantarel - 2017
 
 variables:
-  GIT_SUBMODULE_STRATEGY: recursive
+  GIT_SUBMODULE_STRATEGY: normal
 
 before_script:
+  - module load astrocyte/1.0.0
   - module load nextflow/20.01.0
-  - module load singularity/3.0.2
+  - module load singularity/3.5.3
 
 stages:
-  - integration
+  - check
+  - test_human
+  - test_mouse
+
+# This performs validation of the astrocyte_pkg.yml file
+astrocyte_check:
+  stage: check
+  script:
+  - astrocyte_cli check "$CI_PROJECT_DIR" 
 
 test_human:
-  stage: integration
+  stage: test_human
   script:
-    - nextflow run -c nextflow.config -with-dag flowchart.png -with-timeline human_timeline.html -with-report human_report.html workflow/main.nf --design  /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq/design.rnaseq.txt --input  /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq --output human_output
+    - nextflow run -c workflow/configs/nextflow.config -with-dag flowchart.png -with-timeline human_timeline.html -with-report human_report.html workflow/main.nf --design  /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq/design.rnaseq.txt --input  /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq --output human_output
   artifacts:
     expire_in: 2 days
 
 test_mouse:
-  stage: integration
+  stage: test_mouse
   script:
-    - nextflow run -c nextflow.config -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html  workflow/main.nf --input  /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq --design /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq/mouse_se.design.txt --pairs se --fusion skip --genome /project/shared/bicf_workflow_ref/mouse/GRCm38 --markdups null --output mouse_output
+    - nextflow run -c workflow/configs/nextflow.config -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html  workflow/main.nf --input  /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq --design /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq/mouse_se.design.txt --pairs se --fusion skip --genome /project/shared/bicf_workflow_ref/mouse/GRCm38 --markdups null --output mouse_output
   artifacts:
     expire_in: 2 days

.gitmodules

+[submodule "workflow/process_scripts"]
+	path = workflow/process_scripts
+	url = git@git.biohpc.swmed.edu:ngsclialab/process_scripts.git

README.md

@@ -69,7 +69,7 @@ module load nextflow/20.01.0 singularity/3.5.3
 base=$repoClonedDirectory
 datadir='/project/shared/bicf_workflow_ref/workflow_testdata/rnaseq'
 
-nextflow -C ${base}/nextflow.config run ${base}/workflow/main.nf --design  ${datadir}/design.rnaseq.txt --input ${datadir} --output analysis
+nextflow -C ${base}/workflow/configs/nextflow.config run ${base}/workflow/main.nf --design  ${datadir}/design.rnaseq.txt --input ${datadir} --output analysis
 
 ```
 
@@ -80,7 +80,7 @@ module load nextflow/20.01.0 singularity/3.5.3
 base=$repoClonedDirectory
 datadir='/project/shared/bicf_workflow_ref/workflow_testdata/rnaseq'
 
-nextflow -C ${base}/nextflow.config run -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html ${base}/workflow/main.nf --design  ${datadir}/mouse_se.design.txt --input ${datadir} --pairs se --output analysis
+nextflow -C ${base}/workflow/configs/nextflow.config run -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html ${base}/workflow/main.nf --design  ${datadir}/mouse_se.design.txt --input ${datadir} --pairs se --output analysis
 
 ```
 

astrocyte_pkg.yml

@@ -20,6 +20,20 @@ description: |
   It implements differential expression analysis, gene set enrichment analysis,
   gene fusion analysis and variant identification using RNASeq data.
 
+# The minimum Astrocyte version that requires to run this workflow. For old pipelines, which do not have this label 
+# a default value of 0.3.1 will be assigned automatically. A request of minimum version less than 0.4.0 will be ignored.
+minimum_astrocyte_version: '0.4.1'
+# The Nextflow version that requires to run this workflow.  For old pipelines, which do not have this label 
+# a default value of 0.31.0 will be assigned automatically. Please make sure the requested nextflow version is available
+# in the module list.
+nextflow_version: '20.01.0'
+# (Optional) The Nextflow config file to use for this workflow. If provided, the file should exist in workflow/configs
+nextflow_config: 'nextflow.config'
+# The container to use for this workflow, none/singularity. If omitted, the default value 'none' will be used.
+container: 'singularity'
+# The version of singularity to use. This is required if container == 'singularity'
+singularity_version: '3.5.3'
+
 
 # -----------------------------------------------------------------------------
 # DOCUMENTATION
@@ -50,6 +64,22 @@ workflow_modules:
   - 'stringtie/1.1.2-intel'
   - 'speedseq/20160506'
   - 'python/2.7.x-anaconda'
+
+# A list of container images requires to run this workflow.
+# Specify full path and version names to ensure reproducability.
+# This keyword is required when 'container' is specified in Astrocyte 0.4.1 and above
+workflow_containers:
+  - docker://goalconsortium/trim_galore:1.0.9
+  - docker://goalconsortium/abra2:1.0.9
+  - docker://goalconsortium/profiling_qc:1.0.9
+  - docker://goalconsortium/dna_alignment:1.0.9
+  - docker://goalconsortium/variantcalling:1.0.9
+  - docker://goalconsortium/structuralvariant:1.1.4
+  - docker://goalconsortium/starfusion:1.0.9
+  - docker://goalconsortium/rna_alignment:1.0.9
+  - docker://goalconsortium/rna_gene_abundance:1.1.3
+  - docker://goalconsortium/rna_statanal:1.1.4
+
 # A list of parameters used by the workflow, defining how to present them,
 # options etc in the web interface. For each parameter:
 #

testing/run_human_pe_test.sh

@@ -4,4 +4,4 @@ module load nextflow/20.01.0 singularity/3.5.3
 base='/project/BICF/BICF_Core/s166458/rnaseq_astrocyte'
 datadir='/project/shared/bicf_workflow_ref/workflow_testdata/rnaseq'
 
-nextflow -C ${base}/nextflow.config run ${base}/workflow/main.nf --design  ${datadir}/design.rnaseq.txt --input ${datadir} --output analysis
+nextflow -C ${base}/workflow/configs/nextflow.config run ${base}/workflow/main.nf --design  ${datadir}/design.rnaseq.txt --input ${datadir} --output analysis

testing/run_mouse_se_test.sh

@@ -4,4 +4,4 @@ module load nextflow/20.01.0 singularity/3.5.3
 base='/project/BICF/BICF_Core/s166458/rnaseq_astrocyte'
 datadir='/project/shared/bicf_workflow_ref/workflow_testdata/rnaseq'
 
-nextflow -C ${base}/nextflow.config run -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html ${base}/workflow/main.nf --design  ${datadir}/mouse_se.design.txt --input ${datadir} --pairs se --output analysis
+nextflow -C ${base}/workflow/configs/nextflow.config run -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html ${base}/workflow/main.nf --design  ${datadir}/mouse_se.design.txt --input ${datadir} --pairs se --output analysis

nextflow.config → workflow/configs/nextflow.config

 params {
        repoDir='/seqprg'
 }
+
 process {
   executor = 'slurm'
   clusterOptions = '--hold --no-kill'
   queue = '128GB,256GB,256GBv1'
   withLabel: trim {
-    container = 'goalconsortium/trim_galore:1.0.9'
+    container = 'goalconsortium/trim_galore:1.1.3'
   }
   withLabel: abra2 {
     container = 'goalconsortium/abra2:1.0.9'
   }
   withLabel: profiling_qc {
-    container = 'goalconsortium/profiling_qc:1.0.9'
+    container = 'goalconsortium/profiling_qc:1.1.3'
   }
   withLabel: dnaalign {
-     container = 'goalconsortium/dna_alignment:1.0.9'
+     container = 'goalconsortium/dna_alignment:1.1.3'
   }
   withLabel: variantcalling  {
-    container = 'goalconsortium/variantcalling:1.0.9'
+    container = 'goalconsortium/variantcalling:1.1.5'
   }
   withLabel: structuralvariant {
-     container = 'goalconsortium/structuralvariant:1.1.2'
+     container = 'goalconsortium/structuralvariant:1.1.4'
   }
   withLabel: starfusion {
     container = 'goalconsortium/starfusion:1.0.9'
@@ -39,13 +40,13 @@ process {
 
 singularity {
   enabled = true
-  runOptions='--no-home --cleanenv'
-  singularity.cacheDir="$PWD"
+  runOptions='--no-home --cleanenv --disable-cache'
+  cacheDir = "$baseDir/images/singularity"
 }
 
 trace {
   enabled = true
-  file = 'pipeline_trace.txt'
+  file = 'trace.txt'
   field = 'task_id,native_id,process,name,status,exit,submit,start,complete,duration,realtime'
 }
 

workflow/main.nf

@@ -23,9 +23,13 @@ indel="$params.genome/GoldIndels.vcf.gz"
 knownindel=file(indel)
 dbsnp=file(dbsnp)
 
-repoDir=workflow.projectDir
+
 if (params.repoDir) {
-   repoDir=params.repoDir
+  repoDir=params.repoDir
+} else if (workflow.projectDir) {
+  repoDir=workflow.projectDir
+} else {
+  repoDir="$baseDir"
 }
 
 // params genome is the directory
@@ -45,6 +49,8 @@ process checkdesignfile {
 	stdout spltnames
 	script:
 	"""
+  # convert dos \\r to unix format
+  sed -i 's/\\x0D\$//' design.ori.txt
 	bash $repoDir/process_scripts/design_file/checkdesignfile.sh ${params.pairs} design.ori.txt
 	"""
 }
@@ -90,12 +96,15 @@ process ralign {
   errorStrategy 'ignore'
   label 'ralign'
   publishDir "$params.output", mode: 'copy'
+  
   input:
   set pair_id, file(fqs) from trimread
+  
   output:
   set pair_id, file("${pair_id}.bam") into aligned
   set pair_id, file("${pair_id}.bam") into aligned2
   file("${pair_id}.alignerout.txt") into hsatout
+  
   script:
   """
   bash $repoDir/process_scripts/alignment/rnaseqalign.sh -a $params.align -p ${pair_id} -r ${index_path} ${fqs}

process_scripts @ e5e905a0

+Subproject commit e5e905a0ada2d2a93ff0ed59b2fdf022a56c7464