Merge branch '40-BUG.thread.error' into 'develop'

Resolve "BUG: thread error on Astrocyte"

Closes #39 and #40

See merge request !59

.gitlab-ci.yml

@@ -40,7 +40,7 @@ simple_1:
       - develop
       - master
   script:
-  - nextflow run workflow/main.nf --fastq "test_data/hu.v3s1r500/*.fastq.gz" --designFile "test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.1.0' -with-tower
+  - nextflow run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s1r500/*.fastq.gz" --designFile "test_data/hu.v3s1r500/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'three' --version '3.1.0' -with-tower
   - pytest -m count310
   artifacts:
     name: "$CI_JOB_NAME"
@@ -65,7 +65,7 @@ simple_2:
       - master
       - tags
   script:
-  - nextflow run workflow/main.nf --fastq "test_data/mu.v3s1r500/*.fastq.gz" --designFile "test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.1.0' -with-tower
+  - nextflow run workflow/main.nf -profile biohpc,cluster --fastq "test_data/mu.v3s1r500/*.fastq.gz" --designFile "test_data/mu.v3s1r500/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.1.0' -with-tower
   - pytest -m count310
   artifacts:
     name: "$CI_JOB_NAME"
@@ -89,7 +89,7 @@ detailed_1:
     refs:
       - tags
   script:
-  - nextflow run workflow/main.nf --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0' -with-tower
+  - nextflow run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.1.0' -with-tower
   - pytest -m count310
   artifacts:
     name: "$CI_JOB_NAME"
@@ -114,7 +114,7 @@ detailed_2:
     refs:
       - tags
   script:
-  - nextflow run workflow/main.nf --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.0.2' -with-tower
+  - nextflow run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v3s2r10k/*.fastq.gz" --designFile "test_data/hu.v3s2r10k/design.csv" --genome 'GRCh38-3.0.0' --kitVersion 'auto' --version '3.0.2' -with-tower
   - pytest -m count302
   artifacts:
     name: "$CI_JOB_NAME"
@@ -138,7 +138,7 @@ detailed_3:
     refs:
       - tags
   script:
-  - nextflow run workflow/main.nf --fastq "test_data/mu.v3s2r10k/*.fastq.gz" --designFile "test_data/mu.v3s2r10k/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.1' -with-tower
+  - nextflow run workflow/main.nf -profile biohpc,cluster --fastq "test_data/mu.v3s2r10k/*.fastq.gz" --designFile "test_data/mu.v3s2r10k/design.csv" --genome 'mm10-3.0.0' --kitVersion 'three' --version '3.0.1' -with-tower
   - pytest -m count301
   artifacts:
     name: "$CI_JOB_NAME"
@@ -162,7 +162,7 @@ detailed_4:
     refs:
       - tags
   script:
-  - nextflow run workflow/main.nf --fastq "test_data/hu.v2s2r10k/*.fastq.gz" --designFile "test_data/hu.v2s2r10k/design.csv" --genome 'GRCh38-1.2.0' --kitVersion 'two' --version '2.1.1' -with-tower
+  - nextflow run workflow/main.nf -profile biohpc,cluster --fastq "test_data/hu.v2s2r10k/*.fastq.gz" --designFile "test_data/hu.v2s2r10k/design.csv" --genome 'GRCh38-1.2.0' --kitVersion 'two' --version '2.1.1' -with-tower
   - pytest -m count211
   artifacts:
     name: "$CI_JOB_NAME"

CHANGELOG.md

-# v1.3.0 (in development)
+# v1.2.1 (in development)
 **User Facing**
 * Check Design File for spaces in name and file contents
+* Attempt to preven thredding error (which appears to only happen on 256GBv1 nodes)
 
 **Background**
 * Add Nextflow Tower integration into CI (GHH's profile)
+* Add new layered config folders, including prepare for awsifying
 
 *Known Bugs*
 

nextflow.config

-profiles {
-  standard {
-    includeConfig 'workflow/conf/biohpc.config'
-  }
-}

workflow/conf/aws.config

+workDir = 's3://'
+aws.client.storageEncryption = 'AES256'
+aws {
+  region = ''
+  batch {
+    cliPath = '/home/ec2-user/miniconda/bin/aws'
+  }
+}
+
+process {
+  executor = 'awsbatch'
+  queue = 'default-'
+  cpus = 1
+  memory = '10 GB'
+}
\ No newline at end of file

workflow/conf/biohpc.config

-process {
-  executor = 'slurm'
-  queue = 'super'
-  clusterOptions = '--hold'
-
-  withLabel: checkDesignFile {
-    module = ['python/3.6.1-2-anaconda']
-    executor = 'local'
-  }
-  withLabel: count211 {
-    module = ['cellranger/2.1.1']
-    queue = '128GB,256GB,256GBv1,384GB'
-  }
-  withLabel: count301 {
-    module = ['cellranger/3.0.1']
-    queue = '128GB,256GB,256GBv1,384GB'
-  }
-  withLabel: count302 {
-    module = ['cellranger/3.0.2']
-    queue = '128GB,256GB,256GBv1,384GB'
-  }
-  withLabel: count310 {
-    module = ['cellranger/3.1.0']
-    queue = '128GB,256GB,256GBv1,384GB'
-  }
-  withLabel: versions {
-    module = ['python/3.6.1-2-anaconda','pandoc/2.7','multiqc/1.7']
-    executor = 'local'
-  }
-  withLabel: multiqc {
-    module = ['multiqc/1.7']
-    executor = 'local'
-  }
-}
-
 params {
   // Reference file paths on BioHPC
   genomes {
@@ -82,25 +47,4 @@ params {
       param = 'SC3Pv3'
     }
   }
-}
-
-trace {
-  enabled = true
-  file = 'pipeline_trace.txt'
-  fields = 'task_id,native_id,process,name,status,exit,submit,start,complete,duration,realtime,%cpu,%mem,rss'
-}
-
-timeline {
-  enabled = true
-  file = 'timeline.html'
-}
-
-report {
-  enabled = true
-  file = 'report.html'
-}
-
-tower {
-  accessToken = '3ade8f325d4855434b49aa387421a44c63e3360f'
-  enabled = true
-}
+}
\ No newline at end of file

workflow/conf/biohpc_cluster.config

+process {
+  executor = 'slurm'
+  queue = 'super'
+  clusterOptions = '--hold'
+
+  withLabel: checkDesignFile {
+    module = ['python/3.6.1-2-anaconda']
+    executor = 'local'
+  }
+  withLabel: count211 {
+    module = ['cellranger/2.1.1']
+    queue = '128GB,256GB,256GBv1,384GB'
+  }
+  withLabel: count301 {
+    module = ['cellranger/3.0.1']
+    queue = '128GB,256GB,256GBv1,384GB'
+  }
+  withLabel: count302 {
+    module = ['cellranger/3.0.2']
+    queue = '128GB,256GB,256GBv1,384GB'
+  }
+  withLabel: count310 {
+    module = ['cellranger/3.1.0']
+    queue = '128GB,256GB,256GBv1,384GB'
+  }
+  withLabel: versions {
+    module = ['python/3.6.1-2-anaconda','pandoc/2.7','multiqc/1.7']
+    executor = 'local'
+  }
+  withLabel: multiqc {
+    module = ['multiqc/1.7']
+    executor = 'local'
+  }
+}
\ No newline at end of file

workflow/conf/biohpc_local.config

+process {
+  executor = 'local'
+
+  withLabel: checkDesignFile {
+    module = ['python/3.6.1-2-anaconda']
+  }
+  withLabel: count211 {
+    module = ['cellranger/2.1.1']
+  }
+  withLabel: count301 {
+    module = ['cellranger/3.0.1']
+  }
+  withLabel: count302 {
+    module = ['cellranger/3.0.2']
+  }
+  withLabel: count310 {
+    module = ['cellranger/3.1.0']
+  }
+  withLabel: versions {
+    module = ['python/3.6.1-2-anaconda','pandoc/2.7','multiqc/1.7']
+  }
+  withLabel: multiqc {
+    module = ['multiqc/1.7']
+  }
+}
\ No newline at end of file

workflow/main.nf

@@ -242,6 +242,7 @@ process count302 {
     if (forceCells302 == 0) {
       """
       hostname
+      ulimit -u 16384
       ulimit -a
       bash ${baseDir}/scripts/filename_check.sh -r ${ref}
       cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --expect-cells=${expectCells302} --chemistry=${chemistryParam302}
@@ -251,6 +252,7 @@ process count302 {
     else {
       """
       hostname
+      ulimit -u 16384
       ulimit -a
       bash ${baseDir}/scripts/filename_check.sh -r ${ref}
       cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells302} --chemistry=${chemistryParam302}
@@ -295,6 +297,7 @@ process count310 {
     else {
       """
       hostname
+      ulimit -u 16384
       ulimit -a
       bash ${baseDir}/scripts/filename_check.sh -r ${ref}
       cellranger count --id=${sample} --transcriptome=./${ref} --fastqs=. --sample=${sample} --force-cells=${forceCells310} --chemistry=${chemistryParam310}

workflow/nextflow.config

 profiles {
-  standard {
+  biohpc {
     includeConfig 'conf/biohpc.config'
   }
+  local {
+    includeConfig 'conf/biohpc_local.config'
+  }
+  cluster {
+    includeConfig 'conf/biohpc_cluster.config'
+  }
+  aws {
+    includeConfig 'conf/aws.config'
+  }
+}
+
+trace {
+  enabled = true
+  file = 'pipeline_trace.txt'
+  fields = 'task_id,native_id,process,name,status,exit,submit,start,complete,duration,realtime,%cpu,%mem,rss'
+}
+
+timeline {
+  enabled = true
+  file = 'timeline.html'
+}
+	
+report {
+  enabled = true
+  file = 'report.html'
+}
+
+tower {
+  accessToken = '3ade8f325d4855434b49aa387421a44c63e3360f'
+  enabled = true
 }
+
+manifest {
+  homePage = 'https://git.biohpc.swmed.edu/BICF/Astrocyte/cellranger_count'
+  description = 'This pipeline is a wrapper for the cellranger count tool from 10x Genomics. It takes fastq files from 10x Genomics Single Cell Gene Expression libraries, performs alignment, filtering, barcode counting, and UMI counting. It uses the Chromium cellular barcodes to generate gene-barcode matrices, determine clusters, and perform gene expression analysis.'
+  mainScript = 'main.nf'
+  version = 'v1.2.1_indev'
+  nextflowVersion = '>=0.31.0'
+}
\ No newline at end of file