Layer configs

.gitlab-ci.yml

@@ -27,7 +27,7 @@ astrocyte_check:
 simple_1FC:
   stage: simple
   script:
-    - nextflow run workflow/main.nf --bcl "test_data/simple1/*.tar.gz" --designFile "test_data/simple1/cellranger-tiny-bcl-simple-1_2_0.csv"
+    - nextflow run workflow/main.nf -config biohpc,cluster --bcl "test_data/simple1/*.tar.gz" --designFile "test_data/simple1/cellranger-tiny-bcl-simple-1_2_0.csv" --ci true
     ##- pytest -m simple1
   artifacts:
     name: "$CI_JOB_NAME"
@@ -44,7 +44,7 @@ simple_1FC:
 simple_2FC:
   stage: simple
   script:
-    - nextflow run workflow/main.nf --bcl "test_data/simple2/*.tar.gz" --designFile "test_data/simple2/cellranger-tiny-bcl-simple-1_2_0.csv"
+    - nextflow run workflow/main.nf -config biohpc,cluster --bcl "test_data/simple2/*.tar.gz" --designFile "test_data/simple2/cellranger-tiny-bcl-simple-1_2_0.csv" --ci true
     ##- pytest -m simple2
   artifacts:
     name: "$CI_JOB_NAME"

workflow/configs/aws_spot.config → workflow/configs/aws.config

@@ -8,9 +8,8 @@ aws {
 }
 
 process {
-  executor = 'awsbatch'
-  queue = 'default-3278a8b0-1fc8-11ea-b1ac-021e2396e2cc'
-  container = 'bicf/bicfbase:1.4'
+  executor = 'awsbatch'=
+  container = 'bicf/bicfbase:2.0.0'
   cpus = 1
   memory = '1 GB'
 

workflow/configs/aws_ondemand.config

-workDir = 's3://gudmap.rbk/work'
-aws.client.storageEncryption = 'AES256'
-aws {
-  region = 'us-east-2'
-  batch {
-    cliPath = '/home/ec2-user/miniconda/bin/aws'
-  }
-}
-
-process {
-  executor = 'awsbatch'
-  queue = 'highpriority-3278a8b0-1fc8-11ea-b1ac-021e2396e2cc'
-  container = 'bicf/bicfbase:1.4'
-  cpus = 1
-  memory = '1 GB'
-
-  withName:checkDesignFile {
-    container = 'bicf/python3:1.3'
-    cpus = 4
-  }
-  withName:mkfastq {
-    container = 'bicf/cellranger3.1.0:1.0'
-    cpus = 6
-    memory = '2 GB'
-  }
-  withName:fastqc {
-    container = 'bicf/fastqc:1.5'
-    cpus = 30
-    memory = '15 GB'
-  }
-  withName:versions {
-    container = 'bicf/python3:1.3'
-    cpus = 6
-  }
-  withName:multiqc {
-    container = 'bicf/multiqc:1.4'
-  }
-}
\ No newline at end of file

workflow/configs/biohpc.config

 process {
-  executor = 'slurm'
-  queue = 'super'
-  clusterOptions = '--hold'
-  container = 'docker://bicf/bicfbase:1.4'
+  container = 'docker://bicf/bicfbase:2.0.0'
 
-
-  withName:trackStart {
-    executor = 'local'
-  }
   withName:checkDesignFile {
     container = 'docker://bicf/python3:1.3'
-    executor = 'local'
-  }
-  withName:untarBCL {
-    queue = 'super'
   }
   withName:mkfastq {
     container = 'docker://bicf/cellranger3.1.0:1.0'
-    queue = '128GB,256GB,256GBv1,384GB'
-  }
-  withName:countDesign {
-    executor = 'local'
   }
   withName:fastqc {
     container = 'docker://bicf/fastqc:1.5'
-    queue = 'super'
   }
   withName:versions {
     container = 'docker://bicf/python3:1.3'
-    executor = 'local'
   }
   withName:multiqc {
     container = 'docker://bicf/multiqc:1.4'
-    executor = 'local'
   }
 }
 

workflow/configs/biohpc_local.config

-process {
-  executor = 'local'
-  container = 'docker://bicf/bicfbase:1.3'
-
-  withName:checkDesignFile {
-    container = 'docker://bicf/python3:1.3'
-  }
-  withName:mkfastq {
-    container = 'docker://bicf/cellranger3.1.0:1.0'
-  }
-  withName:fastqc {
-    container = 'docker://bicf/fastqc:1.5'
-  }
-  withName:versions {
-    container = 'docker://bicf/python3:1.3'
-  }
-  withName:multiqc {
-    container = 'docker://bicf/multiqc:1.4'
-  }
-}
-
-singularity {
-  enabled = true
-  cacheDir = '/project/shared/bicf_workflow_ref/singularity_images/'
-}
-
-env {
-  http_proxy = 'http://proxy.swmed.edu:3128'
-  https_proxy = 'http://proxy.swmed.edu:3128'
-  all_proxy = 'http://proxy.swmed.edu:3128'
-}
\ No newline at end of file

workflow/configs/cluster.config

+process {
+  executor = 'slurm'
+  queue = 'super'
+  clusterOptions = '--hold'
+
+  withName:trackStart {
+    executor = 'local'
+  }
+  withName:checkDesignFile {
+    executor = 'local'
+  }
+  withName:untarBCL {
+    queue = 'super'
+  }
+  withName:mkfastq {
+    queue = '128GB,256GB,256GBv1,384GB'
+  }
+  withName:countDesign {
+    executor = 'local'
+  }
+  withName:fastqc {
+    queue = 'super'
+  }
+  withName:versions {
+    executor = 'local'
+  }
+  withName:multiqc {
+    executor = 'local'
+  }
+}

workflow/configs/local.config

+process {
+  executor = 'local'
+}

workflow/configs/ondemand.config

+process {
+  queue = 'highpriority-3278a8b0-1fc8-11ea-b1ac-021e2396e2cc'
+}

workflow/configs/spot.config

+process {
+  queue = 'default-3278a8b0-1fc8-11ea-b1ac-021e2396e2cc'
+}

workflow/nextflow.config

@@ -2,14 +2,20 @@ profiles {
   standard {
     includeConfig 'configs/biohpc.config'
   }
-  biohpc_local {
-    includeConfig 'configs/biohpc_local.config'
+  local {
+    includeConfig 'configs/local.config'
   }
-  aws_ondemand {
-    includeConfig 'configs/aws_ondemand.config'
+  local {
+    includeConfig 'configs/cluster.config'
   }
-  aws_spot {
-    includeConfig 'configs/aws_spot.config'
+  aws {
+    includeConfig 'configs/aws.config'
+  }
+  ondemand {
+    includeConfig 'configs/ondemand.config'
+  }
+  spot {
+    includeConfig 'configs/spot.config'
   }
 }