From 9295191553830818fd1bf0cfbf7188c320cf5adc Mon Sep 17 00:00:00 2001
From: "Gervaise H. Henry" <gervaise.henry@utsouthwestern.edu>
Date: Tue, 14 Jan 2020 16:17:23 -0600
Subject: [PATCH] Update nextflow configs to include aws
---
workflow/conf/aws_ondemand.config | 26 ++++++++++++++++++
workflow/conf/aws_spot.config | 26 ++++++++++++++++++
workflow/conf/biohpc.config | 44 +------------------------------
workflow/nextflow.config | 35 ++++++++++++++++++++++++
4 files changed, 88 insertions(+), 43 deletions(-)
create mode 100755 workflow/conf/aws_ondemand.config
create mode 100755 workflow/conf/aws_spot.config
diff --git a/workflow/conf/aws_ondemand.config b/workflow/conf/aws_ondemand.config
new file mode 100755
index 0000000..4c27b6b
--- /dev/null
+++ b/workflow/conf/aws_ondemand.config
@@ -0,0 +1,26 @@
+workDir = 's3://gudmap.rbk/work'
+aws.client.storageEncryption = 'AES256'
+aws {
+ region = 'us-east-2'
+ batch {
+ cliPath = '/home/ec2-user/miniconda/bin/aws'
+ }
+}
+
+process {
+ executor = 'awsbatch'
+ queue = 'highpriority-3278a8b0-1fc8-11ea-b1ac-021e2396e2cc'
+ cpus = 1
+ memory = '1 GB'
+
+ withName:getBag {
+ container = 'bicf/gudmaprbkfilexfer:1.1'
+ }
+ withName:getData {
+ container = 'bicf/gudmaprbkfilexfer:1.1'
+ }
+ withName:trimData {
+ container = 'bicf/trimgalore:1.1'
+ cpus = 15
+ }
+}
\ No newline at end of file
diff --git a/workflow/conf/aws_spot.config b/workflow/conf/aws_spot.config
new file mode 100755
index 0000000..33b2561
--- /dev/null
+++ b/workflow/conf/aws_spot.config
@@ -0,0 +1,26 @@
+workDir = 's3://gudmap.rbk/work'
+aws.client.storageEncryption = 'AES256'
+aws {
+ region = 'us-east-2'
+ batch {
+ cliPath = '/home/ec2-user/miniconda/bin/aws'
+ }
+}
+
+process {
+ executor = 'awsbatch'
+ queue = 'default-3278a8b0-1fc8-11ea-b1ac-021e2396e2cc'
+ cpus = 1
+ memory = '1 GB'
+
+ withName:getBag {
+ container = 'bicf/gudmaprbkfilexfer:1.1'
+ }
+ withName:getData {
+ container = 'bicf/gudmaprbkfilexfer:1.1'
+ }
+ withName:trimData {
+ container = 'bicf/trimgalore:1.1'
+ cpus = 15
+ }
+}
diff --git a/workflow/conf/biohpc.config b/workflow/conf/biohpc.config
index 0fea2a6..2990fe6 100755
--- a/workflow/conf/biohpc.config
+++ b/workflow/conf/biohpc.config
@@ -3,7 +3,6 @@ process {
queue = 'super'
clusterOptions = '--hold'
- // Process specific configuration
withName:getBag {
executor = 'local'
container = 'docker://bicf/gudmaprbkfilexfer:1.1'
@@ -16,45 +15,4 @@ process {
container = 'docker://bicf/trimgalore:1.1'
queue = '256GB,256GBv1,384GB'
}
-}
-
-
-trace {
- enabled = true
- file = 'pipeline_trace.txt'
- fields = 'task_id,native_id,process,name,status,exit,submit,start,complete,duration,realtime,%cpu,%mem,rss'
-}
-
-timeline {
- enabled = true
- file = 'timeline.html'
-}
-
-report {
- enabled = true
- file = 'report.html'
-}
-
-tower {
- accessToken = '3ade8f325d4855434b49aa387421a44c63e3360f'
- enabled = true
-}
-
-singularity {
- enabled = true
- cacheDir = '/project/shared/bicf_workflow_ref/singularity_images/'
-}
-
-env {
- http_proxy = 'http://proxy.swmed.edu:3128'
- https_proxy = 'http://proxy.swmed.edu:3128'
- all_proxy = 'http://proxy.swmed.edu:3128'
-}
-
-manifest {
- homePage = 'https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq'
- description = 'This pipeline was created to analize RNA-sequencing data from GUDMAP/RBK consortiums.'
- mainScript = 'rna-seq.nf'
- version = 'v0.0.1_indev'
- nextflowVersion = '>=19.09.0'
-}
+}
\ No newline at end of file
diff --git a/workflow/nextflow.config b/workflow/nextflow.config
index 30e47ea..3758499 100644
--- a/workflow/nextflow.config
+++ b/workflow/nextflow.config
@@ -2,4 +2,39 @@ profiles {
standard {
includeConfig 'conf/biohpc.config'
}
+ aws_ondemand {
+ includeConfig 'conf/aws_ondemand.config'
+ }
+ aws_spot {
+ includeConfig 'conf/aws_spot.config'
+ }
+}
+
+trace {
+ enabled = true
+ file = 'pipeline_trace.txt'
+ fields = 'task_id,native_id,process,name,status,exit,submit,start,complete,duration,realtime,%cpu,%mem,rss'
+}
+
+timeline {
+ enabled = true
+ file = 'timeline.html'
+}
+
+report {
+ enabled = true
+ file = 'report.html'
+}
+
+tower {
+ accessToken = '3ade8f325d4855434b49aa387421a44c63e3360f'
+ enabled = true
}
+
+manifest {
+ homePage = 'https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq'
+ description = 'This pipeline was created to be a standard mRNA-sequencing analysis pipeline which integrates with the GUDMAP and RBK consortium data-hub.'
+ mainScript = 'rna-seq.nf'
+ version = 'v0.0.1_indev'
+ nextflowVersion = '>=19.09.0'
+}
\ No newline at end of file
--
GitLab