main.nf 7.9 KB
Newer Older
Gervaise Henry's avatar
Gervaise Henry committed
1
#!/usr/bin/env nextflow
Gervaise Henry's avatar
Gervaise Henry committed
2
3
4
5
6
7
8
9
/*
main.nf
*
* --------------------------------------------------------------------------
* Licensed under MIT (https://git.biohpc.swmed.edu/BICF/Astrocyte/cellranger_mkfastq/blob/develop/LICENSE)
* --------------------------------------------------------------------------
*
*/
Gervaise Henry's avatar
Gervaise Henry committed
10

11
12
13
14
15
16
17
18
//  ########  ####  ######  ######## 
//  ##     ##  ##  ##    ## ##       
//  ##     ##  ##  ##       ##       
//  ########   ##  ##       ######   
//  ##     ##  ##  ##       ##       
//  ##     ##  ##  ##    ## ##       
//  ########  ####  ######  ##       

Gervaise Henry's avatar
Gervaise Henry committed
19
// Define input variables
20
params.name = "run"
21
22
params.bcl = "${baseDir}/../test_data/simple1/*.tar.gz"
params.designFile = "${baseDir}/../test_data/single1/cellranger-tiny-bcl-simple-1_2_0.csv"
23
params.mask = ""
Gervaise Henry's avatar
Gervaise Henry committed
24
params.astrocyte = false
Holly Ruess's avatar
Holly Ruess committed
25
26
params.outDir = "${baseDir}/output"

Gervaise Henry's avatar
Gervaise Henry committed
27
// Define list of files
Gervaise Henry's avatar
Gervaise Henry committed
28
29
tarList = Channel
  .fromPath( params.bcl )
30
bclCount = Channel
Gervaise Henry's avatar
Gervaise Henry committed
31
32
  .fromPath( params.bcl )
  .count()
Gervaise Henry's avatar
Gervaise Henry committed
33
34

// Define regular variables
Gervaise Henry's avatar
Gervaise Henry committed
35
pipelineVersion = "2.2.0-indev"
Gervaise Henry's avatar
Gervaise Henry committed
36
name = params.name
37
38
39
designLocation = Channel
  .fromPath(params.designFile)
  .ifEmpty { exit 1, "design file not found: ${params.designFile}" }
40
mask = params.mask
41
outDir = params.outDir
Gervaise Henry's avatar
Gervaise Henry committed
42

Gervaise Henry's avatar
Gervaise Henry committed
43
44
45
46
47
// Define script files
check_designScript = Channel.fromPath("$baseDir/scripts/check_design.py")
untarBCLScript = Channel.fromPath("$baseDir/scripts/untarBCL.sh")
countDesignScript = Channel.fromPath("$baseDir/scripts/countDesign.sh")
fastqcScript = Channel.fromPath("$baseDir/scripts/fastqc.sh")
48
49
50
versionsScript = Channel.fromPath("$baseDir/scripts/generate_versions.py")
referencesScript = Channel.fromPath("$baseDir/scripts/generate_references.py")
versions_pythonScript = Channel.fromPath("$baseDir/scripts/versions_python.sh")
Gervaise Henry's avatar
Gervaise Henry committed
51
//versions_pigzScript = Channel.fromPath("$baseDir/scripts/versions_pigz.sh")
52
53
54
55
56
versions_cellrangerScript = Channel.fromPath("$baseDir/scripts/versions_cellranger.sh")
versions_bcl2fastqScript = Channel.fromPath("$baseDir/scripts/versions_bcl2fastq.sh")
versions_fastqcScript = Channel.fromPath("$baseDir/scripts/versions_fastqc.sh")

// Define report files
Gervaise Henry's avatar
Gervaise Henry committed
57
multiqcConf = Channel.fromPath("${baseDir}/configs/multiqc_config.yaml")
58
59
references = Channel.fromPath("${baseDir}/../docs/references.md")

Gervaise Henry's avatar
Gervaise Henry committed
60

Gervaise Henry's avatar
Gervaise Henry committed
61
62
63
64
/*
 * trackStart: track start of pipeline
 */
params.ci = false
65
params.dev = false
Gervaise Henry's avatar
Gervaise Henry committed
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
process trackStart {
  script:
  """
  hostname
  ulimit -a
  export https_proxy=\${http_proxy}
 
  curl -H 'Content-Type: application/json' -X PUT -d '{ \
      "sessionId": "${workflow.sessionId}", \
      "pipeline": "cellranger_mkfastq", \
      "pipelineVersion": "${pipelineVersion}", \
      "start": "${workflow.start}", \
      "astrocyte": ${params.astrocyte}, \
      "status": "started", \
      "nextflowVersion": "${workflow.nextflow.version}",
81
      "ci": ${params.ci},
Gervaise Henry's avatar
Gervaise Henry committed
82
      "dev": ${params.dev}}' \
Gervaise Henry's avatar
Gervaise Henry committed
83
84
85
86
87
  "https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking"
  """
}


Gervaise Henry's avatar
Gervaise Henry committed
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
log.info """\
  
  BICF cellranger_mkfastq Pipeline
  ================================
  Run name        : ${params.name}
  bcl             : ${params.bcl}
  Design File     : ${params.designFile}
  Output Directory: ${params.outDir}
  --------------------------------
  Nextflow Version: ${workflow.nextflow.version}
  Pipeline Version: ${pipelineVersion}
  Session ID      : ${workflow.sessionId}
  --------------------------------
  Astrocyte      : ${params.astrocyte}
  CI             : ${params.ci}
  Development    : ${params.dev}
  """
Holly Ruess's avatar
Holly Ruess committed
105

Gervaise Henry's avatar
Gervaise Henry committed
106
process checkDesignFile {
Holly Ruess's avatar
Holly Ruess committed
107
  tag "${name}"
108
  module 'python/3.6.1-2-anaconda'
Gervaise Henry's avatar
Gervaise Henry committed
109
110

  input:
Gervaise Henry's avatar
Gervaise Henry committed
111
    file check_designScript
112
    file versions_pythonScript
Holly Ruess's avatar
Holly Ruess committed
113
    file designLocation
Gervaise Henry's avatar
Gervaise Henry committed
114
115

  output:
Holly Ruess's avatar
Holly Ruess committed
116
117
    file("design.checked.csv") into designPaths
    file("design.checked.csv") into designCount
118
119
    file("version_pipeline.txt") into version_pipeline
    file("version_nextflow.txt") into version_nextflow
120
    file("version_python.txt") into version_python
Gervaise Henry's avatar
Gervaise Henry committed
121
122

  script:
Holly Ruess's avatar
Holly Ruess committed
123
124
125
    """
    hostname
    ulimit -a
126
127
128
129
    noSpaceDesign=\$(echo "${designLocation}" | tr -d ' ')
    if [[ "\${noSpaceDesign}" != "${designLocation}" ]]; then
      mv "${designLocation}" "\${noSpaceDesign}"
    fi
Gervaise Henry's avatar
Gervaise Henry committed
130
    python3 check_design.py -d \${noSpaceDesign}
131
    bash versions_python.sh > version_python.txt
132
    echo "${workflow.nextflow.version}" > version_nextflow.txt
133
    echo "${pipelineVersion}" > version_pipeline.txt
Holly Ruess's avatar
Holly Ruess committed
134
    """
Gervaise Henry's avatar
Gervaise Henry committed
135
}
136
137
138
/* nextflow workflow manifest version calls that aren't compatible with Asrcocyte
    echo "${workflow.manifest.version}" > version_pipeline.txt
*/
Gervaise Henry's avatar
Gervaise Henry committed
139

140
141
process untarBCL {  
  tag "${tar.simpleName}"
142
  module 'pigz/2.4'
Gervaise Henry's avatar
Gervaise Henry committed
143
144

  input:
Gervaise Henry's avatar
Gervaise Henry committed
145
    file untarBCLScript
Gervaise Henry's avatar
Gervaise Henry committed
146
    //file versions_pigzScript
147
    each file(tar) from tarList
Gervaise Henry's avatar
Gervaise Henry committed
148
149

  output:
150
    file("*[!version_pigz.txt]") into bclPaths mode flatten
Gervaise Henry's avatar
Gervaise Henry committed
151
    //file("version_pigz.txt") into version_pigz
Gervaise Henry's avatar
Gervaise Henry committed
152
153

  script:
Holly Ruess's avatar
Holly Ruess committed
154
155
    """
    hostname
156
    ulimit -u 16384
Holly Ruess's avatar
Holly Ruess committed
157
    ulimit -a
158
    bash untarBCL.sh -t ${tar}
Gervaise Henry's avatar
Gervaise Henry committed
159
    #bash versions_pigz.sh > version_pigz.txt
Holly Ruess's avatar
Holly Ruess committed
160
161
    """

Gervaise Henry's avatar
Gervaise Henry committed
162
163
}

Gervaise Henry's avatar
Gervaise Henry committed
164

Gervaise Henry's avatar
Gervaise Henry committed
165
process mkfastq {
166
  tag "${bcl.simpleName}"
Holly Ruess's avatar
Holly Ruess committed
167
  publishDir "${outDir}/${task.process}", mode: 'copy', pattern: "{*/outs/**/*.fastq.gz}"
168
  queue '128GB,256GB,256GBv1,384GB'
Gervaise Henry's avatar
Gervaise Henry committed
169
  module 'cellranger/4.0.0:bcl2fastq/2.19.1'
Gervaise Henry's avatar
Gervaise Henry committed
170
171

  input:
172
173
    file versions_cellrangerScript
    file versions_bcl2fastqScript
174
    each file(bcl) from bclPaths.collect()
Holly Ruess's avatar
Holly Ruess committed
175
    file design from designPaths
Gervaise Henry's avatar
Gervaise Henry committed
176
177

  output:
178
179
180
    file("fq/${bcl.simpleName}/*.fastq.gz") into fastqPaths
    val "${bcl.simpleName}" into bclName
    file("**/outs/**/*.fastq.gz") into cellrangerCount mode flatten
Gervaise Henry's avatar
Gervaise Henry committed
181
    file("**/outs/fastq_path/Stats/*") into bqcPaths
182
183
    file("version_cellranger.txt") into version_cellranger
    file("version_bcl2fastq.txt") into version_bcl2fastq
Gervaise Henry's avatar
Gervaise Henry committed
184
185

  script:
Holly Ruess's avatar
Holly Ruess committed
186
187
    """
    hostname
188
189
    ulimit -u 16384
    ulimit -a
Gervaise Henry's avatar
Gervaise Henry committed
190
    cellranger mkfastq --id=mkfastq_${bcl.simpleName} --run=${bcl} --csv=${design} ${mask}
191
192
193
194
195
    mkdir fq
    mkdir "fq/${bcl.simpleName}"
    find . -name "*.fastq.gz" -exec cp {} fq/${bcl.simpleName}/ \\;
    bash versions_cellranger.sh > version_cellranger.txt
    bash versions_bcl2fastq.sh > version_bcl2fastq.txt
Holly Ruess's avatar
Holly Ruess committed
196
    """
Gervaise Henry's avatar
Gervaise Henry committed
197
}
Gervaise Henry's avatar
Gervaise Henry committed
198

Gervaise Henry's avatar
Gervaise Henry committed
199

200
201
if (bclCount.value == 1) {
  process countDesign {
Holly Ruess's avatar
Holly Ruess committed
202
    tag "${name}"
203
    publishDir "${outDir}/${task.process}/${name}", mode: 'copy'
204
205

    input:
Gervaise Henry's avatar
Gervaise Henry committed
206
      file countDesignScript
Holly Ruess's avatar
Holly Ruess committed
207
208
      file fastqs from cellrangerCount.collect()
      file design from designCount
209
210

    output:
Holly Ruess's avatar
Holly Ruess committed
211
      file("Cellranger_Count_Design.csv") into CountDesign
212
213

    script:
Holly Ruess's avatar
Holly Ruess committed
214
      """
215
216
      hostname
      ulimit -a
Gervaise Henry's avatar
Gervaise Henry committed
217
      bash countDesign.sh
Holly Ruess's avatar
Holly Ruess committed
218
      """
219
220
221
  }
}

Gervaise Henry's avatar
Gervaise Henry committed
222

223
process fastqc {
224
  tag "${bcl}"
225
  module 'fastqc/0.11.5:parallel'
226
227

  input:
228
229
230
231
    file fastqcScript    
    file versions_fastqcScript
    file(fastq) from fastqPaths
    each bcl from bclName
232
233

  output:
234
235
    file("*fastqc.zip") into fqcPaths mode flatten
    file("version_fastqc.txt") into version_fastqc
236
237

  script:
Holly Ruess's avatar
Holly Ruess committed
238
239
    """
    hostname
240
    ulimit -u 16384
Holly Ruess's avatar
Holly Ruess committed
241
    ulimit -a
242
    find *.fastq.gz -exec mv {} ${bcl}.{} \\;
Gervaise Henry's avatar
Gervaise Henry committed
243
    bash fastqc.sh
244
    bash versions_fastqc.sh > version_fastqc.txt
Holly Ruess's avatar
Holly Ruess committed
245
    """
246
247
}

248

249
process versions {
Holly Ruess's avatar
Holly Ruess committed
250
  tag "${name}"
251
  module 'python/3.6.1-2-anaconda:cellranger/3.1.0:bcl2fastq/2.19.1:fastqc/0.11.5:pandoc/2.7'
252
253

  input:
254
255
  file versionsScript
  file referencesScript
256
257
  file version_pipeline
  file version_nextflow
258
  file version_python
Gervaise Henry's avatar
Gervaise Henry committed
259
  //file version_pigz
260
261
262
263
  file version_cellranger
  file version_bcl2fastq
  file version_fastqc
  file references
264
265

  output:
Holly Ruess's avatar
Holly Ruess committed
266
    file("*.yaml") into yamlPaths
267
268

  script:
Holly Ruess's avatar
Holly Ruess committed
269
270
271
    """
    hostname
    ulimit -a
272
273
    python3 generate_versions.py -f version_*.txt -o versions
    python3 generate_references.py -r ${references} -o references
Holly Ruess's avatar
Holly Ruess committed
274
    """
275
276
}

Gervaise Henry's avatar
Gervaise Henry committed
277

Gervaise Henry's avatar
Gervaise Henry committed
278
process multiqc {
Holly Ruess's avatar
Holly Ruess committed
279
280
  tag "${name}"
  publishDir "${outDir}/${task.process}/${name}", mode: 'copy', pattern: "{multiqc*}"
281
  module 'multiqc/1.7'
Gervaise Henry's avatar
Gervaise Henry committed
282
283

  input:
284
    file multiqcConf
Holly Ruess's avatar
Holly Ruess committed
285
    file bqc name "bqc/?/*" from bqcPaths.collect()
286
    file fqc name "fqc/?/*" from fqcPaths.collect()
Holly Ruess's avatar
Holly Ruess committed
287
    file yamlPaths
Gervaise Henry's avatar
Gervaise Henry committed
288
289

  output:
Holly Ruess's avatar
Holly Ruess committed
290
    file("multiqc_report.html") into mqcPaths
Gervaise Henry's avatar
Gervaise Henry committed
291
292

  script:
Holly Ruess's avatar
Holly Ruess committed
293
294
295
    """
    hostname
    ulimit -a
296
297
    #export LC_ALL=C.UTF-8
    #export LANG=C.UTF-8
Holly Ruess's avatar
Holly Ruess committed
298
299
    multiqc -c ${multiqcConf} .
    """
Gervaise Henry's avatar
Gervaise Henry committed
300
}