main.nf 7.93 KB
Newer Older
Gervaise Henry's avatar
Gervaise Henry committed
1
#!/usr/bin/env nextflow
Gervaise Henry's avatar
Gervaise Henry committed
2
3
4
5
6
7
8
9
/*
main.nf
*
* --------------------------------------------------------------------------
* Licensed under MIT (https://git.biohpc.swmed.edu/BICF/Astrocyte/cellranger_mkfastq/blob/develop/LICENSE)
* --------------------------------------------------------------------------
*
*/
Gervaise Henry's avatar
Gervaise Henry committed
10

11
12
13
14
15
16
17
18
//  ########  ####  ######  ######## 
//  ##     ##  ##  ##    ## ##       
//  ##     ##  ##  ##       ##       
//  ########   ##  ##       ######   
//  ##     ##  ##  ##       ##       
//  ##     ##  ##  ##    ## ##       
//  ########  ####  ######  ##       

Gervaise Henry's avatar
Gervaise Henry committed
19
// Define input variables
20
params.name = "run"
21
22
params.bcl = "${baseDir}/../test_data/simple1/*.tar.gz"
params.designFile = "${baseDir}/../test_data/single1/cellranger-tiny-bcl-simple-1_2_0.csv"
23
params.mask = ""
Gervaise Henry's avatar
Gervaise Henry committed
24
params.astrocyte = false
Holly Ruess's avatar
Holly Ruess committed
25
26
params.outDir = "${baseDir}/output"

Gervaise Henry's avatar
Gervaise Henry committed
27
// Define list of files
Gervaise Henry's avatar
Gervaise Henry committed
28
29
tarList = Channel
  .fromPath( params.bcl )
30
bclCount = Channel
Gervaise Henry's avatar
Gervaise Henry committed
31
32
  .fromPath( params.bcl )
  .count()
Gervaise Henry's avatar
Gervaise Henry committed
33
34

// Define regular variables
Gervaise Henry's avatar
Gervaise Henry committed
35
pipelineVersion = "2.2.0-indev"
Gervaise Henry's avatar
Gervaise Henry committed
36
name = params.name
37
38
39
designLocation = Channel
  .fromPath(params.designFile)
  .ifEmpty { exit 1, "design file not found: ${params.designFile}" }
40
mask = params.mask
41
outDir = params.outDir
Gervaise Henry's avatar
Gervaise Henry committed
42

Gervaise Henry's avatar
Gervaise Henry committed
43
44
45
46
47
// Define script files
check_designScript = Channel.fromPath("$baseDir/scripts/check_design.py")
untarBCLScript = Channel.fromPath("$baseDir/scripts/untarBCL.sh")
countDesignScript = Channel.fromPath("$baseDir/scripts/countDesign.sh")
fastqcScript = Channel.fromPath("$baseDir/scripts/fastqc.sh")
48
49
50
versionsScript = Channel.fromPath("$baseDir/scripts/generate_versions.py")
referencesScript = Channel.fromPath("$baseDir/scripts/generate_references.py")
versions_pythonScript = Channel.fromPath("$baseDir/scripts/versions_python.sh")
Gervaise Henry's avatar
Gervaise Henry committed
51
//versions_pigzScript = Channel.fromPath("$baseDir/scripts/versions_pigz.sh")
52
53
54
55
56
versions_cellrangerScript = Channel.fromPath("$baseDir/scripts/versions_cellranger.sh")
versions_bcl2fastqScript = Channel.fromPath("$baseDir/scripts/versions_bcl2fastq.sh")
versions_fastqcScript = Channel.fromPath("$baseDir/scripts/versions_fastqc.sh")

// Define report files
Gervaise Henry's avatar
Gervaise Henry committed
57
multiqcConf = Channel.fromPath("${baseDir}/configs/multiqc_config.yaml")
58
59
references = Channel.fromPath("${baseDir}/../docs/references.md")

Gervaise Henry's avatar
Gervaise Henry committed
60

Gervaise Henry's avatar
Gervaise Henry committed
61
62
63
64
/*
 * trackStart: track start of pipeline
 */
params.ci = false
65
params.dev = false
Gervaise Henry's avatar
Gervaise Henry committed
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
process trackStart {
  script:
  """
  hostname
  ulimit -a
  export https_proxy=\${http_proxy}
 
  curl -H 'Content-Type: application/json' -X PUT -d '{ \
      "sessionId": "${workflow.sessionId}", \
      "pipeline": "cellranger_mkfastq", \
      "pipelineVersion": "${pipelineVersion}", \
      "start": "${workflow.start}", \
      "astrocyte": ${params.astrocyte}, \
      "status": "started", \
      "nextflowVersion": "${workflow.nextflow.version}",
81
      "ci": ${params.ci},
Gervaise Henry's avatar
Gervaise Henry committed
82
      "dev": ${params.dev}}' \
Gervaise Henry's avatar
Gervaise Henry committed
83
84
85
86
87
  "https://xku43pcwnf.execute-api.us-east-1.amazonaws.com/ProdDeploy/pipeline-tracking"
  """
}


Gervaise Henry's avatar
Gervaise Henry committed
88
log.info """\
Gervaise Henry's avatar
Gervaise Henry committed
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
================================
BICF cellranger_mkfastq Pipeline
================================
Run name        : ${params.name}
bcl             : ${params.bcl}
Design File     : ${params.designFile}
Output Directory: ${params.outDir}
--------------------------------
Nextflow Version: ${workflow.nextflow.version}
Pipeline Version: ${pipelineVersion}
Session ID      : ${workflow.sessionId}
--------------------------------
Astrocyte      : ${params.astrocyte}
CI             : ${params.ci}
Development    : ${params.dev}
--------------------------------
"""
Holly Ruess's avatar
Holly Ruess committed
106

Gervaise Henry's avatar
Gervaise Henry committed
107
process checkDesignFile {
Holly Ruess's avatar
Holly Ruess committed
108
  tag "${name}"
109
  module 'python/3.6.1-2-anaconda'
Gervaise Henry's avatar
Gervaise Henry committed
110
111

  input:
Gervaise Henry's avatar
Gervaise Henry committed
112
    file check_designScript
113
    file versions_pythonScript
Holly Ruess's avatar
Holly Ruess committed
114
    file designLocation
Gervaise Henry's avatar
Gervaise Henry committed
115
116

  output:
Holly Ruess's avatar
Holly Ruess committed
117
118
    file("design.checked.csv") into designPaths
    file("design.checked.csv") into designCount
119
120
    file("version_pipeline.txt") into version_pipeline
    file("version_nextflow.txt") into version_nextflow
121
    file("version_python.txt") into version_python
Gervaise Henry's avatar
Gervaise Henry committed
122
123

  script:
Holly Ruess's avatar
Holly Ruess committed
124
125
126
    """
    hostname
    ulimit -a
127
128
129
130
    noSpaceDesign=\$(echo "${designLocation}" | tr -d ' ')
    if [[ "\${noSpaceDesign}" != "${designLocation}" ]]; then
      mv "${designLocation}" "\${noSpaceDesign}"
    fi
Gervaise Henry's avatar
Gervaise Henry committed
131
    python3 check_design.py -d \${noSpaceDesign}
132
    bash versions_python.sh > version_python.txt
133
    echo "${workflow.nextflow.version}" > version_nextflow.txt
134
    echo "${pipelineVersion}" > version_pipeline.txt
Holly Ruess's avatar
Holly Ruess committed
135
    """
Gervaise Henry's avatar
Gervaise Henry committed
136
}
137
138
139
/* nextflow workflow manifest version calls that aren't compatible with Asrcocyte
    echo "${workflow.manifest.version}" > version_pipeline.txt
*/
Gervaise Henry's avatar
Gervaise Henry committed
140

141
142
process untarBCL {  
  tag "${tar.simpleName}"
143
  module 'pigz/2.4'
Gervaise Henry's avatar
Gervaise Henry committed
144
145

  input:
Gervaise Henry's avatar
Gervaise Henry committed
146
    file untarBCLScript
Gervaise Henry's avatar
Gervaise Henry committed
147
    //file versions_pigzScript
148
    each file(tar) from tarList
Gervaise Henry's avatar
Gervaise Henry committed
149
150

  output:
151
    file("*[!version_pigz.txt]") into bclPaths mode flatten
Gervaise Henry's avatar
Gervaise Henry committed
152
    //file("version_pigz.txt") into version_pigz
Gervaise Henry's avatar
Gervaise Henry committed
153
154

  script:
Holly Ruess's avatar
Holly Ruess committed
155
156
    """
    hostname
157
    ulimit -u 16384
Holly Ruess's avatar
Holly Ruess committed
158
    ulimit -a
159
    bash untarBCL.sh -t ${tar}
Gervaise Henry's avatar
Gervaise Henry committed
160
    #bash versions_pigz.sh > version_pigz.txt
Holly Ruess's avatar
Holly Ruess committed
161
162
    """

Gervaise Henry's avatar
Gervaise Henry committed
163
164
}

Gervaise Henry's avatar
Gervaise Henry committed
165

Gervaise Henry's avatar
Gervaise Henry committed
166
process mkfastq {
167
  tag "${bcl.simpleName}"
Holly Ruess's avatar
Holly Ruess committed
168
  publishDir "${outDir}/${task.process}", mode: 'copy', pattern: "{*/outs/**/*.fastq.gz}"
169
  queue '128GB,256GB,256GBv1,384GB'
Gervaise Henry's avatar
Gervaise Henry committed
170
  module 'cellranger/4.0.0:bcl2fastq/2.19.1'
Gervaise Henry's avatar
Gervaise Henry committed
171
172

  input:
173
174
    file versions_cellrangerScript
    file versions_bcl2fastqScript
175
    each file(bcl) from bclPaths.collect()
Holly Ruess's avatar
Holly Ruess committed
176
    file design from designPaths
Gervaise Henry's avatar
Gervaise Henry committed
177
178

  output:
179
180
181
    file("fq/${bcl.simpleName}/*.fastq.gz") into fastqPaths
    val "${bcl.simpleName}" into bclName
    file("**/outs/**/*.fastq.gz") into cellrangerCount mode flatten
Gervaise Henry's avatar
Gervaise Henry committed
182
    file("**/outs/fastq_path/Stats/*") into bqcPaths
183
184
    file("version_cellranger.txt") into version_cellranger
    file("version_bcl2fastq.txt") into version_bcl2fastq
Gervaise Henry's avatar
Gervaise Henry committed
185
186

  script:
Holly Ruess's avatar
Holly Ruess committed
187
188
    """
    hostname
189
190
    ulimit -u 16384
    ulimit -a
Gervaise Henry's avatar
Gervaise Henry committed
191
    cellranger mkfastq --id=mkfastq_${bcl.simpleName} --run=${bcl} --csv=${design} ${mask}
192
193
194
195
196
    mkdir fq
    mkdir "fq/${bcl.simpleName}"
    find . -name "*.fastq.gz" -exec cp {} fq/${bcl.simpleName}/ \\;
    bash versions_cellranger.sh > version_cellranger.txt
    bash versions_bcl2fastq.sh > version_bcl2fastq.txt
Holly Ruess's avatar
Holly Ruess committed
197
    """
Gervaise Henry's avatar
Gervaise Henry committed
198
}
Gervaise Henry's avatar
Gervaise Henry committed
199

Gervaise Henry's avatar
Gervaise Henry committed
200

201
202
if (bclCount.value == 1) {
  process countDesign {
Holly Ruess's avatar
Holly Ruess committed
203
    tag "${name}"
204
    publishDir "${outDir}/${task.process}/${name}", mode: 'copy'
205
206

    input:
Gervaise Henry's avatar
Gervaise Henry committed
207
      file countDesignScript
Holly Ruess's avatar
Holly Ruess committed
208
209
      file fastqs from cellrangerCount.collect()
      file design from designCount
210
211

    output:
Holly Ruess's avatar
Holly Ruess committed
212
      file("Cellranger_Count_Design.csv") into CountDesign
213
214

    script:
Holly Ruess's avatar
Holly Ruess committed
215
      """
216
217
      hostname
      ulimit -a
Gervaise Henry's avatar
Gervaise Henry committed
218
      bash countDesign.sh
Holly Ruess's avatar
Holly Ruess committed
219
      """
220
221
222
  }
}

Gervaise Henry's avatar
Gervaise Henry committed
223

224
process fastqc {
225
  tag "${bcl}"
226
  module 'fastqc/0.11.5:parallel'
227
228

  input:
229
230
231
232
    file fastqcScript    
    file versions_fastqcScript
    file(fastq) from fastqPaths
    each bcl from bclName
233
234

  output:
235
236
    file("*fastqc.zip") into fqcPaths mode flatten
    file("version_fastqc.txt") into version_fastqc
237
238

  script:
Holly Ruess's avatar
Holly Ruess committed
239
240
    """
    hostname
241
    ulimit -u 16384
Holly Ruess's avatar
Holly Ruess committed
242
    ulimit -a
243
    find *.fastq.gz -exec mv {} ${bcl}.{} \\;
Gervaise Henry's avatar
Gervaise Henry committed
244
    bash fastqc.sh
245
    bash versions_fastqc.sh > version_fastqc.txt
Holly Ruess's avatar
Holly Ruess committed
246
    """
247
248
}

249

250
process versions {
Holly Ruess's avatar
Holly Ruess committed
251
  tag "${name}"
252
  module 'python/3.6.1-2-anaconda:cellranger/3.1.0:bcl2fastq/2.19.1:fastqc/0.11.5:pandoc/2.7'
253
254

  input:
255
256
  file versionsScript
  file referencesScript
257
258
  file version_pipeline
  file version_nextflow
259
  file version_python
Gervaise Henry's avatar
Gervaise Henry committed
260
  //file version_pigz
261
262
263
264
  file version_cellranger
  file version_bcl2fastq
  file version_fastqc
  file references
265
266

  output:
Holly Ruess's avatar
Holly Ruess committed
267
    file("*.yaml") into yamlPaths
268
269

  script:
Holly Ruess's avatar
Holly Ruess committed
270
271
272
    """
    hostname
    ulimit -a
273
274
    python3 generate_versions.py -f version_*.txt -o versions
    python3 generate_references.py -r ${references} -o references
Holly Ruess's avatar
Holly Ruess committed
275
    """
276
277
}

Gervaise Henry's avatar
Gervaise Henry committed
278

Gervaise Henry's avatar
Gervaise Henry committed
279
process multiqc {
Holly Ruess's avatar
Holly Ruess committed
280
281
  tag "${name}"
  publishDir "${outDir}/${task.process}/${name}", mode: 'copy', pattern: "{multiqc*}"
282
  module 'multiqc/1.7'
Gervaise Henry's avatar
Gervaise Henry committed
283
284

  input:
285
    file multiqcConf
Holly Ruess's avatar
Holly Ruess committed
286
    file bqc name "bqc/?/*" from bqcPaths.collect()
287
    file fqc name "fqc/?/*" from fqcPaths.collect()
Holly Ruess's avatar
Holly Ruess committed
288
    file yamlPaths
Gervaise Henry's avatar
Gervaise Henry committed
289
290

  output:
Holly Ruess's avatar
Holly Ruess committed
291
    file("multiqc_report.html") into mqcPaths
Gervaise Henry's avatar
Gervaise Henry committed
292
293

  script:
Holly Ruess's avatar
Holly Ruess committed
294
295
296
    """
    hostname
    ulimit -a
297
298
    #export LC_ALL=C.UTF-8
    #export LANG=C.UTF-8
Holly Ruess's avatar
Holly Ruess committed
299
300
    multiqc -c ${multiqcConf} .
    """
Gervaise Henry's avatar
Gervaise Henry committed
301
}