Merge branch '1-process_getData' into 'develop'

Resolve "process_getData"

See merge request !1

.gitignore

@@ -297,6 +297,7 @@ $RECYCLE.BIN/
 
 # nextflow analysis folders/files
 /test_data/*
+/workflow/docker/images/*
 /workflow/.nextflow/*
 /workflow/work/*
 /workflow/output/*

cleanup.sh

+rm *.out
+rm pipeline_trace*.txt*
+rm report*.html*
+rm timeline*.html*
+rm .nextflow*.log*
+rm -r .nextflow/
+rm -r work/

nextflow.config

+profiles {
+  standard {
+    includeConfig 'workflow/conf/biohpc.config'
+  }
+}

workflow/conf/biohpc.config

+process {
+  executor = 'slurm'
+  queue='super'
+
+  // Process specific configuration
+  withLabel:getData {
+    executor = 'super'
+  }
+}
+
+
+trace {
+  enabled = true
+  file = 'pipeline_trace.txt'
+  fields = 'task_id,native_id,process,name,status,exit,submit,start,complete,duration,realtime,%cpu,%mem,rss'
+}
+
+timeline {
+  enabled = true
+  file = 'timeline.html'
+}
+
+report {
+  enabled = true
+  file = 'report.html'
+}
+
+tower {
+  accessToken = '3ade8f325d4855434b49aa387421a44c63e3360f'
+  enabled = true
+}
\ No newline at end of file

workflow/conf/conda.env.bdbag.yml

+name: bdbag
+dependencies:
+  - pandas=0.23.3=py36_0
+  - pip:
+    - bdbag==1.5.5

workflow/docker/temp

+
+
+RUN apt-get install -y python3.7 python3-pip
+
+RUN wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+  bash Miniconda3-latest-Linux-x86_64.sh -p /miniconda -b && \
+  rm Miniconda3-latest-Linux-x86_64.sh
+ENV PATH=/miniconda/bin:${PATH}                                                                                      
+RUN conda config --add channels defaults && \
+  conda config --add channels bioconda && \
+  conda config --add channels conda-forge && \
+  conda update -n base -c defaults -y conda
+
+RUN pip install --upgrade pip

workflow/main.nf

-#!/usr/bin/env nextflow

workflow/nextflow.config

+profiles {
+  standard {
+    includeConfig 'conf/biohpc.config'
+  }
+}

workflow/rna-seq.nf

+#!/usr/bin/env nextflow
+
+// Define input variables
+params.bdbag = "${baseDir}/../test_data/Study_Q-Y4H0.zip"
+
+params.outDir = "${baseDir}/../output"
+
+// Parse input variables
+bdbag = Channel
+  .fromPath(params.bdbag)
+  .ifEmpty { exit 1, "bdbag zip file not found: ${params.bdbag}" }
+
+outDir = params.outDir
+
+
+/*
+ * getData: fetch study files from consortium with downloaded bdbag.zip
+ * python must be loaded prior to nextflow run, because conda env create from .yml doesn't work with nextflow loaded module (either process in-line, or config file)
+ */
+ process getData {
+     publishDir "${outDir}/temp/getData", mode: "symlink"
+     conda "${baseDir}/conf/conda.env.bdbag.yml"
+
+     input:
+        file bdbag
+
+    output:
+        file("**/*.R*.fastq.gz") into fastqPaths
+        file("**/File.csv") into filePaths
+        file("**/Experiment Settings.csv") into experimentSettingsPaths
+        file("**/Experiment.csv") into experimentPaths
+
+    script:
+        """
+        hostname
+        ulimit -a
+        study=\$(echo "${bdbag}" | cut -d'.' -f1)
+        echo LOG: \${study}
+        unzip ${bdbag}
+        echo LOG: bdgag unzipped
+        python3 ${baseDir}/scripts/modifyFetch.py --fetchFile \${study}
+        echo LOG: fetch file filtered for only .fastq.gz
+        #bdbag --materialize "\$(echo "${bdbag}" | cut -d'.' -f1)"
+        sh ${baseDir}/scripts/bdbagFetch.sh \${study}
+        echo LOG: bdbag fetched
+        sh ${baseDir}/scripts/renameFastq.sh \${study}
+        echo LOG: fastq.gz files renamed to replicate RID
+        """
+ }
\ No newline at end of file

workflow/scripts/bdbagFetch.sh

+#!/bin
+
+bdbag --resolve-fetch all --fetch-filter filename\$*fastq.gz $1
\ No newline at end of file

workflow/scripts/modifyFetch.py

+import argparse
+import pandas as pd
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--fetchFile',help="The fetch file from bdgap.zip.",required=True)
+    args = parser.parse_args()
+    return args
+
+def main():
+    args = get_args()
+    fetch = pd.read_csv(args.fetchFile+"/fetch.txt",sep="\t",header=None)
+    fetch_filtered = fetch[fetch[2].str[-9:]==".fastq.gz"]
+    fetch_filtered.to_csv(args.fetchFile+"/fetch.txt",sep="\t",header=False,index=False)
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file

workflow/scripts/renameFastq.sh

+#!/bin
+
+while read loc checksum fileLocation
+do
+    file=$(echo ${fileLocation##*/})
+    fileName=$(echo ${file%.R*.fastq.gz})
+    fileExt=$(echo ${file##${fileName}.})
+    while IFS="," read RID Study_RID Experiment_RID Replicate_RID Caption File_Type File_Name URI File_size MD5 GEO_Archival_URL dbGaP_Accession_ID Processed Notes Principal_Investigator Consortium Release_Date RCT RMT Legacy_File_RID GUDMAP_NGF_OID GUDMAP_NGS_OID
+    do
+        if [ ${file} == ${File_Name} ]
+        then
+            find . -type f -name ${file} -execdir mv {} ${Replicate_RID}.${fileExt} ';'
+        fi
+    done < $1/data/File.csv
+done < $1/fetch.txt
\ No newline at end of file