Merge branch '15-auto.bagit.download' into 'develop'

Resolve "Add automated download of bagit through deriva"

Closes #15

See merge request !8

workflow/conf/biohpc.config

@@ -4,10 +4,12 @@ process {
   clusterOptions = '--hold'
 
   // Process specific configuration
-  withName:splitData {
+  withName:getBag {
+    executor = 'local'
     container = 'docker://bicf/gudmaprbkfilexfer:1.1'
   }
   withName:getData {
+    executor = 'local'
     container = 'docker://bicf/gudmaprbkfilexfer:1.1'
   }
   withName:trimData {
@@ -48,3 +50,11 @@ env {
   https_proxy = 'http://proxy.swmed.edu:3128'
   all_proxy = 'http://proxy.swmed.edu:3128'
 }
+
+manifest {
+  homePage = 'https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq'
+  description = 'This pipeline was created to analize RNA-sequencing data from GUDMAP/RBK consortiums.'
+  mainScript = 'rna-seq.nf'
+  version = 'v0.0.1_indev'
+  nextflowVersion = '>=19.09.0'
+}

workflow/conf/replicate_export_config.json

+{
+  "bag": {
+    "bag_name": "Replicate_{rid}",
+    "bag_algorithms": [
+      "md5"
+    ],
+    "bag_archiver": "zip"
+  },
+  "catalog": {
+    "query_processors": [
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Study",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Study_RID)=(RNASeq:Study:RID)/Study_RID:=RID,Internal_ID,Title,Summary,Overall_Design,GEO_Series_Accession_ID,GEO_Platform_Accession_ID,Funding,Pubmed_ID,Principal_Investigator,Consortium,Release_Date,RCT,RMT?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Experiment",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Experiment_RID)=(RNASeq:Experiment:RID)/Experiment_RID:=RID,Study_RID,Internal_ID,Name,Description,Experiment_Method,Sequencing_Type,Species,Specimen_Type,Molecule_Type,Pooled_Sample,Pool_Size,Markers,Cell_Count,Treatment_Protocol,Treatment_Protocol_Reference,Isolation_Protocol,Isolation_Protocol_Reference,Growth_Protocol,Growth_Protocol_Reference,Label_Protocol,Label_Protocol_Reference,Hybridization_Protocol,Hybridization_Protocol_Reference,Scan_Protocol,Scan_Protocol_Reference,Data_Processing,Value_Definition,Notes,Principal_Investigator,Consortium,Release_Date,RCT,RMT?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Experiment Antibodies",
+          "query_path": "/entity/M:=RNASeq:Replicate/RID={rid}/(Experiment_RID)=(RNASeq:Experiment_Antibodies:Experiment_RID)?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Experiment Custom Metadata",
+          "query_path": "/entity/M:=RNASeq:Replicate/RID={rid}/(Experiment_RID)=(RNASeq:Experiment_Custom_Metadata:Experiment_RID)?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Experiment Settings",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Experiment_RID)=(RNASeq:Experiment_Settings:Experiment_RID)/RID,Experiment_RID,Alignment_Format,Aligner,Aligner_Version,Reference_Genome,Sequence_Trimming,Duplicate_Removal,Pre-alignment_Sequence_Removal,Junction_Reads,Library_Type,Protocol_Reference,Library_Selection,Quantification_Format,Quantification_Software,Expression_Metric,Transcriptome_Model,Sequencing_Platform,Paired_End,Read_Length,Has_Strand_Specific_Information,Used_Spike_Ins,Spike_Ins_Amount,Visualization_Format,Visualization_Software,Visualization_Version,Visualization_Setting,Notes,RCT,RMT?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Replicate",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/RID,Study_RID,Experiment_RID,Biological_Replicate_Number,Technical_Replicate_Number,Specimen_RID,Collection_Date,Mapped_Reads,GEO_Sample_Accession_ID,Notes,Principal_Investigator,Consortium,Release_Date,RCT,RMT?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Specimen",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/S:=(Specimen_RID)=(Gene_Expression:Specimen:RID)/T:=left(Stage_ID)=(Vocabulary:Developmental_Stage:ID)/$S/RID,Title,Species,Stage_ID,Stage_Name:=T:Name,Stage_Detail,Assay_Type,Strain,Wild_Type,Sex,Passage,Phenotype,Cell_Line,Parent_Specimen,Upload_Notes,Preparation,Fixation,Embedding,Internal_ID,Principal_Investigator,Consortium,Release_Date,RCT,RMT,GUDMAP2_Accession_ID?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Specimen_Anatomical_Source",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Specimen_RID)=(Gene_Expression:Specimen:RID)/(RID)=(Gene_Expression:Specimen_Tissue:Specimen_RID)/RID,Specimen_RID,Tissue,RCT,RMT?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Specimen_Cell_Types",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Specimen_RID)=(Gene_Expression:Specimen:RID)/(RID)=(Gene_Expression:Specimen_Cell_Type:Specimen)/RID,Specimen_RID:=Specimen,Cell_Type,RCT,RMT?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "Single Cell Metrics",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(RID)=(RNASeq:Single_Cell_Metrics:Replicate_RID)/RID,Study_RID,Experiment_RID,Replicate_RID,Reads_%28Millions%29,Reads%2FCell,Detected_Gene_Count,Genes%2FCell,UMI%2FCell,Estimated_Cell_Count,Principal_Investigator,Consortium,Release_Date,RCT,RMT?limit=none"
+        }
+      },
+      {
+        "processor": "csv",
+        "processor_params": {
+          "output_path": "File",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(RID)=(RNASeq:File:Replicate_RID)/RID,Study_RID,Experiment_RID,Replicate_RID,Caption,File_Type,File_Name,URI,File_size,MD5,GEO_Archival_URL,dbGaP_Accession_ID,Processed,Notes,Principal_Investigator,Consortium,Release_Date,RCT,RMT,Legacy_File_RID,GUDMAP_NGF_OID,GUDMAP_NGS_OID?limit=none"
+        }
+      },
+      {
+        "processor": "fetch",
+        "processor_params": {
+          "output_path": "assets/Study/{Study_RID}/Experiment/{Experiment_RID}/Replicate/{Replicate_RID}",
+          "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(RID)=(RNASeq:File:Replicate_RID)/url:=URI,length:=File_size,filename:=File_Name,md5:=MD5,Study_RID,Experiment_RID,Replicate_RID?limit=none"
+        }
+      }
+    ]
+  }
+}

workflow/rna-seq.nf

 #!/usr/bin/env nextflow
 
 // Define input variables
-params.deriva = "/project/BICF/BICF_Core/shared/gudmap/cookies/deriva-cookies.txt"
-params.bdbag = "${baseDir}/../test_data/Study_Q-Y4H0.zip"
+params.deriva = "${baseDir}/../test_data/credential.json"
+params.bdbag = "${baseDir}/../test_data/cookies.txt"
+//params.repRID = "16-1ZX4"
+params.repRID = "Q-Y5JA"
 
 params.outDir = "${baseDir}/../output"
 
 // Parse input variables
-deriva = file(params.deriva, checkIfExists: 'true')
+deriva = Channel
+  .fromPath(params.deriva)
+  .ifEmpty { exit 1, "deriva credential file not found: ${params.deriva}" }
 bdbag = Channel
   .fromPath(params.bdbag)
-  .ifEmpty { exit 1, "bdbag zip file not found: ${params.bdbag}" }
+  .ifEmpty { exit 1, "deriva cookie file for bdbag not found: ${params.bdbag}" }
+
+Channel.from(params.repRID)
+  .into {
+    repRID_getBag
+    repRID_getData
+    repRID_trimData
+  }
 
 outDir = params.outDir
 logsDir = "${outDir}/Logs"
 
+// Define fixed files
+derivaConfig = Channel.fromPath("${baseDir}/conf/replicate_export_config.json")
+
 /*
- * splitData: split bdbag files by replicate so fetch can occure in parallel, and rename files to replicate rid
+ * getData: get bagit file from consortium
  */
-process splitData {
-  tag "${bdbag.baseName}"
+process getBag {
   executor 'local'
-  publishDir "${logsDir}/splitData", mode: 'symlink', pattern: "${bdbag.baseName}.splitData.err"
+  tag "${repRID_getBag}"
+  publishDir "${logsDir}/getBag", mode: 'symlink', pattern: "${repRID_getBag}.getBag.err"
 
   input:
-    file bdbag
-    path cookies, stageAs: 'cookies.txt' from deriva
+    val repRID_getBag
+    path credential, stageAs: 'credential.json' from deriva
+    path derivaConfig
 
   output:
-    file("Replicate_*.zip") into bdbagSplit mode flatten
-    file("${bdbag.baseName}/data/File.csv") into fileMeta
-    file("${bdbag.baseName}/data/Experiment Settings.csv") into experimentSettingsMeta
-    file("${bdbag.baseName}/data/Experiment.csv") into experimentMeta
-    file ("${bdbag.baseName}.splitData.err")
+    path ("Replicate_*.zip") into bagit
+    file ("${repRID_getBag}.getBag.err")
 
   script:
     """
-    hostname >> ${bdbag.baseName}.splitData.err
-    ulimit -a >> ${bdbag.baseName}.splitData.err
-    ln -sf `readlink -e cookies.txt` ~/.bdbag/deriva-cookies.txt 2>>${bdbag.baseName}.splitData.err
-    echo "LOG: deriva cookie linked" >> ${bdbag.baseName}.splitData.err 
-    study=`echo "${bdbag}" | cut -d '.' -f1` 2>>${bdbag.baseName}.splitData.err
-    echo "LOG: \${study}" >> ${bdbag.baseName}.splitData.err
-    unzip ${bdbag} 2>>${bdbag.baseName}.splitData.err
-    echo "LOG: bdgag unzipped" >> ${bdbag.baseName}.splitData.err
-    python3 ${baseDir}/scripts/modifyFetch.py --fetchFile \${study} 2>>${bdbag.baseName}.splitData.err
-    echo "LOG: fetch file filtered for only .fastq.gz" >> ${bdbag.baseName}.splitData.err
-    python3 ${baseDir}/scripts/splitFetch.py --fetchFile \${study} 2>>${bdbag.baseName}.splitData.err
-    echo "LOG: fetch file split by replicates" >> ${bdbag.baseName}.splitData.err
-    sh ${baseDir}/scripts/splitBag.sh \${study} 2>>${bdbag.baseName}.splitData.err
-    echo "LOG: bag recreated with replicate split fetch file" >> ${bdbag.baseName}.splitData.err
+    hostname >>${repRID_getBag}.getBag.err
+    ulimit -a >>${repRID_getBag}.getBag.err
+    ln -sf `readlink -e credential.json` ~/.deriva/credential.json 2>>${repRID_getBag}.getBag.err
+    echo "LOG: deriva credentials linked" >>${repRID_getBag}.getBag.err
+    deriva-download-cli dev.gudmap.org --catalog 2 ${derivaConfig} . rid=${repRID_getBag} 2>>${repRID_getBag}.getBag.err
     """
 }
 
@@ -57,26 +60,36 @@ process splitData {
  * getData: fetch study files from consortium with downloaded bdbag.zip
  */
 process getData {
-  tag "${rep.baseName}"
-  publishDir "${logsDir}/getData", mode: 'symlink', pattern: "${rep.baseName}.getData.err"
+  tag "${repRID_getData}"
+  publishDir "${logsDir}/getData", mode: 'symlink', pattern: "${repRID_getData}.getData.err"
 
   input:
-    each rep from bdbagSplit
+    val repRID_getData
+    executor 'local'
+    path cookies, stageAs: 'deriva-cookies.txt' from bdbag
+    path bagit
 
   output:
-    set val ("${rep.baseName}"), file ("*.R{1,2}.fastq.gz") into trimming
+    path ("*.R{1,2}.fastq.gz") into fastqs
+    file("**/File.csv") into fileMeta
+    file("**/Experiment Settings.csv") into experimentSettingsMeta
+    file("**/Experiment.csv") into experimentMeta
+    file ("${repRID_getData}.getData.err")
+
 
   script:
     """
-    hostname >>${rep.baseName}.getData.err
-    ulimit -a >>${rep.baseName}.getData.err
+    hostname >>${repRID_getData}.getData.err
+    ulimit -a >>${repRID_getData}.getData.err
     export https_proxy=\${http_proxy}
-    replicate=\$(basename "${rep}" | cut -d '.' -f1)
-    echo "LOG: \${replicate}" >>${rep.baseName}.getData.err
-    unzip ${rep} 2>>${rep.baseName}.getData.err
-    echo "LOG: replicate bdbag unzipped" >>${rep.baseName}.getData.err
-    sh ${baseDir}/scripts/bdbagFetch.sh \${replicate} 2>>${rep.baseName}.getData.err
-    echo "LOG: replicate bdbag fetched" >>${rep.baseName}.getData.err
+    ln -sf `readlink -e deriva-cookies.txt` ~/.bdbag/deriva-cookies.txt >>${repRID_getData}.getData.err
+    echo "LOG: deriva cookie linked" >>${repRID_getData}.getData.err
+    replicate=\$(basename "${bagit}" | cut -d '.' -f1)
+    echo "LOG: \${replicate}" >>${repRID_getData}.getData.err
+    unzip ${bagit} 2>>${repRID_getData}.getData.err
+    echo "LOG: replicate bdbag unzipped" >>${repRID_getData}.getData.err
+    sh ${baseDir}/scripts/bdbagFetch.sh \${replicate} ${repRID_getData} 2>>${repRID_getData}.getData.err
+    echo "LOG: replicate bdbag fetched" >>${repRID_getData}.getData.err
     """
 }
 
@@ -84,19 +97,34 @@ process getData {
  * trimData: trims any adapter or non-host sequences from the data
 */
 process trimData {
-  tag "trim-${repID}"
-  publishDir "${outDir}/tempOut/trimmed", mode: "symlink", pattern: "*_val_{1,2}.fq.gz"
-  publishDir "${logsDir}/trimData", mode: 'symlink', pattern: "\${rep}.trimData.*"
+  tag "${repRID_trimData}"
+  publishDir "${logsDir}/trimData", mode: 'symlink', pattern: "\${repRID_trimData}.trimData.*"
 
   input:
-    set repID, reads from trimming
+    val repRID_trimData
+    file(fastq) from fastqs
 
   output:
-    path ("*_val_{1,2}.fq.gz", type: 'file', maxDepth: '0')
+    path ("*.fq.gz") into fastqs_trimmed
+    val ends
+    file ("${repRID_trimData}.trimData.log")
+    file ("${repRID_trimData}.trimData.err")
 
   script:
     """
-    rep=`echo ${repID} | cut -f2- -d '_'`;
-    trim_galore --gzip --max_n 1 --paired --basename \${rep} -j `nproc` ${reads[0]} ${reads[1]} 1>>\${rep}.trimData.log 2>>\${rep}.trimData.err;
+    if [ `nproc` -gt 8 ]
+    then
+      ncore=8
+    else
+      ncore=`nproc`
+    fi
+    if [ '${fastq[1]}' == 'null' ]
+    then
+      ends='se'
+      trim_galore --gzip -q 25 --illumina --length 35 --basename ${repRID_trimData} -j \${ncore} ${fastq[0]} 1>>${repRID_trimData}.trimData.log 2>>${repRID_trimData}.trimData.err;
+    else
+      ends='pe'
+      trim_galore --gzip -q 25 --illumina --length 35 --paired --basename ${repRID_trimData} -j \${ncore} ${fastq[0]} ${fastq[1]} 1>>${repRID_trimData}.trimData.log 2>>${repRID_trimData}.trimData.err;
+    fi
     """
-}
+}
\ No newline at end of file

workflow/scripts/bdbagFetch.sh

 #!/bin/bash
 
-bdbag --resolve-fetch all --fetch-filter filename\$*fastq.gz $1 &&
+bdbag --resolve-fetch all --fetch-filter filename\$*fastq.gz $1
 for i in $(find */ -name "*.R*.fastq.gz"); do
-  mv ${i} .;
-done;
+    path=${2}$(echo ${i##*/} | grep -o "\.R.\.fastq\.gz");
+    mv ${i} ./${path}
+done;
\ No newline at end of file

workflow/scripts/splitBag.sh

-#!/bin
-
-for i in $(ls -d Replicate_*)
-do
-rsync -r $1/ ${i} --exclude=fetch.txt
-zip -r ${i}.zip ${i}
-done
\ No newline at end of file