diff --git a/workflow/conf/biohpc.config b/workflow/conf/biohpc.config index 13368ad2ec3ba4a6696dac0731de81f00bb3c479..0fea2a6227bfa6fe87e5da5bfbdc261e7db9aa7b 100755 --- a/workflow/conf/biohpc.config +++ b/workflow/conf/biohpc.config @@ -4,10 +4,12 @@ process { clusterOptions = '--hold' // Process specific configuration - withName:splitData { + withName:getBag { + executor = 'local' container = 'docker://bicf/gudmaprbkfilexfer:1.1' } withName:getData { + executor = 'local' container = 'docker://bicf/gudmaprbkfilexfer:1.1' } withName:trimData { @@ -48,3 +50,11 @@ env { https_proxy = 'http://proxy.swmed.edu:3128' all_proxy = 'http://proxy.swmed.edu:3128' } + +manifest { + homePage = 'https://git.biohpc.swmed.edu/gudmap_rbk/rna-seq' + description = 'This pipeline was created to analize RNA-sequencing data from GUDMAP/RBK consortiums.' + mainScript = 'rna-seq.nf' + version = 'v0.0.1_indev' + nextflowVersion = '>=19.09.0' +} diff --git a/workflow/conf/replicate_export_config.json b/workflow/conf/replicate_export_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ff17fa513c5bc130a2e2bdaf9aa41b070c99b290 --- /dev/null +++ b/workflow/conf/replicate_export_config.json @@ -0,0 +1,97 @@ +{ + "bag": { + "bag_name": "Replicate_{rid}", + "bag_algorithms": [ + "md5" + ], + "bag_archiver": "zip" + }, + "catalog": { + "query_processors": [ + { + "processor": "csv", + "processor_params": { + "output_path": "Study", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Study_RID)=(RNASeq:Study:RID)/Study_RID:=RID,Internal_ID,Title,Summary,Overall_Design,GEO_Series_Accession_ID,GEO_Platform_Accession_ID,Funding,Pubmed_ID,Principal_Investigator,Consortium,Release_Date,RCT,RMT?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "Experiment", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Experiment_RID)=(RNASeq:Experiment:RID)/Experiment_RID:=RID,Study_RID,Internal_ID,Name,Description,Experiment_Method,Sequencing_Type,Species,Specimen_Type,Molecule_Type,Pooled_Sample,Pool_Size,Markers,Cell_Count,Treatment_Protocol,Treatment_Protocol_Reference,Isolation_Protocol,Isolation_Protocol_Reference,Growth_Protocol,Growth_Protocol_Reference,Label_Protocol,Label_Protocol_Reference,Hybridization_Protocol,Hybridization_Protocol_Reference,Scan_Protocol,Scan_Protocol_Reference,Data_Processing,Value_Definition,Notes,Principal_Investigator,Consortium,Release_Date,RCT,RMT?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "Experiment Antibodies", + "query_path": "/entity/M:=RNASeq:Replicate/RID={rid}/(Experiment_RID)=(RNASeq:Experiment_Antibodies:Experiment_RID)?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "Experiment Custom Metadata", + "query_path": "/entity/M:=RNASeq:Replicate/RID={rid}/(Experiment_RID)=(RNASeq:Experiment_Custom_Metadata:Experiment_RID)?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "Experiment Settings", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Experiment_RID)=(RNASeq:Experiment_Settings:Experiment_RID)/RID,Experiment_RID,Alignment_Format,Aligner,Aligner_Version,Reference_Genome,Sequence_Trimming,Duplicate_Removal,Pre-alignment_Sequence_Removal,Junction_Reads,Library_Type,Protocol_Reference,Library_Selection,Quantification_Format,Quantification_Software,Expression_Metric,Transcriptome_Model,Sequencing_Platform,Paired_End,Read_Length,Has_Strand_Specific_Information,Used_Spike_Ins,Spike_Ins_Amount,Visualization_Format,Visualization_Software,Visualization_Version,Visualization_Setting,Notes,RCT,RMT?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "Replicate", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/RID,Study_RID,Experiment_RID,Biological_Replicate_Number,Technical_Replicate_Number,Specimen_RID,Collection_Date,Mapped_Reads,GEO_Sample_Accession_ID,Notes,Principal_Investigator,Consortium,Release_Date,RCT,RMT?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "Specimen", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/S:=(Specimen_RID)=(Gene_Expression:Specimen:RID)/T:=left(Stage_ID)=(Vocabulary:Developmental_Stage:ID)/$S/RID,Title,Species,Stage_ID,Stage_Name:=T:Name,Stage_Detail,Assay_Type,Strain,Wild_Type,Sex,Passage,Phenotype,Cell_Line,Parent_Specimen,Upload_Notes,Preparation,Fixation,Embedding,Internal_ID,Principal_Investigator,Consortium,Release_Date,RCT,RMT,GUDMAP2_Accession_ID?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "Specimen_Anatomical_Source", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Specimen_RID)=(Gene_Expression:Specimen:RID)/(RID)=(Gene_Expression:Specimen_Tissue:Specimen_RID)/RID,Specimen_RID,Tissue,RCT,RMT?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "Specimen_Cell_Types", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(Specimen_RID)=(Gene_Expression:Specimen:RID)/(RID)=(Gene_Expression:Specimen_Cell_Type:Specimen)/RID,Specimen_RID:=Specimen,Cell_Type,RCT,RMT?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "Single Cell Metrics", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(RID)=(RNASeq:Single_Cell_Metrics:Replicate_RID)/RID,Study_RID,Experiment_RID,Replicate_RID,Reads_%28Millions%29,Reads%2FCell,Detected_Gene_Count,Genes%2FCell,UMI%2FCell,Estimated_Cell_Count,Principal_Investigator,Consortium,Release_Date,RCT,RMT?limit=none" + } + }, + { + "processor": "csv", + "processor_params": { + "output_path": "File", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(RID)=(RNASeq:File:Replicate_RID)/RID,Study_RID,Experiment_RID,Replicate_RID,Caption,File_Type,File_Name,URI,File_size,MD5,GEO_Archival_URL,dbGaP_Accession_ID,Processed,Notes,Principal_Investigator,Consortium,Release_Date,RCT,RMT,Legacy_File_RID,GUDMAP_NGF_OID,GUDMAP_NGS_OID?limit=none" + } + }, + { + "processor": "fetch", + "processor_params": { + "output_path": "assets/Study/{Study_RID}/Experiment/{Experiment_RID}/Replicate/{Replicate_RID}", + "query_path": "/attribute/M:=RNASeq:Replicate/RID={rid}/(RID)=(RNASeq:File:Replicate_RID)/url:=URI,length:=File_size,filename:=File_Name,md5:=MD5,Study_RID,Experiment_RID,Replicate_RID?limit=none" + } + } + ] + } +} diff --git a/workflow/rna-seq.nf b/workflow/rna-seq.nf index 035faa8ee8b3f2b95f298e4edc8f074a0b695587..08052962f4e07a2115745aaaa5f5679b6c3fa1e4 100755 --- a/workflow/rna-seq.nf +++ b/workflow/rna-seq.nf @@ -1,55 +1,58 @@ #!/usr/bin/env nextflow // Define input variables -params.deriva = "/project/BICF/BICF_Core/shared/gudmap/cookies/deriva-cookies.txt" -params.bdbag = "${baseDir}/../test_data/Study_Q-Y4H0.zip" +params.deriva = "${baseDir}/../test_data/credential.json" +params.bdbag = "${baseDir}/../test_data/cookies.txt" +//params.repRID = "16-1ZX4" +params.repRID = "Q-Y5JA" params.outDir = "${baseDir}/../output" // Parse input variables -deriva = file(params.deriva, checkIfExists: 'true') +deriva = Channel + .fromPath(params.deriva) + .ifEmpty { exit 1, "deriva credential file not found: ${params.deriva}" } bdbag = Channel .fromPath(params.bdbag) - .ifEmpty { exit 1, "bdbag zip file not found: ${params.bdbag}" } + .ifEmpty { exit 1, "deriva cookie file for bdbag not found: ${params.bdbag}" } + +Channel.from(params.repRID) + .into { + repRID_getBag + repRID_getData + repRID_trimData + } outDir = params.outDir logsDir = "${outDir}/Logs" +// Define fixed files +derivaConfig = Channel.fromPath("${baseDir}/conf/replicate_export_config.json") + /* - * splitData: split bdbag files by replicate so fetch can occure in parallel, and rename files to replicate rid + * getData: get bagit file from consortium */ -process splitData { - tag "${bdbag.baseName}" +process getBag { executor 'local' - publishDir "${logsDir}/splitData", mode: 'symlink', pattern: "${bdbag.baseName}.splitData.err" + tag "${repRID_getBag}" + publishDir "${logsDir}/getBag", mode: 'symlink', pattern: "${repRID_getBag}.getBag.err" input: - file bdbag - path cookies, stageAs: 'cookies.txt' from deriva + val repRID_getBag + path credential, stageAs: 'credential.json' from deriva + path derivaConfig output: - file("Replicate_*.zip") into bdbagSplit mode flatten - file("${bdbag.baseName}/data/File.csv") into fileMeta - file("${bdbag.baseName}/data/Experiment Settings.csv") into experimentSettingsMeta - file("${bdbag.baseName}/data/Experiment.csv") into experimentMeta - file ("${bdbag.baseName}.splitData.err") + path ("Replicate_*.zip") into bagit + file ("${repRID_getBag}.getBag.err") script: """ - hostname >> ${bdbag.baseName}.splitData.err - ulimit -a >> ${bdbag.baseName}.splitData.err - ln -sf `readlink -e cookies.txt` ~/.bdbag/deriva-cookies.txt 2>>${bdbag.baseName}.splitData.err - echo "LOG: deriva cookie linked" >> ${bdbag.baseName}.splitData.err - study=`echo "${bdbag}" | cut -d '.' -f1` 2>>${bdbag.baseName}.splitData.err - echo "LOG: \${study}" >> ${bdbag.baseName}.splitData.err - unzip ${bdbag} 2>>${bdbag.baseName}.splitData.err - echo "LOG: bdgag unzipped" >> ${bdbag.baseName}.splitData.err - python3 ${baseDir}/scripts/modifyFetch.py --fetchFile \${study} 2>>${bdbag.baseName}.splitData.err - echo "LOG: fetch file filtered for only .fastq.gz" >> ${bdbag.baseName}.splitData.err - python3 ${baseDir}/scripts/splitFetch.py --fetchFile \${study} 2>>${bdbag.baseName}.splitData.err - echo "LOG: fetch file split by replicates" >> ${bdbag.baseName}.splitData.err - sh ${baseDir}/scripts/splitBag.sh \${study} 2>>${bdbag.baseName}.splitData.err - echo "LOG: bag recreated with replicate split fetch file" >> ${bdbag.baseName}.splitData.err + hostname >>${repRID_getBag}.getBag.err + ulimit -a >>${repRID_getBag}.getBag.err + ln -sf `readlink -e credential.json` ~/.deriva/credential.json 2>>${repRID_getBag}.getBag.err + echo "LOG: deriva credentials linked" >>${repRID_getBag}.getBag.err + deriva-download-cli dev.gudmap.org --catalog 2 ${derivaConfig} . rid=${repRID_getBag} 2>>${repRID_getBag}.getBag.err """ } @@ -57,26 +60,36 @@ process splitData { * getData: fetch study files from consortium with downloaded bdbag.zip */ process getData { - tag "${rep.baseName}" - publishDir "${logsDir}/getData", mode: 'symlink', pattern: "${rep.baseName}.getData.err" + tag "${repRID_getData}" + publishDir "${logsDir}/getData", mode: 'symlink', pattern: "${repRID_getData}.getData.err" input: - each rep from bdbagSplit + val repRID_getData + executor 'local' + path cookies, stageAs: 'deriva-cookies.txt' from bdbag + path bagit output: - set val ("${rep.baseName}"), file ("*.R{1,2}.fastq.gz") into trimming + path ("*.R{1,2}.fastq.gz") into fastqs + file("**/File.csv") into fileMeta + file("**/Experiment Settings.csv") into experimentSettingsMeta + file("**/Experiment.csv") into experimentMeta + file ("${repRID_getData}.getData.err") + script: """ - hostname >>${rep.baseName}.getData.err - ulimit -a >>${rep.baseName}.getData.err + hostname >>${repRID_getData}.getData.err + ulimit -a >>${repRID_getData}.getData.err export https_proxy=\${http_proxy} - replicate=\$(basename "${rep}" | cut -d '.' -f1) - echo "LOG: \${replicate}" >>${rep.baseName}.getData.err - unzip ${rep} 2>>${rep.baseName}.getData.err - echo "LOG: replicate bdbag unzipped" >>${rep.baseName}.getData.err - sh ${baseDir}/scripts/bdbagFetch.sh \${replicate} 2>>${rep.baseName}.getData.err - echo "LOG: replicate bdbag fetched" >>${rep.baseName}.getData.err + ln -sf `readlink -e deriva-cookies.txt` ~/.bdbag/deriva-cookies.txt >>${repRID_getData}.getData.err + echo "LOG: deriva cookie linked" >>${repRID_getData}.getData.err + replicate=\$(basename "${bagit}" | cut -d '.' -f1) + echo "LOG: \${replicate}" >>${repRID_getData}.getData.err + unzip ${bagit} 2>>${repRID_getData}.getData.err + echo "LOG: replicate bdbag unzipped" >>${repRID_getData}.getData.err + sh ${baseDir}/scripts/bdbagFetch.sh \${replicate} ${repRID_getData} 2>>${repRID_getData}.getData.err + echo "LOG: replicate bdbag fetched" >>${repRID_getData}.getData.err """ } @@ -84,19 +97,34 @@ process getData { * trimData: trims any adapter or non-host sequences from the data */ process trimData { - tag "trim-${repID}" - publishDir "${outDir}/tempOut/trimmed", mode: "symlink", pattern: "*_val_{1,2}.fq.gz" - publishDir "${logsDir}/trimData", mode: 'symlink', pattern: "\${rep}.trimData.*" + tag "${repRID_trimData}" + publishDir "${logsDir}/trimData", mode: 'symlink', pattern: "\${repRID_trimData}.trimData.*" input: - set repID, reads from trimming + val repRID_trimData + file(fastq) from fastqs output: - path ("*_val_{1,2}.fq.gz", type: 'file', maxDepth: '0') + path ("*.fq.gz") into fastqs_trimmed + val ends + file ("${repRID_trimData}.trimData.log") + file ("${repRID_trimData}.trimData.err") script: """ - rep=`echo ${repID} | cut -f2- -d '_'`; - trim_galore --gzip --max_n 1 --paired --basename \${rep} -j `nproc` ${reads[0]} ${reads[1]} 1>>\${rep}.trimData.log 2>>\${rep}.trimData.err; + if [ `nproc` -gt 8 ] + then + ncore=8 + else + ncore=`nproc` + fi + if [ '${fastq[1]}' == 'null' ] + then + ends='se' + trim_galore --gzip -q 25 --illumina --length 35 --basename ${repRID_trimData} -j \${ncore} ${fastq[0]} 1>>${repRID_trimData}.trimData.log 2>>${repRID_trimData}.trimData.err; + else + ends='pe' + trim_galore --gzip -q 25 --illumina --length 35 --paired --basename ${repRID_trimData} -j \${ncore} ${fastq[0]} ${fastq[1]} 1>>${repRID_trimData}.trimData.log 2>>${repRID_trimData}.trimData.err; + fi """ -} +} \ No newline at end of file diff --git a/workflow/scripts/bdbagFetch.sh b/workflow/scripts/bdbagFetch.sh index 9af4eb46c0e716e0e1db7cb66e9f027f63611218..902222a2ebb6aa7e978f0a820ad3c04472395848 100644 --- a/workflow/scripts/bdbagFetch.sh +++ b/workflow/scripts/bdbagFetch.sh @@ -1,6 +1,7 @@ #!/bin/bash -bdbag --resolve-fetch all --fetch-filter filename\$*fastq.gz $1 && +bdbag --resolve-fetch all --fetch-filter filename\$*fastq.gz $1 for i in $(find */ -name "*.R*.fastq.gz"); do - mv ${i} .; -done; + path=${2}$(echo ${i##*/} | grep -o "\.R.\.fastq\.gz"); + mv ${i} ./${path} +done; \ No newline at end of file diff --git a/workflow/scripts/splitBag.sh b/workflow/scripts/splitBag.sh deleted file mode 100644 index 3f6f6cdb610c684bdb57f666822dc0deb864fb04..0000000000000000000000000000000000000000 --- a/workflow/scripts/splitBag.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin - -for i in $(ls -d Replicate_*) -do -rsync -r $1/ ${i} --exclude=fetch.txt -zip -r ${i}.zip ${i} -done \ No newline at end of file