diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5d9a3be3d46027621fc4a0fc7cf88d53079f83a1..eb44569784c41e056da3f7d23181c8a6550f4849 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,34 +2,25 @@ # Brandi L. Cantarel - 2017 variables: - GIT_SUBMODULE_STRATEGY: normal + GIT_SUBMODULE_STRATEGY: recursive before_script: - - module load astrocyte/1.0.0 - module load nextflow/20.01.0 - - module load singularity/3.5.3 + - module load singularity/3.0.2 stages: - - check - - test_human - - test_mouse - -# This performs validation of the astrocyte_pkg.yml file -astrocyte_check: - stage: check - script: - - astrocyte_cli check "$CI_PROJECT_DIR" + - integration test_human: - stage: test_human + stage: integration script: - - nextflow run -c workflow/configs/nextflow.config -with-dag flowchart.png -with-timeline human_timeline.html -with-report human_report.html workflow/main.nf --design /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq/design.rnaseq.txt --input /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq --output human_output + - nextflow run -c nextflow.config -with-dag flowchart.png -with-timeline human_timeline.html -with-report human_report.html workflow/main.nf --design /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq/design.rnaseq.txt --input /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq --output human_output artifacts: expire_in: 2 days test_mouse: - stage: test_mouse + stage: integration script: - - nextflow run -c workflow/configs/nextflow.config -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html workflow/main.nf --input /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq --design /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq/mouse_se.design.txt --pairs se --fusion skip --genome /project/shared/bicf_workflow_ref/mouse/GRCm38 --markdups null --output mouse_output + - nextflow run -c nextflow.config -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html workflow/main.nf --input /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq --design /project/shared/bicf_workflow_ref/workflow_testdata/rnaseq/mouse_se.design.txt --pairs se --fusion skip --genome /project/shared/bicf_workflow_ref/mouse/GRCm38 --markdups null --output mouse_output artifacts: expire_in: 2 days diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 0242dee412031e1290cd19266bcb68035939fcaf..0000000000000000000000000000000000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "workflow/process_scripts"] - path = workflow/process_scripts - url = git@git.biohpc.swmed.edu:ngsclialab/process_scripts.git diff --git a/README.md b/README.md index 7b6ac564cc5bd18887916764ae90aecf1086a67b..93561bb73bf59d23fbe58af7c6ef5555b9f96105 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ module load nextflow/20.01.0 singularity/3.5.3 base=$repoClonedDirectory datadir='/project/shared/bicf_workflow_ref/workflow_testdata/rnaseq' -nextflow -C ${base}/workflow/configs/nextflow.config run ${base}/workflow/main.nf --design ${datadir}/design.rnaseq.txt --input ${datadir} --output analysis +nextflow -C ${base}/nextflow.config run ${base}/workflow/main.nf --design ${datadir}/design.rnaseq.txt --input ${datadir} --output analysis ``` @@ -80,7 +80,7 @@ module load nextflow/20.01.0 singularity/3.5.3 base=$repoClonedDirectory datadir='/project/shared/bicf_workflow_ref/workflow_testdata/rnaseq' -nextflow -C ${base}/workflow/configs/nextflow.config run -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html ${base}/workflow/main.nf --design ${datadir}/mouse_se.design.txt --input ${datadir} --pairs se --output analysis +nextflow -C ${base}/nextflow.config run -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html ${base}/workflow/main.nf --design ${datadir}/mouse_se.design.txt --input ${datadir} --pairs se --output analysis ``` diff --git a/astrocyte_pkg.yml b/astrocyte_pkg.yml index a7b3480013c7b355b6169b5a7c73bbfd1057c7d0..47962cd4987298242134038d30852a0a66188825 100644 --- a/astrocyte_pkg.yml +++ b/astrocyte_pkg.yml @@ -20,20 +20,6 @@ description: | It implements differential expression analysis, gene set enrichment analysis, gene fusion analysis and variant identification using RNASeq data. -# The minimum Astrocyte version that requires to run this workflow. For old pipelines, which do not have this label -# a default value of 0.3.1 will be assigned automatically. A request of minimum version less than 0.4.0 will be ignored. -minimum_astrocyte_version: '0.4.1' -# The Nextflow version that requires to run this workflow. For old pipelines, which do not have this label -# a default value of 0.31.0 will be assigned automatically. Please make sure the requested nextflow version is available -# in the module list. -nextflow_version: '20.01.0' -# (Optional) The Nextflow config file to use for this workflow. If provided, the file should exist in workflow/configs -nextflow_config: 'nextflow.config' -# The container to use for this workflow, none/singularity. If omitted, the default value 'none' will be used. -container: 'singularity' -# The version of singularity to use. This is required if container == 'singularity' -singularity_version: '3.5.3' - # ----------------------------------------------------------------------------- # DOCUMENTATION @@ -64,22 +50,6 @@ workflow_modules: - 'stringtie/1.1.2-intel' - 'speedseq/20160506' - 'python/2.7.x-anaconda' - -# A list of container images requires to run this workflow. -# Specify full path and version names to ensure reproducability. -# This keyword is required when 'container' is specified in Astrocyte 0.4.1 and above -workflow_containers: - - docker://goalconsortium/trim_galore:1.0.9 - - docker://goalconsortium/abra2:1.0.9 - - docker://goalconsortium/profiling_qc:1.0.9 - - docker://goalconsortium/dna_alignment:1.0.9 - - docker://goalconsortium/variantcalling:1.0.9 - - docker://goalconsortium/structuralvariant:1.1.4 - - docker://goalconsortium/starfusion:1.0.9 - - docker://goalconsortium/rna_alignment:1.0.9 - - docker://goalconsortium/rna_gene_abundance:1.1.3 - - docker://goalconsortium/rna_statanal:1.1.4 - # A list of parameters used by the workflow, defining how to present them, # options etc in the web interface. For each parameter: # diff --git a/workflow/configs/nextflow.config b/nextflow.config similarity index 77% rename from workflow/configs/nextflow.config rename to nextflow.config index c0eec91496b730abec3ee662fc608fab67ebb61b..150748a34379cf059276d0144009eb1943283927 100644 --- a/workflow/configs/nextflow.config +++ b/nextflow.config @@ -1,28 +1,27 @@ params { repoDir='/seqprg' } - process { executor = 'slurm' clusterOptions = '--hold --no-kill' queue = '128GB,256GB,256GBv1' withLabel: trim { - container = 'goalconsortium/trim_galore:1.1.3' + container = 'goalconsortium/trim_galore:1.0.9' } withLabel: abra2 { container = 'goalconsortium/abra2:1.0.9' } withLabel: profiling_qc { - container = 'goalconsortium/profiling_qc:1.1.3' + container = 'goalconsortium/profiling_qc:1.0.9' } withLabel: dnaalign { - container = 'goalconsortium/dna_alignment:1.1.3' + container = 'goalconsortium/dna_alignment:1.0.9' } withLabel: variantcalling { - container = 'goalconsortium/variantcalling:1.1.5' + container = 'goalconsortium/variantcalling:1.0.9' } withLabel: structuralvariant { - container = 'goalconsortium/structuralvariant:1.1.4' + container = 'goalconsortium/structuralvariant:1.1.2' } withLabel: starfusion { container = 'goalconsortium/starfusion:1.0.9' @@ -40,13 +39,13 @@ process { singularity { enabled = true - runOptions='--no-home --cleanenv --disable-cache' - cacheDir = "$baseDir/images/singularity" + runOptions='--no-home --cleanenv' + singularity.cacheDir="$PWD" } trace { enabled = true - file = 'trace.txt' + file = 'pipeline_trace.txt' field = 'task_id,native_id,process,name,status,exit,submit,start,complete,duration,realtime' } diff --git a/testing/run_human_pe_test.sh b/testing/run_human_pe_test.sh index ade5af35acd8489732831ce49b85764075781518..e8624c09d8caa4d85134c0c3f2ddbe7f55b9e952 100644 --- a/testing/run_human_pe_test.sh +++ b/testing/run_human_pe_test.sh @@ -4,4 +4,4 @@ module load nextflow/20.01.0 singularity/3.5.3 base='/project/BICF/BICF_Core/s166458/rnaseq_astrocyte' datadir='/project/shared/bicf_workflow_ref/workflow_testdata/rnaseq' -nextflow -C ${base}/workflow/configs/nextflow.config run ${base}/workflow/main.nf --design ${datadir}/design.rnaseq.txt --input ${datadir} --output analysis +nextflow -C ${base}/nextflow.config run ${base}/workflow/main.nf --design ${datadir}/design.rnaseq.txt --input ${datadir} --output analysis diff --git a/testing/run_mouse_se_test.sh b/testing/run_mouse_se_test.sh index 2d3f06b22f936cb4cffa9387882f97f9b0614b01..2c105e3bcdb151a336dc6e0302704828291c4570 100644 --- a/testing/run_mouse_se_test.sh +++ b/testing/run_mouse_se_test.sh @@ -4,4 +4,4 @@ module load nextflow/20.01.0 singularity/3.5.3 base='/project/BICF/BICF_Core/s166458/rnaseq_astrocyte' datadir='/project/shared/bicf_workflow_ref/workflow_testdata/rnaseq' -nextflow -C ${base}/workflow/configs/nextflow.config run -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html ${base}/workflow/main.nf --design ${datadir}/mouse_se.design.txt --input ${datadir} --pairs se --output analysis +nextflow -C ${base}/nextflow.config run -with-dag flowchart.png -with-timeline mouse_timeline.html -with-report mouse_report.html ${base}/workflow/main.nf --design ${datadir}/mouse_se.design.txt --input ${datadir} --pairs se --output analysis diff --git a/workflow/main.nf b/workflow/main.nf index c6a508d4c32cc5a440ae464e7bb8d6fd887ac11c..ef33763fcf301bd64566fe467137e4069db5e659 100644 --- a/workflow/main.nf +++ b/workflow/main.nf @@ -23,13 +23,9 @@ indel="$params.genome/GoldIndels.vcf.gz" knownindel=file(indel) dbsnp=file(dbsnp) - +repoDir=workflow.projectDir if (params.repoDir) { - repoDir=params.repoDir -} else if (workflow.projectDir) { - repoDir=workflow.projectDir -} else { - repoDir="$baseDir" + repoDir=params.repoDir } // params genome is the directory @@ -49,8 +45,6 @@ process checkdesignfile { stdout spltnames script: """ - # convert dos \\r to unix format - sed -i 's/\\x0D\$//' design.ori.txt bash $repoDir/process_scripts/design_file/checkdesignfile.sh ${params.pairs} design.ori.txt """ } @@ -96,15 +90,12 @@ process ralign { errorStrategy 'ignore' label 'ralign' publishDir "$params.output", mode: 'copy' - input: set pair_id, file(fqs) from trimread - output: set pair_id, file("${pair_id}.bam") into aligned set pair_id, file("${pair_id}.bam") into aligned2 file("${pair_id}.alignerout.txt") into hsatout - script: """ bash $repoDir/process_scripts/alignment/rnaseqalign.sh -a $params.align -p ${pair_id} -r ${index_path} ${fqs} diff --git a/workflow/process_scripts b/workflow/process_scripts deleted file mode 160000 index e5e905a0ada2d2a93ff0ed59b2fdf022a56c7464..0000000000000000000000000000000000000000 --- a/workflow/process_scripts +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e5e905a0ada2d2a93ff0ed59b2fdf022a56c7464