0.0.3

• TPM table:
  • Add Ensembl Gene ID
  • Rename columns: GENCODE_Gene_Symbol, Ensembl_GeneID, NCBI_GeneID
• MultiQC output custom tables (html+JSON):
  • Run table: Session ID and Pipeline Version
  • Reference Table: Species, Genome Reference Consortium Build, Genome Reference Consortium Patch, GENCODE Annotation Release (outputs both human and mouse versions)
• Add inputBag override param (inputBagForce) [*.zip]
  • Uses provided inputBag instead of downloading from data-hub
  • Still requires matching repRID input param
• Add fastq override param (fastqsForce) [R1,R2]
  • Uses provided fastq instead of downloading from data-hub
  • Still requires matching repRID input param and will pull inputBag from data-hub to access submitted metadata for reporting
• Add species override param (speciesForce) [Mus musculus or Homo sapiens]
  • forces the use of the provided species
  • ignores inferred ambiguous species
• Add GeneSymbol/EnsemblID/EntrezID translation files to references

PR checklist

• This comment contains a description of changes (with reason)
• If you've fixed a bug or added code that should be tested, add tests!
• Documentation in docs is updated
• Replace dag.png with the most recent CI pipleine integrated_pe artifact
• CHANGELOG.md is updated
• README.md is updated
• LICENSE.md is updated with new contributors
• Docker images moved to production release and changed in pipeline
• Docker images used in the CI unit tests match those used in pipeline

• Close issue
  Closes #61 (closed) Closes #71 (closed) Closes #72 (closed) Closes #73 (closed) Closes #74 (closed)

/cc @ghenry @venkat.malladi

.gitlab-ci.yml

@@ -14,6 +14,11 @@ stages:
 
 getBag:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - ln -sfn `readlink -e ./test_data/auth/credential.json` ~/.deriva/credential.json
   - singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' deriva-download-cli dev.gudmap.org --catalog 2 ./workflow/conf/replicate_export_config.json . rid=Q-Y5F6
@@ -21,6 +26,11 @@ getBag:
 
 getData:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
   - unzip ./test_data/bag/Replicate_Q-Y5F6.zip
@@ -29,6 +39,11 @@ getData:
 
 parseMetadata:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - rep=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p repRID)
   - exp=$(singularity run 'docker://bicf/python3:2.0.1_indev' python3 ./workflow/scripts/parseMeta.py -r Replicate_RID -m "./test_data/meta/metaTest.csv" -p expRID)
@@ -44,6 +59,11 @@ parseMetadata:
 
 inferMetadata:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - >
     align=$(echo $(grep "Overall alignment rate" ./test_data/meta/Q-Y5F6_1M.se.alignSummary.txt | cut -f2 -d ':' | cut -f2 -d ' ' | tr -d '%')) &&
@@ -56,6 +76,11 @@ inferMetadata:
 
 getRef:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - mkdir -p hu
   - mkdir -p mo
@@ -64,6 +89,11 @@ getRef:
 
 trimData:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - singularity run 'docker://bicf/trimgalore:1.1' trim_galore --gzip -q 25 --length 35 --basename Q-Y5F6_1M.se ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz
   - singularity run 'docker://bicf/trimgalore:1.1' trim_galore --gzip -q 25 --length 35 --paired --basename Q-Y5F6_1M.pe ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz ./test_data/fastq/small/Q-Y5F6_1M.R2.fastq.gz
@@ -73,12 +103,22 @@ trimData:
 
 downsampleData:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - singularity run 'docker://bicf/seqtk:2.0.1_indev' seqtk sample -s100 ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz 1000 1> sampled.1.fq
   - pytest -m downsampleData
 
 alignData:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - singularity run 'docker://bicf/gudmaprbkaligner:2.0.1_indev' hisat2 -p 20 --add-chrname --un-gz Q-Y5F6_1M.se.unal.gz -S Q-Y5F6_1M.se.sam -x /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/hisat2/genome --rna-strandness F -U ./test_data/fastq/small/Q-Y5F6_1M.se_trimmed.fq.gz --summary-file Q-Y5F6_1M.se.alignSummary.txt --new-summary
   - singularity run 'docker://bicf/gudmaprbkaligner:2.0.1_indev' samtools view -1 -@ 20 -F 4 -F 8 -F 256 -o Q-Y5F6_1M.se.bam Q-Y5F6_1M.se.sam
@@ -92,6 +132,11 @@ alignData:
 
 dedupData:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - singularity run 'docker://bicf/gudmaprbkdedup:2.0.0' java -jar /picard/build/libs/picard.jar MarkDuplicates I=./test_data/bam/small/Q-Y5F6_1M.se.sorted.bam O=Q-Y5F6_1M.se.deduped.bam M=Q-Y5F6_1M.se.deduped.Metrics.txt REMOVE_DUPLICATES=true
   - singularity run 'docker://bicf/gudmaprbkdedup:2.0.0' samtools sort -@ 20 -O BAM -o Q-Y5F6_1M.se.sorted.deduped.bam ./test_data/bam/small/Q-Y5F6_1M.se.deduped.bam
@@ -104,6 +149,11 @@ dedupData:
 
 countData:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/geneID.tsv
   - ln -s /project/BICF/BICF_Core/shared/gudmap/references/GRCh38.p12.v31/Entrez.tsv
@@ -115,18 +165,33 @@ countData:
 
 makeBigWig:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - singularity run 'docker://bicf/deeptools3.3:2.0.1_indev' bamCoverage -p 20 -b ./test_data/bam/small/Q-Y5F6_1M.se.sorted.deduped.bam -o Q-Y5F6_1M.se.bw
   - pytest -m makeBigWig
 
 fastqc:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - singularity run 'docker://bicf/fastqc:2.0.1_indev' fastqc ./test_data/fastq/small/Q-Y5F6_1M.R1.fastq.gz -o .
   - pytest -m fastqc
 
 dataQC:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - echo -e  "geneID\tchrom\ttx_start\ttx_end\tTIN" > Q-Y5F6_1M.se.sorted.deduped.tin.xls
   - for i in {"chr8","chr4","chrY"}; do
@@ -135,6 +200,11 @@ dataQC:
 
 outputBag:
   stage: unit
+  only:
+    - push
+    - tags
+  except:
+    - merge_requests
   script:
   - mkdir test
   - singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' bdbag test --archiver zip
@@ -143,6 +213,11 @@ outputBag:
 
 integration_se:
   stage: integration
+  only:
+    - merge_requests
+  except:
+    refs:
+      - master
   script:
   - hostname
   - ulimit -a
@@ -156,9 +231,18 @@ integration_se:
       - output/report/
       - SE_multiqc_data.json
     expire_in: 7 days
+  retry:
+    max: 1
+    when:
+      - always
 
 integration_pe:
   stage: integration
+  only:
+    - merge_requests
+  except:
+    refs:
+      - master
   script:
   - hostname
   - ulimit -a
@@ -173,9 +257,18 @@ integration_pe:
       - output/report/
       - PE_multiqc_data.json
     expire_in: 7 days
+  retry:
+    max: 1
+    when:
+      - always
 
 override_inputBag:
   stage: integration
+  only:
+    - merge_requests
+  except:
+    refs:
+      - master
   script:
   - hostname
   - ulimit -a
@@ -187,9 +280,18 @@ override_inputBag:
     paths:
       - inputBagOverride_PE_multiqc_data.json
     expire_in: 7 days
+  retry:
+    max: 1
+    when:
+      - always
 
 override_fastq:
   stage: integration
+  only:
+    - merge_requests
+  except:
+    refs:
+      - master
   script:
   - hostname
   - ulimit -a
@@ -201,9 +303,18 @@ override_fastq:
     paths:
       - fastqOverride_PE_multiqc_data.json
     expire_in: 7 days
+  retry:
+    max: 1
+    when:
+      - always
     
 override_species:
   stage: integration
+  only:
+    - merge_requests
+  except:
+    refs:
+      - master
   script:
   - hostname
   - ulimit -a
@@ -215,9 +326,19 @@ override_species:
     paths:
       - speciesOverride_PE_multiqc_data.json
     expire_in: 7 days
+  retry:
+    max: 1
+    when:
+      - always
+  
 
 consistency:
   stage: consistency
+  only:
+    - merge_requests
+  except:
+    refs:
+      - master
   script:
   - grep -m 1 \"Assigned\":.[0-9] SE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedSE.txt
   - grep -m 1 \"Assigned\":.[0-9] PE_multiqc_data.json | grep -oe '\([0-9.]*\)' > assignedPE.txt