Check for old mRNA QC and delete catalog entry

.gitlab-ci.yml

@@ -45,8 +45,8 @@ getData:
   script:
   - singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' bdbag --version > version_bdbag.txt
   - ln -sfn `readlink -e ./test_data/auth/cookies.txt` ~/.bdbag/deriva-cookies.txt
-  - unzip ./test_data/bag/staging/Replicate_Q-Y5F6.zip
-  - singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' bash ./workflow/scripts/bdbagFetch.sh Replicate_Q-Y5F6 Replicate_Q-Y5F6 TEST
+  - unzip ./test_data/bag/staging/Q-Y5F6_inputBag.zip
+  - singularity run 'docker://bicf/gudmaprbkfilexfer:2.0.1_indev' bash ./workflow/scripts/bdbagFetch.sh Q-Y5F6_inputBag Q-Y5F6_inputBag TEST
   - pytest -m getData
   artifacts:
     name: "$CI_JOB_NAME"

workflow/rna-seq.nf

@@ -1459,19 +1459,22 @@ process uploadQC {
 
   cookie=\$(cat credential.json | grep -A 1 '\\"${source}\\": {' | grep -o '\\"cookie\\": \\".*\\"')
   cookie=\${cookie:11:-1}
-
+singularity run 'docker://gudmaprbk/deriva1.3:1.0.0' python3 ./workflow/scripts/
   exist=\$(curl -s https://${source}/ermrest/catalog/2/entity/RNASeq:mRNA_QC/Replicate=${repRID}/Execution_Run=${executionRunRID})
-  if [ "\${exist}" == "[]" ]
+  if [ "\${exist}" != "[]" ]
   then
-    qc_rid=\$(python3 uploadQC.py -r ${repRID} -e ${executionRunRID} -p "\${end}" -s ${stranded} -l ${length} -w ${rawCount} -f ${finalCount} -o ${source} -c \${cookie} -u F)
-    echo LOG: mRNA QC RID uploaded - \${qc_rid} >> ${repRID}.uploadQC.log
-  else
-    rid=\$(echo \${exist} | grep -o '\\"RID\\":\\".*\\",\\"RCT')
-    rid=\${rid:7:-6}
-    qc_rid=\$(python3 uploadQC.py -r ${repRID} -e ${executionRunRID} -p "\${end}" -s ${stranded} -l ${length} -w ${rawCount} -f ${finalCount} -o ${source} -c \${cookie} -u \${rid})
-    echo LOG: mRNA QC RID updated - \${qc_rid} >> ${repRID}.uploadQC.log
+    rids=\$(echo $exist | grep -o '\\"RID\\":\\".\\{7\\}' | sed 's/^.\\{7\\}//')
+    for rid in \${rids}
+    do
+      python3 deleteEntry.py -r \${rid} -t mRNA_QC -o ${source} -c \${cookie}
+      echo LOG: old mRNA QC RID deleted - \${rid} >> ${repRID}.uploadQC.log
+    done
+    echo LOG: all old mRNA QC RIDs deleted >> ${repRID}.uploadQC.log
   fi
 
+  qc_rid=\$(python3 uploadQC.py -r ${repRID} -e ${executionRunRID} -p "\${end}" -s ${stranded} -l ${length} -w ${rawCount} -f ${finalCount} -o ${source} -c \${cookie} -u F)
+  echo LOG: mRNA QC RID uploaded - \${qc_rid} >> ${repRID}.uploadQC.log
+
   echo \${qc_rid} > qcRID.csv
   """
 }

workflow/scripts/deleteEntry.py

+import argparse
+from deriva.core import ErmrestCatalog, get_credential, BaseCLI
+import sys
+import csv
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-r', '--RID', help="replicate RID", required=True)
+    parser.add_argument('-t', '--table', help="source table", required=True)
+    parser.add_argument('-o', '--host', help="datahub host", required=True)
+    parser.add_argument('-c', '--cookie', help="cookie token", required=True)
+    args = parser.parse_args()
+    return args
+
+def main(hostname, catalog_number, credential):
+    catalog = ErmrestCatalog('https', hostname, catalog_number, credential)
+    pb = catalog.getPathBuilder()
+    if args.table == 'mRNA_QC':
+        run_table = pb.RNASeq.mRNA_QC
+    elif args.table == "Processed_File":
+        run_table = pb.RNASeq.Processed_File
+
+    path = run_table.filter(run_table.RID == args.RID)
+    path.delete()
+    rid = args.RID
+    
+
+    print(rid + "deleted")
+
+
+if __name__ == '__main__':
+    args = get_args()
+    cli = BaseCLI("Custom RNASeq query", None, 1)
+    cli.remove_options(["--config-file"])
+    host = args.host
+    credentials = {"cookie": args.cookie}
+    main(host, 2, credentials)
\ No newline at end of file

workflow/tests/test_getBag.py

@@ -12,4 +12,4 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
 @pytest.mark.getBag
 def test_getBag():
     assert os.path.exists(os.path.join(
-        test_output_path, 'Replicate_Q-Y5F6.zip'))
+        test_output_path, 'Q-Y5F6_inputBag.zip'))

workflow/tests/test_getData.py

@@ -12,6 +12,6 @@ test_output_path = os.path.dirname(os.path.abspath(__file__)) + \
 @pytest.mark.getData
 def test_getData():
     assert os.path.exists(os.path.join(
-        test_output_path, 'Replicate_Q-Y5F6/bagit.txt'))
+        test_output_path, 'Q-Y5F6_inputBag/bagit.txt'))
     assert os.path.exists(os.path.join(
-        test_output_path, 'Replicate_Q-Y5F6/data/assets/Study/Q-Y4GY/Experiment/Q-Y4DP/Replicate/Q-Y5F6/mMARIS_Six2-#3.gene.rpkm.txt'))
+        test_output_path, 'Q-Y5F6_inputBag/data/assets/Study/Q-Y4GY/Experiment/Q-Y4DP/Replicate/Q-Y5F6/mMARIS_Six2-#3.gene.rpkm.txt'))