Merge branch '9-surgery' into 'master'

Resolve "Surgery Table"

Closes #9

See merge request !11

scripts/sql_queries/surgery.sql

+select E.MRN, D.*, A.START_DATE, A.ENCOUNTER_NUM, A.PROVIDER_ID, A.LOCATION_CD, A.OBSERVATION_BLOB
+from kidney_obs_fact A
+right join (select C.CPT, B.c_name
+            from I2B2_TERMINOLOGY B, SURGERY_MAP C
+            where B.c_basecode = C.CPT)
+            D
+on A.concept_cd = D.CPT
+inner join KIDNEY_MRN_LIST E
+on A.PATIENT_NUM = E.PATIENT_NUM
+inner join "MRN_Filter_KCE" F
+on E.MRN = F.MRN;

scripts/transform_surgery.py

+#!/usr/bin/env python3
+
+'''Generate Surgery Table'''
+
+import argparse
+import datetime
+import os
+import pandas as pd
+
+
+EPILOG = '''
+For more details:
+        %(prog)s --help
+'''
+
+
+def get_args():
+    '''Define arguments.'''
+
+    parser = argparse.ArgumentParser(
+        description=__doc__, epilog=EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    parser.add_argument('-f', '--file',
+                        help="Surgery File (csv format).",
+                        required=True)
+
+    parser.add_argument('-d', '--date',
+                        help="Date Shift (tsv format).",
+                        required=True)
+
+    parser.add_argument('-s', '--surgery',
+                        help="CPT Filter file (tsv format).",
+                        required=True)
+
+    parser.add_argument('-o', '--out',
+                        help="The output path (csv format).",
+                        required=True)
+
+    args = parser.parse_args()
+    return args
+
+
+def reformat_record(surgery):
+    '''Reformat surgery table'''
+
+
+    surgery_reformat = surgery[['MRN', 'CPT', 'C_NAME', 'START_DATE']]
+
+    return surgery_reformat
+
+
+def convert_surgery(surgery_reformt, surgery_map):
+    '''Convert to surgery table format'''
+
+    # Group surgeries
+    surgery_group = surgery_reformt.groupby(['MRN', 'START_DATE'])
+
+    # Set empty dataframes
+    surgery_table = pd.DataFrame(columns=['MRN', 'SurgeryDate', 'NephrectomyType',
+                            'NephrectomyApproach', 'Ablation', 'RoboticAssisted' ])
+    conflicting_surgery = pd.DataFrame()
+
+    # Filter for Surgery types
+    for key,group_df in surgery_group:
+        mrn = key[0]
+        surgery_date = key[1]
+
+        nephrectomy = False
+
+        if len(group_df) == 1:
+            continue
+
+        # Filter for Robotic
+        if 'CPT:S2900' in group_df.CPT.values:
+            robotic = 'Yes'
+            group_df = group_df[group_df.CPT != 'CPT:S2900']
+        else:
+            robotic = 'No'
+        group_df_filtered = group_df.drop_duplicates()
+
+        # Check from Nephrectomy or appliation
+        surgery_map_nephrectomy = surgery_map[surgery_map.NephrectomyAproach != "Not Applicable"]
+        surgery_map_ablation = surgery_map[surgery_map.Ablation == 'Yes']
+
+        if group_df_filtered.CPT.values in surgery_map_nephrectomy.Surgery_code.values:
+            group_df_nephrectomy = group_df_filtered[group_df_filtered.CPT.isin(surgery_map_nephrectomy.Surgery_code)]
+            if len(group_df_nephrectomy) > 1:
+                temp_table = conflicting_surgery.append(group_df_filtered)
+                conflicting_surgery = temp_table
+                continue
+            else:
+                temp = surgery_map_nephrectomy[surgery_map_nephrectomy.Surgery_code.isin(group_df_nephrectomy.CPT)]
+                nephrectomy_type = temp.NephrectomyType.values[0]
+                nephrectomy_approach = temp.NephrectomyAproach.values[0]
+                ablation = temp.Ablation.values[0]
+                nephrectomy = True
+
+        if group_df_filtered.CPT.values in surgery_map_ablation.Surgery_code.values:
+            group_df_ablation = group_df_filtered[group_df_filtered.CPT.isin(surgery_map_ablation.Surgery_code)]
+            if len(group_df_ablation) > 1:
+                temp_table = conflicting_surgery.append(group_df_filtered)
+                conflicting_surgery = temp_table
+                continue
+            if nephrectomy == False:
+                temp = surgery_map_ablation[surgery_map_ablation.Surgery_code.isin(group_df_ablation.CPT)]
+                nephrectomy_type = temp.NephrectomyType.values[0]
+                nephrectomy_approach = temp.NephrectomyAproach.values[0]
+                ablation = temp.Ablation.values[0]
+
+        # Add values to
+        temp_table = surgery_table.append({ 'MRN': mrn ,
+                                 'SurgeryDate': surgery_date,
+                                 'NephrectomyType': nephrectomy_type,
+                                 'NephrectomyApproach': nephrectomy_approach,
+                                 'Ablation': ablation,
+                                 'RoboticAssisted': robotic} , ignore_index=True)
+        surgery_table = temp_table
+
+    return surgery_table, conflicting_surgery
+
+
+def calculate_shift(surgery_reformat, date_shift):
+    '''Shift Date for start date'''
+
+    # Convert to DateTime
+    date_shift['Shift'] = pd.to_timedelta(date_shift['Shift'], unit='s')
+    surgery_reformat['SurgeryDate'] = pd.to_datetime(surgery_reformat['SurgeryDate'])
+
+    # Merge data
+    merged = surgery_reformat.merge(date_shift, left_on='MRN', right_on='MRN', how='inner')
+
+    # Calculate Date Shift
+    merged['SurgeryDate'] = merged['SurgeryDate'] + merged['Shift']
+
+    merged['SurgeryDate'] = merged['SurgeryDate'].dt.date
+
+    # Drop Shift and Clinic column
+    merged.drop(['Shift'], axis=1, inplace=True)
+
+    # Int MRN
+    merged.MRN = merged.MRN.astype(int)
+
+    # Rename columns
+    #surgery_columns = ["mrn", "cpt", "type","start_date",]
+    #merged.columns = imagining_columns
+
+    return merged
+
+
+def main():
+    args = get_args()
+    surgery = args.file
+    date = args.date
+    surgery_map = args.surgery
+    out_path = args.out
+
+    # Make output files
+    surgery_table = os.path.join(out_path + 'surgery_table.csv')
+    unmapped_table = os.path.join(out_path + 'unmapped_surgery_table.csv')
+
+    # Read in files
+    surgery_df = pd.read_csv(surgery)
+    date_shift = pd.read_csv(date)
+    surgery_map = pd.read_csv(surgery_map, sep='\t')
+
+    # Reformat data
+    surgery_reformat = reformat_record(surgery_df)
+
+    # Convert Surgery
+    surgery_convert, conflicting_surgery = convert_surgery(surgery_reformat, surgery_map)
+
+    # Calculate Date Shift
+    shifted_df = calculate_shift(surgery_convert, date_shift)
+
+    # Write out surgery table
+    shifted_df.to_csv(surgery_table, index=False)
+    conflicting_surgery.to_csv(unmapped_table, index=False)
+
+if __name__ == '__main__':
+    main()

standards/surgery_map.tsv

+Surgery_code	Description	Ablation	Robotic	NephrectomyAproach	NephrectomyType	Biopsy
+CPT:50593	"Ablation, renal tumor(s), unilateral, percutaneous, cryotherapy"	Yes	No	Not Applicable	Not Applicable	Not Applicable
+CPT:50250	"Ablation, open, 1 or more renal mass lesion(s), cryosurgical, including intraoperative ultrasound guidance and monitoring, if performed"	Yes	No	Not Applicable	Not Applicable	Not Applicable
+CPT:50592	"Ablation, 1 or more renal tumor(s), percutaneous, unilateral, radiofrequency"	Yes	No	Not Applicable	Not Applicable	Not Applicable
+CPT:50542	"Laparoscopy, surgical; ablation of renal mass lesion(s), including intraoperative ultrasound guidance and monitoring, when performed"	Yes	No	Not Applicable	Not Applicable	Not Applicable
+CPT:50546	"Laparoscopy, surgical; nephrectomy, including partial ureterectomy"	No	No	Laparoscopic	Total	Not Applicable
+CPT:50543	"Laparoscopy, surgical; partial nephrectomy"	No	No	Laparoscopic	Partial	Not Applicable
+CPT:50230	"Nephrectomy, including partial ureterectomy, any open approach including rib resection; radical, with regional lymphadenectomy and/or vena caval thrombectomy"	No	No	Open	Total	Not Applicable
+CPT:50220	"Nephrectomy, including partial ureterectomy, any open approach including rib resection"	No	No	Open	Total	Not Applicable
+CPT:50240	"Nephrectomy, partial"	No	No	Open	Partial	Not Applicable
+CPT:50225	"Nephrectomy, including partial ureterectomy, any open approach including rib resection; complicated because of previous surgery on same kidney"	No	No	Open	Total	Not Applicable
+CPT:50545	"Laparoscopy, surgical; radical nephrectomy (includes removal of Gerota's fascia and surrounding fatty tissue, removal of regional lymph nodes, and adrenalectomy)"	No	No	Laparoscopic	Total	Not Applicable
+CPT:50205	Renal biopsy; by surgical exposure of kidney	No	No	Not Applicable	Not Applicable	Yes
+CPT:50200	"Renal biopsy; percutaneous, by trocar or needle"	No	No	Not Applicable	Not Applicable	Yes
+CPT:S2900	Robotic surgical system	No	Yes	Not Applicable	Not Applicable	Not Applicable
\ No newline at end of file