Merge branch '4-medications' into 'master'

Resolve "Cancer Drug Table"

Closes #4

See merge request !10

scripts/sql_queries/medications.sql

+select E.MRN, B.*, A.CONCEPT_CD, A.START_DATE, A.END_DATE, A.VALTYPE_CD, A.TVAL_CHAR, A.NVAL_NUM, A.VALUEFLAG_CD, A.UNITS_CD, A.MODIFIER_CD, A.OBSERVATION_BLOB
+from kidney_obs_fact A
+right join (select D.name,D.generic_name, D.medication_id, D.investigatl_med_yn
+            from clarity_medication D
+            where (lower(D.name) like '%aldesleukin%' or lower(D.generic_name) like '%aldesleukin%'
+                or lower(D.name) like '%atezolizumab%' or lower(D.generic_name) like '%atezolizumab%'
+                or lower(D.name) like '%avelumab%' or lower(D.generic_name) like '%avelumab%'
+                or lower(D.name) like '%axitinib%' or lower(D.generic_name) like '%axitinib%'
+                or lower(D.name) like '%bevacizumab%' or lower(D.generic_name) like '%bevacizumab%'
+                or lower(D.name) like '%cabozantinib%' or lower(D.generic_name) like '%cabozantinib%'
+                or lower(D.name) like '%everolimus%' or lower(D.generic_name) like '%everolimus%'
+                or lower(D.name) like 'interferon alfa-2%' or lower(D.generic_name) like 'interferon alfa-2%'
+                or lower(D.name) like '%ipilimumab%' or lower(D.generic_name) like '%ipilimumab%'
+                or lower(D.name) like '%lenvatinib%' or lower(D.generic_name) like '%lenvatinib%'
+                or lower(D.name) like '%nivolumab%' or lower(D.generic_name) like '%nivolumab%'
+                or lower(D.name) like '%pazopanib%' or lower(D.generic_name) like '%pazopanib%'
+                or lower(D.name) like '%pembrolizumab%' or lower(D.generic_name) like '%pembrolizumab%'
+                or lower(D.name) like '%sirolimus%' or lower(D.generic_name) like '%sirolimus%'
+                or lower(D.name) like '%sorafenib%' or lower(D.generic_name) like '%sorafenib%'
+                or lower(D.name) like '%sunitinib%' or lower(D.generic_name) like '%sunitinib%'
+                or lower(D.name) like '%cpi-444%' or lower(D.name) like '%nktr%'
+                or lower(D.name) like '%pt2385%' or lower(D.name) like '%rad001%'
+                or lower(D.name) like '%denosumab%'or lower(D.generic_name) like '%denosumab%'
+                or lower(D.name) like '%zoledronic acid%'or lower(D.generic_name) like '%zoledronic acid%')
+            ) B
+on A.concept_cd = 'MED:'||B.medication_id
+inner join KIDNEY_MRN_LIST E
+on A.PATIENT_NUM = E.PATIENT_NUM
+inner join "MRN_Filter_KCE" F
+on E.MRN = F.MRN;

scripts/transform_medications.py

+#!/usr/bin/env python3
+
+'''Generate Medications Tables'''
+
+import argparse
+import datetime
+import os
+import pandas as pd
+import numpy as np
+from bs4 import BeautifulSoup
+
+
+EPILOG = '''
+For more details:
+        %(prog)s --help
+'''
+
+
+def get_args():
+    '''Define arguments.'''
+
+    parser = argparse.ArgumentParser(
+        description=__doc__, epilog=EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    parser.add_argument('-f', '--file',
+                        help="Epic File (csv format).",
+                        required=True)
+
+    parser.add_argument('-s', '--supportive',
+                        help="Supportive Meds Filter (tsv format).",
+                        required=True)
+
+    parser.add_argument('-m', '--medsmap',
+                        help="Medications mapping (csv format).",
+                        required=True)
+
+    parser.add_argument('-d', '--date',
+                        help="Date Shift (tsv format).",
+                        required=True)
+
+    parser.add_argument('-o', '--out',
+                        help="The output path (csv format).",
+                        required=True)
+
+    args = parser.parse_args()
+    return args
+
+
+def filter_supplemental(epic, sup, meds_map):
+    '''Filter by supplemental med name'''
+
+    # Filter for columns
+    sup_list = list(sup['ID'])
+    sup_filter = epic[epic['CONCEPT_CD'].isin(sup_list)]
+    onco_list = list(meds_map['ID'])
+    onco_filter = epic[epic['CONCEPT_CD'].isin(onco_list)]
+    missing_meds = epic[~(epic['CONCEPT_CD'].isin(onco_list) | epic['CONCEPT_CD'].isin(sup_list)) ]
+
+    # Merge supp meds mapping
+    meds_named = pd.merge(onco_filter, meds_map, left_on = 'CONCEPT_CD', right_on = 'ID' )
+    supmeds_named = pd.merge(sup_filter, sup, left_on = 'CONCEPT_CD', right_on = 'ID' )
+
+    return supmeds_named, meds_named, missing_meds
+
+
+def filter_supp(supp_meds):
+    '''Filter supportive meds'''
+
+    # Filter Adminstered drugs
+    meds_admin = supp_meds[supp_meds['MODIFIER_CD'].isin(['RX|ADMIN', 'RX|FILLED'])]
+
+    # Filter for Meds taken
+    meds_taken = meds_admin[meds_admin['TVAL_CHAR'].isin(['Given', 'New Bag', 'IV Started', 'IV Stopped'])]
+    meds_non_taken = meds_admin[~meds_admin['TVAL_CHAR'].isin(['Given', 'New Bag', 'IV Started', 'IV Stopped'])]
+
+    # Make date
+    meds_taken['START_DATE'] = pd.to_datetime(meds_taken['START_DATE'])
+
+    #Filter columns
+    supp_meds_reformated = meds_taken[['MRN', 'Meds', 'START_DATE']]
+    supp_meds_reformated.columns = ['mrn', 'name', 'Start_date']
+
+    return supp_meds_reformated, meds_non_taken
+
+
+def parse_blob(meds):
+    '''Parse observation blob for values in blob'''
+
+    meds_tups = []
+    for index, row in meds.iterrows():
+        mrn = row['MRN']
+        name = row['Meds']
+        order_type = row['MODIFIER_CD']
+        date = row['START_DATE']
+        ordering_date = ''
+        start_date = ''
+        end_date = ''
+        quantity = ''
+        sig = ''
+        refills = ''
+        frequency = ''
+        pcori_freq = ''
+        dispense_amt = ''
+        dispense_date = ''
+        dispense_sup = ''
+        ndc = ''
+        if pd.notnull(row['OBSERVATION_BLOB']):
+            soup = BeautifulSoup(row['OBSERVATION_BLOB'], 'html.parser')
+            tags = [tag.name for tag in soup.find_all()]
+
+            if 'odering_date' in tags:
+                ordering_date = soup.ordering_date.text
+            if 'start_date' in tags:
+                start_date = soup.start_date.text
+            if 'end_date' in tags:
+                end_date = soup.end_date.text
+            if 'quantity' in tags:
+                quantity = soup.quantity.text
+            if 'sig' in tags:
+                sig = soup.sig.text
+            if 'refills' in tags:
+                refills = soup.refills.text
+            if 'frequency' in tags:
+                frequency = soup.frequency.text
+            if 'pcori_freq' in tags:
+                pcori_freq = soup.pcori_freq.text
+            if 'dispense_amt' in tags:
+                dispense_amt = soup.dispense_amt.text
+            if 'dispense_date' in tags:
+                dispense_date = soup.dispense_date.text
+            if 'dispense_sup' in tags:
+                dispense_sup = soup.dispense_sup.text
+            if 'ndc' in tags:
+                ndc = soup.ndc.text
+
+        meds_tups.append((
+            mrn,
+            name,
+            order_type,
+            date,
+            ordering_date,
+            start_date,
+            end_date,
+            quantity,
+            sig,
+            refills,
+            frequency, pcori_freq, dispense_date, dispense_amt, dispense_sup, ndc ))
+
+    meds_columns = ["mrn", 'name', 'order_type', "date",'ordering_date',
+                   "start", "end", "quantity", 'sig', 'refills', 'frequency', 'pcori_freq', 'dispense_date',
+                   'dispanse_amt', 'dispense_sup', 'ndc']
+    meds_transformed = pd.DataFrame(meds_tups, columns=meds_columns)
+    return meds_transformed
+
+
+def calculate_regime(meds):
+    '''Calculate main meds regime'''
+
+    # Filter for columns
+    meds_transformed = parse_blob(meds)
+
+    # Conver to dates
+    meds_transformed['date'] = pd.to_datetime(meds_transformed['date'])
+    meds_transformed['ordering_date'] = pd.to_datetime(meds_transformed['ordering_date'])
+    meds_transformed['start'] = pd.to_datetime(meds_transformed['start'])
+    meds_transformed['end'] = pd.to_datetime(meds_transformed['end'])
+    meds_transformed['dispense_date'] = pd.to_datetime(meds_transformed['dispense_date'])
+
+    # Format Oncological Meds
+    meds_filtered_uniq = meds_transformed.groupby(['mrn', 'name'], as_index=False).agg({'date' : [np.min, np.max], 'start' : [np.min], 'end' : [np.max]   })
+    meds_filtered_uniq.columns = ['mrn', 'name','date_start', 'date_end', 'start', 'end']
+    meds_filtered_uniq['Start_date'] = meds_filtered_uniq[['date_start', 'date_end', 'start', 'end']].min(axis=1)
+    meds_filtered_uniq['End_date'] = meds_filtered_uniq[['date_start', 'date_end', 'start', 'end']].max(axis=1)
+    meds_filtered_uniq['Duration'] = (meds_filtered_uniq['End_date'] - meds_filtered_uniq['Start_date']).dt.days
+    onco_meds_reformated = meds_filtered_uniq[['mrn', 'name', 'Start_date', 'End_date', 'Duration']]
+
+    # Reset duration for 0
+    onco_meds_reformated.loc[onco_meds_reformated.Duration == 0, 'End_date'] = None
+    onco_meds_reformated.loc[onco_meds_reformated.Duration == 0, 'Duration'] = None
+
+    # If End Date > thatn current pull reset to null
+    onco_meds_reformated.loc[onco_meds_reformated.End_date > datetime.datetime(2019, 1, 1), 'End_date'] = None
+    onco_meds_reformated.loc[onco_meds_reformated.End_date > datetime.datetime(2019, 1, 1), 'Duration'] = None
+
+    # If End_date is null
+    onco_meds_reformated.loc[onco_meds_reformated.End_date.isnull(), 'Duration'] = None
+
+    onco_meds_reformated = onco_meds_reformated.reset_index(drop=True)
+
+    return onco_meds_reformated
+
+
+def calculate_shift(meds, date_shift):
+    '''Shift Date for dates'''
+
+    # Convert to DateTime
+    date_shift['Shift'] = pd.to_timedelta(date_shift['Shift'], unit='s')
+    meds['Start_date'] = pd.to_datetime(meds['Start_date'])
+
+    # Merge data
+    merged = meds.merge(date_shift, left_on='mrn', right_on='MRN', how='inner')
+
+    # Calculate Date Shift
+    merged['Start_date'] = merged['Start_date'] + merged['Shift']
+
+    if 'End_date' in merged.columns:
+        merged['End_date'] = merged['End_date'] + merged['Shift']
+
+    # Drop Shift column
+    merged.drop(['Shift'], axis=1, inplace=True)
+
+    # Int MRN
+    merged.MRN = merged.MRN.astype(int)
+
+    return merged
+
+
+def main():
+    args = get_args()
+    epic = args.file
+    sup = args.supportive
+    medsmap = args.medsmap
+    date = args.date
+    out_path = args.out
+
+    # Make output files
+    meds_table = os.path.join(out_path + 'med_table.csv')
+    unmapped_table = os.path.join(out_path + 'unmapped_med_table.csv')
+    suppmeds_table = os.path.join(out_path + 'suppmed_table.csv')
+    not_given_suppmeds_table = os.path.join(out_path + 'not_given_suppmed_table.csv')
+
+    # Read in files
+    epic_df = pd.read_csv(epic)
+    supmedsmap_df = pd.read_csv(sup)
+    medsmap_df = pd.read_csv(medsmap)
+    date_shift = pd.read_csv(date)
+
+    # Filter for supplemental meds
+    supp_meds, onco_meds, missing_meds = filter_supplemental(epic_df, supmedsmap_df, medsmap_df)
+
+    # Find Supplemental Meds Frequency
+    supp_meds_regime, filtered_supp_meds = filter_supp(supp_meds)
+
+    # Calculate  start and end dates
+    # Need to include Follow-up Date
+    onco_meds_regime = calculate_regime(onco_meds)
+
+    # Calculate Date Shift
+    supp_meds_regime_shifted_df = calculate_shift(supp_meds_regime, date_shift)
+    onco_meds_shifted_df = calculate_shift(onco_meds_regime, date_shift)
+
+    # Write out meds tables
+    supp_meds_regime_shifted_df.to_csv(suppmeds_table, index=False)
+    onco_meds_shifted_df.to_csv(meds_table, index=False)
+    missing_meds.to_csv(unmapped_table, index=False)
+    filtered_supp_meds.to_csv(not_given_suppmeds_table, index=False)
+
+
+if __name__ == '__main__':
+    main()

standards/cancer_meds_map.csv

+Meds,ID
+Aldesleukin,MED:7846
+Aldesleukin,MED:10672
+Aldesleukin,MED:43802
+Aldesleukin,MED:61700
+Aldesleukin,MED:105635
+Aldesleukin,MED:121140
+Aldesleukin,MED:161431
+Aldesleukin,MED:730442
+Aldesleukin,MED:852524
+Aldesleukin,MED:852525
+Aldesleukin,MED:852526
+Atezolizumab,MED:731401
+Atezolizumab,MED:853026
+Atezolizumab,MED:853120
+Avelumab,MED:731859
+Avelumab,MED:852013
+Avelumab,MED:853172
+Axitinib,MED:247563
+Axitinib,MED:247564
+Axitinib,MED:247682
+Axitinib,MED:247683
+Axitinib,MED:247876
+Axitinib,MED:247974
+Axitinib,MED:853099
+Bevacizumab,MED:69065
+Bevacizumab,MED:69067
+Bevacizumab,MED:69324
+Bevacizumab,MED:730448
+Bevacizumab,MED:730888
+Bevacizumab,MED:852245
+Bevacizumab,MED:852278
+Bevacizumab,MED:852573
+Cabozantinib,MED:253271
+Cabozantinib,MED:253273
+Cabozantinib,MED:253392
+Cabozantinib,MED:253394
+Cabozantinib,MED:253578
+Cabozantinib,MED:253596
+Cabozantinib,MED:267418
+Cabozantinib,MED:267419
+Cabozantinib,MED:267420
+Cabozantinib,MED:267422
+Cabozantinib,MED:267423
+Cabozantinib,MED:267424
+Cabozantinib,MED:267528
+CPI-444,MED:853024
+Everolimus,MED:234836
+Everolimus,MED:234837
+Everolimus,MED:234843
+Everolimus,MED:234844
+Everolimus,MED:235136
+Everolimus,MED:235199
+Everolimus,MED:239612
+Everolimus,MED:239613
+Everolimus,MED:239614
+Everolimus,MED:239615
+Everolimus,MED:239616
+Everolimus,MED:239617
+Everolimus,MED:240573
+Everolimus,MED:240574
+Everolimus,MED:248182
+Everolimus,MED:248239
+Everolimus,MED:254968
+Everolimus,MED:254976
+Everolimus,MED:852158
+Everolimus,MED:852427
+IL-2,MED:10672
+IL-2,MED:730442
+IL-2,MED:852526
+Interferon,MED:12108
+Interferon,MED:12109
+Interferon,MED:155209
+Interferon,MED:155215
+Interferon,MED:220970
+Interferon,MED:23465
+Interferon,MED:23497
+Interferon,MED:27586
+Interferon,MED:27595
+Interferon,MED:28326
+Interferon,MED:28327
+Interferon,MED:28328
+Interferon,MED:30197
+Interferon,MED:4753
+Interferon,MED:4754
+Interferon,MED:4755
+Interferon,MED:54858
+Interferon,MED:54860
+Interferon,MED:852532
+Interferon,MED:97446
+Interferon,MED:98144
+Ipilimumab,MED:243828
+Ipilimumab,MED:244146
+Ipilimumab,MED:244227
+Ipilimumab,MED:249027
+Ipilimumab,MED:852537
+Ipilimumab,MED:852766
+Ipilimumab,MED:853032
+Ipilimumab,MED:853174
+Ipilimumab,MED:853253
+Lenvatinib,MED:262146
+Lenvatinib,MED:262147
+Lenvatinib,MED:262159
+Lenvatinib,MED:262160
+Lenvatinib,MED:267829
+Lenvatinib,MED:267830
+Lenvatinib,MED:267856
+Lenvatinib,MED:267857
+Lenvatinib,MED:275251
+Nivolumab,MED:261428
+Nivolumab,MED:261429
+Nivolumab,MED:261432
+Nivolumab,MED:261496
+Nivolumab,MED:261501
+Nivolumab,MED:273507
+Nivolumab,MED:731719
+Nivolumab,MED:852764
+Nivolumab,MED:852907
+Nivolumab,MED:852977
+Nivolumab,MED:852988
+Nivolumab,MED:853030
+Nivolumab,MED:853140
+Nivolumab,MED:853167
+Nivolumab,MED:853250
+NKTR-214,MED:853361
+NKTR-262,MED:853372
+Pazopanib,MED:237450
+Pazopanib,MED:237453
+Pazopanib,MED:237614
+Pazopanib,MED:237654
+Pazopanib,MED:852748
+Pembrolizumab,MED:260612
+Pembrolizumab,MED:260627
+Pembrolizumab,MED:261829
+Pembrolizumab,MED:731190
+Pembrolizumab,MED:852984
+Pembrolizumab,MED:852994
+Pembrolizumab,MED:853084
+Pembrolizumab,MED:853098
+Pembrolizumab,MED:853322
+PT2385,MED:852791
+Sirolimus,MED:107783
+Sirolimus,MED:239512
+Sirolimus,MED:239534
+Sirolimus,MED:30774
+Sirolimus,MED:35686
+Sirolimus,MED:36020
+Sirolimus,MED:40437
+Sirolimus,MED:40441
+Sirolimus,MED:62768
+Sirolimus,MED:64482
+Sorafenib,MED:226829
+Sorafenib,MED:79714
+Sorafenib,MED:79722
+Sorafenib,MED:79825
+Sunitinib,MED:227223
+Sunitinib,MED:259330
+Sunitinib,MED:259336
+Sunitinib,MED:80076
+Sunitinib,MED:80077
+Sunitinib,MED:80082
+Sunitinib,MED:80083
+Sunitinib,MED:80084
+Sunitinib,MED:80155
+Sunitinib,MED:80276
+Sunitinib,MED:853096
+Temsirolimus,MED:730376
+Temsirolimus,MED:92164
+Temsirolimus,MED:92331
+Temsirolimus,MED:92570

standards/supportive_meds_filter.tsv

+Meds,ID
+Denosumab,MED:239980
+Denosumab,MED:239981
+Denosumab,MED:240075
+Denosumab,MED:240208
+Denosumab,MED:240377
+Denosumab,MED:242251
+Denosumab,MED:242257
+Denosumab,MED:242491
+Denosumab,MED:852340
+Denosumab,MED:852341
+Zoledronic Acid,MED:35688
+Zoledronic Acid,MED:36138
+Zoledronic Acid,MED:40851
+Zoledronic Acid,MED:40854
+Zoledronic Acid,MED:68981
+Zoledronic Acid,MED:68987
+Zoledronic Acid,MED:90803
+Zoledronic Acid,MED:91157
+Zoledronic Acid,MED:91335
+Zoledronic Acid,MED:115080
+Zoledronic Acid,MED:155942
+Zoledronic Acid,MED:155998
+Zoledronic Acid,MED:163172
+Zoledronic Acid,MED:164546
+Zoledronic Acid,MED:229262
+Zoledronic Acid,MED:246007
+Zoledronic Acid,MED:246013
+Zoledronic Acid,MED:253243
+Zoledronic Acid,MED:256033
+Zoledronic Acid,MED:256768
+Zoledronic Acid,MED:271306
+Zoledronic Acid,MED:730414
+Zoledronic Acid,MED:731784
+Zoledronic Acid,MED:50080112
+Zoledronic Acid,MED:50080155

tests/transform/med_table.csv

+mrn,name,Start_date,End_date,Duration,MRN
+88,Sorafenib,1859-07-23 00:00:00,1860-05-15 00:00:00,297.0,88
+313,Everolimus,1856-03-03 00:00:00,1857-03-25 21:00:00,387.0,313
+313,Sunitinib,1855-09-05 00:00:00,1856-07-19 00:00:00,318.0,313
+313,Temsirolimus,1854-12-27 00:00:00,1855-10-17 08:37:00,294.0,313
+822,Aldesleukin,1901-08-19 00:00:00,1902-03-13 05:07:00,206.0,822
+822,Everolimus,1902-10-22 00:00:00,1903-06-11 00:00:00,232.0,822
+822,IL-2,1901-08-19 00:00:00,1902-03-13 05:07:00,206.0,822
+822,Nivolumab,1903-04-16 00:00:00,1903-09-24 00:00:00,161.0,822
+822,Sunitinib,1902-08-04 00:00:00,1902-12-10 00:00:00,128.0,822
+903,Sunitinib,1822-05-25 00:00:00,1823-09-01 00:00:00,464.0,903
+934,Pazopanib,1860-06-11 00:00:00,1861-06-11 00:00:00,365.0,934

tests/transform/not_given_suppmed_table.csv

+MRN,NAME,GENERIC_NAME,CONCEPT_CD,START_DATE,END_DATE,MODIFIER_CD,TVAL_CHAR,UNITS_CD,OBSERVATION_BLOB,Meds,ID
+903,Zolderonic Acid,ZOLEDRONIC ACID INFUSION 730414,MED:730414,7/14/15 10:56,,RX|ADMIN,Stopped,@,,Zoledronic Acid,MED:730414
+903,Zolderonic Acid,ZOLEDRONIC ACID INFUSION 730414,MED:730414,9/17/15 15:30,,RX|ADMIN,Due,@,,Zoledronic Acid,MED:730414

tests/transform/suppmed_table.csv

+mrn,name,Start_date,MRN
+313,Denosumab,1855-01-10 15:20:00,313
+313,Denosumab,1855-05-30 15:20:00,313
+903,Zoledronic Acid,1822-07-06 10:45:00,903

tests/transform/unmapped_med_table.csv

+MRN,NAME,GENERIC_NAME,CONCEPT_CD,START_DATE,END_DATE,MODIFIER_CD,TVAL_CHAR,UNITS_CD,OBSERVATION_BLOB