Merge branch '1-radiation' into 'master'

Resolve "Radiation Table"

Closes #1

See merge request !9

scripts/transform_radiation_history.py

+#!/usr/bin/env python3
+
+'''Generate Radiation History'''
+
+import argparse
+import datetime
+import os
+import pandas as pd
+
+
+EPILOG = '''
+For more details:
+        %(prog)s --help
+'''
+
+
+def get_args():
+    '''Define arguments.'''
+
+    parser = argparse.ArgumentParser(
+        description=__doc__, epilog=EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    parser.add_argument('-f', '--file',
+                        help="Radiation History File (csv format).",
+                        required=True)
+
+    parser.add_argument('-s', '--sites',
+                        help="Site Map File (csv format).",
+                        required=True)
+
+    parser.add_argument('-i', '--icd',
+                        help="ICD Filter file (tsv format).",
+                        required=True)
+
+    parser.add_argument('-d', '--date',
+                        help="Date Shift (tsv format).",
+                        required=True)
+
+    parser.add_argument('-o', '--out',
+                        help="The output path (csv format).",
+                        required=True)
+
+    args = parser.parse_args()
+    return args
+
+
+def calcuate_dosage(radiation):
+    '''Calculate dosage and filter columns'''
+
+    # Calculate Dosage
+    radiation['DOSE_FRACTION_ACTUAL'] = radiation['DOSE_CGY_ACTUAL']/radiation['FRACTIONS_ACTUAL']
+    radiation['DOSE_FRACTION_PLANNED'] = radiation['DOSE_CGY_PLANNED']/radiation['FRACTIONS_PLANNED']
+
+    # Filter for columns
+    filter_col = ["MRN", "SITE", "TECHNIQUE",
+                "MODALITY", "DOSE_CGY_ACTUAL",
+                "DOSE_CGY_PLANNED", "FRACTIONS_ACTUAL",
+                "FRACTIONS_PLANNED", "DOSE_FRACTION_ACTUAL",
+                "DOSE_FRACTION_PLANNED", "STARTDATE",
+                "LASTDATE", "ICD", "DIAGNOSIS", "TREATINTENT"]
+    df_filter = radiation.loc[:, filter_col]
+
+    return df_filter
+
+
+def filter_icd(radiation, icd):
+    '''Filter by ICD code for primary site at kidney therapy'''
+
+    # Filter for columns
+    icd_list = list(icd['ICD'])
+    radiation_filter = radiation[radiation['ICD'].isin(icd_list)]
+
+    return radiation_filter
+
+
+def convert_sites(radiation, sites):
+    '''Converts speicifc sites to apporpriate site'''
+
+    # Make index
+    sites.index = sites.MRN.map(str) + sites.SITE_SPECIFIC
+    radiation.index = radiation.MRN.map(str) + radiation.SITE
+
+    # Filter Columns Sites
+    sites_filtered = sites.loc[:, ['SITE_SPECIFIC', 'SITE_GENERAL']]
+    radiation_sites = radiation.merge(sites_filtered, left_index=True, right_index=True, how='outer')
+
+    # Find sites that don't match known site map
+    radiation_unmapped = radiation_sites[radiation_sites.SITE_SPECIFIC.isna()]
+    radiation_mapped = radiation_sites[~radiation_sites.SITE_SPECIFIC.isna()]
+
+    # Filter for columns
+    filter_col = ["MRN", "SITE_SPECIFIC", "SITE_GENERAL",
+                "DOSE_CGY_ACTUAL", "FRACTIONS_ACTUAL",
+                "STARTDATE", "LASTDATE", "TREATINTENT"]
+    radiation_filtered = radiation_mapped.loc[:, filter_col]
+
+    # Remove commas in
+    radiation_filtered['SITE_SPECIFIC'] = radiation_filtered['SITE_SPECIFIC'].str.replace(',', '')
+
+    return radiation_filtered, radiation_unmapped
+
+
+def convert_treatment(merged):
+    '''Converts treament to apporpriate string for database'''
+
+    merged.loc[merged['TREATINTENT'].isna(), 'TREATINTENT'] = 'Not Available'
+
+    return merged
+
+
+def calculate_shift(radiation, date_shift):
+    '''Shift Date for start and last date'''
+
+    # Convert to DateTime
+    date_shift['Shift'] = pd.to_timedelta(date_shift['Shift'], unit='s')
+    radiation['STARTDATE'] = pd.to_datetime(radiation['STARTDATE'])
+    radiation['LASTDATE'] = pd.to_datetime(radiation['LASTDATE'])
+
+    # Merge data
+    merged = radiation.merge(date_shift, left_on='MRN', right_on='MRN', how='inner')
+
+    # Calculate Date Shift
+    merged['STARTDATE'] = merged['STARTDATE'] + merged['Shift']
+    merged['STARTDATE'] = merged['STARTDATE'].dt.date
+
+    merged['LASTDATE'] = merged['LASTDATE'] + merged['Shift']
+    merged['LASTDATE'] = merged['LASTDATE'].dt.date
+
+
+    # Drop Shift column
+    merged.drop(['Shift'], axis=1, inplace=True)
+
+    # Int MRN
+    merged.MRN = merged.MRN.astype(int)
+
+    #TODO: Add radiation units
+
+    # Rename columns
+    radiation_columns = ["mrn", "site_specific", "site_general",
+                   "dose", "fractions", 'start_date', 'end_date', 'treatment_intention']
+    merged.columns = radiation_columns
+
+    return merged
+
+
+def main():
+    args = get_args()
+    rthistory = args.file
+    sites = args.sites
+    icd = args.icd
+    date = args.date
+    out_path = args.out
+
+    # Make output files
+    radiation_table = os.path.join(out_path + 'radiation_table.csv')
+    unmapped_table = os.path.join(out_path + 'unmapped_radiation_table.csv')
+
+    # Read in files
+    radiation = pd.read_csv(rthistory)
+    site_map = pd.read_csv(sites)
+    icd_filter = pd.read_csv(icd, sep='\t')
+    date_shift = pd.read_csv(date)
+
+    # Calculate Dosage
+    fix_dosage = calcuate_dosage(radiation)
+
+    # Filter for ICD Code
+    fix_icd = filter_icd(fix_dosage, icd_filter)
+
+    # Map Specific Sites to General Sites
+    fix_sites, unmapped_sites = convert_sites(fix_icd, site_map)
+
+    # Convert missing Treatment
+    fix_treatment = convert_treatment(fix_sites)
+
+    # Calculate Date Shift
+    shifted_df = calculate_shift(fix_treatment, date_shift)
+
+    # Write out radiation table
+    shifted_df.to_csv(radiation_table, index=False)
+
+    # Write out unmapped sites if there are any
+    if unmapped_sites.shape[0] > 1:
+        unmapped_sites.to_csv(unmapped_table, index=False)
+
+
+if __name__ == '__main__':
+    main()

standards/icd_filter.tsv

+ICD	Long Description	Version
+189.1	Malignant neoplasm of renal pelvis	ICD9
+171.5	Malignant neoplasm of connective and other soft tissue of abdomen 	ICD9
+171.9	Malignant neoplasm of connective and other soft tissue, site unspecified 	ICD9
+189.0	Malignant neoplasm of kidney, except pelvis	ICD9
+196.0	Secondary and unspecified malignant neoplasm of lymph nodes of head, face, and neck	ICD9
+196.1	Secondary and unspecified malignant neoplasm of intrathoracic lymph nodes	ICD9
+196.2	Secondary and unspecified malignant neoplasm of intra-abdominal lymph nodes 	ICD9
+196.6	Secondary and unspecified malignant neoplasm of intrapelvic lymph nodes 	ICD9
+196.8	Secondary and unspecified malignant neoplasm of lymph nodes of multiple sites	ICD9
+196.9	Secondary and unspecified malignant neoplasm of lymph nodes, site unspecified	ICD9
+197.0	Secondary malignant neoplasm of lung	ICD9
+197.6	Secondary malignant neoplasm of retroperitoneum and peritoneum	ICD9
+197.7	Malignant neoplasm of liver, secondary	ICD9
+197.8	Secondary malignant neoplasm of other digestive organs and spleen	ICD9
+198.3	Secondary malignant neoplasm of brain and spinal cord 	ICD9
+198.5	Secondary malignant neoplasm of bone and bone marrow	ICD9
+198.7	Secondary malignant neoplasm of adrenal gland 	ICD9
+198.89	Secondary malignant neoplasm of other specified sites 	ICD9
+C64.1	Malignant neoplasm of right kidney, except renal pelvis	ICD10
+C64.2	Malignant neoplasm of left kidney, except renal pelvis	ICD10
+C77.1	Secondary and unspecified malignant neoplasm of intrathoracic lymph nodes	ICD10
+C77.5	Secondary and unspecified malignant neoplasm of intrapelvic lymph nodes	ICD10
+C78.01	Secondary malignant neoplasm of right lung	ICD10
+C78.02	Secondary malignant neoplasm of left lung	ICD10
+C79.31	Secondary malignant neoplasm of brain	ICD10
+C79.51	Secondary malignant neoplasm of bone	ICD10
+C79.89	Secondary malignant neoplasm of other specified sites	ICD10

tests/data/radiation_history.csv

+MRN,LASTNAME,FIRSTNAME,DOB,RADIATIONONCOLOGIST,SITE,TECHNIQUE,MODALITY,DOSE_CGY_ACTUAL,DOSE_CGY_PLANNED,FRACTIONS_ACTUAL,FRACTIONS_PLANNED,STARTDATE,LASTDATE,ICD,DIAGNOSIS,TREATINTENT
+934,Smith,Erin,6/27/1957,Hannan,Liver,SBRT,x06,3000,3000,5,5,1/5/2013,1/20/2013,197.7,"Liver, specified as secondary",Curative
+313,Smithe,Tammi,3/14/1954,Timmerman,Femur,SBRT,x10,2000,2000,1,1,8/25/2011,8/25/2011,198.5,Bone and bone marrow,Palliative
+313,Smithe,Tammi,3/14/1954,Timmerman,Spine,SBRT,x10,2000,2000,1,1,8/25/2011,8/25/2011,198.5,Bone and bone marrow,Palliative
+399,Silver,Drew,7/18/1928,Hannan,LLL,Arc IMRT,6X,3600,3600,3,3,12/2/2011,12/12/2011,197,Lung,Definitive
+903,Silverton,Paula,9/17/1992,Timmerman,0A:1 L frontal,Gamma Knife,Co-60,1300,1300,1,1,2/4/2017,2/4/2017,C79.31,Secondary malignant neoplasm of brain,
+903,Silverton,Paula,9/17/1992,Timmerman,0B:2 R Cor Callos,Gamma Knife,Co-60,1300,1300,1,1,2/4/2017,2/4/2017,C79.31,Secondary malignant neoplasm of brain,
+903,Silverton,Paula,9/17/1992,Timmerman,0C:3 R ventral po,Gamma Knife,Co-60,1300,1300,1,1,2/4/2017,2/4/2017,C79.31,Secondary malignant neoplasm of brain,
+903,Silverton,Paula,9/17/1992,Timmerman,0D:4 R eye,Gamma Knife,Co-60,1300,1300,1,1,2/4/2017,2/4/2017,C79.31,Secondary malignant neoplasm of brain,
+822,Singleton,Danielle,7/29/1913,Kim,Abdomen,IMRT,x06,2500,2500,5,5,8/26/2014,9/15/2014,171.5,"ConNECtive, subcutaneous and other soft tissue of abdomen",
+687,Stephens,Greg,7/15/1958,Desai,prostate gland,IMRT,x10,2700,2700,15,15,6/27/2009,7/11/2009,185,Prostate gland,Curative
+687,Stephens,Greg,7/15/1958,Timmerman,PTV RUL 40Gy5fx,SBRT,6x,4000,4000,5,5,8/18/2013,9/9/2013,C78.01,Secondary malignant neoplasm of right lung,Consolidative
+423,Stevens,Georgia,12/5/1944,Albuquerque,RtAdrenal,3Dconformal,18x,3600,3600,3,3,4/29/2015,4/29/2015,C64.1,"Malignant neoplasm of right kidney, except renal pelvis",Consolidative
+88,Stewart,Deborah,9/14/1950,Kim,L shoulder,AP/PA,10x,3000,3000,10,10,7/4/2009,7/28/2009,198.5,Bone and bone marrow,
+88,Stewart,Deborah,9/14/1950,Kim,R Hip,IMRT,x06,3000,3000,5,5,2/20/2010,3/1/2010,198.5,Bone and bone marrow,De-Bulking

tests/data/radiation_sitemap.csv

+MRN,SITE_SPECIFIC,SITE_GENERAL,DOSE_CGY_ACTUAL,FRACTIONS_ACTUAL,STARTDATE,TREATINTENT,Sites
+934,Liver,Liver,3000,5,1/5/2013,Curative,Liver
+313,Femur,Bone,2000,1,8/25/2011,Palliative,Bone/Bone marrow/Spine
+313,Spine,Spine,2000,1,8/25/2011,Palliative,Bone/Bone marrow/Spine
+903,0A:1 L frontal,Brain,1300,1,2/4/2017,Not Available,Brain
+903,0B:2 R Cor Callos,Brain,1300,1,2/4/2017,Not Available,Brain
+903,0C:3 R ventral po,Brain,1300,1,2/4/2017,Not Available,Brain
+903,0D:4 R eye,Brain,1300,1,2/4/2017,Not Available,Brain
+822,Abdomen,Retroperitoneum & peritoneum,2500,5,8/26/2014,Not Available,Other
+687,prostate gland,Prostate gland,2700,15,6/27/2009,Curative,Other
+687,PTV RUL 40Gy5fx,"Lung, right",4000,5,8/18/2013,Consolidative,Lung
+423,RtAdrenal,"Adrenal gland, right",3600,3,4/29/2015,Consolidative,Adrenal
+88,L shoulder,Bone,3000,10,7/4/2009,Not Available,Bone/Bone marrow/Spine
+88,R Hip,Bone,3000,5,2/20/2010,De-Bulking,Bone/Bone marrow/Spine

tests/transform/radiation_table.csv

+mrn,site_specific,site_general,dose_cgy_actual,fractions_actual,start_date,end_date,treatment_intention
+313,Femur,Bone,2000.0,1.0,1857-06-14,1857-06-14,Palliative
+313,Spine,Spine,2000.0,1.0,1857-06-14,1857-06-14,Palliative
+423,RtAdrenal,"Adrenal gland, right",3600.0,3.0,1870-05-26,1870-05-26,Consolidative
+687,PTV RUL 40Gy5fx,"Lung, right",4000.0,5.0,1855-02-05,1855-02-27,Consolidative
+822,Abdomen,Retroperitoneum & peritoneum,2500.0,5.0,1901-01-30,1901-02-19,Not Available
+88,L shoulder,Bone,3000.0,10.0,1858-10-22,1858-11-15,Not Available
+88,R Hip,Bone,3000.0,5.0,1859-06-10,1859-06-19,De-Bulking
+903,0A:1 L frontal,Brain,1300.0,1.0,1824-05-21,1824-05-21,Not Available
+903,0B:2 R Cor Callos,Brain,1300.0,1.0,1824-05-21,1824-05-21,Not Available
+903,0C:3 R ventral po,Brain,1300.0,1.0,1824-05-21,1824-05-21,Not Available
+903,0D:4 R eye,Brain,1300.0,1.0,1824-05-21,1824-05-21,Not Available
+934,Liver,Liver,3000.0,5.0,1855-07-13,1855-07-28,Curative