Commit 044599ae authored by Venkat Malladi's avatar Venkat Malladi
Browse files

Merge branch '1-radiation' into 'master'

Resolve "Radiation Table"

Closes #1

See merge request !9
parents 6bc28655 78bb7145
Pipeline #5277 failed with stage
in 1 second
#!/usr/bin/env python3
'''Generate Radiation History'''
import argparse
import datetime
import os
import pandas as pd
EPILOG = '''
For more details:
%(prog)s --help
'''
def get_args():
'''Define arguments.'''
parser = argparse.ArgumentParser(
description=__doc__, epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-f', '--file',
help="Radiation History File (csv format).",
required=True)
parser.add_argument('-s', '--sites',
help="Site Map File (csv format).",
required=True)
parser.add_argument('-i', '--icd',
help="ICD Filter file (tsv format).",
required=True)
parser.add_argument('-d', '--date',
help="Date Shift (tsv format).",
required=True)
parser.add_argument('-o', '--out',
help="The output path (csv format).",
required=True)
args = parser.parse_args()
return args
def calcuate_dosage(radiation):
'''Calculate dosage and filter columns'''
# Calculate Dosage
radiation['DOSE_FRACTION_ACTUAL'] = radiation['DOSE_CGY_ACTUAL']/radiation['FRACTIONS_ACTUAL']
radiation['DOSE_FRACTION_PLANNED'] = radiation['DOSE_CGY_PLANNED']/radiation['FRACTIONS_PLANNED']
# Filter for columns
filter_col = ["MRN", "SITE", "TECHNIQUE",
"MODALITY", "DOSE_CGY_ACTUAL",
"DOSE_CGY_PLANNED", "FRACTIONS_ACTUAL",
"FRACTIONS_PLANNED", "DOSE_FRACTION_ACTUAL",
"DOSE_FRACTION_PLANNED", "STARTDATE",
"LASTDATE", "ICD", "DIAGNOSIS", "TREATINTENT"]
df_filter = radiation.loc[:, filter_col]
return df_filter
def filter_icd(radiation, icd):
'''Filter by ICD code for primary site at kidney therapy'''
# Filter for columns
icd_list = list(icd['ICD'])
radiation_filter = radiation[radiation['ICD'].isin(icd_list)]
return radiation_filter
def convert_sites(radiation, sites):
'''Converts speicifc sites to apporpriate site'''
# Make index
sites.index = sites.MRN.map(str) + sites.SITE_SPECIFIC
radiation.index = radiation.MRN.map(str) + radiation.SITE
# Filter Columns Sites
sites_filtered = sites.loc[:, ['SITE_SPECIFIC', 'SITE_GENERAL']]
radiation_sites = radiation.merge(sites_filtered, left_index=True, right_index=True, how='outer')
# Find sites that don't match known site map
radiation_unmapped = radiation_sites[radiation_sites.SITE_SPECIFIC.isna()]
radiation_mapped = radiation_sites[~radiation_sites.SITE_SPECIFIC.isna()]
# Filter for columns
filter_col = ["MRN", "SITE_SPECIFIC", "SITE_GENERAL",
"DOSE_CGY_ACTUAL", "FRACTIONS_ACTUAL",
"STARTDATE", "LASTDATE", "TREATINTENT"]
radiation_filtered = radiation_mapped.loc[:, filter_col]
# Remove commas in
radiation_filtered['SITE_SPECIFIC'] = radiation_filtered['SITE_SPECIFIC'].str.replace(',', '')
return radiation_filtered, radiation_unmapped
def convert_treatment(merged):
'''Converts treament to apporpriate string for database'''
merged.loc[merged['TREATINTENT'].isna(), 'TREATINTENT'] = 'Not Available'
return merged
def calculate_shift(radiation, date_shift):
'''Shift Date for start and last date'''
# Convert to DateTime
date_shift['Shift'] = pd.to_timedelta(date_shift['Shift'], unit='s')
radiation['STARTDATE'] = pd.to_datetime(radiation['STARTDATE'])
radiation['LASTDATE'] = pd.to_datetime(radiation['LASTDATE'])
# Merge data
merged = radiation.merge(date_shift, left_on='MRN', right_on='MRN', how='inner')
# Calculate Date Shift
merged['STARTDATE'] = merged['STARTDATE'] + merged['Shift']
merged['STARTDATE'] = merged['STARTDATE'].dt.date
merged['LASTDATE'] = merged['LASTDATE'] + merged['Shift']
merged['LASTDATE'] = merged['LASTDATE'].dt.date
# Drop Shift column
merged.drop(['Shift'], axis=1, inplace=True)
# Int MRN
merged.MRN = merged.MRN.astype(int)
#TODO: Add radiation units
# Rename columns
radiation_columns = ["mrn", "site_specific", "site_general",
"dose", "fractions", 'start_date', 'end_date', 'treatment_intention']
merged.columns = radiation_columns
return merged
def main():
args = get_args()
rthistory = args.file
sites = args.sites
icd = args.icd
date = args.date
out_path = args.out
# Make output files
radiation_table = os.path.join(out_path + 'radiation_table.csv')
unmapped_table = os.path.join(out_path + 'unmapped_radiation_table.csv')
# Read in files
radiation = pd.read_csv(rthistory)
site_map = pd.read_csv(sites)
icd_filter = pd.read_csv(icd, sep='\t')
date_shift = pd.read_csv(date)
# Calculate Dosage
fix_dosage = calcuate_dosage(radiation)
# Filter for ICD Code
fix_icd = filter_icd(fix_dosage, icd_filter)
# Map Specific Sites to General Sites
fix_sites, unmapped_sites = convert_sites(fix_icd, site_map)
# Convert missing Treatment
fix_treatment = convert_treatment(fix_sites)
# Calculate Date Shift
shifted_df = calculate_shift(fix_treatment, date_shift)
# Write out radiation table
shifted_df.to_csv(radiation_table, index=False)
# Write out unmapped sites if there are any
if unmapped_sites.shape[0] > 1:
unmapped_sites.to_csv(unmapped_table, index=False)
if __name__ == '__main__':
main()
ICD Long Description Version
189.1 Malignant neoplasm of renal pelvis ICD9
171.5 Malignant neoplasm of connective and other soft tissue of abdomen ICD9
171.9 Malignant neoplasm of connective and other soft tissue, site unspecified ICD9
189.0 Malignant neoplasm of kidney, except pelvis ICD9
196.0 Secondary and unspecified malignant neoplasm of lymph nodes of head, face, and neck ICD9
196.1 Secondary and unspecified malignant neoplasm of intrathoracic lymph nodes ICD9
196.2 Secondary and unspecified malignant neoplasm of intra-abdominal lymph nodes ICD9
196.6 Secondary and unspecified malignant neoplasm of intrapelvic lymph nodes ICD9
196.8 Secondary and unspecified malignant neoplasm of lymph nodes of multiple sites ICD9
196.9 Secondary and unspecified malignant neoplasm of lymph nodes, site unspecified ICD9
197.0 Secondary malignant neoplasm of lung ICD9
197.6 Secondary malignant neoplasm of retroperitoneum and peritoneum ICD9
197.7 Malignant neoplasm of liver, secondary ICD9
197.8 Secondary malignant neoplasm of other digestive organs and spleen ICD9
198.3 Secondary malignant neoplasm of brain and spinal cord ICD9
198.5 Secondary malignant neoplasm of bone and bone marrow ICD9
198.7 Secondary malignant neoplasm of adrenal gland ICD9
198.89 Secondary malignant neoplasm of other specified sites ICD9
C64.1 Malignant neoplasm of right kidney, except renal pelvis ICD10
C64.2 Malignant neoplasm of left kidney, except renal pelvis ICD10
C77.1 Secondary and unspecified malignant neoplasm of intrathoracic lymph nodes ICD10
C77.5 Secondary and unspecified malignant neoplasm of intrapelvic lymph nodes ICD10
C78.01 Secondary malignant neoplasm of right lung ICD10
C78.02 Secondary malignant neoplasm of left lung ICD10
C79.31 Secondary malignant neoplasm of brain ICD10
C79.51 Secondary malignant neoplasm of bone ICD10
C79.89 Secondary malignant neoplasm of other specified sites ICD10
MRN,LASTNAME,FIRSTNAME,DOB,RADIATIONONCOLOGIST,SITE,TECHNIQUE,MODALITY,DOSE_CGY_ACTUAL,DOSE_CGY_PLANNED,FRACTIONS_ACTUAL,FRACTIONS_PLANNED,STARTDATE,LASTDATE,ICD,DIAGNOSIS,TREATINTENT
934,Smith,Erin,6/27/1957,Hannan,Liver,SBRT,x06,3000,3000,5,5,1/5/2013,1/20/2013,197.7,"Liver, specified as secondary",Curative
313,Smithe,Tammi,3/14/1954,Timmerman,Femur,SBRT,x10,2000,2000,1,1,8/25/2011,8/25/2011,198.5,Bone and bone marrow,Palliative
313,Smithe,Tammi,3/14/1954,Timmerman,Spine,SBRT,x10,2000,2000,1,1,8/25/2011,8/25/2011,198.5,Bone and bone marrow,Palliative
399,Silver,Drew,7/18/1928,Hannan,LLL,Arc IMRT,6X,3600,3600,3,3,12/2/2011,12/12/2011,197,Lung,Definitive
903,Silverton,Paula,9/17/1992,Timmerman,0A:1 L frontal,Gamma Knife,Co-60,1300,1300,1,1,2/4/2017,2/4/2017,C79.31,Secondary malignant neoplasm of brain,
903,Silverton,Paula,9/17/1992,Timmerman,0B:2 R Cor Callos,Gamma Knife,Co-60,1300,1300,1,1,2/4/2017,2/4/2017,C79.31,Secondary malignant neoplasm of brain,
903,Silverton,Paula,9/17/1992,Timmerman,0C:3 R ventral po,Gamma Knife,Co-60,1300,1300,1,1,2/4/2017,2/4/2017,C79.31,Secondary malignant neoplasm of brain,
903,Silverton,Paula,9/17/1992,Timmerman,0D:4 R eye,Gamma Knife,Co-60,1300,1300,1,1,2/4/2017,2/4/2017,C79.31,Secondary malignant neoplasm of brain,
822,Singleton,Danielle,7/29/1913,Kim,Abdomen,IMRT,x06,2500,2500,5,5,8/26/2014,9/15/2014,171.5,"ConNECtive, subcutaneous and other soft tissue of abdomen",
687,Stephens,Greg,7/15/1958,Desai,prostate gland,IMRT,x10,2700,2700,15,15,6/27/2009,7/11/2009,185,Prostate gland,Curative
687,Stephens,Greg,7/15/1958,Timmerman,PTV RUL 40Gy5fx,SBRT,6x,4000,4000,5,5,8/18/2013,9/9/2013,C78.01,Secondary malignant neoplasm of right lung,Consolidative
423,Stevens,Georgia,12/5/1944,Albuquerque,RtAdrenal,3Dconformal,18x,3600,3600,3,3,4/29/2015,4/29/2015,C64.1,"Malignant neoplasm of right kidney, except renal pelvis",Consolidative
88,Stewart,Deborah,9/14/1950,Kim,L shoulder,AP/PA,10x,3000,3000,10,10,7/4/2009,7/28/2009,198.5,Bone and bone marrow,
88,Stewart,Deborah,9/14/1950,Kim,R Hip,IMRT,x06,3000,3000,5,5,2/20/2010,3/1/2010,198.5,Bone and bone marrow,De-Bulking
MRN,SITE_SPECIFIC,SITE_GENERAL,DOSE_CGY_ACTUAL,FRACTIONS_ACTUAL,STARTDATE,TREATINTENT,Sites
934,Liver,Liver,3000,5,1/5/2013,Curative,Liver
313,Femur,Bone,2000,1,8/25/2011,Palliative,Bone/Bone marrow/Spine
313,Spine,Spine,2000,1,8/25/2011,Palliative,Bone/Bone marrow/Spine
903,0A:1 L frontal,Brain,1300,1,2/4/2017,Not Available,Brain
903,0B:2 R Cor Callos,Brain,1300,1,2/4/2017,Not Available,Brain
903,0C:3 R ventral po,Brain,1300,1,2/4/2017,Not Available,Brain
903,0D:4 R eye,Brain,1300,1,2/4/2017,Not Available,Brain
822,Abdomen,Retroperitoneum & peritoneum,2500,5,8/26/2014,Not Available,Other
687,prostate gland,Prostate gland,2700,15,6/27/2009,Curative,Other
687,PTV RUL 40Gy5fx,"Lung, right",4000,5,8/18/2013,Consolidative,Lung
423,RtAdrenal,"Adrenal gland, right",3600,3,4/29/2015,Consolidative,Adrenal
88,L shoulder,Bone,3000,10,7/4/2009,Not Available,Bone/Bone marrow/Spine
88,R Hip,Bone,3000,5,2/20/2010,De-Bulking,Bone/Bone marrow/Spine
mrn,site_specific,site_general,dose_cgy_actual,fractions_actual,start_date,end_date,treatment_intention
313,Femur,Bone,2000.0,1.0,1857-06-14,1857-06-14,Palliative
313,Spine,Spine,2000.0,1.0,1857-06-14,1857-06-14,Palliative
423,RtAdrenal,"Adrenal gland, right",3600.0,3.0,1870-05-26,1870-05-26,Consolidative
687,PTV RUL 40Gy5fx,"Lung, right",4000.0,5.0,1855-02-05,1855-02-27,Consolidative
822,Abdomen,Retroperitoneum & peritoneum,2500.0,5.0,1901-01-30,1901-02-19,Not Available
88,L shoulder,Bone,3000.0,10.0,1858-10-22,1858-11-15,Not Available
88,R Hip,Bone,3000.0,5.0,1859-06-10,1859-06-19,De-Bulking
903,0A:1 L frontal,Brain,1300.0,1.0,1824-05-21,1824-05-21,Not Available
903,0B:2 R Cor Callos,Brain,1300.0,1.0,1824-05-21,1824-05-21,Not Available
903,0C:3 R ventral po,Brain,1300.0,1.0,1824-05-21,1824-05-21,Not Available
903,0D:4 R eye,Brain,1300.0,1.0,1824-05-21,1824-05-21,Not Available
934,Liver,Liver,3000.0,5.0,1855-07-13,1855-07-28,Curative
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment