Commit 00e3c7f0 authored by Venkat Malladi's avatar Venkat Malladi
Browse files

Add in tumor registry diagnosis date.

parent 6667d2e7
Pipeline #7802 failed with stage
in 1 minute and 37 seconds
...@@ -42,17 +42,14 @@ def filter_table(demographics, tumor_registry): ...@@ -42,17 +42,14 @@ def filter_table(demographics, tumor_registry):
# Column filters # Column filters
demo_filter_col = ["MRN", "GENDER", "ETHNICITY", "RACE", "BIRTH_DATE", demo_filter_col = ["MRN", "GENDER", "ETHNICITY", "RACE", "BIRTH_DATE",
"DEATH_DATE", "DEATH_SOURCE"] "DEATH_DATE", "DEATH_SOURCE"]
tumor_filter_col = ["Medical Record Number", "Date of Last Contact-Date", "Vital Status"] tumor_filter_col = ["Medical Record Number", "Date of Last Contact-Date", "Vital Status", 'Date of Diagnosis-Date']
# Filter # Filter
demo_filter = demographics.loc[:, demo_filter_col] demo_filter = demographics.loc[:, demo_filter_col]
tumor_filter = tumor_registry.loc[:, tumor_filter_col] tumor_filter = tumor_registry.loc[:, tumor_filter_col]
# Filter for dead status
tumor_dead = tumor_filter[tumor_filter["Vital Status"] == 0]
# merge # merge
df_merged = demo_filter.merge(tumor_dead, left_on='MRN', df_merged = demo_filter.merge(tumor_filter, left_on='MRN',
right_on='Medical Record Number', how='left') right_on='Medical Record Number', how='left')
return df_merged return df_merged
...@@ -95,6 +92,8 @@ def calculate_shift(merged): ...@@ -95,6 +92,8 @@ def calculate_shift(merged):
merged['BIRTH_DATE'] = pd.to_datetime(merged['BIRTH_DATE']) merged['BIRTH_DATE'] = pd.to_datetime(merged['BIRTH_DATE'])
merged['DEATH_DATE'] = pd.to_datetime(merged['DEATH_DATE']) merged['DEATH_DATE'] = pd.to_datetime(merged['DEATH_DATE'])
merged['Date of Last Contact-Date'] = pd.to_datetime(merged['Date of Last Contact-Date']) merged['Date of Last Contact-Date'] = pd.to_datetime(merged['Date of Last Contact-Date'])
merged['Date of Diagnosis-Date'] = pd.to_datetime(merged['Date of Diagnosis-Date'])
# Calculate date shift relative to 1800/01/01 # Calculate date shift relative to 1800/01/01
shift_date = datetime.datetime(1800, 1, 1) shift_date = datetime.datetime(1800, 1, 1)
...@@ -109,6 +108,7 @@ def calculate_shift(merged): ...@@ -109,6 +108,7 @@ def calculate_shift(merged):
# Calculate date Shift for death date # Calculate date Shift for death date
merged['DEATH_DATE'] = merged['DEATH_DATE'] + merged['Shift'] merged['DEATH_DATE'] = merged['DEATH_DATE'] + merged['Shift']
merged['DIAGNOSIS_DATE'] = merged['Date of Diagnosis-Date'] + merged['Shift']
# Shift to total seconds # Shift to total seconds
time_delta_series = merged['Shift'] time_delta_series = merged['Shift']
...@@ -149,8 +149,8 @@ def main(): ...@@ -149,8 +149,8 @@ def main():
# Write out patients table # Write out patients table
shifted_filtered = shifted_df[["MRN", "GENDER", "ETHNICITY", "RACE", shifted_filtered = shifted_df[["MRN", "GENDER", "ETHNICITY", "RACE",
'DEATH_DATE', 'DEATH_SOURCE']] 'DEATH_DATE', 'DEATH_SOURCE', 'DIAGNOSIS_DATE']]
shifted_filtered.columns = ['mrn', 'sex', 'ethnicity', 'race', 'death_date', 'death_source'] shifted_filtered.columns = ['mrn', 'sex', 'ethnicity', 'race', 'death_date', 'death_source', 'diagnosis_date']
shifted_filtered.to_csv(patients, index=False) shifted_filtered.to_csv(patients, index=False)
# Write out shift table # Write out shift table
......
This diff is collapsed.
...@@ -2,20 +2,28 @@ MRN,Shift ...@@ -2,20 +2,28 @@ MRN,Shift
934,-4969728000.0 934,-4969728000.0
313,-4865961600.0 313,-4865961600.0
398,-4432060800.0 398,-4432060800.0
398,-4432060800.0
215,-5721235200.0 215,-5721235200.0
904,-5552841600.0 904,-5552841600.0
411,-4322073600.0 411,-4322073600.0
266,-4628880000.0 266,-4628880000.0
266,-4628880000.0
707,-4058726400.0 707,-4058726400.0
350,-5616604800.0 350,-5616604800.0
350,-5616604800.0
399,-4056393600.0
399,-4056393600.0 399,-4056393600.0
903,-6081350400.0 903,-6081350400.0
903,-6081350400.0
628,-5599238400.0 628,-5599238400.0
558,-5419612800.0 558,-5419612800.0
590,-5277312000.0 590,-5277312000.0
590,-5277312000.0
822,-3583958400.0 822,-3583958400.0
532,-5058979200.0 532,-5058979200.0
687,-5002819200.0 687,-5002819200.0
687,-5002819200.0
423,-4573411200.0
423,-4573411200.0 423,-4573411200.0
33,-4528396800.0 33,-4528396800.0
88,-4755628800.0 88,-4755628800.0
mrn,sex,ethnicity,race,death_date,death_source mrn,sex,ethnicity,race,death_date,death_source,diagnosis_date
934,Female,Hispanic,White,, 934,Female,Hispanic,White,,,
313,Female,Non-hispanic,Black,1858-01-14,UTSouthwestern Hospital 313,Female,Non-hispanic,Black,1858-01-14 00:00:00.000000000,UTSouthwestern Hospital,1815-10-22 00:00:00.020090219
398,Female,Non-hispanic,Black,, 398,Female,Non-hispanic,Black,,,1829-07-22 00:00:00.020120816
215,Female,Non-hispanic,White,, 398,Female,Non-hispanic,Black,,,1829-07-22 00:00:00.020140208
904,Male,Non-hispanic,White,, 215,Female,Non-hispanic,White,,,1788-09-13 00:00:00.020150511
411,Female,Unknown,Unknown,, 904,Male,Non-hispanic,White,,,1794-01-14 00:00:00.020090701
266,Male,Hispanic,White,, 411,Female,Unknown,Other,,,1833-01-15 00:00:00.020150525
707,Male,Non-hispanic,American Indian,, 266,Male,Hispanic,White,,,1823-04-27 00:00:00.020080930
350,Female,Unknown,Unknown,, 266,Male,Hispanic,White,,,1823-04-27 00:00:00.020080910
399,Male,Non-hispanic,Asian,, 707,Male,Non-hispanic,American Indian,,,1841-05-21 00:00:00.020100615
903,Female,Non-hispanic,White,1824-09-11,UTSouthwestern Hospital 350,Female,Unknown,Other,,,1792-01-07 00:00:00.020050724
628,Male,Non-hispanic,White,, 350,Female,Unknown,Other,,,1792-01-07 00:00:00.020050703
558,Male,Hispanic,White,, 399,Male,Non-hispanic,Asian,,,1841-06-17 00:00:00.020060621
590,Female,Hispanic,Other,, 399,Male,Non-hispanic,Asian,,,1841-06-17 00:00:00.020060608
822,Female,Unknown,Unknown,, 903,Female,Non-hispanic,White,1824-09-11 00:00:00.000000000,UTSouthwestern Hospital,1777-04-16 00:00:00.020150122
532,Male,Declined,Declined,, 903,Female,Non-hispanic,White,1824-09-11 00:00:00.000000000,UTSouthwestern Hospital,1777-04-16 00:00:00.020150203
687,Male,Non-hispanic,White,, 628,Male,Non-hispanic,White,,,1792-07-26 00:00:00.019980115
423,Female,Declined,White,, 558,Male,Hispanic,White,,,1798-04-05 00:00:00.020150307
33,Male,Non-hispanic,Asian,, 590,Female,Hispanic,Other,,,1802-10-09 00:00:00.020050716
88,Female,Non-hispanic,Hawaiian Pacific Islander,1860-09-18,UTSouthwestern Hospital 590,Female,Hispanic,Other,,,1802-10-09 00:00:00.020050721
822,Female,Unknown,Other,,,1856-06-06 00:00:00.020150216
532,Male,Declined,Declined,,,1809-09-09 00:00:00.020070715
687,Male,Non-hispanic,White,,,1811-06-21 00:00:00.020120429
687,Male,Non-hispanic,White,,,1811-06-21 00:00:00.020120503
423,Female,Declined,White,,,1825-01-28 00:00:00.020150505
423,Female,Declined,White,,,1825-01-28 00:00:00.020160627
33,Male,Non-hispanic,Asian,,,1826-07-03 00:00:00.020100807
88,Female,Non-hispanic,Hawaiian Pacific Islander,1860-09-18 00:00:00.000000000,UTSouthwestern Hospital,1819-04-21 00:00:00.020100401
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment