Merge branch '15-image' into 'master'

Imaging table filtering.

Closes #15

See merge request !7

scripts/sql_queries/images.sql

+select E.MRN, D.*, A.START_DATE, A.OBSERVATION_BLOB
+from kidney_obs_fact A
+right join (select C.CPT, C.TYPE, B.c_name
+            from I2B2_TERMINOLOGY B, IMAGE_MAP C
+            where B.c_basecode = C.CPT)
+            D
+on A.concept_cd = D.CPT
+inner join KIDNEY_MRN_LIST E
+on A.PATIENT_NUM = E.PATIENT_NUM
+inner join "MRN_Filter_KCE" F
+on E.MRN = F.MRN;

scripts/transform_imaging.py

+#!/usr/bin/env python3
+
+'''Generate Imaging Table'''
+
+import argparse
+import datetime
+import os
+import pandas as pd
+
+
+EPILOG = '''
+For more details:
+        %(prog)s --help
+'''
+
+
+def get_args():
+    '''Define arguments.'''
+
+    parser = argparse.ArgumentParser(
+        description=__doc__, epilog=EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    parser.add_argument('-f', '--file',
+                        help="Imaging File (csv format).",
+                        required=True)
+
+    parser.add_argument('-d', '--date',
+                        help="Date Shift (tsv format).",
+                        required=True)
+
+    parser.add_argument('-o', '--out',
+                        help="The output path (csv format).",
+                        required=True)
+
+    args = parser.parse_args()
+    return args
+
+
+def reformat_record(imaging):
+    '''Reformat imaging table'''
+
+
+    imaging_reformat = imaging[["MRN", "CPT", "TYPE",
+              "START_DATE"]]
+
+    return imaging_reformat
+
+
+def calculate_shift(image_reformat, date_shift):
+    '''Shift Date for start date'''
+
+    # Convert to DateTime
+    date_shift['Shift'] = pd.to_timedelta(date_shift['Shift'], unit='s')
+    image_reformat['START_DATE'] = pd.to_datetime(image_reformat['START_DATE'])
+
+    # Merge data
+    merged = image_reformat.merge(date_shift, left_on='MRN', right_on='MRN', how='inner')
+
+    # Calculate Date Shift
+    merged['START_DATE'] = merged['START_DATE'] + merged['Shift']
+
+    merged['START_DATE'] = merged['START_DATE'].dt.date
+
+    # Drop Shift and Clinic column
+    merged.drop(['Shift'], axis=1, inplace=True)
+
+    # Int MRN
+    merged.MRN = merged.MRN.astype(int)
+
+    # Rename columns
+    imagining_columns = ["mrn", "cpt", "type",
+                   "start_date",]
+    merged.columns = imagining_columns
+
+    return merged
+
+
+def main():
+    args = get_args()
+    image = args.file
+    date = args.date
+    out_path = args.out
+
+    # Make output files
+    imaging_table = os.path.join(out_path + 'imaging_table.csv')
+
+    # Read in files
+    image_df = pd.read_csv(image)
+    date_shift = pd.read_csv(date)
+
+    # Reformat data
+    image_reformat = reformat_record(image_df)
+
+    # Calculate Date Shift
+    shifted_df = calculate_shift(image_reformat, date_shift)
+
+    # Write out radiation table
+    shifted_df.to_csv(imaging_table, index=False)
+
+
+if __name__ == '__main__':
+    main()

tests/data/image.csv

+MRN,CPT,TYPE,C_NAME,START_DATE,OBSERVATION_BLOB
+934,CPT:74160,CT,"(CPT:74160)Computed tomography, abdomen; with contrast material(s)",9/20/17 0:00,"AVM_VISIT_NUMBER:13348936;ICD9CODE:789.5; IDX PROCEDURE_NAME:CT, ABD, W/CONTRAST"
+934,CPT:74181,MR,"(CPT:74181)Magnetic resonance (eg, proton) imaging, abdomen; without contrast material(s)",9/25/17 0:00,;ICD9CODE:228.04; IDX PROCEDURE_NAME:MRI ABD
+313,CPT:74150,CT,"(CPT:74150)Computed tomography, abdomen; without contrast material",2/20/09 0:00,";ICD9CODE:189.9,192.0; IDX PROCEDURE_NAME:CT LIMITED ABDOMINAL WO CONTRAST"
+398,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",9/1/12 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
+904,CPT:74183,MR,"(CPT:74183)Magnetic resonance (eg, proton) imaging, abdomen; without contrast material(s), followed by with contrast material(s) and further sequences",6/6/09 0:00,AVM_VISIT_NUMBER:8604463;ICD9CODE:593.9; IDX PROCEDURE_NAME:MRI ABDOMEN W & WO CONTRAST.
+904,CPT:74175,CT,"(CPT:74175)Computed tomographic angiography, abdomen, with contrast material(s), including noncontrast images, if performed, and image postprocessing",7/5/09 0:00,MODIFIER:26\\\; DX:6193\6173\\\\
+411,CPT:74177,CT,"(CPT:74177)Computed tomography, abdomen and pelvis; with contrast material(s)",4/3/15 0:00,MODIFIER:26\\\; DX:6193\1829\5515\5877\\
+266,CPT:74176,CT,"(CPT:74176)Computed tomography, abdomen and pelvis; without contrast material",10/1/08 0:00,MODIFIER:26\\\; DX:306494\1741\\\\
+350,CPT:74178,CT,"(CPT:74178)Computed tomography, abdomen and pelvis; without contrast material in one or both body regions, followed by contrast material(s) and further sections in one or both body regions",6/30/05 0:00,MODIFIER:26\\\; DX:5191\5199\\\\
+399,CPT:74174,CT,"(CPT:74174)Computed tomographic angiography, abdomen and pelvis, with contrast material(s), including noncontrast images, if performed, and image postprocessing",6/20/06 0:00,";ICD9CODE:189.9,192.0; IDX PROCEDURE_NAME:CT LIMITED ABDOMINAL WO CONTRAST"
+903,CPT:74150,CT,"(CPT:74150)Computed tomography, abdomen; without contrast material",2/1/15 0:00,"AVM_VISIT_NUMBER:13348936;ICD9CODE:789.5; IDX PROCEDURE_NAME:CT, ABD, W/CONTRAST"
+558,CPT:74160,CT,"(CPT:74160)Computed tomography, abdomen; with contrast material(s)",2/20/15 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
+558,CPT:78815,PET,(CPT:78815)Positron emission tomography (PET) with concurrently acquired computed tomography (CT) for attenuation correction and anatomical localization imaging; skull base to mid-thigh,12/14/15 0:00,AVM_VISIT_NUMBER:10219982;ICD9CODE:174.9; IDX PROCEDURE_NAME:TUMOR IMAG PET W/CT; SKULL MID THI
+590,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",7/5/05 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
+822,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",2/25/15 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
+532,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",8/1/07 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
+687,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",4/1/12 0:00,";ICD9CODE:189.9,192.0; IDX PROCEDURE_NAME:CT LIMITED ABDOMINAL WO CONTRAST"
+423,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",4/23/15 0:00,
+88,CPT:74150,CT,"(CPT:74150)Computed tomography, abdomen; without contrast material",3/20/10 0:00,
+88,CPT:78816,PET,(CPT:78816)Positron emission tomography (PET) with concurrently acquired computed tomography (CT) for attenuation correction and anatomical localization imaging; whole body,3/20/10 0:00,AVM_VISIT_NUMBER:8823896;ICD9CODE:189.0; IDX PROCEDURE_NAME:TUMOR IMAG PET W/CT; WHOLE BDY
\ No newline at end of file

tests/transform/imaging_table.csv

+mrn,cpt,type,start_date
+934,CPT:74160,CT,1860-03-27
+934,CPT:74181,MR,1860-04-01
+313,CPT:74150,CT,1854-12-11
+398,CPT:74170,CT,1872-03-22
+904,CPT:74183,MR,1833-06-20
+904,CPT:74175,CT,1833-07-19
+411,CPT:74177,CT,1878-04-17
+266,CPT:74176,CT,1862-01-25
+350,CPT:74178,CT,1827-07-07
+399,CPT:74174,CT,1877-12-04
+903,CPT:74150,CT,1822-05-18
+558,CPT:74160,CT,1843-05-26
+558,CPT:78815,PET,1844-03-18
+590,CPT:74170,CT,1838-04-12
+822,CPT:74170,CT,1901-08-01
+532,CPT:74170,CT,1847-04-09
+687,CPT:74170,CT,1853-09-19
+423,CPT:74170,CT,1870-05-20
+88,CPT:74150,CT,1859-07-08
+88,CPT:78816,PET,1859-07-08