Commit 78bb7145 authored by Venkat Malladi's avatar Venkat Malladi
Browse files

Merge branch 'master' into 1-radiation

parents ae4c9465 6bc28655
Pipeline #5276 failed with stage
in 1 second
before_script:
- module add python/2.7.14-anaconda
- env
- module load python/3.7.x-anaconda
- conda env create --file environment.yml
stages:
- unit
user_configuration:
unit_tests:
stage: unit
script:
- source activate kce_etl
- pytest -m unit
- conda activate kce_etl
- pytest -m unit --cov=./scripts
after_script:
- conda deactivate
- conda env remove --name kce_etl -y
name: kce_etl
channels:
- conda-forge
- bioconda
- defaults
dependencies:
- conda-forge::pandas=0.24.2
pip:
- pytest-pythonpath==0.7.1
- pytest-cov==2.5.1
- _libgcc_mutex=0.1=main
- ca-certificates=2019.11.28=hecc5488_0
- certifi=2019.11.28=py38_0
- ld_impl_linux-64=2.33.1=h53a641e_7
- libblas=3.8.0=14_openblas
- libcblas=3.8.0=14_openblas
- libffi=3.2.1=he1b5a44_1006
- libgcc-ng=9.2.0=hdf63c60_0
- libgfortran-ng=7.3.0=hdf63c60_2
- liblapack=3.8.0=14_openblas
- libopenblas=0.3.7=h5ec1e0e_5
- libstdcxx-ng=9.2.0=hdf63c60_0
- ncurses=6.1=hf484d3e_1002
- numpy=1.17.3=py38h95a1406_0
- openssl=1.1.1d=h516909a_0
- pandas=0.24.2=py38hb3f55d8_1
- pip=19.3.1=py38_0
- python=3.8.0=h357f687_5
- python-dateutil=2.8.1=py_0
- pytz=2019.3=py_0
- readline=8.0=hf8c457e_0
- setuptools=42.0.2=py38_0
- six=1.13.0=py38_0
- sqlite=3.30.1=hcee41ef_0
- tk=8.6.10=hed695b0_0
- wheel=0.33.6=py38_0
- xz=5.2.4=h14c3975_1001
- zlib=1.2.11=h516909a_1006
- pip:
- attrs==19.3.0
- coverage==4.5.4
- more-itertools==8.0.2
- packaging==19.2
- pluggy==0.13.1
- py==1.8.0
- pyparsing==2.4.5
- pytest==5.3.1
- pytest-cov==2.5.1
- pytest-pythonpath==0.7.1
- wcwidth==0.1.7
select E.MRN, D.*, A.START_DATE, A.OBSERVATION_BLOB
from kidney_obs_fact A
right join (select C.CPT, C.TYPE, B.c_name
from I2B2_TERMINOLOGY B, IMAGE_MAP C
where B.c_basecode = C.CPT)
D
on A.concept_cd = D.CPT
inner join KIDNEY_MRN_LIST E
on A.PATIENT_NUM = E.PATIENT_NUM
inner join "MRN_Filter_KCE" F
on E.MRN = F.MRN;
#!/usr/bin/env python3
'''Generate Imaging Table'''
import argparse
import datetime
import os
import pandas as pd
EPILOG = '''
For more details:
%(prog)s --help
'''
def get_args():
'''Define arguments.'''
parser = argparse.ArgumentParser(
description=__doc__, epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-f', '--file',
help="Imaging File (csv format).",
required=True)
parser.add_argument('-d', '--date',
help="Date Shift (tsv format).",
required=True)
parser.add_argument('-o', '--out',
help="The output path (csv format).",
required=True)
args = parser.parse_args()
return args
def reformat_record(imaging):
'''Reformat imaging table'''
imaging_reformat = imaging[["MRN", "CPT", "TYPE",
"START_DATE"]]
return imaging_reformat
def calculate_shift(image_reformat, date_shift):
'''Shift Date for start date'''
# Convert to DateTime
date_shift['Shift'] = pd.to_timedelta(date_shift['Shift'], unit='s')
image_reformat['START_DATE'] = pd.to_datetime(image_reformat['START_DATE'])
# Merge data
merged = image_reformat.merge(date_shift, left_on='MRN', right_on='MRN', how='inner')
# Calculate Date Shift
merged['START_DATE'] = merged['START_DATE'] + merged['Shift']
merged['START_DATE'] = merged['START_DATE'].dt.date
# Drop Shift and Clinic column
merged.drop(['Shift'], axis=1, inplace=True)
# Int MRN
merged.MRN = merged.MRN.astype(int)
# Rename columns
imagining_columns = ["mrn", "cpt", "type",
"start_date",]
merged.columns = imagining_columns
return merged
def main():
args = get_args()
image = args.file
date = args.date
out_path = args.out
# Make output files
imaging_table = os.path.join(out_path + 'imaging_table.csv')
# Read in files
image_df = pd.read_csv(image)
date_shift = pd.read_csv(date)
# Reformat data
image_reformat = reformat_record(image_df)
# Calculate Date Shift
shifted_df = calculate_shift(image_reformat, date_shift)
# Write out radiation table
shifted_df.to_csv(imaging_table, index=False)
if __name__ == '__main__':
main()
......@@ -71,7 +71,6 @@ def convert_race(merged):
merged.loc[merged['RACE'] == 'AmIndAN', 'RACE'] = 'American Indian'
merged.loc[merged['RACE'] == 'HawPacIs', 'RACE'] = 'Hawaiian Pacific Islander'
merged.loc[merged['RACE'] == 'Unknown', 'RACE'] = 'Other'
return merged
......
MRN,CPT,TYPE,C_NAME,START_DATE,OBSERVATION_BLOB
934,CPT:74160,CT,"(CPT:74160)Computed tomography, abdomen; with contrast material(s)",9/20/17 0:00,"AVM_VISIT_NUMBER:13348936;ICD9CODE:789.5; IDX PROCEDURE_NAME:CT, ABD, W/CONTRAST"
934,CPT:74181,MR,"(CPT:74181)Magnetic resonance (eg, proton) imaging, abdomen; without contrast material(s)",9/25/17 0:00,;ICD9CODE:228.04; IDX PROCEDURE_NAME:MRI ABD
313,CPT:74150,CT,"(CPT:74150)Computed tomography, abdomen; without contrast material",2/20/09 0:00,";ICD9CODE:189.9,192.0; IDX PROCEDURE_NAME:CT LIMITED ABDOMINAL WO CONTRAST"
398,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",9/1/12 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
904,CPT:74183,MR,"(CPT:74183)Magnetic resonance (eg, proton) imaging, abdomen; without contrast material(s), followed by with contrast material(s) and further sequences",6/6/09 0:00,AVM_VISIT_NUMBER:8604463;ICD9CODE:593.9; IDX PROCEDURE_NAME:MRI ABDOMEN W & WO CONTRAST.
904,CPT:74175,CT,"(CPT:74175)Computed tomographic angiography, abdomen, with contrast material(s), including noncontrast images, if performed, and image postprocessing",7/5/09 0:00,MODIFIER:26\\\; DX:6193\6173\\\\
411,CPT:74177,CT,"(CPT:74177)Computed tomography, abdomen and pelvis; with contrast material(s)",4/3/15 0:00,MODIFIER:26\\\; DX:6193\1829\5515\5877\\
266,CPT:74176,CT,"(CPT:74176)Computed tomography, abdomen and pelvis; without contrast material",10/1/08 0:00,MODIFIER:26\\\; DX:306494\1741\\\\
350,CPT:74178,CT,"(CPT:74178)Computed tomography, abdomen and pelvis; without contrast material in one or both body regions, followed by contrast material(s) and further sections in one or both body regions",6/30/05 0:00,MODIFIER:26\\\; DX:5191\5199\\\\
399,CPT:74174,CT,"(CPT:74174)Computed tomographic angiography, abdomen and pelvis, with contrast material(s), including noncontrast images, if performed, and image postprocessing",6/20/06 0:00,";ICD9CODE:189.9,192.0; IDX PROCEDURE_NAME:CT LIMITED ABDOMINAL WO CONTRAST"
903,CPT:74150,CT,"(CPT:74150)Computed tomography, abdomen; without contrast material",2/1/15 0:00,"AVM_VISIT_NUMBER:13348936;ICD9CODE:789.5; IDX PROCEDURE_NAME:CT, ABD, W/CONTRAST"
558,CPT:74160,CT,"(CPT:74160)Computed tomography, abdomen; with contrast material(s)",2/20/15 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
558,CPT:78815,PET,(CPT:78815)Positron emission tomography (PET) with concurrently acquired computed tomography (CT) for attenuation correction and anatomical localization imaging; skull base to mid-thigh,12/14/15 0:00,AVM_VISIT_NUMBER:10219982;ICD9CODE:174.9; IDX PROCEDURE_NAME:TUMOR IMAG PET W/CT; SKULL MID THI
590,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",7/5/05 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
822,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",2/25/15 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
532,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",8/1/07 0:00,"AVM_VISIT_NUMBER:16376339;ICD9CODE:593.9; IDX PROCEDURE_NAME:ABDOMEN,CT SCAN W/WO CONTRAST"
687,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",4/1/12 0:00,";ICD9CODE:189.9,192.0; IDX PROCEDURE_NAME:CT LIMITED ABDOMINAL WO CONTRAST"
423,CPT:74170,CT,"(CPT:74170)Computed tomography, abdomen; without contrast material, followed by contrast material(s) and further sections",4/23/15 0:00,
88,CPT:74150,CT,"(CPT:74150)Computed tomography, abdomen; without contrast material",3/20/10 0:00,
88,CPT:78816,PET,(CPT:78816)Positron emission tomography (PET) with concurrently acquired computed tomography (CT) for attenuation correction and anatomical localization imaging; whole body,3/20/10 0:00,AVM_VISIT_NUMBER:8823896;ICD9CODE:189.0; IDX PROCEDURE_NAME:TUMOR IMAG PET W/CT; WHOLE BDY
\ No newline at end of file
#!/usr/bin/env python3
import pytest
import os
import pandas as pd
from StringIO import StringIO
import transform_imaging
IMAGE_STRING = """MRN,CPT,TYPE,C_NAME,START_DATE,OBSERVATION_BLOB
45,CPT:74160,CT,"(CPT:74160)Computed tomography, abdomen; with contrast material(s)",9/20/17 0:00,"AVM_VISIT_NUMBER:13348936;ICD9CODE:789.5; IDX PROCEDURE_NAME:CT, ABD, W/CONTRAST"
"""
@pytest.fixture
def image():
image_file = StringIO(IMAGE_STRING)
image_df = pd.read_csv(image_file)
return image_df
@pytest.mark.unit
def test_check_reformmat(image):
transform_immage = transform_imaging.reformat_record(image)
assert transform_immage.shape[1] == 4
......@@ -77,7 +77,7 @@ def test_check_race_hawpacis(demographic_2):
def test_check_race_unknown(demographic_2):
demographic_2.loc[934, 'RACE'] = 'Unknown'
check_race_hawpacis = transform_patients.convert_race(demographic_2)
assert check_race_hawpacis.loc[934,'RACE'] == 'Other'
assert check_race_hawpacis.loc[934,'RACE'] == 'Unknown'
@pytest.mark.unit
......
mrn,cpt,type,start_date
934,CPT:74160,CT,1860-03-27
934,CPT:74181,MR,1860-04-01
313,CPT:74150,CT,1854-12-11
398,CPT:74170,CT,1872-03-22
904,CPT:74183,MR,1833-06-20
904,CPT:74175,CT,1833-07-19
411,CPT:74177,CT,1878-04-17
266,CPT:74176,CT,1862-01-25
350,CPT:74178,CT,1827-07-07
399,CPT:74174,CT,1877-12-04
903,CPT:74150,CT,1822-05-18
558,CPT:74160,CT,1843-05-26
558,CPT:78815,PET,1844-03-18
590,CPT:74170,CT,1838-04-12
822,CPT:74170,CT,1901-08-01
532,CPT:74170,CT,1847-04-09
687,CPT:74170,CT,1853-09-19
423,CPT:74170,CT,1870-05-20
88,CPT:74150,CT,1859-07-08
88,CPT:78816,PET,1859-07-08
......@@ -4,16 +4,16 @@ mrn,gender,ethnicity,race,age,age_units
398,Female,Non-hispanic,Black,72,years
215,Female,Non-hispanic,White,34,years
904,Male,Non-hispanic,White,34,years
411,Female,Unknown,Other,79,years
411,Female,Unknown,Unknown,79,years
266,Male,Hispanic,White,62,years
707,Male,Non-hispanic,American Indian,82,years
350,Female,Unknown,Other,28,years
350,Female,Unknown,Unknown,28,years
399,Male,Non-hispanic,Asian,78,years
903,Female,Non-hispanic,White,23,years
628,Male,Non-hispanic,White,21,years
558,Male,Hispanic,White,44,years
590,Female,Hispanic,Other,38,years
822,Female,Unknown,Other,90 or above,years
822,Female,Unknown,Unknown,90 or above,years
532,Male,Declined,Declined,47,years
687,Male,Non-hispanic,White,54,years
423,Female,Declined,White,71,years
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment