From c8ac3fed8e84fe41ad1d2733dd9e2bfb4887a02e Mon Sep 17 00:00:00 2001 From: Farid Seifi Date: Mon, 28 Oct 2024 11:16:57 -0400 Subject: [PATCH 1/3] changes --- computable_phenotypes/script.sql | 335 +++++++++++------------- computable_phenotypes/utils/Patients.py | 39 ++- computable_phenotypes/utils/db.py | 150 +++++------ computable_phenotypes/utils/utils.py | 48 ++-- test.db | Bin 0 -> 16384 bytes 5 files changed, 274 insertions(+), 298 deletions(-) create mode 100644 test.db diff --git a/computable_phenotypes/script.sql b/computable_phenotypes/script.sql index 074bf31..1f997ba 100644 --- a/computable_phenotypes/script.sql +++ b/computable_phenotypes/script.sql @@ -1,5 +1,4 @@ --use PlayGround; -GO @@ -7,133 +6,132 @@ GO --Preliminary step: Create reference table from spreadsheet of codes related to inclusions and exclusiodbo. --Create table to contain diagnosis codes. -if OBJECT_ID('dbo.Ref_NS_Codes', 'U') is not NULL - drop table dbo.Ref_NS_Codes; +drop table IF EXISTS Ref_NS_Codes; -CREATE TABLE dbo.Ref_NS_Codes +CREATE TABLE Ref_NS_Codes ( Code_Type nvarchar(10) NULL, Diag_Code nvarchar(10) NULL, Incl_Excl nvarchar(10) NULL, Incl_Excl_Type nvarchar(30) NULL ) -GO + --Insert diagnosis code values into table -insert into dbo.Ref_NS_Codes values('ICD-9','583.89','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','582.89','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','583','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','V08','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','42','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','42.1','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','42.2','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','42.8','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','42.9','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.2','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.21','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.22','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.23','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.3','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.31','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.32','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.33','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.41','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.44','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.51','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.54','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.7','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','70.71','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','287','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','580','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','580.4','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','593.73','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','741.9','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','741','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','596.54','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','277.87','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','593.73','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','593.7','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','593.70','Exclusion','Exclude_Encounter'); - -insert into dbo.Ref_NS_Codes values('ICD-9','582.9','Exclusion','Neph5829_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','583.2','Exclusion','Neph5832_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','582','Exclusion','Neph5820_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','277.39','Exclusion','Amyloidosis_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','277.3','Exclusion','Amyloidosis_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','277.3','Exclusion','Amyloidosis_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','250.4','Exclusion','Diabetes2_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','250.43','Exclusion','Diabetes2_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','250.41','Exclusion','Diabetes1_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','250.43','Exclusion','Diabetes1_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','710','Exclusion','Lupus_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','710','Exclusion','Lupus_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','581.1','Inclusion','PrimaryNS_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','581.3','Inclusion','PrimaryNS_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','582.1','Inclusion','PrimaryNS_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','583.1','Inclusion','PrimaryNS_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-9','581.9','Inclusion','NSNOS_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N05.1','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N06.1','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N07.1','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N03.8','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N05.9','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','Z21','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B20','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B16.2','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B191.1','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B160','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B18.1','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B180','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B16.9','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B191.0','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B161','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B18.1','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B18.0','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B17.11','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B18.2','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B17.10','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B18.2','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B19.20','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','B192.1','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','D69.0','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N00.3','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N01.3','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N13.729','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','Q05.8','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','Q05.4','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N31.9','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E884.0','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E884.1','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E884.2','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E884.9','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','H49819','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N13.729','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N13.70','Exclusion','Exclude_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N03.9','Exclusion','Neph5829_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N05.5','Exclusion','Neph5832_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N03.2','Exclusion','Neph5820_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E85.1','Exclusion','Amyloidosis_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E853','Exclusion','Amyloidosis_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E858','Exclusion','Amyloidosis_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E08.21','Exclusion','Diabetes2_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E08.22','Exclusion','Diabetes2_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E112.9','Exclusion','Diabetes2_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','E102.9','Exclusion','Diabetes1_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','M32.10','Exclusion','Lupus_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N02.2','Inclusion','PrimaryNS_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N04.0','Inclusion','PrimaryNS_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N03.3','Inclusion','PrimaryNS_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N05.2','Inclusion','PrimaryNS_Encounter'); -insert into dbo.Ref_NS_Codes values('ICD-10','N04.9','Inclusion','NSNOS_Encounter'); +insert into Ref_NS_Codes values('ICD-9','583.89','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','582.89','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','583','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','V08','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','42','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','42.1','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','42.2','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','42.8','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','42.9','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.2','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.21','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.22','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.23','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.3','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.31','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.32','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.33','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.41','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.44','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.51','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.54','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.7','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','70.71','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','287','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','580','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','580.4','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','593.73','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','741.9','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','741','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','596.54','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','277.87','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','593.73','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','593.7','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-9','593.70','Exclusion','Exclude_Encounter'); + +insert into Ref_NS_Codes values('ICD-9','582.9','Exclusion','Neph5829_Encounter'); +insert into Ref_NS_Codes values('ICD-9','583.2','Exclusion','Neph5832_Encounter'); +insert into Ref_NS_Codes values('ICD-9','582','Exclusion','Neph5820_Encounter'); +insert into Ref_NS_Codes values('ICD-9','277.39','Exclusion','Amyloidosis_Encounter'); +insert into Ref_NS_Codes values('ICD-9','277.3','Exclusion','Amyloidosis_Encounter'); +insert into Ref_NS_Codes values('ICD-9','277.3','Exclusion','Amyloidosis_Encounter'); +insert into Ref_NS_Codes values('ICD-9','250.4','Exclusion','Diabetes2_Encounter'); +insert into Ref_NS_Codes values('ICD-9','250.43','Exclusion','Diabetes2_Encounter'); +insert into Ref_NS_Codes values('ICD-9','250.41','Exclusion','Diabetes1_Encounter'); +insert into Ref_NS_Codes values('ICD-9','250.43','Exclusion','Diabetes1_Encounter'); +insert into Ref_NS_Codes values('ICD-9','710','Exclusion','Lupus_Encounter'); +insert into Ref_NS_Codes values('ICD-9','710','Exclusion','Lupus_Encounter'); +insert into Ref_NS_Codes values('ICD-9','581.1','Inclusion','PrimaryNS_Encounter'); +insert into Ref_NS_Codes values('ICD-9','581.3','Inclusion','PrimaryNS_Encounter'); +insert into Ref_NS_Codes values('ICD-9','582.1','Inclusion','PrimaryNS_Encounter'); +insert into Ref_NS_Codes values('ICD-9','583.1','Inclusion','PrimaryNS_Encounter'); +insert into Ref_NS_Codes values('ICD-9','581.9','Inclusion','NSNOS_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N05.1','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N06.1','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N07.1','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N03.8','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N05.9','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','Z21','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B20','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B16.2','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B191.1','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B160','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B18.1','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B180','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B16.9','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B191.0','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B161','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B18.1','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B18.0','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B17.11','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B18.2','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B17.10','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B18.2','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B19.20','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','B192.1','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','D69.0','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N00.3','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N01.3','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N13.729','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','Q05.8','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','Q05.4','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N31.9','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E884.0','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E884.1','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E884.2','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E884.9','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','H49819','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N13.729','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N13.70','Exclusion','Exclude_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N03.9','Exclusion','Neph5829_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N05.5','Exclusion','Neph5832_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N03.2','Exclusion','Neph5820_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E85.1','Exclusion','Amyloidosis_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E853','Exclusion','Amyloidosis_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E858','Exclusion','Amyloidosis_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E08.21','Exclusion','Diabetes2_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E08.22','Exclusion','Diabetes2_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E112.9','Exclusion','Diabetes2_Encounter'); +insert into Ref_NS_Codes values('ICD-10','E102.9','Exclusion','Diabetes1_Encounter'); +insert into Ref_NS_Codes values('ICD-10','M32.10','Exclusion','Lupus_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N02.2','Inclusion','PrimaryNS_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N04.0','Inclusion','PrimaryNS_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N03.3','Inclusion','PrimaryNS_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N05.2','Inclusion','PrimaryNS_Encounter'); +insert into Ref_NS_Codes values('ICD-10','N04.9','Inclusion','NSNOS_Encounter'); --fake data --if OBJECT_ID('dbo.Diagnosis', 'U') is not NULL --- drop table dbo.Diagnosis; ---CREATE TABLE dbo.Diagnosis +-- drop table Diagnosis; +--CREATE TABLE Diagnosis --( -- PATID int, -- DIAGNOSISID nvarchar(10) NULL, @@ -145,8 +143,8 @@ insert into dbo.Ref_NS_Codes values('ICD-10','N04.9','Inclusion','NSNOS_Encounte --GO --if OBJECT_ID('dbo.Encounter', 'U') is not NULL --- drop table dbo.Encounter; ---CREATE TABLE dbo.Encounter +-- drop table Encounter; +--CREATE TABLE Encounter --( -- PATID int, -- ENCOUNTERID nvarchar(30) NULL, @@ -158,8 +156,8 @@ insert into dbo.Ref_NS_Codes values('ICD-10','N04.9','Inclusion','NSNOS_Encounte --GO --if OBJECT_ID('dbo.Demographic', 'U') is not NULL --- drop table dbo.Demographic; ---CREATE TABLE dbo.Demographic +-- drop table Demographic; +--CREATE TABLE Demographic --( -- PATID int, -- BIRTH_DATE datetime, @@ -172,9 +170,7 @@ insert into dbo.Ref_NS_Codes values('ICD-10','N04.9','Inclusion','NSNOS_Encounte --Part 2---------------------------------------------------------------------------------------------------- --Get Diagnosis records and set flags for conditions related to each diagnosis record - -if object_id('dbo.NS_EncounterConditions', 'U') is not NULL - drop table dbo.NS_EncounterConditions; +drop table IF EXISTS NS_EncounterConditions; select D.PATID, D.ENCOUNTERID , max(case when C.Incl_Excl_Type = 'NEPH5829_Encounter' then 1 else 0 end) as NEPH5829_Flag @@ -187,9 +183,9 @@ select D.PATID, D.ENCOUNTERID , max(case when C.Incl_Excl_Type = 'PrimaryNS_Encounter' then 1 else 0 end) as PrimaryNS_Flag , max(case when C.Incl_Excl_Type = 'NSNOS_Encounter' then 1 else 0 end) as NSNOS_Flag , max(case when C.Incl_Excl_Type = 'Exclude_Encounter' then 1 else 0 end) as Exclude_Code_Flag -into dbo.NS_EncounterConditions -from dbo.Diagnosis as D - join dbo.Ref_NS_Codes as C +into NS_EncounterConditions +from Diagnosis as D + join Ref_NS_Codes as C on D.DX = C.Diag_Code and ( @@ -202,13 +198,12 @@ group by D.PATID, D.ENCOUNTERID --Create Index ---create clustered columnstore index CCSIX_NS_EncounterConditions on dbo.NS_EncounterConditions with (DROP_EXISTING = OFF); --Error 1: can not add index +--create clustered columnstore index CCSIX_NS_EncounterConditions on NS_EncounterConditions with (DROP_EXISTING = OFF); --Error 1: can not add index --Part 3-------------------------------------------------------------------------------------------------------------------- --Add encounter and demographic data to the diagnosis data -if OBJECT_ID('dbo.NS_Encounter_Level', 'U') is not NULL - drop table dbo.NS_Encounter_Level; +drop table IF EXISTS NS_Encounter_Level; select E.ENCOUNTERID @@ -232,11 +227,11 @@ select ,DX.PrimaryNS_Flag ,DX.NSNOS_Flag ,DX.Exclude_Code_Flag -into dbo.NS_Encounter_Level -from dbo.Encounter as E - JOIN dbo.NS_EncounterConditions as DX +into NS_Encounter_Level +from Encounter as E + JOIN NS_EncounterConditions as DX on E.ENCOUNTERID = DX.ENCOUNTERID - JOIN dbo.Demographic as D + JOIN Demographic as D on E.PATID = D.PATID where E.ADMIT_DATE IS NOT NULL @@ -246,14 +241,13 @@ where --Part 4--------------------------------------------------------------------------------------------------------------- --Sort the encounters and assign a row number. -if OBJECT_ID('dbo.NS_Encounter_Sort', 'U') is not NULL - drop table dbo.NS_Encounter_Sort; +drop table IF EXISTS NS_Encounter_Sort; select * , ROW_NUMBER() OVER (PARTITION BY PATID ORDER BY DISCHARGE_DATE, Admit_Date, EncounterID) AS Encounter_Row -into dbo.NS_Encounter_Sort -from dbo.NS_Encounter_Level +into NS_Encounter_Sort +from NS_Encounter_Level ; @@ -261,15 +255,14 @@ from dbo.NS_Encounter_Level --Part 5---------------------------------------------------------------------------------------------------------------- --Add dates for next visit -if OBJECT_ID('dbo.NS_Encounter_Sort_2', 'U') is not NULL - drop table dbo.NS_Encounter_Sort_2; +drop table IF EXISTS NS_Encounter_Sort_2; Select Cur.* ,N.ADMIT_DATE AS Next_Admit_Date ,N.Discharge_Date AS Next_Discharge_Date -into dbo.NS_Encounter_Sort_2 -from dbo.NS_Encounter_Sort as Cur - left JOIN dbo.NS_Encounter_Sort as N +into NS_Encounter_Sort_2 +from NS_Encounter_Sort as Cur + left JOIN NS_Encounter_Sort as N on Cur.Encounter_Row = N.Encounter_Row - 1 and Cur.patid=N.patid ; @@ -278,15 +271,14 @@ from dbo.NS_Encounter_Sort as Cur --Part 6------------------------------------------------------------------------------------------------------------------ --Add dates for prior visit -if OBJECT_ID('dbo.NS_Encounter_Sort_3', 'U') is not NULL - drop table dbo.NS_Encounter_Sort_3; +drop table IF EXISTS NS_Encounter_Sort_3; Select Cur.* ,N.ADMIT_DATE AS Prior_Admit_Date ,N.Discharge_Date AS Prior_Discharge_Date -into dbo.NS_Encounter_Sort_3 -from dbo.NS_Encounter_Sort_2 as Cur - left JOIN dbo.NS_Encounter_Sort_2 as N +into NS_Encounter_Sort_3 +from NS_Encounter_Sort_2 as Cur + left JOIN NS_Encounter_Sort_2 as N on Cur.Encounter_Row = N.Encounter_Row + 1 and Cur.patid=N.patid ; @@ -294,8 +286,7 @@ from dbo.NS_Encounter_Sort_2 as Cur --Part 7-------------------------------------------------------------------------------------------------------------- --Get sums of inclusion/exclusion variables -if OBJECT_ID('dbo.NS_Encounter_Sums', 'U') is not NULL - drop table dbo.NS_Encounter_Sums; +drop table IF EXISTS NS_Encounter_Sums; select PATID @@ -333,8 +324,8 @@ select ,Next_Discharge_Date ,Prior_Admit_Date ,Prior_Discharge_Date -into dbo.NS_Encounter_Sums -from dbo.NS_Encounter_Sort_3 +into NS_Encounter_Sums +from NS_Encounter_Sort_3 ; @@ -342,8 +333,7 @@ from dbo.NS_Encounter_Sort_3 --Part 8------------------------------------------------------------------------------------------- --Identify Possible Inclusions based on the criteria specified in the WHERE clause. -if OBJECT_ID('dbo.NS_Possible_Inclusions', 'U') is not NULL - drop table dbo.NS_Possible_Inclusions; +drop table IF EXISTS NS_Possible_Inclusions; select PATID @@ -352,9 +342,9 @@ select ,PrimaryNS_Total ,NSNOS_Total ,1 as Possible_Inclusion -into dbo.NS_Possible_Inclusions +into NS_Possible_Inclusions from - dbo.NS_Encounter_Sums + NS_Encounter_Sums where PrimaryNS_Total > 1 or ((PrimaryNS_Total + NSNOS_Total) > 1 and Age < 20) @@ -365,8 +355,7 @@ where --Part 9-------------------------------------------------------------------------------------------- --Identify Possible Exclusions based on the criteria specified in the WHERE clause. -if OBJECT_ID('dbo.NS_Possible_Exclusions', 'U') is not NULL - drop table dbo.NS_Possible_Exclusions; +drop table IF EXISTS NS_Possible_Exclusions; select PATID @@ -381,9 +370,9 @@ select ,Lupus_Total ,Exclude_Code_Total ,1 as Possible_Exclusion -into dbo.NS_Possible_Exclusions +into NS_Possible_Exclusions from - dbo.NS_Encounter_Sums + NS_Encounter_Sums where NEPH5829_Total > 1 or NEPH5832_Total > 1 @@ -400,8 +389,7 @@ where --Part 10--------------------------------------------------------------------------------------------- --Join possible inclusions and exclusions with sum data -if OBJECT_ID('dbo.NS_Possible', 'U') is not NULL - drop table dbo.NS_Possible; +drop table IF EXISTS NS_Possible; select Sums.* @@ -411,11 +399,11 @@ select ,case when ex.possible_exclusion = 1 then 1 else 0 end as Possible_Exclusion -into dbo.NS_Possible -from dbo.NS_Encounter_Sums as Sums - left join dbo.NS_Possible_Inclusions as I +into NS_Possible +from NS_Encounter_Sums as Sums + left join NS_Possible_Inclusions as I on Sums.encounterid = I.encounterid - left join dbo.NS_Possible_Exclusions as Ex + left join NS_Possible_Exclusions as Ex on Sums.ENCOUNTERID = Ex.EncounterID ; @@ -424,15 +412,14 @@ from dbo.NS_Encounter_Sums as Sums --Part 11------------------------------------------------------------------------------------- --Flag final inclusions -if OBJECT_ID('dbo.NS_Final_Inclusions_Flag', 'U') is not NULL - drop table dbo.NS_Final_Inclusions_Flag; +drop table IF EXISTS NS_Final_Inclusions_Flag; select P.* ,1 as Final_Inclusion_Flag ,P.Prior_Discharge_Date AS Entry_Date -into dbo.NS_Final_Inclusions_Flag -from dbo.NS_Possible as P +into NS_Final_Inclusions_Flag +from NS_Possible as P where P.possible_inclusion = 1 and P.possible_exclusion != 1 ; @@ -440,8 +427,7 @@ where P.possible_inclusion = 1 and P.possible_exclusion != 1 --Part 12------------------------------------------------------------------------------------ --Identify first qualified inclusion (once included, always included) -if OBJECT_ID('dbo.NS_Final_Inclusions', 'U') is not NULL - drop table dbo.NS_Final_Inclusions; +drop table IF EXISTS NS_Final_Inclusions; select @@ -455,11 +441,11 @@ select , RACE as Race , SEX as Sex , Hispanic -into dbo.NS_Final_Inclusions +into NS_Final_Inclusions from ( select *, ROW_NUMBER() OVER(PARTITION BY patid ORDER BY Encounter_Row) as RowNum - from dbo.NS_Final_Inclusions_Flag + from NS_Final_Inclusions_Flag ) as OrderedSet where RowNum = 1 ; @@ -469,8 +455,7 @@ where RowNum = 1 --Part 13------------------------------------------------------------------------------------------ --Identify inclusions based on a specified date -if OBJECT_ID('dbo.NS_Final_Inclusions_Output', 'U') is not NULL - drop table dbo.NS_Final_Inclusions_Output; +drop table IF EXISTS NS_Final_Inclusions_Output; DECLARE @MyDate date; SET @MyDate = '01-01-2024'; -- As of this date, who has already been included? @@ -486,8 +471,8 @@ select , Race , Sex , Hispanic -into dbo.NS_Final_Inclusions_Output -from dbo.NS_Final_Inclusions +into NS_Final_Inclusions_Output +from NS_Final_Inclusions where Entry_Date <= @MyDate ; @@ -509,8 +494,8 @@ sqlcmd -S SERVERNAME -d DATABASENAME -E -o "PATH\Nephrotic_Syndrome_Patients.csv /***** ALTERNATIVE: If you have the appropriate permissions, you can run the command below from SQL Server instead: ****** Again change SERVERNAME, DATABASENAME, PATH for output, and SCHEMA prefix for table name.*/ ---Exec xp_cmdshell 'sqlcmd -S (localdb)\MSSQLLocalDB -d PlayGround -E -o "c:\test\Nephrotic_Syndrome_Patients.csv" -Q "Set NOCOUNT ON; Select * from dbo.NS_Final_Inclusions_Output" -W -w 999 -s","' -select * from dbo.NS_Final_Inclusions_Output +--Exec xp_cmdshell 'sqlcmd -S (localdb)\MSSQLLocalDB -d PlayGround -E -o "c:\test\Nephrotic_Syndrome_Patients.csv" -Q "Set NOCOUNT ON; Select * from NS_Final_Inclusions_Output" -W -w 999 -s","' +select * from NS_Final_Inclusions_Output --END of script--------------------------------------------------------------------------------------------- diff --git a/computable_phenotypes/utils/Patients.py b/computable_phenotypes/utils/Patients.py index 607d547..aec276c 100644 --- a/computable_phenotypes/utils/Patients.py +++ b/computable_phenotypes/utils/Patients.py @@ -4,15 +4,12 @@ class Patients: - def __init__(self, db=None, verbose=False): + def __init__(self, database_name=None, verbose=False): self.patients = {} self.loaded = False self.verbose = verbose - self.db = db - if db is None: - self.write = False - else: - self.write = True + self.database_name = database_name + def add_patient(self, birthday, race=None, sex=None, hisp=None, pat_id=None): """ @@ -41,10 +38,10 @@ def add_patient(self, birthday, race=None, sex=None, hisp=None, pat_id=None): "encounters": {}, } base[pat_id] = patient_info - if self.write: - pat_instr = f"""insert into dbo.Demographic values({patient_info['pat_id']},{patient_info['birthdate']},{patient_info['sex']},{patient_info['hispanic']},{patient_info['race']});""" - self.log(pat_instr) - execute(self.db, pat_instr) + + pat_instr = f"""insert into Demographic values({patient_info['pat_id']},{patient_info['birthdate']},{patient_info['sex']},{patient_info['hispanic']},{patient_info['race']});""" + self.log(pat_instr) + execute(self.database_name, pat_instr) return pat_id def add_enc( @@ -79,10 +76,10 @@ def add_enc( "diagnosisList": {}, } base[enc_id] = enc_info - if self.write: - enc_instr = f"""insert into dbo.Encounter values({pat_id},{enc_info['enc_id']},{enc_info['admit_date']},{enc_info['enc_type']},{enc_info['raw_enc_type']},{enc_info['discharge_date']});""" - self.log(enc_instr) - execute(self.db, enc_instr) + + enc_instr = f"""insert into Encounter values({pat_id},{enc_info['enc_id']},{enc_info['admit_date']},{enc_info['enc_type']},{enc_info['raw_enc_type']},{enc_info['discharge_date']});""" + self.log(enc_instr) + execute(self.database_name, enc_instr) return enc_id def add_dx(self, pat_id, enc_id, code, dx_id=None, dx_source=None, db=None): @@ -109,10 +106,10 @@ def add_dx(self, pat_id, enc_id, code, dx_id=None, dx_source=None, db=None): "dx_source": nullify(dx_source), } base[dx_id] = dx_info - if self.write: - dx_instr = f"""insert into dbo.Diagnosis values({pat_id},{dx_info['dx_id']},{dx_info['dx']},{dx_info['dx_type']},{dx_info['dx_source']},{enc_id});""" - self.log(dx_instr) - execute(self.db, dx_instr) + + dx_instr = f"""insert into Diagnosis values({pat_id},{dx_info['dx_id']},{dx_info['dx']},{dx_info['dx_type']},{dx_info['dx_source']},{enc_id});""" + self.log(dx_instr) + execute(self.database_name, dx_instr) return dx_id def print(self): @@ -132,15 +129,15 @@ def batch_load(self, db_connection): for patid, pat in self.patients: # load patient - pat_instr = f"""insert into dbo.Demographic values({pat['pat_id']},{pat['birthdate']},{pat['sex']},{pat['hispanic']},{pat['race']});""" + pat_instr = f"""insert into Demographic values({pat['pat_id']},{pat['birthdate']},{pat['sex']},{pat['hispanic']},{pat['race']});""" self.log(pat_instr) execute(db_connection, pat_instr) for enc_id, encs in pat["encounters"]: - enc_instr = f"""insert into dbo.Encounter values({pat['pat_id']},{encs['enc_id']},{encs['admit_date']},{encs['enc_type']},{encs['raw_enc_type']},{encs['discharge_date']});""" + enc_instr = f"""insert into Encounter values({pat['pat_id']},{encs['enc_id']},{encs['admit_date']},{encs['enc_type']},{encs['raw_enc_type']},{encs['discharge_date']});""" self.log(enc_instr) execute(db_connection, enc_instr) for dx_id, dx in encs["diagnosis_list"]: - dx_instr = f"""insert into dbo.Diagnosis values({pat['pat_id']},{dx['dx_id']},{dx['dx']},{dx['dx_type']},{dx['dx_source']},{encs['enc_id']});""" + dx_instr = f"""insert into Diagnosis values({pat['pat_id']},{dx['dx_id']},{dx['dx']},{dx['dx_type']},{dx['dx_source']},{encs['enc_id']});""" self.log(dx_instr) execute(db_connection, dx_instr) diff --git a/computable_phenotypes/utils/db.py b/computable_phenotypes/utils/db.py index 813663b..8be0da1 100644 --- a/computable_phenotypes/utils/db.py +++ b/computable_phenotypes/utils/db.py @@ -1,89 +1,89 @@ +import json import os import urllib from subprocess import PIPE, run -import pyodbc +import sqlite3 from dotenv import load_dotenv from sqlalchemy import create_engine, text -def create_database(connection, database_name): - connection.execution_options(isolation_level="AUTOCOMMIT").execute( - text(f"CREATE DATABASE {database_name};") - ) +def create_database(database_name): + connection = sqlite3.connect(database_name) + cursor = connection.cursor() + return cursor -def delete_database(connection, database_name): - execute( - connection, - f"""IF DB_ID('{database_name}') IS NOT NULL - DROP DATABASE {database_name};""", - allow_transaction=False, # Disable transaction for this command - ) +def delete_database(database_name): + if os.path.exists(database_name): + os.remove(database_name) -def execute(connection, command, allow_transaction=True): - if allow_transaction: - # For normal commands, use transactions - with connection.begin(): - connection.execute(text(command)) - else: - # For commands like CREATE DATABASE or DROP DATABASE, use AUTOCOMMIT - connection.execution_options(isolation_level="AUTOCOMMIT").execute( - text(command) - ) - - -def run_script(sql_script, db, output): - host, user, password = load_env() - process = run( - [f"sqlcmd -C -U {user} -P {password} -d {db} -i {sql_script} "], - stderr=PIPE, - stdout=PIPE, - stdin=PIPE, - shell=True, - ) # -o {output} - # process.stdin.write(password) - # print(process.stderr) - # print(process.stdout) +def execute(database_name, command): + connection = sqlite3.connect(database_name) + cursor = connection.cursor() + cursor.execute(command) + connection.commit() +def run_script(sql_script, database_name, output): + connection = sqlite3.connect(database_name) + cursor = connection.cursor() + with open(sql_script, 'r') as file: + sql_script_content = file.read() + cursor.executescript(sql_script_content) + connection.commit() + connection.close() def connect(database=None): - host, user, password = load_env() - database_conn = "" - if database: - database_conn = ";database=" + database + # host, user, password = load_env() + # database_conn = "" + # if database: + # database_conn = ";database=" + database - conn_str = f"DRIVER={{ODBC Driver 18 for SQL Server}};SERVER={host};UID={user};PWD={password};TrustServerCertificate=yes{database_conn}" - params = urllib.parse.quote_plus(conn_str) - engine = create_engine(f"mssql+pyodbc:///?odbc_connect={params}") - - return engine - - -def load_env(): - pyodbc.pooling = False - load_dotenv() - sql_host = os.getenv("MSSQL_HOST") - sql_username = os.getenv("MSSQL_USERNAME") - sql_password = os.getenv("MSSQL_PASSWORD") - return sql_host, sql_username, sql_password - - -def remove_table(db_conn, table_name): - execute( - db_conn, - f"""if OBJECT_ID('{table_name}', 'U') is not NULL - drop table {table_name};""", - ) - - -def create_tables(db_conn): - remove_table(db_conn, "dbo.Encounter") - remove_table(db_conn, "dbo.Diagnosis") - remove_table(db_conn, "dbo.Demographic") - remove_table(db_conn, "dbo.PCOR_Encounters") - encounter = """CREATE TABLE dbo.Encounter ( + # conn_str = f"DRIVER={{ODBC Driver 18 for SQL Server}};SERVER={host};UID={user};PWD={password};TrustServerCertificate=yes{database_conn}" + # params = urllib.parse.quote_plus(conn_str) + # engine = create_engine(f"mssql+pyodbc:///?odbc_connect={params}") + connection = sqlite3.connect(database) + cursor = connection.cursor() + + return cursor + +def fetch(database_name, query): + connection = sqlite3.connect(database_name) + cursor = connection.cursor() + + # Run the SELECT query + query = query + cursor.execute(query) + + # Fetch the results and column names + rows = cursor.fetchall() + column_names = [description[0] for description in cursor.description] + + # Convert to a list of dictionaries + data = [dict(zip(column_names, row)) for row in rows] + + # Convert to JSON + json_data = json.dumps(data, indent=4) + + # Close the database connection + connection.close() + return json_data + +def remove_table(database_name, table_name): + connection = sqlite3.connect(database_name) + cursor = connection.cursor() + cursor.execute("DROP TABLE IF EXISTS " + table_name ) + + connection.commit() + + +def create_tables(database_name): + remove_table(database_name, "Encounter") + remove_table(database_name, "Diagnosis") + remove_table(database_name, "Demographic") + remove_table(database_name, "PCOR_Encounters") + encounter = """CREATE TABLE Encounter ( PATID int, ENCOUNTERID nvarchar(30) NULL, ADMIT_DATE datetime NULL, @@ -92,7 +92,7 @@ def create_tables(db_conn): DISCHARGE_DATE datetime NULL );""" diagnosis = """ - CREATE TABLE dbo.Diagnosis + CREATE TABLE Diagnosis ( PATID int, DIAGNOSISID nvarchar(10) NULL, @@ -102,7 +102,7 @@ def create_tables(db_conn): ENCOUNTERID nvarchar(30) NULL );""" demo = """ - CREATE TABLE dbo.Demographic + CREATE TABLE Demographic ( PATID int, BIRTH_DATE datetime, @@ -110,6 +110,6 @@ def create_tables(db_conn): HISPANIC nvarchar(2), RACE nvarchar(2) );""" - execute(db_conn, encounter) - execute(db_conn, diagnosis) - execute(db_conn, demo) + execute(database_name, encounter) + execute(database_name, diagnosis) + execute(database_name, demo) diff --git a/computable_phenotypes/utils/utils.py b/computable_phenotypes/utils/utils.py index f79d49c..40ce033 100644 --- a/computable_phenotypes/utils/utils.py +++ b/computable_phenotypes/utils/utils.py @@ -10,12 +10,13 @@ create_tables, delete_database, run_script, + fetch ) from computable_phenotypes.utils.Patients import Patients -def read_json(patients_list, connection): - patients = Patients(db=connection) +def read_json(patients_list, database_name): + patients = Patients(database_name=database_name) for pat in patients_list: try: @@ -37,42 +38,35 @@ def read_json(patients_list, connection): def process_json(patients_list: list[dict]): - db_engine = connect() database_name = "test" # connection.autocommit=True try: logger.info("Deleting DB") - with db_engine.connect() as conn: - delete_database(conn, database_name) + delete_database(database_name) logger.info("Creating DB") - with db_engine.connect() as conn: - create_database(conn, database_name) + create_database(database_name) + + create_tables(database_name) + logger.info("Reading Input") + read_json(patients_list, database_name) + # tables_conn.commit() + logger.info("Running Script") + run_script( + "./computable_phenotypes/script.sql", + database_name, + "./script_output.txt", + ) + - logger.info("Creating tables") - tables_engine = connect(database_name) - with tables_engine.connect() as tables_conn: - create_tables(tables_conn) - logger.info("Reading Input") - read_json(patients_list, tables_conn) - # tables_conn.commit() - logger.info("Running Script") - run_script( - "./computable_phenotypes/script.sql", - database_name, - "./script_output.txt", - ) - res = pd.read_sql_query( - "select * from dbo.NS_Final_Inclusions", tables_conn - ) - output = res.to_json(orient="records", indent=2, date_format="iso") - tables_engine.dispose() + output = fetch(database_name,"select * from dbo.NS_Final_Inclusions") + except Exception as e: + print(f"An error occurred: {e}") finally: logger.info("Deleting DB") - with db_engine.connect() as drop_conn: - delete_database(drop_conn, database_name) + delete_database(database_name) return output diff --git a/test.db b/test.db new file mode 100644 index 0000000000000000000000000000000000000000..eb49cc2bbf7be41afc0123d2f02376e83c092e93 GIT binary patch literal 16384 zcmeI#(NBUv90za^H&@N-%|~zZMYh&5|A7#0N>enz)k|Y!q-+TevDTyZk9Trmn8thV zd)tlU?zsE?KF7;hcU?dB$V(Va-Iz3mb3qh^8$tv@kXS9UnwH=1VmgrXl!VLs+cKLV zm3zY3$*IU5C=h@E1Rwwb2tWV=5P$##{=dMoA(bk%nz)i<_vzKsylFU&+}Xq*Wtv6R zq>4?6t+cw7WCo52sk2JRK(Td=_(6P;ytZ`H?l{`|n`bxnVt?xK0gFD6;LVLj6E~_~ z%e=FrTLZ<=)!h+OQE9d}SzKO~3Z+W5DtE1oHBSLcbn+a literal 0 HcmV?d00001 From 0d9d391077eccf42d76dbbdcf69930cc36b5cae9 Mon Sep 17 00:00:00 2001 From: Farid Seifi Date: Wed, 30 Oct 2024 15:50:12 -0400 Subject: [PATCH 2/3] fixed sqlite3 implementation --- computable_phenotypes/script.sql | 37 +++++++++++----------------- computable_phenotypes/utils/utils.py | 2 +- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/computable_phenotypes/script.sql b/computable_phenotypes/script.sql index 1f997ba..001ce20 100644 --- a/computable_phenotypes/script.sql +++ b/computable_phenotypes/script.sql @@ -15,7 +15,7 @@ CREATE TABLE Ref_NS_Codes Diag_Code nvarchar(10) NULL, Incl_Excl nvarchar(10) NULL, Incl_Excl_Type nvarchar(30) NULL -) +); --Insert diagnosis code values into table @@ -171,7 +171,7 @@ insert into Ref_NS_Codes values('ICD-10','N04.9','Inclusion','NSNOS_Encounter'); --Part 2---------------------------------------------------------------------------------------------------- --Get Diagnosis records and set flags for conditions related to each diagnosis record drop table IF EXISTS NS_EncounterConditions; - +CREATE TABLE NS_EncounterConditions AS select D.PATID, D.ENCOUNTERID , max(case when C.Incl_Excl_Type = 'NEPH5829_Encounter' then 1 else 0 end) as NEPH5829_Flag , max(case when C.Incl_Excl_Type = 'NEPH5832_Encounter' then 1 else 0 end) as NEPH5832_Flag @@ -183,7 +183,6 @@ select D.PATID, D.ENCOUNTERID , max(case when C.Incl_Excl_Type = 'PrimaryNS_Encounter' then 1 else 0 end) as PrimaryNS_Flag , max(case when C.Incl_Excl_Type = 'NSNOS_Encounter' then 1 else 0 end) as NSNOS_Flag , max(case when C.Incl_Excl_Type = 'Exclude_Encounter' then 1 else 0 end) as Exclude_Code_Flag -into NS_EncounterConditions from Diagnosis as D join Ref_NS_Codes as C on D.DX = C.Diag_Code @@ -204,7 +203,7 @@ group by D.PATID, D.ENCOUNTERID --Add encounter and demographic data to the diagnosis data drop table IF EXISTS NS_Encounter_Level; - +CREATE TABLE NS_Encounter_Level AS select E.ENCOUNTERID ,E.PATID @@ -213,7 +212,7 @@ select ,E.ENC_TYPE ,E.Raw_Enc_Type ,D.BIRTH_DATE - ,(0 + Convert(Char(8),E.ADMIT_DATE,112) - Convert(Char(8),D.BIRTH_DATE,112)) / 10000 AS Age + ,(strftime('%Y', E.ADMIT_DATE) - strftime('%Y', D.BIRTH_DATE)) as Age ,D.SEX ,D.HISPANIC ,D.RACE @@ -227,7 +226,6 @@ select ,DX.PrimaryNS_Flag ,DX.NSNOS_Flag ,DX.Exclude_Code_Flag -into NS_Encounter_Level from Encounter as E JOIN NS_EncounterConditions as DX on E.ENCOUNTERID = DX.ENCOUNTERID @@ -243,10 +241,9 @@ where drop table IF EXISTS NS_Encounter_Sort; - +CREATE TABLE NS_Encounter_Sort AS select * , ROW_NUMBER() OVER (PARTITION BY PATID ORDER BY DISCHARGE_DATE, Admit_Date, EncounterID) AS Encounter_Row -into NS_Encounter_Sort from NS_Encounter_Level ; @@ -257,10 +254,10 @@ from NS_Encounter_Level drop table IF EXISTS NS_Encounter_Sort_2; +CREATE TABLE NS_Encounter_Sort_2 AS Select Cur.* ,N.ADMIT_DATE AS Next_Admit_Date ,N.Discharge_Date AS Next_Discharge_Date -into NS_Encounter_Sort_2 from NS_Encounter_Sort as Cur left JOIN NS_Encounter_Sort as N on Cur.Encounter_Row = N.Encounter_Row - 1 and Cur.patid=N.patid @@ -273,10 +270,10 @@ from NS_Encounter_Sort as Cur drop table IF EXISTS NS_Encounter_Sort_3; +CREATE TABLE NS_Encounter_Sort_3 AS Select Cur.* ,N.ADMIT_DATE AS Prior_Admit_Date ,N.Discharge_Date AS Prior_Discharge_Date -into NS_Encounter_Sort_3 from NS_Encounter_Sort_2 as Cur left JOIN NS_Encounter_Sort_2 as N on Cur.Encounter_Row = N.Encounter_Row + 1 and Cur.patid=N.patid @@ -288,6 +285,7 @@ from NS_Encounter_Sort_2 as Cur drop table IF EXISTS NS_Encounter_Sums; +CREATE TABLE NS_Encounter_Sums AS select PATID ,ENCOUNTERID @@ -324,7 +322,6 @@ select ,Next_Discharge_Date ,Prior_Admit_Date ,Prior_Discharge_Date -into NS_Encounter_Sums from NS_Encounter_Sort_3 ; @@ -335,6 +332,7 @@ from NS_Encounter_Sort_3 drop table IF EXISTS NS_Possible_Inclusions; +CREATE TABLE NS_Possible_Inclusions AS select PATID ,EncounterID @@ -342,7 +340,6 @@ select ,PrimaryNS_Total ,NSNOS_Total ,1 as Possible_Inclusion -into NS_Possible_Inclusions from NS_Encounter_Sums where @@ -357,6 +354,7 @@ where drop table IF EXISTS NS_Possible_Exclusions; +CREATE TABLE NS_Possible_Exclusions AS select PATID ,EncounterID @@ -370,7 +368,6 @@ select ,Lupus_Total ,Exclude_Code_Total ,1 as Possible_Exclusion -into NS_Possible_Exclusions from NS_Encounter_Sums where @@ -391,6 +388,7 @@ where drop table IF EXISTS NS_Possible; +CREATE TABLE NS_Possible AS select Sums.* ,case @@ -399,7 +397,6 @@ select ,case when ex.possible_exclusion = 1 then 1 else 0 end as Possible_Exclusion -into NS_Possible from NS_Encounter_Sums as Sums left join NS_Possible_Inclusions as I on Sums.encounterid = I.encounterid @@ -414,11 +411,11 @@ from NS_Encounter_Sums as Sums drop table IF EXISTS NS_Final_Inclusions_Flag; +CREATE TABLE NS_Final_Inclusions_Flag AS select P.* ,1 as Final_Inclusion_Flag ,P.Prior_Discharge_Date AS Entry_Date -into NS_Final_Inclusions_Flag from NS_Possible as P where P.possible_inclusion = 1 and P.possible_exclusion != 1 ; @@ -429,7 +426,7 @@ where P.possible_inclusion = 1 and P.possible_exclusion != 1 drop table IF EXISTS NS_Final_Inclusions; - +CREATE TABLE NS_Final_Inclusions AS select patid AS PatientID , ENCOUNTERID as Anchor_EncounterID @@ -441,7 +438,6 @@ select , RACE as Race , SEX as Sex , Hispanic -into NS_Final_Inclusions from ( select *, ROW_NUMBER() OVER(PARTITION BY patid ORDER BY Encounter_Row) as RowNum @@ -457,9 +453,7 @@ where RowNum = 1 drop table IF EXISTS NS_Final_Inclusions_Output; -DECLARE @MyDate date; -SET @MyDate = '01-01-2024'; -- As of this date, who has already been included? - +CREATE TABLE NS_Final_Inclusions_Output AS select PatientID , Entry_Date as Cohort_Entry_Date @@ -471,9 +465,8 @@ select , Race , Sex , Hispanic -into NS_Final_Inclusions_Output from NS_Final_Inclusions -where Entry_Date <= @MyDate +where Entry_Date <= '2024-01-01' ; diff --git a/computable_phenotypes/utils/utils.py b/computable_phenotypes/utils/utils.py index 40ce033..7c71121 100644 --- a/computable_phenotypes/utils/utils.py +++ b/computable_phenotypes/utils/utils.py @@ -61,7 +61,7 @@ def process_json(patients_list: list[dict]): - output = fetch(database_name,"select * from dbo.NS_Final_Inclusions") + output = fetch(database_name,"select * from NS_Final_Inclusions") except Exception as e: print(f"An error occurred: {e}") finally: From 3b5b50d0a824100120262a629c8d25c9f39dcbf2 Mon Sep 17 00:00:00 2001 From: Farid Seifi Date: Thu, 31 Oct 2024 10:40:32 -0400 Subject: [PATCH 3/3] fixing dates --- computable_phenotypes/utils/Patients.py | 16 ++++++++++------ test | Bin 0 -> 86016 bytes 2 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 test diff --git a/computable_phenotypes/utils/Patients.py b/computable_phenotypes/utils/Patients.py index aec276c..3e16963 100644 --- a/computable_phenotypes/utils/Patients.py +++ b/computable_phenotypes/utils/Patients.py @@ -1,3 +1,4 @@ +from datetime import datetime import pprint from computable_phenotypes.utils.db import execute @@ -29,9 +30,11 @@ def add_patient(self, birthday, race=None, sex=None, hisp=None, pat_id=None): if base.get(pat_id) is not None: return pat_id # Add Patient + date_obj=datetime.strptime(birthday, "%m/%d/%Y") + birthdate=date_obj.strftime("\'%Y-%m-%d\'") patient_info = { "pat_id": pat_id, - "birthdate": date_parser(birthday), + "birthdate": birthdate, "race": race, "sex": sex, "hispanic": hisp, @@ -67,10 +70,14 @@ def add_enc( if base.get(enc_id) is not None: return enc_id # Add Enc + date_obj=datetime.strptime(admit_date, "%m/%d/%Y") + admitdate=date_obj.strftime("\'%Y-%m-%d\'") + date_obj=datetime.strptime(dis_date, "%m/%d/%Y") + disdate=date_obj.strftime("\'%Y-%m-%d\'") enc_info = { "enc_id": enc_id, - "admit_date": date_parser(admit_date), - "discharge_date": date_parser(dis_date), + "admit_date": admitdate, + "discharge_date": disdate, "enc_type": nullify(enc_type), "raw_enc_type": nullify(raw_enc_type), "diagnosisList": {}, @@ -157,6 +164,3 @@ def nullify(val): return string_wrap(val) -def date_parser(date): - split = date.split("/") - return f'\'{"-".join(split)}\'' diff --git a/test b/test new file mode 100644 index 0000000000000000000000000000000000000000..47a94a037380b10911aecd9649a6ae9aadadf4c6 GIT binary patch literal 86016 zcmeI5Yiu0HeZY73-me{=DCtZ~Jdw2?Jd$_X2k)_totPp|5*_J8k@SnQ%hJ?YxP_!TVAxPakkOXm@!uimmML!gI071XFDG-;pnpS4fl8$q^Eje|FwBjuWv;e1Ra%>I-UUFM5Wj;jn)M;{!Dq5lZ{Z{gjMzd#9#fewS`5`-A z*Ub1zQMKY}D1V<5PYVU<%SHI#3FUKkJd;B4riu#XXod3o?Rb^i2dQP>3g!1X@zknP zRE#1ksTCUDYgdmJ3WPN%0T1N|o#eFo5MM1Sv>B{WevcimTT*CT3IgJ3D4(_CS(BSm zT6`;%AF$(9N2)sM)u~}(7te|=7ggGKAUO@?Whc3nJxC$eidXU(D_&BeNmbhN0Dy;v z)0TOO3!$EtVkMullWR1-%rd3%Rw$oz;-Pq!6q>;f(itvXO)C%PzPCPWJ6;E?A<7@e_6VIjGQgt|F z7mo^|YeL_krME%{rC5TUcxJ}~wAOjWLI*`FmNpD@MkpF>LMue)|3mU#f&7xZOTI_m zB7a8yko-1jk;lm_IZh6enEapeyYhGCzmPvKe@=cuepYVC56VX&2_8rQ2_OL^fCP{L z57nnqDe{Y>+xWhFDsa>M06U1aF6>sLU(1l1gRN#laG*#3gZC0yIH5Wov5sAoWj} zs=Aerrf0z^K{^)ElXVtOav8S zg|216tze3(QwkGRv*1ZEMU}E)l&XqR&4Tm56hS2zfoc}~1*T}qvQ^zfT?DLPx?Qg7 z0QC#dEH$d7L-_}&FMvu^OQO>Gf0(>Q=l?&Z^Z&cN>tgYd01`j~NB{{S0VIF~kN^@u z0!RP}+&cua;iw23ZG^CDRA37NEaj`&kSPjj3fmb%MUp9!V5b1=wxC)AVl+V$s)k{u zb3L0M!)NpFoi2hkK>|ns2_OL^fCP{L5d@h$Op#S3Cl6rjkxj)|utL%Boh+{`RI2hmXUl2Lzg?aKC6Yh%69ymq-<(~On+ z`r5|!db_zf@nmFZ=+JRXS=C|7g`>>^7n@vp@`=dM0gtlAZFgx;Jnk}mddjG&LCTN0 zlMY7>vT5TAg@p-MsRG3Qk-X#tkU zPrFKSLWM0^y=ymd%44`%QiBYiybZ(kPQwA!8=vhk98`#z4&4B|g%gpXeRj`0p;iJ^ zC#F5+HPoQC8-K)8hDuP6FMPsNhFVb7KJGCK+u?dQT;XAl*-C)fiHAC?7GQV$W1ccp zf;!LGgOQ;@2X9c995Y;+%UV$WHCO%(7$AZMld4BoF9i$`;}wrCboAbA$IBkwN}%qP zM;Ep$_O^RWkK}TPCg?>5_5;E=Ik#?KkQN)~7WoBcLZgbhr3l*?9mV<0*45R`>kYTE z>Q;tfBv4s#E5pDSsC?Y39BBDSDUutsDpz%lB4@U*ZEx|a9VzZeZE`Y_8?p#!Wu-L5 zN9fts>a)#ub4%q$Cnq|zc_a8~k9TNy+t#@4i2{efO5(JyZQ7+=9HYur)^J+2>bh1; zU%h^1qjhm(tL1`GJa${^rK7aqF6;c<@HIbngtBILK+xa#RlvIaQFtZC57UC!n!4+x zH+6Ma9mgMV>uRnOwEMPd{MTrS*{y+5HiGY}3ZvAnOVrf;Tvq`K2YJ0f(Z#Dvb^L%kf4~(`ufqM@Y`F$! zNbhPL8{$O;)d0o&xS|#Ws*vNESCvvVp!#F`BRT6DY*5wSxfk|%j0V9k?DeV!X*{;`-xGbs0{M>C*CMFLT+1ktFLsb^Lf6X`mfVv z7Upt=3c@4%j{C~j<63;itIlWgzUrl?DROSzwRqsGKXHNTyOy+hRl9w<6U!dua;fZ; zz1yiv=l@airvmv8^3UY&$zPMN!w)==01`j~NB{{S0VIF~kN^@u0!RP}Ab}5x0EuQq z61R>-Trx<+E2xpMS5O@YWyGv2iKfd^CSe;%#N(PQdVaI_|3&4m3GnW}*T@qjFaH#N z;DH2?01`j~NB{{S0VIF~kN^@u0!ZLjN1z(rPsCCBfk}A$Ti~-?gTCO-{Gy=ilY#uD zcd=Szf7UOdrqh(>Cj?y|3*;#y)v$k-H7gW4;zslN&8OULl1^?zAgi?zt2Fbh5WNk z^qu_Iq@3^MU+g#gm#U(F%EZp*|8;@9Nw&!m`EB`zT>r@SHLN%iKmter2_OL^fCP{L z59BI%wemk79`c{NkB;tO_U>8EhrI!G zVZRabFCr||_np(5_R2-owZq^+X~=(`F9>~S`e#M|v{06QbA7)p1Z@33^PWK7AkUDL z{JQ)Q{J;YVAOR$R1dsp{Kmter2_OL^fCTPd0_UR#{HNKlpMJFeEZuKf+>G=vISR+_ z=I(y8<>$f&dd=r|J)?(wx_05uhy1IGYFUjf58+UIf$C4ufB!C2v01`j~NB{{S0VIF~?im6X zqK8ESkB566fp;E-7vphwG=6kcGGKn&wJr`a>+$qhIBZU|z0a?2MEV)06ML!VmzEok zghKGcFZZq|?XQIoi`l^1ung%0IPfujsn_H9U32`WLP1r8=j_=;TB<^ZU1@n0X$89~ zvn(A>^k|v{G>p|WeP{i?)BY#L-nOC8RIqW#KNhN{i7BMrKOmdWigr_k&a$mxjm4Gv`r5|!db_!~vbeF?UeSC+shNfP^iq9fY5K&O`iPIhSaM{f z-k3S}SYxTaFgH80RDW`55@hq!OYm#1!G287o}F7-nWcsrkDaB_v#qVQ%d4B0nk%!b z?Ix9gk(H(E*P4!nh1C~m3BAG-a|=tSJ(i{~HSKH`>rXnVPtPsRPdDaf9Kpi$3{=A^ zU86pKx?IzYm6KOiFVWmhfv$B3G{q6PKh0~W;-3!gl&DTR^RviyJpa10Eiu0gZU15) z>%G`qnc0AU9HG-r3mY%6(l%dguS{RO+G;a=Hq-_cd&wpXHy??`a>tL0w~qTadsmw# zgPR;1yxrX5{lxA9?_YyM6Kc^|PF2NQK4E{*qWJg!3<&@;yR4Z?TNFqbTR`s3kP?#L1G#wQrB zh30cBK!=WlTkdeg!^Rzx97`tHu*t3>>(8%l(kslEsuW;+Jac9;nSgWk60?yu00-BRHsokip z^)?;oM@C#1yo+$9Y`3nmtL4ZDz1TUW&DFh)J>Bp+n-$XUsAK zp!JQd))pnj0wHgCZ81x2me;4uR5Uv`eX4P8ac&Xr0-S7lTTm5x%(^P!7u$ojSzFTp z+`MdWt~Gfsc9U==_qpK|##S>)e0ozfN$8A(b8F-TBo7Hk+}&yk?+sB`erB;;w?uZk zTHTb^%DEZpk=k=_)TJ|GW-&XrICFY>;Z)tcU-D#QR+`TLg@F&xEh?4-2_OL^fCP{L z5gQEFduK9P$WAz2E6;2t<`93F_}rl-Z5i_D<| zzQmKQ_0=mY)XlfOMV+rJ=eFC|w%fg9_IX@Bz4e$ccD3DVuDAJ`*UZLcx~NuPZ*N|A z7P+R^*Fee_-dqc2R#f+@juqiu*P$Ms6RpkmWzPbTYb|VH6MJMdI2x%m^ zspEvL3f`>t!sJu$z5T%W5(%ihFQpF}#jPP1wbOSNzScO)G~Q@cY}n@4dZ#w|irlEST^Ae_bGdPEN}2$=kA&dHo)k`ePN4 z01`j~NB{{S0VIF~kN^@u0(T>UXQD@U@Xo(Pe3x(i`>jYn^YkTrVE)d%_ir@zCK=yagMFII+A04o^?!-HCXnyLKX@Pk zB!C2v01`j~NB{{S0VIF~kN^@u0v`y0s+6LhhtW~^kfUHa7x;$=yS_OH9s}!>j*+sM zqAmk7p}EgM(TdbNp{YgCQWYLj4P&aPsPN|i&i_A|0s0;nDc*UM?l|g26X;EN*w?+Az$C`_r$^f0di8My8r+H literal 0 HcmV?d00001