Integrate BigQuery with DOT

.dockerignore

@@ -0,0 +1,8 @@
+# Ignore appsmith container volumes/data
+docker/appsmith/stacks/
+
+# Ignore other things that don't need to be in the Docker image
+.git/
+venv/
+__pycache__/
+*.pyc

.gitignore

@@ -16,4 +16,13 @@ docker/cookie
 .DS_Store
 
 .idea*
-__pycache__
+__pycache__/
+*.py[cod]
+.venv/
+.ipynb_checkpoints/
+
+# Great Expectations local store id, wherever a run creates it
+.ge_store_backend_id
+
+# Local DBeaver scripts, not part of the toolkit
+db/dot/Scripts

README.md

@@ -377,7 +377,7 @@ generated one.
     ```
     'INSERT INTO dot.configured_tests VALUES(TRUE, 'ScanProject1', '0cdc9702-91e0-3499-b6f0-4dec12ad0f08', 'ASSESS-1', 3, '', '', 
     '', 'ancview_pregnancy', 'relationships', 'uuid', '', 
-    $${"name": "danger_signs_with_no_pregnancy", "to": "ref('dot_model__ancview_danger_sign')", "field": "pregnancy_uuid"}$$, 
+    $${"name": "danger_signs_with_no_pregnancy", "to": "ancview_danger_sign", "field": "pregnancy_uuid"}$$, 
     '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'your-name');
     ```
 2. `unique`
@@ -413,7 +413,7 @@ generated one.
     ```
     INSERT INTO dot.configured_tests VALUES(TRUE, 'ScanProject1', '7f78de0e-8268-3da6-8845-9a445457cc9a', 'DUPLICATE-1', 3, '', 
     '', '', 'ancview_pregnancy', 'possible_duplicate_forms', '', '', 
-    $${"table_specific_reported_date": "delivery_date", "table_specific_patient_uuid": "patient_id", "table_specific_uuid": "uuid"}$$, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'your-name');
+    $${"date_column": "delivery_date", "group_column": "patient_id", "id_column": "uuid"}$$, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'your-name');
     ```
 7. `associated_columns_not_null`
     <br><br>
@@ -429,7 +429,7 @@ generated one.
     INSERT INTO dot.configured_tests VALUES(TRUE, 'ScanProject1', '0cdc9702-91e0-3499-b6f0-4dec12ad0f08', 'BIAS-1', 3, 
     'Test for miscalibrated thermometer', '', '', 'ancview_pregnancy', 'expect_similar_means_across_reporters', 
     'child_temperature_pre_chw', '', '{"key": "reported_by","quantity": "child_temperature_pre_chw",
-    "form_name": "dot_model__iccmview_assessment","id_column": "reported_by"}', '2022-01-19 20:00:00.000 -0500', 
+    "data_table": "iccmview_assessment","id_column": "reported_by"}', '2022-01-19 20:00:00.000 -0500', 
     '2022-01-19 20:00:00.000 -0500', 'your-name');
     ```
 9. `expression_is_true`

db/dot/1-schema.sql

@@ -1,7 +1,9 @@
 CREATE SCHEMA dot;
 -- CREATE SCHEMA data;
 
-CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+-- uuid-ossp lives in public; some DOT DBs omit public, so create it first
+CREATE SCHEMA IF NOT EXISTS public;
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA public;
 
 CREATE TABLE IF NOT EXISTS dot.scenarios(
     scenario_id VARCHAR(300) PRIMARY KEY,
@@ -188,7 +190,8 @@ declare
 BEGIN
    -- If you change how this UUID is generated, be sure to also change how it is created in get_test_id in /utils/utils.py
    KEY_STRING := new.project_id || new.test_type || new.entity_id || new.column_name || COALESCE(CAST(new.test_parameters AS VARCHAR),'');
-   NEW.test_id := uuid_generate_v3(uuid_ns_oid(), KEY_STRING);
+   -- Qualify uuid-ossp functions so triggers work even when search_path omits public
+   NEW.test_id := public.uuid_generate_v3(public.uuid_ns_oid(), KEY_STRING);
    new.date_added := NOW();
    new.date_modified := NOW();
    RETURN NEW;
@@ -203,7 +206,7 @@ declare
    KEY_STRING text;
 BEGIN
    KEY_STRING := new.project_id || new.test_type || new.entity_id || new.column_name || COALESCE(CAST(new.test_parameters AS VARCHAR),'');
-   NEW.test_id := uuid_generate_v3(uuid_ns_oid(), KEY_STRING);
+   NEW.test_id := public.uuid_generate_v3(public.uuid_ns_oid(), KEY_STRING);
    new.date_modified := NOW();
    RETURN NEW;
 END;

db/dot/2-upload_static_data.sql

@@ -39,21 +39,21 @@ INSERT INTO dot.test_types VALUES('expression_is_true', 'dbt', 'Test to confirm
 
 -- dot.test_parameters_interface
 -- INSERT INTO dot.test_parameters_interface VALUES('relationships', 'name', 'function_argument', 'Name of the test');
-INSERT INTO dot.test_parameters_interface VALUES('relationships', 'reference', 'view/table', $$ref('dot_model__ancview_pregnancy')$$, 'Referenced field to be checked if missing');
+INSERT INTO dot.test_parameters_interface VALUES('relationships', 'to', 'view/table', 'ancview_pregnancy', 'Referenced entity to be checked if missing');
 INSERT INTO dot.test_parameters_interface VALUES('relationships', 'field', 'entity any field', 'uuid', 'Field being checked');
 -- INSERT INTO dot.test_parameters_interface VALUES('not_negative_string_column', 'name', 'function_argument', 'Name of column to be check3ed for non-negative values');
 INSERT INTO dot.test_parameters_interface VALUES('accepted_values', 'values', 'list of values', $$["dog","cat","ostrich"]$$,'List of accepted values for the field being checked');
-INSERT INTO dot.test_parameters_interface VALUES('possible_duplicate_forms', 'table_specific_reported_date', 'entity date field', 'reported', 'Column which indicates when form created');
-INSERT INTO dot.test_parameters_interface VALUES('possible_duplicate_forms', 'table_specific_patient_uuid', 'entity id field', 'patient_id',  'Column which holds to patient uuid');
-INSERT INTO dot.test_parameters_interface VALUES('possible_duplicate_forms', 'table_specific_uuid', 'entity id field', 'uuid', 'UUID for records in the table (form) being checked');
-INSERT INTO dot.test_parameters_interface VALUES('possible_duplicate_forms', 'table_specific_period', 'one of (hour, day, week)', 'day','Specified period to check for duplicates (hour, day, week)');
+INSERT INTO dot.test_parameters_interface VALUES('possible_duplicate_forms', 'date_column', 'entity date field', 'reported', 'Column which indicates when the record was created or reported');
+INSERT INTO dot.test_parameters_interface VALUES('possible_duplicate_forms', 'group_column', 'entity id field', 'patient_id',  'Column used to group records when checking for duplicates');
+INSERT INTO dot.test_parameters_interface VALUES('possible_duplicate_forms', 'id_column', 'entity id field', 'uuid', 'Primary id for records in the table being checked');
+INSERT INTO dot.test_parameters_interface VALUES('possible_duplicate_forms', 'period', 'one of (hour, day, week)', 'day','Specified period to check for duplicates (hour, day, week)');
 INSERT INTO dot.test_parameters_interface VALUES('custom_sql', 'query', 'sql statement', $$SELECT field1, field2, 'table1' as \"primary_table\", 'field1' as \"primary_table_id_field\" WHERE COLOR='green'$$,'Custom SQL to use to determine test fails, SQL is defined in columns test_parameter');
 -- INSERT INTO dot.test_parameters_interface VALUES('expression_is_true', 'name', 'function_argument', 'Name of the test');
 INSERT INTO dot.test_parameters_interface VALUES('expression_is_true', 'condition', 'entity columns boolean logic', '(patient_age_in_months<24) and (malaria_give_act is not null)','Where clause of rows that are going to be checked');
 INSERT INTO dot.test_parameters_interface VALUES('expression_is_true', 'expression', 'entity columns boolean logic', 'malaria_act_dosage is not null', 'If not true, the row fails the test');
 INSERT INTO dot.test_parameters_interface VALUES('expect_similar_means_across_reporters', 'key', 'entity id field', 'reported_by', 'The grouping field to check means by, ie a person-specific id');
 INSERT INTO dot.test_parameters_interface VALUES('expect_similar_means_across_reporters', 'quantity', 'entity numeric field', 'temperature', 'The name of the numeric field to analyze for variation');
-INSERT INTO dot.test_parameters_interface VALUES('expect_similar_means_across_reporters', 'data_table', 'view/table', 'dot_model__iccmview_assessment', 'The name of entity view where data is');
+INSERT INTO dot.test_parameters_interface VALUES('expect_similar_means_across_reporters', 'data_table', 'view/table', 'iccmview_assessment', 'The name of entity view where data is');
 INSERT INTO dot.test_parameters_interface VALUES('expect_similar_means_across_reporters', 'id_column', 'entity id field', 'reported_by', 'The id column to use to get failed test records');
 
 -- dot.scenario_test_types