diff --git a/openaddr/cache.py b/openaddr/cache.py index c577aadb..c3c74916 100644 --- a/openaddr/cache.py +++ b/openaddr/cache.py @@ -403,10 +403,12 @@ def download(self, source_urls, workdir, source_config): metadata = downloader.get_metadata() - if query_fields is None: - field_names = [f['name'] for f in metadata['fields']] - else: - field_names = query_fields[:] + field_names = [f['name'] for f in metadata['fields']] + + missing_fields = [f for f in query_fields or [] if f not in field_names] + if missing_fields: + _L.warning("Conform fields missing from layer metadata: %s", missing_fields) + field_names += missing_fields if GEOM_FIELDNAME not in field_names: field_names.append(GEOM_FIELDNAME) diff --git a/openaddr/conform.py b/openaddr/conform.py index 92707df7..efd3d6db 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -1012,6 +1012,8 @@ def row_transform_and_convert(source_config, row): c = source_config.data_source["conform"] + raw = row_raw_columns(source_config, row) + "Attribute tags can utilize processing fxns" for k, v in c.items(): if k in source_config.SCHEMA and isinstance(v, list): @@ -1035,11 +1037,28 @@ def row_transform_and_convert(source_config, row): if source_config.layer == "addresses": feat['properties'] = row_canonicalize_unit_and_number(source_config.data_source, feat['properties']) + feat['properties']['raw'] = raw + if feat['geometry'] and len(feat['geometry']['coordinates']) > 0: feat['geometry']['coordinates'] = set_precision(feat['geometry']['coordinates'], 7) return feat +def row_raw_columns(sc, row): + """Copy the source columns that are not already in the output, before any conform function runs. + Columns mapped directly (a column name, or a list of column names) are excluded because their + values are output properties. Columns used only inside functions stay.""" + c = sc.data_source["conform"] + mapped = set() + for key in list(sc.SCHEMA) + ['lat', 'lon']: + v = c.get(key) + if isinstance(v, str): + mapped.add(v) + elif isinstance(v, list): + mapped.update(f for f in v if isinstance(f, str)) + return {k: v for k, v in row.items() + if k not in mapped and not k.lower().startswith('oa:') and v not in (None, '')} + def row_merge(sc, row, key): "Merge multiple columns like 'Maple','St' to 'Maple St'" merge_data = [(row.get(field, '') or '').strip() for field in sc.data_source["conform"][key]] diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 5dfecc58..56621043 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -528,7 +528,7 @@ def test_single_car(self): state = dict(zip(*json.load(file))) self.assertIsNotNone(state['cache']) - self.assertEqual(state['fingerprint'], '4a8047f90dbfe176c2a2b148837dae36') + self.assertEqual(state['fingerprint'], '4bc0e7aa15aed0d183d0485b964fdd4c') self.assertIsNotNone(state['processed']) self.assertIsNotNone(state['preview']) self.assertIsNotNone(state['pmtiles']) @@ -557,7 +557,7 @@ def test_single_car_cached(self): state = dict(zip(*json.load(file))) self.assertIsNotNone(state['cache']) - self.assertEqual(state['fingerprint'], '056bdaab3334e709bf29b0b2f1fcf8c4') + self.assertEqual(state['fingerprint'], '4bc0e7aa15aed0d183d0485b964fdd4c') self.assertIsNotNone(state['processed']) self.assertIsNone(state['preview']) @@ -576,7 +576,7 @@ def test_single_car_old_cached(self): state = dict(zip(*json.load(file))) self.assertIsNotNone(state['cache']) - self.assertEqual(state['fingerprint'], '056bdaab3334e709bf29b0b2f1fcf8c4') + self.assertEqual(state['fingerprint'], '4bc0e7aa15aed0d183d0485b964fdd4c') self.assertIsNotNone(state['processed']) self.assertIsNone(state['preview']) @@ -1019,7 +1019,7 @@ def test_single_tx_waco(self): self.assertEqual(rows[0]['properties']['region'], u'TX') self.assertEqual(rows[0]['properties']['id'], u'') self.assertEqual(rows[0]['properties']['number'], u'308') - self.assertEqual(rows[0]['properties']['hash'], u'd9db73ddf7ed0e2b') + self.assertEqual(rows[0]['properties']['hash'], u'2f31420158da846f') self.assertEqual(rows[0]['properties']['city'], u'Mcgregor') self.assertEqual(rows[0]['geometry']['coordinates'], [-97.3961768, 31.4432706]), self.assertEqual(rows[0]['properties']['street'], u'PULLEN ST') @@ -1081,7 +1081,7 @@ def test_single_ny_orange(self): self.assertEqual(len(rows), 8) self.assertEqual(rows[0]['properties']['id'], u'') self.assertEqual(rows[0]['properties']['number'], u'434') - self.assertEqual(rows[0]['properties']['hash'], u'a0127261b0619522') + self.assertEqual(rows[0]['properties']['hash'], u'41abd95de87dab64') self.assertEqual(rows[0]['properties']['city'], u'MONROE') self.assertEqual(rows[0]['geometry']['coordinates'], [-74.1926686, 41.3187728]) self.assertEqual(rows[0]['properties']['street'], u'') diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 7b22955d..29d57ee9 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -437,6 +437,30 @@ def test_row_fxn_regexp(self): d = row_fxn_regexp(c, d, "street", c.data_source["conform"]["street"]) self.assertEqual(e, d) + def test_raw_properties(self): + d = SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "conform": { + "street": ["s1", "s2"], + "number": {"function": "regexp", "field": "full", "pattern": "^(\\d+)"}, + "id": "pid", + "lon": "x", + "lat": "y" + }, + "fingerprint": "0000" + }] + } + }), "addresses", "default") + + r = row_transform_and_convert(d, { + "s1": "MAPLE", "s2": "ST", "full": "123 MAPLE ST", "pid": "7", "x": "-119.2", "y": "39.3", + "UPRN": "34", "empty": "", "OA:extra": "skip", GEOM_FIELDNAME: "POINT (-119.2 39.3)"}) + self.assertEqual("123", r['properties']['number']) + self.assertEqual({"full": "123 MAPLE ST", "UPRN": "34"}, r['properties']['raw']) + def test_transform_and_convert(self): d = SourceConfig(dict({ "schema": 2, @@ -469,6 +493,7 @@ def test_transform_and_convert(self): 'hash': 'b3af08e447c7ed16', "id": "", "accuracy": "", + "raw": {}, }, 'geometry': { 'coordinates': [-119.2, 39.3], @@ -501,6 +526,7 @@ def test_transform_and_convert(self): "id": "", 'hash': 'd4681f7e1d34e6ed', "accuracy": "", + "raw": {}, }, 'geometry': { "type": "Point", @@ -550,6 +576,7 @@ def test_transform_and_convert(self): "building_name": "", 'hash': '591d7970b5753b0d', "accuracy": "", + "raw": {"s": "123 MAPLE ST"}, } }, r) @@ -590,6 +617,7 @@ def test_transform_and_convert(self): "building_name": "", 'hash': '7b1dc0b74cbc0162', "accuracy": "", + "raw": {}, } }, r)