Add raw property with unmapped source columns

openaddr/cache.py

@@ -403,10 +403,12 @@ def download(self, source_urls, workdir, source_config):
 
             metadata = downloader.get_metadata()
 
-            if query_fields is None:
-                field_names = [f['name'] for f in metadata['fields']]
-            else:
-                field_names = query_fields[:]
+            field_names = [f['name'] for f in metadata['fields']]
+
+            missing_fields = [f for f in query_fields or [] if f not in field_names]
+            if missing_fields:
+                _L.warning("Conform fields missing from layer metadata: %s", missing_fields)
+                field_names += missing_fields
 
             if GEOM_FIELDNAME not in field_names:
                 field_names.append(GEOM_FIELDNAME)

openaddr/conform.py

@@ -1012,6 +1012,8 @@ def row_transform_and_convert(source_config, row):
 
     c = source_config.data_source["conform"]
 
+    raw = row_raw_columns(source_config, row)
+
     "Attribute tags can utilize processing fxns"
     for k, v in c.items():
         if k in source_config.SCHEMA and isinstance(v, list):
@@ -1035,11 +1037,28 @@ def row_transform_and_convert(source_config, row):
     if source_config.layer == "addresses":
         feat['properties'] = row_canonicalize_unit_and_number(source_config.data_source, feat['properties'])
 
+    feat['properties']['raw'] = raw
+
     if feat['geometry'] and len(feat['geometry']['coordinates']) > 0:
         feat['geometry']['coordinates'] = set_precision(feat['geometry']['coordinates'], 7)
 
     return feat
 
+def row_raw_columns(sc, row):
+    """Copy the source columns that are not already in the output, before any conform function runs.
+    Columns mapped directly (a column name, or a list of column names) are excluded because their
+    values are output properties. Columns used only inside functions stay."""
+    c = sc.data_source["conform"]
+    mapped = set()
+    for key in list(sc.SCHEMA) + ['lat', 'lon']:
+        v = c.get(key)
+        if isinstance(v, str):
+            mapped.add(v)
+        elif isinstance(v, list):
+            mapped.update(f for f in v if isinstance(f, str))
+    return {k: v for k, v in row.items()
+            if k not in mapped and not k.lower().startswith('oa:') and v not in (None, '')}
+
 def row_merge(sc, row, key):
     "Merge multiple columns like 'Maple','St' to 'Maple St'"
     merge_data = [(row.get(field, '') or '').strip() for field in sc.data_source["conform"][key]]

openaddr/tests/__init__.py

@@ -528,7 +528,7 @@ def test_single_car(self):
             state = dict(zip(*json.load(file)))
 
         self.assertIsNotNone(state['cache'])
-        self.assertEqual(state['fingerprint'], '4a8047f90dbfe176c2a2b148837dae36')
+        self.assertEqual(state['fingerprint'], '4bc0e7aa15aed0d183d0485b964fdd4c')
         self.assertIsNotNone(state['processed'])
         self.assertIsNotNone(state['preview'])
         self.assertIsNotNone(state['pmtiles'])
@@ -557,7 +557,7 @@ def test_single_car_cached(self):
             state = dict(zip(*json.load(file)))
 
         self.assertIsNotNone(state['cache'])
-        self.assertEqual(state['fingerprint'], '056bdaab3334e709bf29b0b2f1fcf8c4')
+        self.assertEqual(state['fingerprint'], '4bc0e7aa15aed0d183d0485b964fdd4c')
         self.assertIsNotNone(state['processed'])
         self.assertIsNone(state['preview'])
 
@@ -576,7 +576,7 @@ def test_single_car_old_cached(self):
             state = dict(zip(*json.load(file)))
 
         self.assertIsNotNone(state['cache'])
-        self.assertEqual(state['fingerprint'], '056bdaab3334e709bf29b0b2f1fcf8c4')
+        self.assertEqual(state['fingerprint'], '4bc0e7aa15aed0d183d0485b964fdd4c')
         self.assertIsNotNone(state['processed'])
         self.assertIsNone(state['preview'])
 
@@ -1019,7 +1019,7 @@ def test_single_tx_waco(self):
             self.assertEqual(rows[0]['properties']['region'], u'TX')
             self.assertEqual(rows[0]['properties']['id'], u'')
             self.assertEqual(rows[0]['properties']['number'], u'308')
-            self.assertEqual(rows[0]['properties']['hash'], u'd9db73ddf7ed0e2b')
+            self.assertEqual(rows[0]['properties']['hash'], u'2f31420158da846f')
             self.assertEqual(rows[0]['properties']['city'], u'Mcgregor')
             self.assertEqual(rows[0]['geometry']['coordinates'], [-97.3961768, 31.4432706]),
             self.assertEqual(rows[0]['properties']['street'], u'PULLEN ST')
@@ -1081,7 +1081,7 @@ def test_single_ny_orange(self):
             self.assertEqual(len(rows), 8)
             self.assertEqual(rows[0]['properties']['id'], u'')
             self.assertEqual(rows[0]['properties']['number'], u'434')
-            self.assertEqual(rows[0]['properties']['hash'], u'a0127261b0619522')
+            self.assertEqual(rows[0]['properties']['hash'], u'41abd95de87dab64')
             self.assertEqual(rows[0]['properties']['city'], u'MONROE')
             self.assertEqual(rows[0]['geometry']['coordinates'], [-74.1926686, 41.3187728])
             self.assertEqual(rows[0]['properties']['street'], u'')

openaddr/tests/conform.py

@@ -437,6 +437,30 @@ def test_row_fxn_regexp(self):
         d = row_fxn_regexp(c, d, "street", c.data_source["conform"]["street"])
         self.assertEqual(e, d)
 
+    def test_raw_properties(self):
+        d = SourceConfig(dict({
+            "schema": 2,
+            "layers": {
+                "addresses": [{
+                    "name": "default",
+                    "conform": {
+                        "street": ["s1", "s2"],
+                        "number": {"function": "regexp", "field": "full", "pattern": "^(\\d+)"},
+                        "id": "pid",
+                        "lon": "x",
+                        "lat": "y"
+                    },
+                    "fingerprint": "0000"
+                }]
+            }
+        }), "addresses", "default")
+
+        r = row_transform_and_convert(d, {
+            "s1": "MAPLE", "s2": "ST", "full": "123 MAPLE ST", "pid": "7", "x": "-119.2", "y": "39.3",
+            "UPRN": "34", "empty": "", "OA:extra": "skip", GEOM_FIELDNAME: "POINT (-119.2 39.3)"})
+        self.assertEqual("123", r['properties']['number'])
+        self.assertEqual({"full": "123 MAPLE ST", "UPRN": "34"}, r['properties']['raw'])
+
     def test_transform_and_convert(self):
         d = SourceConfig(dict({
             "schema": 2,
@@ -469,6 +493,7 @@ def test_transform_and_convert(self):
                 'hash': 'b3af08e447c7ed16',
                 "id": "",
                 "accuracy": "",
+                "raw": {},
             },
             'geometry': {
                 'coordinates': [-119.2, 39.3],
@@ -501,6 +526,7 @@ def test_transform_and_convert(self):
                 "id": "",
                 'hash': 'd4681f7e1d34e6ed',
                 "accuracy": "",
+                "raw": {},
             },
             'geometry': {
                 "type": "Point",
@@ -550,6 +576,7 @@ def test_transform_and_convert(self):
                 "building_name": "",
                 'hash': '591d7970b5753b0d',
                 "accuracy": "",
+                "raw": {"s": "123 MAPLE ST"},
             }
         }, r)
 
@@ -590,6 +617,7 @@ def test_transform_and_convert(self):
                 "building_name": "",
                 'hash': '7b1dc0b74cbc0162',
                 "accuracy": "",
+                "raw": {},
             }
         }, r)