de_sh: backfill 2023-2025; per-edition column names, German date, decimal-comma area

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 - 
+- DE-SH: make the 2023, 2025 and 2026 editions convert — glob the GeoPackage inside the archive (2023 was written with user_version = 0, so the archive alone matches no driver), parse fachguelti as DD.MM.YYYY, and map the 2023 and upper-case 2025/2026 column spellings that silently dropped determination:datetime and metrics:area (their area is text with a decimal comma)
 - Add DuckDB BaseConverter for efficiently transforming large datasets
 - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR)
 - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import

fiboa_cli/datasets/de_sh.py

@@ -1,14 +1,37 @@
+import pandas as pd
 from vecorel_cli.conversion.admin import AdminConverterMixin
 
 from ..conversion.fiboa_converter import FiboaBaseConverter
 
+URL = (
+    "https://service.gdi-sh.de/SH_OpenGBD/feeds/Atom_SH_Feldblockfinder_OpenGBD/data/"
+    "Feldbloecke_{year}_GPKG.zip"
+)
+
+
+def parse_date(col):
+    # Every edition writes fachguelti as DD.MM.YYYY, which is not an ISO date:
+    # left as text it reaches the STAC step as "Invalid isoformat string".
+    return pd.to_datetime(col, format="%d.%m.%Y")
+
+
+def parse_decimal(col):
+    # 2023, 2025 and 2026 write the area as text with a decimal comma; in 2024
+    # it is a Real and needs no conversion.
+    if pd.api.types.is_numeric_dtype(col):
+        return col
+    return pd.to_numeric(col.str.replace(",", ".", regex=False))
+
 
 class Converter(AdminConverterMixin, FiboaBaseConverter):
+    # Name the GeoPackage inside the archive rather than handing GDAL the
+    # archive itself: the 2023 GeoPackage was written with user_version = 0, so
+    # the GPKG driver identifies it by the .gpkg extension alone and
+    # "/vsizip/Feldbloecke_2023_GPKG.zip" matches no driver at all. The member
+    # is named differently in every edition (Feldbloecke_2023.gpkg, FB_2024.gpkg,
+    # FB_20250101.gpkg, FB_20260101.gpkg), hence the glob.
     variants = {
-        str(
-            y
-        ): f"https://service.gdi-sh.de/SH_OpenGBD/feeds/Atom_SH_Feldblockfinder_OpenGBD/data/Feldbloecke_{y}_GPKG.zip"
-        for y in range(2026, 2023 - 1, -1)
+        str(year): {URL.format(year=year): ["*.gpkg"]} for year in range(2026, 2023 - 1, -1)
     }
     id = "de_sh"
     admin_subdivision_code = "SH"
@@ -18,11 +41,36 @@ class Converter(AdminConverterMixin, FiboaBaseConverter):
     provider = "Land Schleswig-Holstein <https://sh-mis.gdi-sh.de/catalog/#/datasets/iso/21f67269-780f-4f3c-8f66-03dde27acfe7>"
     license = "DL-DE-ZERO-2.0"
     extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml"}
+
+    # The source spellings drift between editions: 2023 has its own set of
+    # names, 2024 is mixed case, 2025 and 2026 are upper case. Without this the
+    # upper-case editions silently lose determination:datetime and metrics:area
+    # ("Column 'fachguelti' not found in dataset, removing from schema").
+    COLUMN_RENAMES = {
+        # 2025, 2026
+        "FACHGUELTI": "fachguelti",
+        "FLAECHE": "Flaeche",
+        # 2023; flgesamt (gross) equals flnetto (net) in all 198,614 rows
+        "flident": "FLIK",
+        "flgesamt": "Flaeche",
+        "hbn": "HBN",
+    }
+
+    def migrate(self, gdf):
+        renames = {old: new for old, new in self.COLUMN_RENAMES.items() if old in gdf.columns}
+        if renames:
+            gdf = gdf.rename(columns=renames)
+        return super().migrate(gdf)
+
     columns = {
         "geometry": "geometry",
         "fachguelti": "determination:datetime",
         "FLIK": ("flik", "id"),
         "Flaeche": "metrics:area",
         "HBN": "hbn",
     }
+    column_migrations = {
+        "fachguelti": parse_date,
+        "Flaeche": parse_decimal,
+    }
     missing_schemas = {"properties": {"hbn": {"type": "string"}}}

tests/test_convert.py

@@ -1,3 +1,4 @@
+import json
 import re
 import sys
 from csv import DictReader
@@ -98,6 +99,25 @@ def _input_files(converter, *names):
 }
 
 
+# Columns a converter must actually deliver.
+#
+# An optional column goes missing silently: the source spelling drifts between
+# editions, the mapping stops matching, the base converter warns once ("Column
+# 'X' not found in dataset, removing from schema") and validation still passes
+# because the field is optional. That is exactly how de_sh published a 2026
+# edition carrying neither determination:datetime nor metrics:area.
+#
+# A value that is constant across the whole edition is written once into the
+# collection metadata rather than as a column, so both places count as
+# delivered -- de_sh's 2026 fixture is a single campaign date.
+#
+# Keyed like extra_convert_parameters, so "<id>#<label>" can state a different
+# expectation per edition where the editions genuinely differ.
+expected_columns = {
+    "de_sh": ("determination:datetime", "metrics:area", "flik", "hbn", "id"),
+}
+
+
 @mark.parametrize("converter", tests)
 @patch("fiboa_cli.datasets.commons.hcat.load_ec_mapping")
 @patch("fiboa_cli.datasets.commons.ec.load_ec_mapping")
@@ -144,6 +164,17 @@ def load_ec(csv_file=None, url=None):
     ValidateData().validate(tmp_parquet_file)
 
     df = pq.read_table(tmp_parquet_file).to_pandas()
+
+    required = expected_columns.get(converter)
+    if required:
+        metadata = pq.ParquetFile(tmp_parquet_file).schema_arrow.metadata or {}
+        constants = json.loads(metadata[b"collection"].decode()) if b"collection" in metadata else {}
+        missing = [c for c in required if c not in df.columns and constants.get(c) is None]
+        assert not missing, (
+            f"{converter} dropped {missing}: absent from the schema and from the "
+            f"collection metadata. Produced columns: {sorted(df.columns)}"
+        )
+
     if "metrics:area" in df.columns and converter_id not in ("de_bb",):
         # Check for accidental hectare conversion; fields should be more than 10 square meters
         assert (df["metrics:area"] > 10).all()