diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e4c2d5e..02a9a6c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] - +- DE-SH: make the 2023, 2025 and 2026 editions convert — glob the GeoPackage inside the archive (2023 was written with user_version = 0, so the archive alone matches no driver), parse fachguelti as DD.MM.YYYY, and map the 2023 and upper-case 2025/2026 column spellings that silently dropped determination:datetime and metrics:area (their area is text with a decimal comma) - Add DuckDB BaseConverter for efficiently transforming large datasets - Fix `use_variant_as_determination`: the determination:datetime column was dropped again because it was not listed in `columns` (affected DK, HR) - Declare the beautifulsoup4 dependency that the ES-PV and ES-VC converters import diff --git a/fiboa_cli/datasets/de_sh.py b/fiboa_cli/datasets/de_sh.py index 5c594fb5..cadca1ea 100644 --- a/fiboa_cli/datasets/de_sh.py +++ b/fiboa_cli/datasets/de_sh.py @@ -1,14 +1,37 @@ +import pandas as pd from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter +URL = ( + "https://service.gdi-sh.de/SH_OpenGBD/feeds/Atom_SH_Feldblockfinder_OpenGBD/data/" + "Feldbloecke_{year}_GPKG.zip" +) + + +def parse_date(col): + # Every edition writes fachguelti as DD.MM.YYYY, which is not an ISO date: + # left as text it reaches the STAC step as "Invalid isoformat string". + return pd.to_datetime(col, format="%d.%m.%Y") + + +def parse_decimal(col): + # 2023, 2025 and 2026 write the area as text with a decimal comma; in 2024 + # it is a Real and needs no conversion. + if pd.api.types.is_numeric_dtype(col): + return col + return pd.to_numeric(col.str.replace(",", ".", regex=False)) + class Converter(AdminConverterMixin, FiboaBaseConverter): + # Name the GeoPackage inside the archive rather than handing GDAL the + # archive itself: the 2023 GeoPackage was written with user_version = 0, so + # the GPKG driver identifies it by the .gpkg extension alone and + # "/vsizip/Feldbloecke_2023_GPKG.zip" matches no driver at all. The member + # is named differently in every edition (Feldbloecke_2023.gpkg, FB_2024.gpkg, + # FB_20250101.gpkg, FB_20260101.gpkg), hence the glob. variants = { - str( - y - ): f"https://service.gdi-sh.de/SH_OpenGBD/feeds/Atom_SH_Feldblockfinder_OpenGBD/data/Feldbloecke_{y}_GPKG.zip" - for y in range(2026, 2023 - 1, -1) + str(year): {URL.format(year=year): ["*.gpkg"]} for year in range(2026, 2023 - 1, -1) } id = "de_sh" admin_subdivision_code = "SH" @@ -18,6 +41,27 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): provider = "Land Schleswig-Holstein " license = "DL-DE-ZERO-2.0" extensions = {"https://fiboa.org/flik-extension/v0.2.0/schema.yaml"} + + # The source spellings drift between editions: 2023 has its own set of + # names, 2024 is mixed case, 2025 and 2026 are upper case. Without this the + # upper-case editions silently lose determination:datetime and metrics:area + # ("Column 'fachguelti' not found in dataset, removing from schema"). + COLUMN_RENAMES = { + # 2025, 2026 + "FACHGUELTI": "fachguelti", + "FLAECHE": "Flaeche", + # 2023; flgesamt (gross) equals flnetto (net) in all 198,614 rows + "flident": "FLIK", + "flgesamt": "Flaeche", + "hbn": "HBN", + } + + def migrate(self, gdf): + renames = {old: new for old, new in self.COLUMN_RENAMES.items() if old in gdf.columns} + if renames: + gdf = gdf.rename(columns=renames) + return super().migrate(gdf) + columns = { "geometry": "geometry", "fachguelti": "determination:datetime", @@ -25,4 +69,8 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): "Flaeche": "metrics:area", "HBN": "hbn", } + column_migrations = { + "fachguelti": parse_date, + "Flaeche": parse_decimal, + } missing_schemas = {"properties": {"hbn": {"type": "string"}}} diff --git a/tests/data-files/convert/de_sh/Feldbloecke_2026_GPKG.zip b/tests/data-files/convert/de_sh/Feldbloecke_2026_GPKG.zip index 39879a99..092b3f23 100644 Binary files a/tests/data-files/convert/de_sh/Feldbloecke_2026_GPKG.zip and b/tests/data-files/convert/de_sh/Feldbloecke_2026_GPKG.zip differ diff --git a/tests/test_convert.py b/tests/test_convert.py index 49c17aa2..af969e12 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -1,3 +1,4 @@ +import json import re import sys from csv import DictReader @@ -98,6 +99,25 @@ def _input_files(converter, *names): } +# Columns a converter must actually deliver. +# +# An optional column goes missing silently: the source spelling drifts between +# editions, the mapping stops matching, the base converter warns once ("Column +# 'X' not found in dataset, removing from schema") and validation still passes +# because the field is optional. That is exactly how de_sh published a 2026 +# edition carrying neither determination:datetime nor metrics:area. +# +# A value that is constant across the whole edition is written once into the +# collection metadata rather than as a column, so both places count as +# delivered -- de_sh's 2026 fixture is a single campaign date. +# +# Keyed like extra_convert_parameters, so "#