diff --git a/docs/content/en/latest/pipelines/ldm_extension/_index.md b/docs/content/en/latest/pipelines/ldm_extension/_index.md index 1a4de18f1..24388c50b 100644 --- a/docs/content/en/latest/pipelines/ldm_extension/_index.md +++ b/docs/content/en/latest/pipelines/ldm_extension/_index.md @@ -27,6 +27,8 @@ ldm_extension_manager = LdmExtensionManager.create(host=host, token=token) To extend the LDM, you need to define the custom datasets and the fields they should contain. The script also checks the validity of analytical objects before and after the update. Updates introducing new invalid relations are automatically rolled back. You can opt out of this behavior by setting the `check_relations` parameter to False. +To create date datasets with the second-based granularities (`SECOND`, `SECOND_OF_MINUTE`, `SECOND_OF_DAY`, `MINUTE_OF_DAY`), set the `enable_second_granularities` parameter to True when creating the LdmExtensionManager. This requires the `enableSecondGranularities` feature flag to be enabled for your GoodData organization. + ### Custom Dataset Definitions The custom dataset represents a new dataset appended to the child LDM. It is defined by the following parameters: diff --git a/packages/gooddata-dbt/README.md b/packages/gooddata-dbt/README.md index eb4f199ff..f318461a3 100644 --- a/packages/gooddata-dbt/README.md +++ b/packages/gooddata-dbt/README.md @@ -50,6 +50,8 @@ The plugin provides the following use cases: - Reads dbt models and profiles - Scans data source (connection props from dbt profiles) through GoodData to get column data types (optional in dbt) - Generates GoodData LDM(Logical Data Model) from dbt models. Can utilize custom gooddata-specific metadata, more below + - With `--gooddata-enable-second-granularities`, date datasets are created with second-based granularities. + Requires the `enableSecondGranularities` feature flag enabled in the GoodData organization. - upload_notification - Invalidates caches for data source - deploy_analytics diff --git a/packages/gooddata-dbt/src/gooddata_dbt/args.py b/packages/gooddata-dbt/src/gooddata_dbt/args.py index d1d1d5c82..f5c316b7b 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/args.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/args.py @@ -67,6 +67,16 @@ def set_gooddata_upper_case_args(parser: argparse.ArgumentParser) -> None: ) +def set_gooddata_enable_second_granularities_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--gooddata-enable-second-granularities", + help="Create date datasets with second-based granularities. " + "Requires the enableSecondGranularities feature flag enabled in the GoodData organization.", + action="store_true", + default=False, + ) + + def set_gooddata_workspace_title_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( "-gwt", "--gooddata-workspace-title", help="Workspace title", default=os.getenv("GOODDATA_WORKSPACE_TITLE") @@ -169,6 +179,7 @@ def parse_arguments(description: str) -> argparse.Namespace: set_dbt_args(deploy_ldm) set_environment_id_arg(deploy_ldm) set_gooddata_upper_case_args(deploy_ldm) + set_gooddata_enable_second_granularities_args(deploy_ldm) deploy_ldm.set_defaults(method="deploy_ldm") upload_notification = subparsers.add_parser("upload_notification") diff --git a/packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py b/packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py index 930ab770a..03f218b16 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py @@ -52,6 +52,13 @@ class GoodDataSortDirection(Enum): "MINUTE_OF_HOUR", "HOUR_OF_DAY", ] +# newly added granularities gated behind `enableSecondGranularities` feature flag +SECOND_TIMESTAMP_GRANULARITIES = [ + "SECOND", + "SECOND_OF_MINUTE", + "SECOND_OF_DAY", + "MINUTE_OF_DAY", +] T = TypeVar("T", bound="Base") DBT_TARGET_DIR = Path("target") diff --git a/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py b/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py index a72582881..0b0c93897 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py @@ -18,6 +18,7 @@ DBT_PATH_TO_MANIFEST, DBT_TARGET_DIR, NUMERIC_DATA_TYPES, + SECOND_TIMESTAMP_GRANULARITIES, TIMESTAMP_DATA_TYPES, TIMESTAMP_GRANULARITIES, Base, @@ -202,9 +203,12 @@ class DbtModelTables: * column_type – Optional if missing call scan """ - def __init__(self, tables: list[DbtModelTable], upper_case: bool) -> None: + def __init__( + self, tables: list[DbtModelTable], upper_case: bool, enable_second_granularities: bool = False + ) -> None: self.upper_case = upper_case self.tables = tables + self._enable_second_granularities = enable_second_granularities @classmethod def from_cloud( @@ -214,22 +218,27 @@ def from_cloud( upper_case: bool, all_model_ids: list[str], path: Union[str, Path] = DBT_TARGET_DIR, + enable_second_granularities: bool = False, ) -> "DbtModelTables": path = path if isinstance(path, Path) else Path(path) dbt_conn.download_manifest(run_id=run_id, path=path) with open(path / "manifest.json") as fp: dbt_catalog = json.load(fp) tables = cls.read_dbt_models(dbt_catalog, upper_case, all_model_ids) - return cls(tables, upper_case) + return cls(tables, upper_case, enable_second_granularities) @classmethod def from_local( - cls, upper_case: bool, all_model_ids: list[str], manifest_path: Union[str, Path] = DBT_PATH_TO_MANIFEST + cls, + upper_case: bool, + all_model_ids: list[str], + manifest_path: Union[str, Path] = DBT_PATH_TO_MANIFEST, + enable_second_granularities: bool = False, ) -> "DbtModelTables": with open(manifest_path) as fp: dbt_catalog = json.load(fp) tables = cls.read_dbt_models(dbt_catalog, upper_case, all_model_ids) - return cls(tables, upper_case) + return cls(tables, upper_case, enable_second_granularities) @staticmethod def read_dbt_models(dbt_catalog: dict, upper_case: bool, all_model_ids: list[str]) -> list[DbtModelTable]: @@ -437,14 +446,17 @@ def make_attributes(self, table: DbtModelTable) -> list[dict]: ) return attributes - @staticmethod - def make_date_datasets(table: DbtModelTable, existing_date_datasets: list[dict]) -> list[dict]: + def make_date_datasets(self, table: DbtModelTable, existing_date_datasets: list[dict]) -> list[dict]: date_datasets = [] for column in table.columns.values(): existing_dataset_ids = [d["id"] for d in existing_date_datasets] if column.is_date() and column.gooddata_ldm_id not in existing_dataset_ids: if column.data_type in TIMESTAMP_DATA_TYPES: - granularities = DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES + granularities = ( + DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES + SECOND_TIMESTAMP_GRANULARITIES + if self._enable_second_granularities + else DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES + ) else: granularities = DATE_GRANULARITIES date_datasets.append( diff --git a/packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py b/packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py index c485966c6..aaad25692 100644 --- a/packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py +++ b/packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py @@ -73,7 +73,11 @@ def deploy_ldm( logger.info("Generate and put LDM") dbt_profiles = DbtProfiles(args) data_source_id = dbt_profiles.data_source_id - dbt_tables = DbtModelTables.from_local(args.gooddata_upper_case, all_model_ids) + dbt_tables = DbtModelTables.from_local( + args.gooddata_upper_case, + all_model_ids, + enable_second_granularities=args.gooddata_enable_second_granularities, + ) generate_and_put_ldm(logger, sdk_wrapper, data_source_id, workspace_id, dbt_tables, model_ids) workspace_url = f"{sdk_wrapper.get_host_from_sdk()}/modeler/#/{workspace_id}" logger.info(f"LDM successfully loaded, verify here: {workspace_url}") diff --git a/packages/gooddata-dbt/tests/test_tables.py b/packages/gooddata-dbt/tests/test_tables.py index c0c9bb9b5..0abb4fcee 100644 --- a/packages/gooddata-dbt/tests/test_tables.py +++ b/packages/gooddata-dbt/tests/test_tables.py @@ -3,7 +3,12 @@ from pathlib import Path from typing import Union -from gooddata_dbt.dbt.tables import DbtModelTables +from gooddata_dbt.dbt.base import ( + DATE_GRANULARITIES, + SECOND_TIMESTAMP_GRANULARITIES, + TIMESTAMP_GRANULARITIES, +) +from gooddata_dbt.dbt.tables import DbtModelColumn, DbtModelTable, DbtModelTables from gooddata_sdk import CatalogDeclarativeModel, CatalogDeclarativeTables _CURR_DIR = Path(__file__).parent @@ -51,6 +56,36 @@ def test_make_ldm(): assert len(ldm.ldm.date_instances) == 4 +def _table_with_date_columns() -> DbtModelTable: + return DbtModelTable( + name="events", + description="", + tags=[], + schema="public", + columns={ + "created_at": DbtModelColumn(name="created_at", description="", tags=[], data_type="TIMESTAMP"), + "created_on": DbtModelColumn(name="created_on", description="", tags=[], data_type="DATE"), + }, + ) + + +def test_make_date_datasets_without_second_granularities(): + tables = DbtModelTables([], upper_case=False) + date_datasets = {d["id"]: d for d in tables.make_date_datasets(_table_with_date_columns(), [])} + assert date_datasets["created_at"]["granularities"] == DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES + assert date_datasets["created_on"]["granularities"] == DATE_GRANULARITIES + + +def test_make_date_datasets_with_second_granularities(): + tables = DbtModelTables([], upper_case=False, enable_second_granularities=True) + date_datasets = {d["id"]: d for d in tables.make_date_datasets(_table_with_date_columns(), [])} + assert ( + date_datasets["created_at"]["granularities"] + == DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES + SECOND_TIMESTAMP_GRANULARITIES + ) + assert date_datasets["created_on"]["granularities"] == DATE_GRANULARITIES + + FAA_MODEL_ID = "faa" diff --git a/packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/input_processor.py b/packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/input_processor.py index 04e8c4bc2..4fa83e746 100644 --- a/packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/input_processor.py +++ b/packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/input_processor.py @@ -82,6 +82,21 @@ class LdmExtensionDataProcessor: "FISCAL_YEAR", ] + # newly added granularities gated behind `enableSecondGranularities` feature flag + _SECOND_DATE_GRANULARITIES: list[str] = [ + "SECOND", + "SECOND_OF_MINUTE", + "SECOND_OF_DAY", + "MINUTE_OF_DAY", + ] + + def __init__(self, enable_second_granularities: bool = False): + self._date_granularities = ( + self.DATE_GRANULARITIES + self._SECOND_DATE_GRANULARITIES + if enable_second_granularities + else self.DATE_GRANULARITIES + ) + @staticmethod def _attribute_from_field( dataset_name: str, @@ -127,7 +142,7 @@ def _date_from_field( title_base="", title_pattern="%titleBase - %granularityTitle", ), - granularities=self.DATE_GRANULARITIES, + granularities=self._date_granularities, description=custom_field.description, tags=_effective_field_tags(dataset_name, custom_field), ) diff --git a/packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/ldm_extension_manager.py b/packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/ldm_extension_manager.py index cd5d797f0..73bc44296 100644 --- a/packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/ldm_extension_manager.py +++ b/packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/ldm_extension_manager.py @@ -33,27 +33,47 @@ class LdmExtensionManager: - """Manager for creating custom datasets and fields in GoodData workspaces.""" + """Manager for creating custom datasets and fields in GoodData workspaces. + + Args: + enable_second_granularities (bool): Whether to use newly added + granularities gated behind `enableSecondGranularities` feature flag. + """ INDENT = " " * 2 @classmethod - def create(cls, host: str, token: str) -> "LdmExtensionManager": - return cls(host=host, token=token) + def create( + cls, host: str, token: str, enable_second_granularities: bool = False + ) -> "LdmExtensionManager": + return cls( + host=host, + token=token, + enable_second_granularities=enable_second_granularities, + ) @classmethod def create_from_profile( cls, profile: str = "default", profiles_path: Path = PROFILES_FILE_PATH, + enable_second_granularities: bool = False, ) -> "LdmExtensionManager": """Creates a provisioner instance using a GoodData profile file.""" content = profile_content(profile, profiles_path) - return cls(host=content["host"], token=content["token"]) + return cls( + host=content["host"], + token=content["token"], + enable_second_granularities=enable_second_granularities, + ) - def __init__(self, host: str, token: str): + def __init__( + self, host: str, token: str, enable_second_granularities: bool = False + ): self._validator = LdmExtensionDataValidator() - self._processor = LdmExtensionDataProcessor() + self._processor = LdmExtensionDataProcessor( + enable_second_granularities=enable_second_granularities + ) self._sdk = GoodDataSdk.create(host_=host, token_=token) self._api = GoodDataApi(host=host, token=token) self.logger = LogObserver() diff --git a/packages/gooddata-pipelines/tests/test_ldm_extension/test_input_processor.py b/packages/gooddata-pipelines/tests/test_ldm_extension/test_input_processor.py index 84476f211..0731e6a5e 100644 --- a/packages/gooddata-pipelines/tests/test_ldm_extension/test_input_processor.py +++ b/packages/gooddata-pipelines/tests/test_ldm_extension/test_input_processor.py @@ -61,6 +61,24 @@ def test_date_from_field(mock_custom_field_date): assert date_ds.tags == ["dataset_name"] +def test_date_from_field_second_granularities_disabled( + mock_custom_field_date, +): + processor = LdmExtensionDataProcessor() + date_ds = processor._date_from_field("dataset_name", mock_custom_field_date) + assert not set(date_ds.granularities) & set( + processor._SECOND_DATE_GRANULARITIES + ) + + +def test_date_from_field_second_granularities_enabled(mock_custom_field_date): + processor = LdmExtensionDataProcessor(enable_second_granularities=True) + date_ds = processor._date_from_field("dataset_name", mock_custom_field_date) + assert set(date_ds.granularities) == set( + processor.DATE_GRANULARITIES + processor._SECOND_DATE_GRANULARITIES + ) + + def test_date_ref_from_field(mock_custom_field_date): ref = LdmExtensionDataProcessor._date_ref_from_field(mock_custom_field_date) assert ref.identifier.id == "date1"