feat: add new granularities to gooddata-dbt and gooddata-pipelines

docs/content/en/latest/pipelines/ldm_extension/_index.md

@@ -27,6 +27,8 @@ ldm_extension_manager = LdmExtensionManager.create(host=host, token=token)
 
 To extend the LDM, you need to define the custom datasets and the fields they should contain. The script also checks the validity of analytical objects before and after the update. Updates introducing new invalid relations are automatically rolled back. You can opt out of this behavior by setting the `check_relations` parameter to False.
 
+To create date datasets with the second-based granularities (`SECOND`, `SECOND_OF_MINUTE`, `SECOND_OF_DAY`, `MINUTE_OF_DAY`), set the `enable_second_granularities` parameter to True when creating the LdmExtensionManager. This requires the `enableSecondGranularities` feature flag to be enabled for your GoodData organization.
+
 ### Custom Dataset Definitions
 
 The custom dataset represents a new dataset appended to the child LDM. It is defined by the following parameters:

packages/gooddata-dbt/README.md

@@ -50,6 +50,8 @@ The plugin provides the following use cases:
   - Reads dbt models and profiles
   - Scans data source (connection props from dbt profiles) through GoodData to get column data types (optional in dbt)
   - Generates GoodData LDM(Logical Data Model) from dbt models. Can utilize custom gooddata-specific metadata, more below
+  - With `--gooddata-enable-second-granularities`, date datasets are created with second-based granularities.
+    Requires the `enableSecondGranularities` feature flag enabled in the GoodData organization.
 - upload_notification
   - Invalidates caches for data source
 - deploy_analytics

packages/gooddata-dbt/src/gooddata_dbt/args.py

@@ -67,6 +67,16 @@ def set_gooddata_upper_case_args(parser: argparse.ArgumentParser) -> None:
     )
 
 
+def set_gooddata_enable_second_granularities_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--gooddata-enable-second-granularities",
+        help="Create date datasets with second-based granularities. "
+        "Requires the enableSecondGranularities feature flag enabled in the GoodData organization.",
+        action="store_true",
+        default=False,
+    )
+
+
 def set_gooddata_workspace_title_args(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "-gwt", "--gooddata-workspace-title", help="Workspace title", default=os.getenv("GOODDATA_WORKSPACE_TITLE")
@@ -169,6 +179,7 @@ def parse_arguments(description: str) -> argparse.Namespace:
     set_dbt_args(deploy_ldm)
     set_environment_id_arg(deploy_ldm)
     set_gooddata_upper_case_args(deploy_ldm)
+    set_gooddata_enable_second_granularities_args(deploy_ldm)
     deploy_ldm.set_defaults(method="deploy_ldm")
 
     upload_notification = subparsers.add_parser("upload_notification")

packages/gooddata-dbt/src/gooddata_dbt/dbt/base.py

@@ -52,6 +52,13 @@ class GoodDataSortDirection(Enum):
     "MINUTE_OF_HOUR",
     "HOUR_OF_DAY",
 ]
+# newly added granularities gated behind `enableSecondGranularities` feature flag
+SECOND_TIMESTAMP_GRANULARITIES = [
+    "SECOND",
+    "SECOND_OF_MINUTE",
+    "SECOND_OF_DAY",
+    "MINUTE_OF_DAY",
+]
 T = TypeVar("T", bound="Base")
 
 DBT_TARGET_DIR = Path("target")

packages/gooddata-dbt/src/gooddata_dbt/dbt/tables.py

@@ -18,6 +18,7 @@
     DBT_PATH_TO_MANIFEST,
     DBT_TARGET_DIR,
     NUMERIC_DATA_TYPES,
+    SECOND_TIMESTAMP_GRANULARITIES,
     TIMESTAMP_DATA_TYPES,
     TIMESTAMP_GRANULARITIES,
     Base,
@@ -202,9 +203,12 @@ class DbtModelTables:
         * column_type – Optional if missing call  scan
     """
 
-    def __init__(self, tables: list[DbtModelTable], upper_case: bool) -> None:
+    def __init__(
+        self, tables: list[DbtModelTable], upper_case: bool, enable_second_granularities: bool = False
+    ) -> None:
         self.upper_case = upper_case
         self.tables = tables
+        self._enable_second_granularities = enable_second_granularities
 
     @classmethod
     def from_cloud(
@@ -214,22 +218,27 @@ def from_cloud(
         upper_case: bool,
         all_model_ids: list[str],
         path: Union[str, Path] = DBT_TARGET_DIR,
+        enable_second_granularities: bool = False,
     ) -> "DbtModelTables":
         path = path if isinstance(path, Path) else Path(path)
         dbt_conn.download_manifest(run_id=run_id, path=path)
         with open(path / "manifest.json") as fp:
             dbt_catalog = json.load(fp)
         tables = cls.read_dbt_models(dbt_catalog, upper_case, all_model_ids)
-        return cls(tables, upper_case)
+        return cls(tables, upper_case, enable_second_granularities)
 
     @classmethod
     def from_local(
-        cls, upper_case: bool, all_model_ids: list[str], manifest_path: Union[str, Path] = DBT_PATH_TO_MANIFEST
+        cls,
+        upper_case: bool,
+        all_model_ids: list[str],
+        manifest_path: Union[str, Path] = DBT_PATH_TO_MANIFEST,
+        enable_second_granularities: bool = False,
     ) -> "DbtModelTables":
         with open(manifest_path) as fp:
             dbt_catalog = json.load(fp)
         tables = cls.read_dbt_models(dbt_catalog, upper_case, all_model_ids)
-        return cls(tables, upper_case)
+        return cls(tables, upper_case, enable_second_granularities)
 
     @staticmethod
     def read_dbt_models(dbt_catalog: dict, upper_case: bool, all_model_ids: list[str]) -> list[DbtModelTable]:
@@ -437,14 +446,17 @@ def make_attributes(self, table: DbtModelTable) -> list[dict]:
                 )
         return attributes
 
-    @staticmethod
-    def make_date_datasets(table: DbtModelTable, existing_date_datasets: list[dict]) -> list[dict]:
+    def make_date_datasets(self, table: DbtModelTable, existing_date_datasets: list[dict]) -> list[dict]:
         date_datasets = []
         for column in table.columns.values():
             existing_dataset_ids = [d["id"] for d in existing_date_datasets]
             if column.is_date() and column.gooddata_ldm_id not in existing_dataset_ids:
                 if column.data_type in TIMESTAMP_DATA_TYPES:
-                    granularities = DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES
+                    granularities = (
+                        DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES + SECOND_TIMESTAMP_GRANULARITIES
+                        if self._enable_second_granularities
+                        else DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES
+                    )
                 else:
                     granularities = DATE_GRANULARITIES
                 date_datasets.append(

packages/gooddata-dbt/src/gooddata_dbt/dbt_plugin.py

@@ -73,7 +73,11 @@ def deploy_ldm(
     logger.info("Generate and put LDM")
     dbt_profiles = DbtProfiles(args)
     data_source_id = dbt_profiles.data_source_id
-    dbt_tables = DbtModelTables.from_local(args.gooddata_upper_case, all_model_ids)
+    dbt_tables = DbtModelTables.from_local(
+        args.gooddata_upper_case,
+        all_model_ids,
+        enable_second_granularities=args.gooddata_enable_second_granularities,
+    )
     generate_and_put_ldm(logger, sdk_wrapper, data_source_id, workspace_id, dbt_tables, model_ids)
     workspace_url = f"{sdk_wrapper.get_host_from_sdk()}/modeler/#/{workspace_id}"
     logger.info(f"LDM successfully loaded, verify here: {workspace_url}")

packages/gooddata-dbt/tests/test_tables.py

@@ -3,7 +3,12 @@
 from pathlib import Path
 from typing import Union
 
-from gooddata_dbt.dbt.tables import DbtModelTables
+from gooddata_dbt.dbt.base import (
+    DATE_GRANULARITIES,
+    SECOND_TIMESTAMP_GRANULARITIES,
+    TIMESTAMP_GRANULARITIES,
+)
+from gooddata_dbt.dbt.tables import DbtModelColumn, DbtModelTable, DbtModelTables
 from gooddata_sdk import CatalogDeclarativeModel, CatalogDeclarativeTables
 
 _CURR_DIR = Path(__file__).parent
@@ -51,6 +56,36 @@ def test_make_ldm():
     assert len(ldm.ldm.date_instances) == 4
 
 
+def _table_with_date_columns() -> DbtModelTable:
+    return DbtModelTable(
+        name="events",
+        description="",
+        tags=[],
+        schema="public",
+        columns={
+            "created_at": DbtModelColumn(name="created_at", description="", tags=[], data_type="TIMESTAMP"),
+            "created_on": DbtModelColumn(name="created_on", description="", tags=[], data_type="DATE"),
+        },
+    )
+
+
+def test_make_date_datasets_without_second_granularities():
+    tables = DbtModelTables([], upper_case=False)
+    date_datasets = {d["id"]: d for d in tables.make_date_datasets(_table_with_date_columns(), [])}
+    assert date_datasets["created_at"]["granularities"] == DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES
+    assert date_datasets["created_on"]["granularities"] == DATE_GRANULARITIES
+
+
+def test_make_date_datasets_with_second_granularities():
+    tables = DbtModelTables([], upper_case=False, enable_second_granularities=True)
+    date_datasets = {d["id"]: d for d in tables.make_date_datasets(_table_with_date_columns(), [])}
+    assert (
+        date_datasets["created_at"]["granularities"]
+        == DATE_GRANULARITIES + TIMESTAMP_GRANULARITIES + SECOND_TIMESTAMP_GRANULARITIES
+    )
+    assert date_datasets["created_on"]["granularities"] == DATE_GRANULARITIES
+
+
 FAA_MODEL_ID = "faa"
 
 

packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/input_processor.py

@@ -82,6 +82,21 @@ class LdmExtensionDataProcessor:
         "FISCAL_YEAR",
     ]
 
+    # newly added granularities gated behind `enableSecondGranularities` feature flag
+    _SECOND_DATE_GRANULARITIES: list[str] = [
+        "SECOND",
+        "SECOND_OF_MINUTE",
+        "SECOND_OF_DAY",
+        "MINUTE_OF_DAY",
+    ]
+
+    def __init__(self, enable_second_granularities: bool = False):
+        self._date_granularities = (
+            self.DATE_GRANULARITIES + self._SECOND_DATE_GRANULARITIES
+            if enable_second_granularities
+            else self.DATE_GRANULARITIES
+        )
+
     @staticmethod
     def _attribute_from_field(
         dataset_name: str,
@@ -127,7 +142,7 @@ def _date_from_field(
                 title_base="",
                 title_pattern="%titleBase - %granularityTitle",
             ),
-            granularities=self.DATE_GRANULARITIES,
+            granularities=self._date_granularities,
             description=custom_field.description,
             tags=_effective_field_tags(dataset_name, custom_field),
         )

packages/gooddata-pipelines/src/gooddata_pipelines/ldm_extension/ldm_extension_manager.py

@@ -33,27 +33,47 @@
 
 
 class LdmExtensionManager:
-    """Manager for creating custom datasets and fields in GoodData workspaces."""
+    """Manager for creating custom datasets and fields in GoodData workspaces.
+
+    Args:
+        enable_second_granularities (bool): Whether to use newly added
+        granularities gated behind `enableSecondGranularities` feature flag.
+    """
 
     INDENT = " " * 2
 
     @classmethod
-    def create(cls, host: str, token: str) -> "LdmExtensionManager":
-        return cls(host=host, token=token)
+    def create(
+        cls, host: str, token: str, enable_second_granularities: bool = False
+    ) -> "LdmExtensionManager":
+        return cls(
+            host=host,
+            token=token,
+            enable_second_granularities=enable_second_granularities,
+        )
 
     @classmethod
     def create_from_profile(
         cls,
         profile: str = "default",
         profiles_path: Path = PROFILES_FILE_PATH,
+        enable_second_granularities: bool = False,
     ) -> "LdmExtensionManager":
         """Creates a provisioner instance using a GoodData profile file."""
         content = profile_content(profile, profiles_path)
-        return cls(host=content["host"], token=content["token"])
+        return cls(
+            host=content["host"],
+            token=content["token"],
+            enable_second_granularities=enable_second_granularities,
+        )
 
-    def __init__(self, host: str, token: str):
+    def __init__(
+        self, host: str, token: str, enable_second_granularities: bool = False
+    ):
         self._validator = LdmExtensionDataValidator()
-        self._processor = LdmExtensionDataProcessor()
+        self._processor = LdmExtensionDataProcessor(
+            enable_second_granularities=enable_second_granularities
+        )
         self._sdk = GoodDataSdk.create(host_=host, token_=token)
         self._api = GoodDataApi(host=host, token=token)
         self.logger = LogObserver()

packages/gooddata-pipelines/tests/test_ldm_extension/test_input_processor.py

@@ -61,6 +61,24 @@ def test_date_from_field(mock_custom_field_date):
     assert date_ds.tags == ["dataset_name"]
 
 
+def test_date_from_field_second_granularities_disabled(
+    mock_custom_field_date,
+):
+    processor = LdmExtensionDataProcessor()
+    date_ds = processor._date_from_field("dataset_name", mock_custom_field_date)
+    assert not set(date_ds.granularities) & set(
+        processor._SECOND_DATE_GRANULARITIES
+    )
+
+
+def test_date_from_field_second_granularities_enabled(mock_custom_field_date):
+    processor = LdmExtensionDataProcessor(enable_second_granularities=True)
+    date_ds = processor._date_from_field("dataset_name", mock_custom_field_date)
+    assert set(date_ds.granularities) == set(
+        processor.DATE_GRANULARITIES + processor._SECOND_DATE_GRANULARITIES
+    )
+
+
 def test_date_ref_from_field(mock_custom_field_date):
     ref = LdmExtensionDataProcessor._date_ref_from_field(mock_custom_field_date)
     assert ref.identifier.id == "date1"