From fc39580448fbe564ef95997a592b546ec07f1886 Mon Sep 17 00:00:00 2001 From: Andrei Markin Date: Mon, 14 Sep 2026 14:17:56 +0400 Subject: [PATCH] feat(io): add duckdb writer --- docs/usage/writers.md | 2 + docs/usage/writers/duckdb-writer.md | 29 +++++++ docs/usage/writers/sql-writer.md | 2 +- libs/io/garf/io/version.py | 2 +- libs/io/garf/io/writers/duckdb_writer.py | 102 +++++++++++++++++++++++ libs/io/pyproject.toml | 6 +- mkdocs.yml | 1 + 7 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 docs/usage/writers/duckdb-writer.md create mode 100644 libs/io/garf/io/writers/duckdb_writer.py diff --git a/docs/usage/writers.md b/docs/usage/writers.md index 698481ea..c05d63ba 100644 --- a/docs/usage/writers.md +++ b/docs/usage/writers.md @@ -13,6 +13,7 @@ | `json` | JsonWriter | `destination-folder`,`format=json|jsonl`| | `bq` | BigQueryWriter | `project`, `dataset`, `location`, `write-disposition` | | `sqldb` | SqlAlchemyWriter | `connection-string`, `if-exists=fail|replace|append` | +| `duckdb` | DuckDBWriter | `db`, `if-exists=create|replace|append` | | `sheets` | SheetsWriter | `share-with`, `credentials-file`, `spreadsheet-url`, `is_append=True|False`| | `elasticsearch`| ElasticsearchWriter| `hosts` | | `excel` | ExcelWriter | `destination-folder`, `file` | @@ -47,6 +48,7 @@ To install specific writers use: * `pip install garf-io[bq]` for BigQuery support * `pip install garf-io[sheets]` for Google spreadsheets support * `pip install garf-io[sqlalchemy]` for SqlAlchemy support +* `pip install garf-io[duckdb]` for DuckDB support * `pip install garf-io[elasticsearch]` for Elasticsearch support * `pip install garf-io[excel]` for Excel support * `pip install garf-io[kafka]` for Kafka support diff --git a/docs/usage/writers/duckdb-writer.md b/docs/usage/writers/duckdb-writer.md new file mode 100644 index 00000000..e96dfe12 --- /dev/null +++ b/docs/usage/writers/duckdb-writer.md @@ -0,0 +1,29 @@ +!!! important + To save data to DuckDB install `garf-io` with DuckDB support + + ```bash + pip install garf-io[duckdb] + ``` + + +`duckdb` writer allows you to save `GarfReport` to DuckDB database. + +/// tab | cli +```bash +garf query.sql --source API_SOURCE \ + --output duckdb +``` +/// + +/// tab | python +```python +from garf.core import report +from garf.io.writers import duckdb_writer + +# Create example report +sample_report = report.GarfReport(results=[[1]], column_names=['one']) + +writer = duckdb_writer.DuckDBWriter(db=DUCKDB_FILE_PATH) +writer.write(sample_report, 'query') +``` +/// diff --git a/docs/usage/writers/sql-writer.md b/docs/usage/writers/sql-writer.md index f6b9d760..a4d11eed 100644 --- a/docs/usage/writers/sql-writer.md +++ b/docs/usage/writers/sql-writer.md @@ -1,5 +1,5 @@ !!! important - To save data to Google Sheets install `garf-io` with SqlAlchemy support + To save data to Sqlalchemy supported DBs install `garf-io` with SqlAlchemy support ```bash pip install garf-io[sqlalchemy] diff --git a/libs/io/garf/io/version.py b/libs/io/garf/io/version.py index 1fa992bd..273b126b 100644 --- a/libs/io/garf/io/version.py +++ b/libs/io/garf/io/version.py @@ -11,4 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '1.3.5' +__version__ = '1.3.6' diff --git a/libs/io/garf/io/writers/duckdb_writer.py b/libs/io/garf/io/writers/duckdb_writer.py new file mode 100644 index 00000000..23cb0f04 --- /dev/null +++ b/libs/io/garf/io/writers/duckdb_writer.py @@ -0,0 +1,102 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for writing data to DuckDB.""" + +from __future__ import annotations + +try: + import duckdb +except ImportError as e: + raise ImportError( + 'Please install garf-io with DuckDB support - `pip install garf-io[duckdb]`' + ) from e + +import logging +from typing import Literal + +import pandas as pd +from garf.core import report as garf_report +from garf.io import exceptions, formatter +from garf.io.telemetry import tracer +from garf.io.writers import abs_writer + +logger = logging.getLogger(__name__) + + +class DuckDBWriterError(exceptions.GarfIoError): + """DuckDBWriterError specific errors.""" + + +class DuckDBWriter(abs_writer.AbsWriter): + """Handles writing GarfReports data to DuckDB. + + Attributes: + db: Database location. + """ + + def __init__( + self, + db: str, + if_exists: str = Literal['replace', 'append', 'create'], + **kwargs, + ): + """Initializes DuckDBWriter based on db file. + + Args: + db: Database location. + if_exists: Behaviour when data already exists in the table. + """ + super().__init__(**kwargs) + self.db = db + self.if_exists = if_exists + self.api_client = duckdb.connect(database=db) + + @tracer.start_as_current_span('duckdb.write') + def write(self, report: garf_report.GarfReport, destination: str) -> None: + """Writes Garf report to the table. + + Args: + report: GarfReport to be written. + destination: Name of the output table. + """ + report = self.format_for_write(report) + destination = formatter.format_extension( + destination, + prefix=self.options.prefix, + suffix=self.options.suffix, + ) + if not report: + df = pd.DataFrame( + data=report.results_placeholder, columns=report.column_names + ).head(0) + else: + df = report.to_pandas() + logger.debug('Writing %d rows of data to %s', len(df), destination) + if self.if_exists == 'replace': + self.api_client.execute( + f'CREATE OR REPLACE TABLE {destination} AS SELECT * FROM df' + ) + elif self.if_exists == 'append': + self.api_client.execute( + f'INSERT INTO {destination} BY NAME (SELECT * FROM df) ' + ) + elif self.if_exists == 'create': + self.api_client.execute( + f'CREATE TABLE IF NOT EXISTS {destination} AS SELECT * FROM df' + ) + else: + raise DuckDBExecutorError( + f'Unsupported overwrite strategy: {self.if_exists}' + ) + logger.debug('Writing to %s is completed', destination) diff --git a/libs/io/pyproject.toml b/libs/io/pyproject.toml index 4f73d6b2..8e4d4a3e 100644 --- a/libs/io/pyproject.toml +++ b/libs/io/pyproject.toml @@ -84,8 +84,11 @@ firestore = [ pushgateway = [ "prometheus-client", ] +duckdb = [ + "duckdb", +] all = [ - "garf-io[bq,sheets,sqlalchemy,excel,opensearch,elasticsearch,kafka,pubsub,mongo,firestore,pushgateway]" + "garf-io[bq,sheets,sqlalchemy,excel,opensearch,elasticsearch,kafka,pubsub,mongo,firestore,pushgateway,duckdb]" ] [project.entry-points.garf_writer] @@ -105,6 +108,7 @@ kafka = "garf.io.writers.kafka_writer" mongo = "garf.io.writers.mongodb_writer" firestore = "garf.io.writers.firestore_writer" pushgateway = "garf.io.writers.pushgateway_writer" +duckdb = "garf.io.writers.duckdb_writer" [tool.setuptools.packages.find] where = ["."] diff --git a/mkdocs.yml b/mkdocs.yml index ba0e0708..d20a2ac9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -68,6 +68,7 @@ nav: - Json: usage/writers/json-writer.md - BigQuery: usage/writers/bq-writer.md - SQL: usage/writers/sql-writer.md + - DuckDB: usage/writers/duckdb-writer.md - Google Sheets: usage/writers/sheets-writer.md - Excel: usage/writers/excel-writer.md - Google PubSub: usage/writers/pubsub-writer.md