From 22aec631d3b169a3421525bc0d3c44692b7ed140 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Thu, 29 May 2025 11:17:34 +0530 Subject: [PATCH 1/3] chore: clickhouse scraper docs --- .../docs/guide/config-db/scrapers/aws.mdx | 1 - .../guide/config-db/scrapers/clickhouse.mdx | 50 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 mission-control/docs/guide/config-db/scrapers/clickhouse.mdx diff --git a/mission-control/docs/guide/config-db/scrapers/aws.mdx b/mission-control/docs/guide/config-db/scrapers/aws.mdx index 1832d89a..16855f8a 100644 --- a/mission-control/docs/guide/config-db/scrapers/aws.mdx +++ b/mission-control/docs/guide/config-db/scrapers/aws.mdx @@ -16,7 +16,6 @@ The Mission Control Registry includes an [AWS](/integration/aws) Helm chart that ::: ```yaml title="aws-scraper.yaml" file=/modules/config-db/fixtures/aws.yaml - ``` | Field | Description | Scheme | Required | diff --git a/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx b/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx new file mode 100644 index 00000000..f41dd4b7 --- /dev/null +++ b/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx @@ -0,0 +1,50 @@ +--- +title: Clickhouse +sidebar_custom_props: + icon: clickhouse +--- + +The Clickhouse scraper will execute a query and then create a catalog item for each returned row. + +The example below creates a new `Order` config for each row, with name as `order_id` and the row in json format as the config. + +```yaml title="clickhouse-scraper.yaml" file=/modules/config-db/fixtures/clickhouse.yaml +``` + +## Scraper + +| Field | Description | Scheme | Required | +| ----------- | ---------------------------------------------------------------------------------- | -------------------------------------------- | -------- | +| `logLevel` | Specify the level of logging. | `string` | | +| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | +| `full` | Set to `true` to extract changes from scraped configurations. Defaults to `false`. | `bool` | | +| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/guide/config-db/concepts/retention) | | +| `clickhouse`| Specifies the list of SQL configurations to scrape. | [`[]Clickhouse`](#clickhouse) | | + +## Clickhouse + +:@:/?param1=value1¶m2=value2", scheme: "string"}, + {field: "query", description: "Query to execute", scheme: "string", required: true} +]}/> + + +## AWSS3 + + + +## AzureBlobStorage + + From 24ee317b49d577a41992b9aef9b3b031f677fa87 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Tue, 3 Jun 2025 11:21:51 +0530 Subject: [PATCH 2/3] chore: address review comments --- .../guide/config-db/scrapers/clickhouse.mdx | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx b/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx index f41dd4b7..9d65796b 100644 --- a/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx +++ b/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx @@ -1,7 +1,7 @@ --- title: Clickhouse sidebar_custom_props: - icon: clickhouse + icon: clickhouse-icon --- The Clickhouse scraper will execute a query and then create a catalog item for each returned row. @@ -11,6 +11,58 @@ The example below creates a new `Order` config for each row, with name as `order ```yaml title="clickhouse-scraper.yaml" file=/modules/config-db/fixtures/clickhouse.yaml ``` +Objects in external storages like AzureBlobStorage or AWS S3 can also be queried + +```yaml title="azure-blob-storage-scraper.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Connection +metadata: + name: azure-prod +spec: + azure: + clientID: + valueFrom: + secretKeyRef: + name: azure-creds + key: CLIENT_ID + clientSecret: + valueFrom: + secretKeyRef: + name: azure-creds + key: CLIENT_SECRET + tenantID: + valueFrom: + secretKeyRef: + name: azure-creds + key: TENANT_ID +--- +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: audit-logs +spec: + clickhouse: + - query: SELECT * FROM azureBlobStorage(azure_prod_audit_data); + type: AuditRecord + id: $.user_id + transform: + expr: "[config].toJSON()" + azureBlobStorage: + connection: azure-prod + account: acme-logs + container: audit + path: audit-data.csv + collection: azure_prod_audit_data +``` + +:::info +An instance of clichkouse server needs to be deployed for this scraper to function. + +Mission Control can deploy an instance by setting `config-db.clichkouse.enabled: true` in the helm chart. + +An external clichkouse server can also be used via the `clickhouseURL` parameter +::: + ## Scraper | Field | Description | Scheme | Required | @@ -46,5 +98,5 @@ The example below creates a new `Order` config for each row, with name as `order {field: "container", description: "Azure Blob Storage container name", scheme: "string"}, {field: "path", description: "Path within the container", scheme: "string"}, {field: "endpointSuffix", description: "Azure endpoint suffix", scheme: "string"}, - {field: "collectionName", description: "Name of the collection in Azure Blob Storage", scheme: "string", required: true} + {field: "collectionName", description: "Name of the collection in Clickhouse. See [Named Collections](https://clickhouse.com/docs/operations/named-collections)", scheme: "string", required: true} ]}/> From ffdcbe428a1934e3a75597424a2be26d50790cf5 Mon Sep 17 00:00:00 2001 From: Yash Mehrotra Date: Wed, 4 Jun 2025 12:29:39 +0530 Subject: [PATCH 3/3] chore: update flanksource iconsl --- .../docs/guide/config-db/scrapers/clickhouse.mdx | 4 ++++ mission-control/package-lock.json | 14 +++++++------- mission-control/package.json | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx b/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx index 9d65796b..be9aaef9 100644 --- a/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx +++ b/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx @@ -1,9 +1,13 @@ --- title: Clickhouse +show_title: false sidebar_custom_props: icon: clickhouse-icon --- + +# Clickhouse + The Clickhouse scraper will execute a query and then create a catalog item for each returned row. The example below creates a new `Order` config for each row, with name as `order_id` and the row in json format as the config. diff --git a/mission-control/package-lock.json b/mission-control/package-lock.json index 20accccf..87882111 100644 --- a/mission-control/package-lock.json +++ b/mission-control/package-lock.json @@ -11,7 +11,7 @@ "@docusaurus/core": "3.7.0", "@docusaurus/plugin-client-redirects": "3.7.0", "@docusaurus/preset-classic": "3.7.0", - "@flanksource/icons": "^1.0.34", + "@flanksource/icons": "^1.0.35", "@floating-ui/react": "^0.26.28", "@iconify/react": "^5.1.0", "@mdx-js/react": "^3.0.0", @@ -3715,9 +3715,9 @@ } }, "node_modules/@flanksource/icons": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/@flanksource/icons/-/icons-1.0.34.tgz", - "integrity": "sha512-U9WVAOM06FfE+HtnmSkBzUi0QPn9M6In9Z2JzPi8IDQ4bpbV6747UbV9NOJ5LRbqdaOVKKxHfP9BLxsRU8gXBg==", + "version": "1.0.35", + "resolved": "https://registry.npmjs.org/@flanksource/icons/-/icons-1.0.35.tgz", + "integrity": "sha512-KqxQQdJFOFtOIynW7GK0sCVJRODeFvdjBCeGoC7QI3WmZ8MnwFpbpaXFnedfavDuq2T9/rNWwbtZ13rHphMqtA==", "peerDependencies": { "react": "*" } @@ -24826,9 +24826,9 @@ "dev": true }, "@flanksource/icons": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/@flanksource/icons/-/icons-1.0.34.tgz", - "integrity": "sha512-U9WVAOM06FfE+HtnmSkBzUi0QPn9M6In9Z2JzPi8IDQ4bpbV6747UbV9NOJ5LRbqdaOVKKxHfP9BLxsRU8gXBg==", + "version": "1.0.35", + "resolved": "https://registry.npmjs.org/@flanksource/icons/-/icons-1.0.35.tgz", + "integrity": "sha512-KqxQQdJFOFtOIynW7GK0sCVJRODeFvdjBCeGoC7QI3WmZ8MnwFpbpaXFnedfavDuq2T9/rNWwbtZ13rHphMqtA==", "requires": {} }, "@floating-ui/core": { diff --git a/mission-control/package.json b/mission-control/package.json index 8e1db754..9f87c30b 100644 --- a/mission-control/package.json +++ b/mission-control/package.json @@ -17,7 +17,7 @@ "@docusaurus/core": "3.7.0", "@docusaurus/plugin-client-redirects": "3.7.0", "@docusaurus/preset-classic": "3.7.0", - "@flanksource/icons": "^1.0.34", + "@flanksource/icons": "^1.0.35", "@floating-ui/react": "^0.26.28", "@iconify/react": "^5.1.0", "@mdx-js/react": "^3.0.0",