diff --git a/mission-control/docs/guide/config-db/scrapers/aws.mdx b/mission-control/docs/guide/config-db/scrapers/aws.mdx index 1832d89a..16855f8a 100644 --- a/mission-control/docs/guide/config-db/scrapers/aws.mdx +++ b/mission-control/docs/guide/config-db/scrapers/aws.mdx @@ -16,7 +16,6 @@ The Mission Control Registry includes an [AWS](/integration/aws) Helm chart that ::: ```yaml title="aws-scraper.yaml" file=/modules/config-db/fixtures/aws.yaml - ``` | Field | Description | Scheme | Required | diff --git a/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx b/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx new file mode 100644 index 00000000..be9aaef9 --- /dev/null +++ b/mission-control/docs/guide/config-db/scrapers/clickhouse.mdx @@ -0,0 +1,106 @@ +--- +title: Clickhouse +show_title: false +sidebar_custom_props: + icon: clickhouse-icon +--- + + +# Clickhouse + +The Clickhouse scraper will execute a query and then create a catalog item for each returned row. + +The example below creates a new `Order` config for each row, with name as `order_id` and the row in json format as the config. + +```yaml title="clickhouse-scraper.yaml" file=/modules/config-db/fixtures/clickhouse.yaml +``` + +Objects in external storages like AzureBlobStorage or AWS S3 can also be queried + +```yaml title="azure-blob-storage-scraper.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Connection +metadata: + name: azure-prod +spec: + azure: + clientID: + valueFrom: + secretKeyRef: + name: azure-creds + key: CLIENT_ID + clientSecret: + valueFrom: + secretKeyRef: + name: azure-creds + key: CLIENT_SECRET + tenantID: + valueFrom: + secretKeyRef: + name: azure-creds + key: TENANT_ID +--- +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: audit-logs +spec: + clickhouse: + - query: SELECT * FROM azureBlobStorage(azure_prod_audit_data); + type: AuditRecord + id: $.user_id + transform: + expr: "[config].toJSON()" + azureBlobStorage: + connection: azure-prod + account: acme-logs + container: audit + path: audit-data.csv + collection: azure_prod_audit_data +``` + +:::info +An instance of clichkouse server needs to be deployed for this scraper to function. + +Mission Control can deploy an instance by setting `config-db.clichkouse.enabled: true` in the helm chart. + +An external clichkouse server can also be used via the `clickhouseURL` parameter +::: + +## Scraper + +| Field | Description | Scheme | Required | +| ----------- | ---------------------------------------------------------------------------------- | -------------------------------------------- | -------- | +| `logLevel` | Specify the level of logging. | `string` | | +| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | +| `full` | Set to `true` to extract changes from scraped configurations. Defaults to `false`. | `bool` | | +| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/guide/config-db/concepts/retention) | | +| `clickhouse`| Specifies the list of SQL configurations to scrape. | [`[]Clickhouse`](#clickhouse) | | + +## Clickhouse + +:@:/?param1=value1¶m2=value2", scheme: "string"}, + {field: "query", description: "Query to execute", scheme: "string", required: true} +]}/> + + +## AWSS3 + + + +## AzureBlobStorage + + diff --git a/mission-control/package-lock.json b/mission-control/package-lock.json index 20accccf..87882111 100644 --- a/mission-control/package-lock.json +++ b/mission-control/package-lock.json @@ -11,7 +11,7 @@ "@docusaurus/core": "3.7.0", "@docusaurus/plugin-client-redirects": "3.7.0", "@docusaurus/preset-classic": "3.7.0", - "@flanksource/icons": "^1.0.34", + "@flanksource/icons": "^1.0.35", "@floating-ui/react": "^0.26.28", "@iconify/react": "^5.1.0", "@mdx-js/react": "^3.0.0", @@ -3715,9 +3715,9 @@ } }, "node_modules/@flanksource/icons": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/@flanksource/icons/-/icons-1.0.34.tgz", - "integrity": "sha512-U9WVAOM06FfE+HtnmSkBzUi0QPn9M6In9Z2JzPi8IDQ4bpbV6747UbV9NOJ5LRbqdaOVKKxHfP9BLxsRU8gXBg==", + "version": "1.0.35", + "resolved": "https://registry.npmjs.org/@flanksource/icons/-/icons-1.0.35.tgz", + "integrity": "sha512-KqxQQdJFOFtOIynW7GK0sCVJRODeFvdjBCeGoC7QI3WmZ8MnwFpbpaXFnedfavDuq2T9/rNWwbtZ13rHphMqtA==", "peerDependencies": { "react": "*" } @@ -24826,9 +24826,9 @@ "dev": true }, "@flanksource/icons": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/@flanksource/icons/-/icons-1.0.34.tgz", - "integrity": "sha512-U9WVAOM06FfE+HtnmSkBzUi0QPn9M6In9Z2JzPi8IDQ4bpbV6747UbV9NOJ5LRbqdaOVKKxHfP9BLxsRU8gXBg==", + "version": "1.0.35", + "resolved": "https://registry.npmjs.org/@flanksource/icons/-/icons-1.0.35.tgz", + "integrity": "sha512-KqxQQdJFOFtOIynW7GK0sCVJRODeFvdjBCeGoC7QI3WmZ8MnwFpbpaXFnedfavDuq2T9/rNWwbtZ13rHphMqtA==", "requires": {} }, "@floating-ui/core": { diff --git a/mission-control/package.json b/mission-control/package.json index 8e1db754..9f87c30b 100644 --- a/mission-control/package.json +++ b/mission-control/package.json @@ -17,7 +17,7 @@ "@docusaurus/core": "3.7.0", "@docusaurus/plugin-client-redirects": "3.7.0", "@docusaurus/preset-classic": "3.7.0", - "@flanksource/icons": "^1.0.34", + "@flanksource/icons": "^1.0.35", "@floating-ui/react": "^0.26.28", "@iconify/react": "^5.1.0", "@mdx-js/react": "^3.0.0",