diff --git a/canary-checker/docs/concepts/transforms.md b/canary-checker/docs/concepts/transforms.md index a00e4ed8..2cc53437 100644 --- a/canary-checker/docs/concepts/transforms.md +++ b/canary-checker/docs/concepts/transforms.md @@ -21,17 +21,14 @@ spec: ignore: - KubeScheduler.* transform: - javascript: | - var out = _.map(results, function(r) { - return { - name: r.name, - labels: r.labels, - icon: 'alert', - message: r.message, - description: r.message, - } - }) - JSON.stringify(out); + expr: | + results.alerts.map(r, { + 'name': r.name, + 'labels': r.labels, + 'icon': 'alert', + 'message': r.message, + 'description': r.message, + }).toJSON() ``` In the above example, the check will return multiple alerts from alertmanager. By default, all of those alerts will be grouped in a single check. diff --git a/mission-control/docs/config-db/concepts/templating.md b/mission-control/docs/config-db/concepts/templating.md index 779e99b7..23dd63db 100644 --- a/mission-control/docs/config-db/concepts/templating.md +++ b/mission-control/docs/config-db/concepts/templating.md @@ -12,6 +12,7 @@ To specify the template to be used for representing your data, the following opt - [Go template](#go-template) - [Javascript](#javascript) +- [CEL Expression](#cel) ### Go Template @@ -57,3 +58,20 @@ file: paths: - fixtures/data/multiple-configs.json ``` + +### CEL + +CEL code can used to transformed the scraped configuration. Below is an example. + +```yaml +file: + - type: Config + id: $[0].id + name: $[0].name + transform: + script: + expr: |+ + config.map(e,e.hello = "world").toJSON() + paths: + - fixtures/data/multiple-configs.json +``` \ No newline at end of file diff --git a/mission-control/docs/config-db/concepts/transform.md b/mission-control/docs/config-db/concepts/transform.md index bb3a067c..963d45a7 100644 --- a/mission-control/docs/config-db/concepts/transform.md +++ b/mission-control/docs/config-db/concepts/transform.md @@ -29,6 +29,30 @@ spec: - fixtures/data/multiple-configs.json ``` +### CEL + +You can also supply a CEL expression to transform the scraped configuration instead of the above JS. Your expression will have access to the special `config` variable which will contain the scraped config. Your script is expected to return a stringified JSON object which will be the new configuration. + +_Example_: The following `Config DB` configuration specifies a transformation that'll add a new field `"hello"` with the value `"world"` to all the scraped configurations. + +```yaml title="file-scraper.yaml" +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: file-scraper +spec: + file: + - type: Config + id: $[0].id + name: $[0].name + transform: + script: + expr: + config.map(e,e.hello = "world").toJSON() + paths: + - fixtures/data/multiple-configs.json +``` + Considering that the `fixtures/data/multiple-configs.json` file contains the following configuration ```json @@ -48,11 +72,11 @@ Considering that the `fixtures/data/multiple-configs.json` file contains the fol ] ``` -The JS transformation will result in two new config items +The JS or CEL transformation will result in two new config items ```json -{"id": 1, "name": "Config1", "added": "a", "secret": "secret_1", "password": "p1"} -{"id": 2, "name": "Config2", "added": "a", "secret": "secret_2", "password": "p2"} +{"id": 1, "name": "Config1", "hello": "world", "secret": "secret_1", "password": "p1"} +{"id": 2, "name": "Config2", "hello": "world", "secret": "secret_2", "password": "p2"} ``` ## Go Templates diff --git a/mission-control/docs/config-db/scrapers/sql.md b/mission-control/docs/config-db/scrapers/sql.md index 70379872..f2747366 100644 --- a/mission-control/docs/config-db/scrapers/sql.md +++ b/mission-control/docs/config-db/scrapers/sql.md @@ -10,65 +10,40 @@ kind: ScrapeConfig metadata: name: sql-scraper spec: + schedule: '@every 30s' sql: - - connection: 'sqlserver://localhost:1433?database=master' - auth: - username: - value: sa - password: - value: password + - connection: 'sqlserver://sa:password@172.18.5.55:1433?database=master' type: MSSQL::Database - id: $.name - + id: $.id transform: - full: true # transform the entire configuration item, and not just the configuration data (row) - script: - javascript: |+ - var dbs = {} - for (var i = 0; i < config.rows.length; i++) { - var db = config.rows[i] - var name = db.DB - if (dbs[db.DB] == null) { - { - config: dbs[db.DB] = { - name: name, - roles: {} - }, - changes: { - - }, - analysis: { - - } - - } - } - dbs[name].roles[db.role] = db.name - } - JSON.stringify(_.values(dbs)) - + expr: |+ + dyn(config).map(e,{ "id": e.DB, + "roles": e.Roles.map(er, + {er.role : er.Principals.map(ep,ep.name)}) + }).toJSON() query: | - declare @mytable table ( + declare @mytable table ( [DB] [nvarchar](128) NULL, - [name] [nvarchar](255) NOT NULL, - [role] [nvarchar](255) NOT NULL - ) - - + [name] [nvarchar](255) NOT NULL, + [role] [nvarchar](255) NOT NULL + ) DECLARE @command varchar(1000) - SELECT @command = - 'USE ?; SELECT DB_NAME() as DB, DP1.name AS [user], - isnull (DP2.name, ''No members'') AS [role] - FROM sys.database_role_members AS DRM - RIGHT OUTER JOIN sys.database_principals AS DP1 - ON DRM.role_principal_id = DP1.principal_id - LEFT OUTER JOIN sys.database_principals AS DP2 - ON DRM.member_principal_id = DP2.principal_id - WHERE DP1.type = ''R'' and DP2.name is not null' - - insert into @mytable EXEC sp_MSforeachdb @command + SELECT @command = 'USE ?; SELECT DB_NAME() as DB, DP1.name AS [user], + isnull (DP2.name, ''No members'') AS [role] + FROM sys.database_role_members AS DRM + RIGHT OUTER JOIN sys.database_Principals AS DP1 + ON DRM.role_principal_id = DP1.principal_id + LEFT OUTER JOIN sys.database_Principals AS DP2 + ON DRM.member_principal_id = DP2.principal_id + WHERE DP1.type = ''R'' and DP2.name is not null' + insert into @mytable EXEC sp_MSforeachdb @command + + select distinct d.DB,Roles.role , Principals.name + from @mytable d + left join @mytable Roles on d.DB = Roles.DB + left join @mytable Principals on Roles.name = Principals.name + and Roles.DB = Principals.DB FOR JSON AUTO - select * from @mytable ``` ## Scraper @@ -81,7 +56,17 @@ spec: | `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | | | `sql` | Specifies the list of SQL configurations to scrape. | [`[]SQL`](#sql-1) | | -### SQL +## Result Variables + +| Name | Description | Scheme | +| ------- | ----------------------- | -------------------------- | +| `config` | | *[]map[string]interface{ }* | +| `result` | | *map[string]interface{ }* | +| `result.config` | | *[]map[string]interface{ }* | +| `result.last_modified`| | *DateTime* | +| `result.last_scraped_time`| | *DateTime* | + +## SQL | Field | Description | Scheme | Required | | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | -------- | diff --git a/mission-control/docs/topology/examples/flux.md b/mission-control/docs/topology/examples/flux.md index 32e8e25c..51cd5827 100644 --- a/mission-control/docs/topology/examples/flux.md +++ b/mission-control/docs/topology/examples/flux.md @@ -34,41 +34,39 @@ spec: - name: config-db-check query: SELECT * FROM config_items WHERE config_class = 'HelmRelease' display: - javascript: | - JSON.stringify(results.results.map(function(r) { - return { - name: r.name, - icon: 'helm', - namespace: r.namespace, - status: r.status, - status_reason: r.description, - selectors: [{ - name: 'pods', - labelSelector: 'app.kubernetes.io/instance='+r.name, + expr: | + dyn(results.results).map(r, { + 'name': r.name, + 'icon': 'helm', + 'namespace': r.namespace, + 'status': r.status, + 'status_reason': r.description, + 'selectors': [{ + 'name': 'pods', + 'labelSelector': 'app.kubernetes.io/instance='+r.name, }], - configs: [ + 'configs': [ { - name: r.name, - type: "HelmRelease", + 'name': r.name, + 'type': "HelmRelease", } ], - properties: [ + 'properties': [ { 'name': 'Message', - text: r.config.status.conditions[0].message, + 'text': r.config.status.conditions[0].message, }, { 'name': 'Version', - text: r.config.status.lastAppliedRevision, - headline: true, + 'text': r.config.status.lastAppliedRevision, + 'headline': true, }, { 'name': 'Last attempted version', - text: r.config.status.lastAttemptedRevision, + 'text': r.config.status.lastAttemptedRevision, } ] - } - })) + }).toJSON() - name: Kustomizations icon: kustomize components: diff --git a/mission-control/docs/topology/examples/prometheus.md b/mission-control/docs/topology/examples/prometheus.md index d5d120d3..b0235642 100644 --- a/mission-control/docs/topology/examples/prometheus.md +++ b/mission-control/docs/topology/examples/prometheus.md @@ -26,33 +26,22 @@ spec: - lookup: prometheus: - display: - javascript: | - var components = []; - for (idx in results) { - var value = parseInt(Number(results[idx].value)) - // CPU can be between 0 & 1, so take ceil for that case - if (value < 1) {value = 1} + expr: | + dyn(results).map(r, { + 'name': r.node, + 'properties': [{'name': 'cpu', 'value': math.Ceil(int(r.value))}] + }).toJSON() - components.push({ - name: results[idx].pod, - properties: [{name: 'cpu', value: value}], - }) - } - JSON.stringify(components) query: 1000 * max by (pod) (rate(container_cpu_usage_seconds_total{container!=""}[5m])) name: cpu - lookup: prometheus: - display: - javascript: | - var components = []; - for (idx in results) { - components.push({ - name: results[idx].pod, - properties: [{name: 'memory', value: parseInt(Number(results[idx].value))}], - }) - } - JSON.stringify(components) + expr: | + dyn(results).map(r, { + 'name': r.node, + 'properties': [{'name': 'memory', 'value': int(r.value)}] + }).toJSON() query: max by (pod) (avg_over_time(container_memory_working_set_bytes{container!=""}[5m])) name: memory properties: