From dd7a1ed04f8345b99155778ae1f2e8accc0fee51 Mon Sep 17 00:00:00 2001 From: warren Date: Thu, 27 Aug 2026 10:24:46 +0800 Subject: [PATCH 1/6] feat(kiro): add S3 collection and plugin APIs Signed-off-by: warren --- backend/Dockerfile.local | 6 + backend/plugins/kiro/api/blueprint_v200.go | 85 +++++ backend/plugins/kiro/api/connection.go | 163 +++++++++ backend/plugins/kiro/api/init.go | 66 ++++ backend/plugins/kiro/api/remote_api.go | 338 ++++++++++++++++++ backend/plugins/kiro/api/s3_slice_api.go | 117 ++++++ backend/plugins/kiro/api/test_connection.go | 204 +++++++++++ backend/plugins/kiro/impl/impl.go | 208 +++++++++++ backend/plugins/kiro/impl/table_info_test.go | 37 ++ backend/plugins/kiro/kiro.go | 55 +++ backend/plugins/kiro/tasks/discovery.go | 158 ++++++++ backend/plugins/kiro/tasks/discovery_test.go | 211 +++++++++++ backend/plugins/kiro/tasks/extractor.go | 204 +++++++++++ backend/plugins/kiro/tasks/extractor_test.go | 184 ++++++++++ backend/plugins/kiro/tasks/identity_client.go | 93 +++++ backend/plugins/kiro/tasks/log_extractor.go | 84 +++++ backend/plugins/kiro/tasks/log_parser.go | 256 +++++++++++++ backend/plugins/kiro/tasks/log_parser_test.go | 319 +++++++++++++++++ backend/plugins/kiro/tasks/s3_client.go | 217 +++++++++++ backend/plugins/kiro/tasks/s3_client_test.go | 237 ++++++++++++ .../plugins/kiro/tasks/s3_file_collector.go | 171 +++++++++ .../kiro/tasks/s3_file_collector_test.go | 215 +++++++++++ backend/plugins/kiro/tasks/task_data.go | 105 ++++++ .../logs/chat_01_empty_prompt.json.gz | Bin 0 -> 274 bytes .../testdata/logs/chat_02_with_prompt.json.gz | Bin 0 -> 403 bytes .../testdata/logs/chat_03_two_records.json.gz | Bin 0 -> 315 bytes .../testdata/logs/chat_04_no_model_id.json.gz | Bin 0 -> 368 bytes .../logs/completion_01_non_empty.json.gz | Bin 0 -> 956 bytes .../testdata/logs/completion_02_empty.json.gz | Bin 0 -> 909 bytes .../kiro/tasks/user_report_extractor.go | 63 ++++ .../schema_e2e/migration_schema_test.go | 2 + backend/plugins/table_info_test.go | 4 +- 32 files changed, 3800 insertions(+), 2 deletions(-) create mode 100644 backend/plugins/kiro/api/blueprint_v200.go create mode 100644 backend/plugins/kiro/api/connection.go create mode 100644 backend/plugins/kiro/api/init.go create mode 100644 backend/plugins/kiro/api/remote_api.go create mode 100644 backend/plugins/kiro/api/s3_slice_api.go create mode 100644 backend/plugins/kiro/api/test_connection.go create mode 100644 backend/plugins/kiro/impl/impl.go create mode 100644 backend/plugins/kiro/impl/table_info_test.go create mode 100644 backend/plugins/kiro/kiro.go create mode 100644 backend/plugins/kiro/tasks/discovery.go create mode 100644 backend/plugins/kiro/tasks/discovery_test.go create mode 100644 backend/plugins/kiro/tasks/extractor.go create mode 100644 backend/plugins/kiro/tasks/extractor_test.go create mode 100644 backend/plugins/kiro/tasks/identity_client.go create mode 100644 backend/plugins/kiro/tasks/log_extractor.go create mode 100644 backend/plugins/kiro/tasks/log_parser.go create mode 100644 backend/plugins/kiro/tasks/log_parser_test.go create mode 100644 backend/plugins/kiro/tasks/s3_client.go create mode 100644 backend/plugins/kiro/tasks/s3_client_test.go create mode 100644 backend/plugins/kiro/tasks/s3_file_collector.go create mode 100644 backend/plugins/kiro/tasks/s3_file_collector_test.go create mode 100644 backend/plugins/kiro/tasks/task_data.go create mode 100644 backend/plugins/kiro/tasks/testdata/logs/chat_01_empty_prompt.json.gz create mode 100644 backend/plugins/kiro/tasks/testdata/logs/chat_02_with_prompt.json.gz create mode 100644 backend/plugins/kiro/tasks/testdata/logs/chat_03_two_records.json.gz create mode 100644 backend/plugins/kiro/tasks/testdata/logs/chat_04_no_model_id.json.gz create mode 100644 backend/plugins/kiro/tasks/testdata/logs/completion_01_non_empty.json.gz create mode 100644 backend/plugins/kiro/tasks/testdata/logs/completion_02_empty.json.gz create mode 100644 backend/plugins/kiro/tasks/user_report_extractor.go diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local index 2c8b7ef1dcb..d4fab0a1480 100644 --- a/backend/Dockerfile.local +++ b/backend/Dockerfile.local @@ -73,6 +73,12 @@ RUN apt-get update && apt-get install -y \ libssh2-1 \ libssl3 \ ca-certificates \ + # gitextractor shells out to the git CLI unless + # USE_GO_GIT_IN_GIT_EXTRACTOR is set, so the binary must be present or every + # clone fails with "git: executable file not found in $PATH". + # debian:bookworm-slim does not ship it; the official image inherits it from + # its python base. + git \ && rm -rf /var/lib/apt/lists/* # Copy libgit2 diff --git a/backend/plugins/kiro/api/blueprint_v200.go b/backend/plugins/kiro/api/blueprint_v200.go new file mode 100644 index 00000000000..ceea9cbaee5 --- /dev/null +++ b/backend/plugins/kiro/api/blueprint_v200.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + + // No domain layer scopes: this plugin writes only to _tool_kiro_* tables. + // Cross-tool AI modelling in the domain layer is separate work. + return plan, []plugin.Scope{}, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.KiroS3Slice, srvhelper.NoScopeConfig], + connection *models.KiroConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + slice := scopeDetail.Scope + + op := &tasks.KiroOptions{ + ConnectionId: slice.ConnectionId, + ScopeId: slice.Id, + AccountId: slice.AccountId, + Year: slice.Year, + Month: slice.Month, + } + + // An empty entity list enables every subtask; the three streams are + // always collected together because they describe the same activity. + task, err := helper.MakePipelinePlanTask("kiro", subtaskMetas, []string{}, op) + if err != nil { + return nil, err + } + + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + plan[i] = append(stage, task) + } + return plan, nil +} diff --git a/backend/plugins/kiro/api/connection.go b/backend/plugins/kiro/api/connection.go new file mode 100644 index 00000000000..075274b1d56 --- /dev/null +++ b/backend/plugins/kiro/api/connection.go @@ -0,0 +1,163 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// PostConnections creates a new connection. +// @Summary create kiro connection +// @Description Create kiro connection +// @Tags plugins/kiro +// @Param body body models.KiroConnection true "json body" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + // Wrapped as BadInput so a struct-tag validation failure reports 400 rather + // than 500 - the difference between "fix your input" and "the server broke". + if err := api.Decode(input.Body, connection, vld); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid connection payload") + } + if err := validateConnection(&connection.KiroConn); err != nil { + return nil, errors.BadInput.Wrap(err, "connection validation failed") + } + if err := connectionHelper.Create(connection, input); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil +} + +// PatchConnection updates an existing connection. +// @Summary patch kiro connection +// @Description Patch kiro connection +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Param body body models.KiroConnection true "json body" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + if err := (&models.KiroConnection{}).MergeFromRequest(connection, input.Body); err != nil { + return nil, errors.Convert(err) + } + if err := validateConnection(&connection.KiroConn); err != nil { + return nil, errors.BadInput.Wrap(err, "connection validation failed") + } + if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil +} + +// DeleteConnection removes a connection. +// @Summary delete a kiro connection +// @Description Delete a kiro connection +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} srvhelper.DsRefs "References exist to this connection" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + conn := &models.KiroConnection{} + output, err := connectionHelper.Delete(conn, input) + if err != nil { + return output, err + } + output.Body = conn.Sanitize() + return output, nil +} + +// ListConnections lists all connections. +// @Summary get all kiro connections +// @Description Get all kiro connections +// @Tags plugins/kiro +// @Success 200 {object} []models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connections []models.KiroConnection + if err := connectionHelper.List(&connections); err != nil { + return nil, err + } + for i := range connections { + connections[i] = connections[i].Sanitize() + } + return &plugin.ApiResourceOutput{Body: connections}, nil +} + +// GetConnection returns one connection. +// @Summary get kiro connection detail +// @Description Get kiro connection detail +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +// validateConnection checks the fields collection cannot proceed without. +// +// Identity Store fields are deliberately not required: they only resolve display +// names, and identity for joining to git history comes from the report's +// User_Email column. Requiring them would block a working setup. +func validateConnection(conn *models.KiroConn) error { + if conn.AccessKeyId == "" { + return errors.BadInput.New("AccessKeyId is required") + } + if conn.SecretAccessKey == "" { + return errors.BadInput.New("SecretAccessKey is required") + } + if conn.Region == "" { + return errors.BadInput.New("Region is required") + } + if conn.Bucket == "" { + return errors.BadInput.New("Bucket is required") + } + // A partial Identity Store configuration is a mistake worth reporting: it + // silently yields no display names, which looks like a data problem rather + // than a configuration one. + if (conn.IdentityStoreId == "") != (conn.IdentityStoreRegion == "") { + return errors.BadInput.New("IdentityStoreId and IdentityStoreRegion must be set together") + } + return nil +} diff --git a/backend/plugins/kiro/api/init.go b/backend/plugins/kiro/api/init.go new file mode 100644 index 00000000000..e4b614a743d --- /dev/null +++ b/backend/plugins/kiro/api/init.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/go-playground/validator/v10" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var ( + vld *validator.Validate + connectionHelper *api.ConnectionApiHelper + basicRes context.BasicRes + // Scope config is NoScopeConfig: the report CSV's meaning is fixed by AWS + // and uniform across an organization, so there is nothing per-scope to + // configure. + dsHelper *api.DsHelper[models.KiroConnection, models.KiroS3Slice, srvhelper.NoScopeConfig] +) + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + connectionHelper = api.NewConnectionHelper( + basicRes, + vld, + p.Name(), + ) + + dsHelper = api.NewDataSourceHelper[ + models.KiroConnection, models.KiroS3Slice, srvhelper.NoScopeConfig, + ]( + basicRes, + p.Name(), + // Searchable scope fields. + []string{"accountId", "name"}, + func(c models.KiroConnection) models.KiroConnection { return c.Sanitize() }, + func(s models.KiroS3Slice) models.KiroS3Slice { return s.Sanitize() }, + nil, + ) + + // Scope browsing and search are implemented directly in remote_api.go rather + // than through the shared DsRemoteApiScopeList/Search helpers. Those build an + // HTTP client from the connection first, and that constructor runs a DNS + // check on the endpoint - a bucket name is not a hostname, so it fails. The + // helpers assume an HTTP data source; this one is S3. +} diff --git a/backend/plugins/kiro/api/remote_api.go b/backend/plugins/kiro/api/remote_api.go new file mode 100644 index 00000000000..1be7635558f --- /dev/null +++ b/backend/plugins/kiro/api/remote_api.go @@ -0,0 +1,338 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +// listKiroRemoteScopes browses the export layout as a tree. +// +// Three levels, mirroring Kiro's own S3 partitioning: +// +// (root) -> one group per AWS account with exported data +// {account} -> one group per year, plus a whole-year scope +// {account}/{y} -> one selectable scope per month +// +// Everything comes from S3 rather than user input. That is the point: a +// hand-typed prefix cannot be validated from the outcome, because a typo and a +// month with no data both produce a successful run that collects nothing. +func listKiroRemoteScopes(connection *models.KiroConnection, groupId string) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], + err errors.Error, +) { + if connection == nil { + return nil, errors.BadInput.New("connection is required") + } + + discovery, err := tasks.NewDiscovery(connection) + if err != nil { + return nil, err + } + + accountId, year, err := parseGroupId(groupId) + if err != nil { + return nil, err + } + + switch { + case accountId == "": + return listAccountGroups(discovery) + case year == 0: + return listYearGroups(discovery, accountId) + default: + return listMonthScopes(discovery, accountId, year) + } +} + +// listAccountGroups is the tree root: the accounts that actually have exports. +func listAccountGroups(discovery *tasks.Discovery) ( + []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], errors.Error, +) { + accounts, err := discovery.ListAccounts() + if err != nil { + return nil, err + } + + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0, len(accounts)) + for _, accountId := range accounts { + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + Id: accountId, + Name: accountId, + FullName: accountId, + }) + } + return entries, nil +} + +// listYearGroups lists the years under an account. +// +// Each year is offered both as a group to expand and as a directly selectable +// scope, because a nil month means "collect the whole year" - which is how a +// year-long backfill is expressed without creating twelve scopes by hand. +func listYearGroups(discovery *tasks.Discovery, accountId string) ( + []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], errors.Error, +) { + years, err := discovery.ListYears(accountId) + if err != nil { + return nil, err + } + + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0, len(years)*2) + for _, year := range years { + groupId := fmt.Sprintf("%s/%04d", accountId, year) + parent := accountId + + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parent, + Id: groupId, + Name: fmt.Sprintf("%04d", year), + FullName: groupId, + }) + + wholeYear := &models.KiroS3Slice{AccountId: accountId, Year: year} + *wholeYear = wholeYear.Sanitize() + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: &parent, + Id: wholeYear.Id, + Name: fmt.Sprintf("%04d (whole year)", year), + FullName: wholeYear.ScopeName(), + Data: wholeYear, + }) + } + return entries, nil +} + +// listMonthScopes lists the months that hold data for an account and year. +func listMonthScopes(discovery *tasks.Discovery, accountId string, year int) ( + []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], errors.Error, +) { + months, err := discovery.ListMonths(accountId, year) + if err != nil { + return nil, err + } + + parent := fmt.Sprintf("%s/%04d", accountId, year) + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0, len(months)) + for _, month := range months { + m := month + slice := &models.KiroS3Slice{AccountId: accountId, Year: year, Month: &m} + *slice = slice.Sanitize() + + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: &parent, + Id: slice.Id, + Name: fmt.Sprintf("%04d-%02d", year, month), + FullName: slice.ScopeName(), + Data: slice, + }) + } + return entries, nil +} + +// searchKiroRemoteScopes filters the discovered months by substring. +// +// Matching is against "{account} {year}-{month}", so "2026-07" or an account +// number both work. The search space is one listing per year, small enough to +// scan without an index. +func searchKiroRemoteScopes( + connection *models.KiroConnection, + query string, + page int, + pageSize int, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], + err errors.Error, +) { + empty := []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{} + if connection == nil { + return empty, nil + } + query = strings.ToLower(strings.TrimSpace(query)) + if query == "" { + return empty, nil + } + + discovery, err := tasks.NewDiscovery(connection) + if err != nil { + return nil, err + } + accounts, err := discovery.ListAccounts() + if err != nil { + return nil, err + } + + matches := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0) + for _, accountId := range accounts { + years, yearErr := discovery.ListYears(accountId) + if yearErr != nil { + return nil, yearErr + } + for _, year := range years { + months, monthErr := discovery.ListMonths(accountId, year) + if monthErr != nil { + return nil, monthErr + } + for _, month := range months { + m := month + slice := &models.KiroS3Slice{AccountId: accountId, Year: year, Month: &m} + *slice = slice.Sanitize() + + if !strings.Contains(strings.ToLower(slice.ScopeName()), query) && + !strings.Contains(strings.ToLower(slice.Id), query) { + continue + } + matches = append(matches, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: slice.Id, + Name: slice.ScopeName(), + FullName: slice.ScopeName(), + Data: slice, + }) + } + } + } + + return paginate(matches, page, pageSize), nil +} + +// paginate applies the requested page window to an in-memory result set. +func paginate( + entries []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], + page int, pageSize int, +) []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice] { + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 50 + } + start := (page - 1) * pageSize + if start >= len(entries) { + return []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{} + } + end := start + pageSize + if end > len(entries) { + end = len(entries) + } + return entries[start:end] +} + +// parseGroupId splits a tree node id into its parts. +// +// "" is the root, "{account}" is an account node, "{account}/{year}" is a year +// node. +func parseGroupId(groupId string) (accountId string, year int, err errors.Error) { + trimmed := strings.Trim(strings.TrimSpace(groupId), "/") + if trimmed == "" { + return "", 0, nil + } + + parts := strings.Split(trimmed, "/") + switch len(parts) { + case 1: + return parts[0], 0, nil + case 2: + parsedYear, convErr := strconv.Atoi(parts[1]) + if convErr != nil { + return "", 0, errors.BadInput.New("invalid year in groupId: " + groupId) + } + return parts[0], parsedYear, nil + default: + return "", 0, errors.BadInput.New("unrecognized groupId: " + groupId) + } +} + +// RemoteScopes browses the Kiro export layout in S3. +// +// Implemented directly rather than through the shared scope-list helper. That +// helper builds an HTTP client from the connection first, and its constructor +// runs a DNS check on the endpoint - which fails here, because a bucket name is +// not a hostname. The helper is built for HTTP data sources; this one is S3. +// @Summary list available kiro scopes discovered from S3 +// @Description Browse accounts, years and months that actually have exported data +// @Tags plugins/kiro +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param groupId query string false "account id, or account/year" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.KiroS3Slice] +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + + children, err := listKiroRemoteScopes(connection, input.Query.Get("groupId")) + if err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: dsmodels.DsRemoteApiScopeList[models.KiroS3Slice]{Children: children}, + }, nil +} + +// SearchRemoteScopes finds discovered scopes by substring. +// +// Implemented directly rather than through the shared search helper: that +// helper's callback receives only an HTTP ApiClient, and discovery here needs +// the connection itself to build an S3 client. +// @Summary search kiro scopes discovered from S3 +// @Description Search the discovered months by account or year-month +// @Tags plugins/kiro +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param search query string false "search" +// @Param page query int false "page number" +// @Param pageSize query int false "page size per page" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.KiroS3Slice] "the parentIds are always null" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/search-remote-scopes [GET] +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + + page, _ := strconv.Atoi(input.Query.Get("page")) + pageSize, _ := strconv.Atoi(input.Query.Get("pageSize")) + + children, err := searchKiroRemoteScopes(connection, input.Query.Get("search"), page, pageSize) + if err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: dsmodels.DsRemoteApiScopeList[models.KiroS3Slice]{Children: children}, + }, nil +} diff --git a/backend/plugins/kiro/api/s3_slice_api.go b/backend/plugins/kiro/api/s3_slice_api.go new file mode 100644 index 00000000000..f8a05fb185f --- /dev/null +++ b/backend/plugins/kiro/api/s3_slice_api.go @@ -0,0 +1,117 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +type PutScopesReqBody = helper.PutScopesReqBody[models.KiroS3Slice] +type ScopeDetail = srvhelper.ScopeDetail[models.KiroS3Slice, srvhelper.NoScopeConfig] + +// PutScopes creates or updates Kiro collection scopes. +// @Summary create or update kiro scopes +// @Description Create or update kiro scopes, each covering one AWS account for one month +// @Tags plugins/kiro +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.KiroS3Slice +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// GetScopeList returns the scopes for a connection. +// @Summary get kiro scopes +// @Description get kiro scopes +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param pageSize query int false "page size" +// @Param page query int false "page number" +// @Param blueprints query bool false "include blueprint references" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope returns a single scope. +// @Summary get one kiro scope +// @Description get one kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// PatchScope updates a scope. +// @Summary patch a kiro scope +// @Description patch a kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param scope body models.KiroS3Slice true "json" +// @Success 200 {object} models.KiroS3Slice +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// DeleteScope removes a scope and optionally its collected data. +// @Summary delete a kiro scope +// @Description delete a kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +// GetScopeLatestSyncState reports the most recent sync for a scope. +// @Summary get the latest sync state of a kiro scope +// @Description get the latest sync state of a kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} []models.LatestSyncState +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/kiro/api/test_connection.go b/backend/plugins/kiro/api/test_connection.go new file mode 100644 index 00000000000..8df055e582f --- /dev/null +++ b/backend/plugins/kiro/api/test_connection.go @@ -0,0 +1,204 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/http" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +// TestConnection validates a connection that has not been saved yet. +// @Summary test kiro connection +// @Description Test kiro connection +// @Tags plugins/kiro +// @Param body body models.KiroConn true "json body" +// @Success 200 {object} ConnectionReport +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.KiroConnection + // Wrapped as BadInput: a struct-tag validation failure is the caller's + // problem, and an unwrapped Decode error surfaces as HTTP 500 - which sends + // the user looking at server logs instead of at their own form. + if err := api.Decode(input.Body, &connection, vld); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid connection payload") + } + if err := validateConnection(&connection.KiroConn); err != nil { + return nil, errors.BadInput.Wrap(err, "connection validation failed") + } + if err := testConnection(&connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: buildConnectionReport(&connection), + Status: http.StatusOK, + }, nil +} + +// TestExistingConnection validates a saved connection, optionally with overrides +// from the request body. +// @Summary test existing kiro connection +// @Description Test existing kiro connection +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Success 200 {object} ConnectionReport +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := api.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid connection payload") + } + if err := testConnection(connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: buildConnectionReport(connection), + Status: http.StatusOK, + }, nil +} + +// ConnectionReport describes what the connection can actually see. +// +// A bare success/failure verdict is not enough to tell whether a configuration +// is right, because a wrong prefix and a genuinely empty period produce the same +// outcome: collection runs, finds nothing, and reports success. Returning the +// discovered accounts and per-stream object counts makes the difference visible +// before any scope is created. +type ConnectionReport struct { + ReportBucket string `json:"reportBucket"` + PromptLogBucket string `json:"promptLogBucket"` + // Accounts are the AWS account ids found under the report prefix. An empty + // list is the clearest sign that the bucket or prefix is wrong. + Accounts []string `json:"accounts"` + // Streams reports object counts for the most recent discovered period. + Streams []tasks.StreamCount `json:"streams"` + // Hint explains an empty result in plain terms. + Hint string `json:"hint,omitempty"` +} + +// connectionReportCountLimit caps counting so the check stays fast on a bucket +// holding hundreds of thousands of objects. The exact number does not matter for +// verifying a path - only whether it is zero. +const connectionReportCountLimit = 500 + +// buildConnectionReport probes the layout and summarizes what was found. +// +// Errors are folded into the report rather than returned: the connection itself +// is already known to work at this point, and a discovery failure is more useful +// shown as an empty result with a hint than as a failed request. +func buildConnectionReport(connection *models.KiroConnection) *ConnectionReport { + report := &ConnectionReport{ + ReportBucket: connection.Bucket, + PromptLogBucket: connection.GetPromptLogBucket(), + Accounts: []string{}, + Streams: []tasks.StreamCount{}, + } + + discovery, err := tasks.NewDiscovery(connection) + if err != nil { + report.Hint = "could not initialise S3 discovery: " + err.Error() + return report + } + + accounts, err := discovery.ListAccounts() + if err != nil { + report.Hint = "could not list accounts under the report prefix: " + err.Error() + return report + } + report.Accounts = accounts + + if len(accounts) == 0 { + report.Hint = fmt.Sprintf( + "no account directories under s3://%s/%s/AWSLogs/ - check the bucket and report prefix", + connection.Bucket, connection.GetReportPrefix()) + return report + } + + // Probe the newest period that exists, since that is where data is most + // likely to be and therefore the most informative check. + accountId := accounts[len(accounts)-1] + years, err := discovery.ListYears(accountId) + if err != nil || len(years) == 0 { + report.Hint = fmt.Sprintf( + "account %s has no year directories - check the region in the connection", accountId) + return report + } + year := years[len(years)-1] + + var month *int + if months, monthErr := discovery.ListMonths(accountId, year); monthErr == nil && len(months) > 0 { + latest := months[len(months)-1] + month = &latest + } + + report.Streams = discovery.CountStreams(accountId, year, month, connectionReportCountLimit) + + period := fmt.Sprintf("%04d", year) + if month != nil { + period = fmt.Sprintf("%04d-%02d", year, *month) + } + report.Hint = fmt.Sprintf("counts are for account %s, period %s", accountId, period) + return report +} + +// testConnection issues a real request against every bucket in use. +// +// Constructing a client proves nothing - the AWS SDK builds one happily from +// invalid credentials, so a test that stops there reports success for a +// connection that cannot read anything. A single-key list is the cheapest call +// that actually exercises credentials and bucket permissions. +// +// Both buckets are checked when reports and logs are separated, since they may +// carry different KMS keys and IAM conditions; a connection that can read +// reports but not logs would otherwise pass and then silently collect nothing. +func testConnection(connection *models.KiroConnection) errors.Error { + clients, err := tasks.NewKiroS3Clients(connection) + if err != nil { + return err + } + + for _, bucket := range clients.Buckets() { + client := clients.Report + if bucket == clients.PromptLog.Bucket { + client = clients.PromptLog + } + if _, listErr := client.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + MaxKeys: aws.Int64(1), + }); listErr != nil { + return errors.BadInput.Wrap(listErr, "cannot access s3 bucket "+bucket) + } + } + + return nil +} diff --git a/backend/plugins/kiro/impl/impl.go b/backend/plugins/kiro/impl/impl.go new file mode 100644 index 00000000000..c6b19708aeb --- /dev/null +++ b/backend/plugins/kiro/impl/impl.go @@ -0,0 +1,208 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.DataSourcePluginBlueprintV200 +} = (*Kiro)(nil) + +// Kiro collects Kiro enterprise usage exports from S3. +// +// This is a separate plugin rather than an evolution of the retired predecessor: +// the old format is frozen, and a plugin that keeps evolving should not depend +// on frozen code. No implementation code is shared, following the same split +// as bitbucket and bitbucket_server. +type Kiro struct{} + +func (p Kiro) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Kiro) Name() string { + return "kiro" +} + +func (p Kiro) Description() string { + return "collect Kiro usage reports and interaction logs from S3" +} + +func (p Kiro) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/kiro" +} + +// GetTablesInfo must list every model or plugins/table_info_test.go fails. +func (p Kiro) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.KiroConnection{}, + &models.KiroS3Slice{}, + &models.KiroS3FileMeta{}, + &models.KiroUserReport{}, + &models.KiroUserModelMessage{}, + &models.KiroChatLog{}, + &models.KiroCompletionLog{}, + } +} + +func (p Kiro) Connection() dal.Tabler { + return &models.KiroConnection{} +} + +func (p Kiro) Scope() plugin.ToolLayerScope { + return &models.KiroS3Slice{} +} + +// ScopeConfig returns nil: the export format is defined by AWS and uniform +// across an organization, so there is nothing per-scope to configure. +func (p Kiro) ScopeConfig() dal.Tabler { + return nil +} + +func (p Kiro) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +// SubTaskMetas lists discovery first, then one extractor per stream. The +// extractors declare their dependency on discovery, so the split is safe and +// gives each stream its own progress reporting - useful when a scope holds tens +// of thousands of log objects and one needs to know which stream is slow. +func (p Kiro) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectKiroS3FilesMeta, + tasks.ExtractKiroUserReportMeta, + tasks.ExtractKiroChatLogMeta, + tasks.ExtractKiroCompletionLogMeta, + } +} + +func (p Kiro) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.KiroOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, err + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("connectionId is required") + } + if op.AccountId == "" { + return nil, errors.BadInput.New("accountId is required") + } + if op.Year <= 0 { + return nil, errors.BadInput.New("year is required") + } + + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + connection := &models.KiroConnection{} + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, err + } + + s3Clients, err := tasks.NewKiroS3Clients(connection) + if err != nil { + return nil, err + } + + // Identity Store is optional and only supplies display names, so a failure + // here degrades presentation rather than collection. + identityClient, identityErr := tasks.NewKiroIdentityClient(connection) + if identityErr != nil { + taskCtx.GetLogger().Warn(identityErr, "identity store unavailable, proceeding without display names") + identityClient = nil + } + + timePath := fmt.Sprintf("%04d", op.Year) + if op.Month != nil { + timePath = fmt.Sprintf("%04d/%02d", op.Year, *op.Month) + } + + return &tasks.KiroTaskData{ + Options: &op, + Connection: connection, + S3Clients: s3Clients, + IdentityClient: identityClient, + Prefixes: tasks.BuildPrefixes(connection, op.AccountId, timePath), + }, nil +} + +func (p Kiro) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Kiro) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + // Scope discovery: lists the accounts, years and months that actually + // have exported data, so a scope is selected instead of hand-entered. + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "GET": api.GetScopeLatestSyncState, + }, + } +} diff --git a/backend/plugins/kiro/impl/table_info_test.go b/backend/plugins/kiro/impl/table_info_test.go new file mode 100644 index 00000000000..cb195602d63 --- /dev/null +++ b/backend/plugins/kiro/impl/table_info_test.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "testing" + + "github.com/apache/incubator-devlake/helpers/unithelper" +) + +// The repo-wide plugins/table_info_test.go performs this same check for every +// plugin, but building that package requires gitextractor and therefore a +// specific libgit2. Running it here as well keeps the feedback local: a model +// added without registering it in GetTablesInfo fails immediately rather than +// only in CI. +func TestKiroTableInfo(t *testing.T) { + checker := unithelper.NewTableInfoChecker(unithelper.TableInfoCheckerConfig{}) + checker.FeedIn("../models", Kiro{}.GetTablesInfo) + if err := checker.Verify(); err != nil { + t.Error(err) + } +} diff --git a/backend/plugins/kiro/kiro.go b/backend/plugins/kiro/kiro.go new file mode 100644 index 00000000000..27c8dff3032 --- /dev/null +++ b/backend/plugins/kiro/kiro.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/kiro/impl" +) + +var PluginEntry impl.Kiro + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "kiro"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "kiro connection id") + accountId := cmd.Flags().StringP("accountId", "a", "", "AWS account id that Kiro exports for") + year := cmd.Flags().IntP("year", "y", 0, "year to collect") + month := cmd.Flags().IntP("month", "m", 0, "month to collect; omit to collect the whole year") + + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("accountId") + _ = cmd.MarkFlagRequired("year") + + cmd.Run = func(cmd *cobra.Command, args []string) { + options := map[string]interface{}{ + "connectionId": *connectionId, + "accountId": *accountId, + "year": *year, + } + // A zero month means the whole year, matching the scope model where a + // nil month widens collection. + if *month > 0 { + options["month"] = *month + } + runner.DirectRun(cmd, args, PluginEntry, options, "") + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/kiro/tasks/discovery.go b/backend/plugins/kiro/tasks/discovery.go new file mode 100644 index 00000000000..1bd6a134759 --- /dev/null +++ b/backend/plugins/kiro/tasks/discovery.go @@ -0,0 +1,158 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "strconv" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// Discovery walks the report prefix to find what data actually exists. +// +// Kiro's S3 layout encodes every scope dimension as a path segment: +// +// {reportPrefix}/AWSLogs/{accountId}/KiroLogs/user_report/{region}/{year}/{month}/ +// +// so a scope never has to be typed by hand. That matters beyond convenience: a +// mistyped prefix produces exactly the same outcome as a month with no data - +// collection succeeds and finds nothing - so hand-entered paths cannot be +// verified from the result. +type Discovery struct { + clients *KiroS3Clients + connection *models.KiroConnection +} + +func NewDiscovery(connection *models.KiroConnection) (*Discovery, errors.Error) { + clients, err := NewKiroS3Clients(connection) + if err != nil { + return nil, err + } + return &Discovery{clients: clients, connection: connection}, nil +} + +// reportRoot is the prefix holding the per-account directories. +func (d *Discovery) reportRoot() string { + return fmt.Sprintf("%s/AWSLogs", d.connection.GetReportPrefix()) +} + +// accountReportPrefix is where one account's report months live. +func (d *Discovery) accountReportPrefix(accountId string) string { + return fmt.Sprintf("%s/AWSLogs/%s/KiroLogs/user_report/%s", + d.connection.GetReportPrefix(), accountId, d.connection.Region) +} + +// ListAccounts returns the AWS account ids that have exported data. +// +// Kiro requires a bucket per account holding subscriptions and does not support +// cross-account buckets, so in practice this is usually one entry - but reading +// it from S3 removes the chance of a typo in a 12-digit number. +func (d *Discovery) ListAccounts() ([]string, errors.Error) { + return d.clients.Report.ListSubPrefixes(d.reportRoot()) +} + +// ListYears returns the years with report data for an account. +func (d *Discovery) ListYears(accountId string) ([]int, errors.Error) { + names, err := d.clients.Report.ListSubPrefixes(d.accountReportPrefix(accountId)) + if err != nil { + return nil, err + } + return parseNumericSegments(names), nil +} + +// ListMonths returns the months with report data for an account and year. +func (d *Discovery) ListMonths(accountId string, year int) ([]int, errors.Error) { + prefix := fmt.Sprintf("%s/%04d", d.accountReportPrefix(accountId), year) + names, err := d.clients.Report.ListSubPrefixes(prefix) + if err != nil { + return nil, err + } + return parseNumericSegments(names), nil +} + +// StreamCount is how many collectable objects one stream holds. +type StreamCount struct { + FileType string `json:"fileType"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + Count int `json:"count"` + // AtLeast is true when counting stopped at the cap, so Count is a floor. + AtLeast bool `json:"atLeast"` + // Error explains why a stream could not be counted, e.g. missing + // permission on the log bucket while the report bucket is readable. + Error string `json:"error,omitempty"` +} + +// CountStreams reports the object count for each of the three streams. +// +// This is the answer to "is my configuration right?". Reporting counts per +// stream distinguishes the three cases that otherwise look identical: a wrong +// prefix (zero everywhere), a genuinely dormant stream (zero for one type - +// inline completions stopped being produced under agentic usage), and a +// permissions gap on one bucket (an error for the log streams only). +// +// countLimit caps the work per stream; pass 0 to count everything. +func (d *Discovery) CountStreams(accountId string, year int, month *int, countLimit int) []StreamCount { + timePath := fmt.Sprintf("%04d", year) + if month != nil { + timePath = fmt.Sprintf("%04d/%02d", year, *month) + } + + specs := BuildPrefixes(d.connection, accountId, timePath) + results := make([]StreamCount, 0, len(specs)) + + for _, spec := range specs { + client := d.clients.ForFileType(spec.FileType) + result := StreamCount{ + FileType: spec.FileType, + Bucket: client.Bucket, + Prefix: spec.Prefix, + } + count, atLeast, err := client.CountObjects(spec.Prefix, countLimit) + if err != nil { + // Recorded rather than returned: one unreadable stream should still + // leave the others' counts visible, since that contrast is what + // identifies a per-bucket permission problem. + result.Error = err.Error() + } else { + result.Count = count + result.AtLeast = atLeast + } + results = append(results, result) + } + + return results +} + +// parseNumericSegments keeps only the segments that are numbers, in order. +// +// S3 prefixes are strings, and a non-numeric directory would otherwise surface +// as a year or month. +func parseNumericSegments(names []string) []int { + values := make([]int, 0, len(names)) + for _, name := range names { + value, err := strconv.Atoi(name) + if err != nil { + continue + } + values = append(values, value) + } + return values +} diff --git a/backend/plugins/kiro/tasks/discovery_test.go b/backend/plugins/kiro/tasks/discovery_test.go new file mode 100644 index 00000000000..785f79cbd58 --- /dev/null +++ b/backend/plugins/kiro/tasks/discovery_test.go @@ -0,0 +1,211 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// prefixMockS3 answers listings from a canned prefix tree, and records the +// requests so tests can assert that a delimiter was used. +type prefixMockS3 struct { + // commonPrefixes maps a queried prefix to the child prefixes returned. + commonPrefixes map[string][]string + // objects maps a queried prefix to the object keys beneath it. + objects map[string][]string + seenDelimiter []string + seenPrefix []string +} + +func (m *prefixMockS3) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + m.seenPrefix = append(m.seenPrefix, prefix) + if input.Delimiter != nil { + m.seenDelimiter = append(m.seenDelimiter, *input.Delimiter) + } else { + m.seenDelimiter = append(m.seenDelimiter, "") + } + + out := &s3.ListObjectsV2Output{IsTruncated: aws.Bool(false)} + for _, child := range m.commonPrefixes[prefix] { + full := prefix + child + "/" + out.CommonPrefixes = append(out.CommonPrefixes, &s3.CommonPrefix{Prefix: aws.String(full)}) + } + for _, key := range m.objects[prefix] { + out.Contents = append(out.Contents, &s3.Object{Key: aws.String(prefix + key)}) + } + return out, nil +} + +func (m *prefixMockS3) GetObject(*s3.GetObjectInput) (*s3.GetObjectOutput, error) { + return nil, nil +} + +func discoveryFixture(svc S3API) *Discovery { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "kiro-export-test", + ReportPrefix: "user-report", + PromptLogPrefix: "logging", + }} + client := &KiroS3Client{S3: svc, Bucket: conn.Bucket} + return &Discovery{ + clients: &KiroS3Clients{Report: client, PromptLog: client}, + connection: conn, + } +} + +// Kiro encodes every scope dimension as a path segment, which is what allows a +// scope to be selected rather than typed. This matters beyond convenience: a +// mistyped prefix and a month with no data produce the same outcome - a +// successful run that collects nothing - so a hand-entered path cannot be +// verified from the result. +func TestDiscovery_ListsLayoutFromS3(t *testing.T) { + // The real layout, as confirmed against the live bucket. + svc := &prefixMockS3{commonPrefixes: map[string][]string{ + "user-report/AWSLogs/": {"123456789012"}, + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/": {"2026"}, + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/2026/": {"02", "03", "07"}, + }} + discovery := discoveryFixture(svc) + + accounts, err := discovery.ListAccounts() + require.Nil(t, err) + assert.Equal(t, []string{"123456789012"}, accounts) + + years, err := discovery.ListYears("123456789012") + require.Nil(t, err) + assert.Equal(t, []int{2026}, years) + + months, err := discovery.ListMonths("123456789012", 2026) + require.Nil(t, err) + // Zero-padded segments must parse to plain integers, and only months that + // actually hold data are offered. + assert.Equal(t, []int{2, 3, 7}, months) + + // A delimiter is required: without it S3 returns every object under the + // prefix instead of just the segment names, which on a log prefix would be + // hundreds of thousands of keys. + for _, delimiter := range svc.seenDelimiter { + assert.Equal(t, "/", delimiter) + } +} + +func TestDiscovery_IgnoresNonNumericSegments(t *testing.T) { + svc := &prefixMockS3{commonPrefixes: map[string][]string{ + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/": {"2026", "unexpected", "2025"}, + }} + years, err := discoveryFixture(svc).ListYears("123456789012") + require.Nil(t, err) + // S3 prefixes are strings; a stray directory must not surface as a year. + assert.Equal(t, []int{2025, 2026}, years) +} + +func TestDiscovery_EmptyLayout(t *testing.T) { + // An empty account list is the clearest signal that the bucket or report + // prefix is wrong, so it must come back as an empty result rather than an + // error. + discovery := discoveryFixture(&prefixMockS3{}) + + accounts, err := discovery.ListAccounts() + require.Nil(t, err) + assert.Empty(t, accounts) +} + +// Per-stream counts separate three situations that a single pass/fail verdict +// cannot: a wrong prefix (zero everywhere), a dormant stream (zero for one type, +// which is what inline completions look like under agentic usage), and a +// permission gap on one bucket. +func TestDiscovery_CountStreams(t *testing.T) { + base := "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/2026/07/" + logBase := "logging/AWSLogs/123456789012/KiroLogs/" + svc := &prefixMockS3{objects: map[string][]string{ + base: {"KIRO_CLI_x_user_report_1.csv", "KIRO_IDE_x_user_report_1.csv"}, + logBase + "GenerateAssistantResponse/us-east-1/2026/07/": {"a.json.gz", "b.json.gz", "c.json.gz"}, + // GenerateCompletions intentionally absent: dormant, not misconfigured. + }} + + month := 7 + counts := discoveryFixture(svc).CountStreams("123456789012", 2026, &month, 0) + require.Len(t, counts, 3) + + byType := map[string]StreamCount{} + for _, c := range counts { + byType[c.FileType] = c + } + assert.Equal(t, 2, byType[models.FileTypeReport].Count) + assert.Equal(t, 3, byType[models.FileTypeChatLog].Count) + assert.Equal(t, 0, byType[models.FileTypeCompletionLog].Count) + + // The prefix is reported alongside the count so a zero can be checked + // against the bucket directly. + assert.Contains(t, byType[models.FileTypeReport].Prefix, "user_report/us-east-1/2026/07") + for _, c := range counts { + assert.NotEmpty(t, c.Bucket) + assert.Empty(t, c.Error) + } +} + +func TestCountObjects(t *testing.T) { + prefix := "p/" + svc := &prefixMockS3{objects: map[string][]string{ + // Only .csv and .json.gz are collectable; the count must match what + // would actually be collected, not every object present. + prefix: {"a.csv", "b.json.gz", "c.txt", "d", "e.zip"}, + }} + client := &KiroS3Client{S3: svc, Bucket: "b"} + + count, atLeast, err := client.CountObjects(prefix, 0) + require.Nil(t, err) + assert.Equal(t, 2, count) + assert.False(t, atLeast) + + // Counting stops at the cap so the check stays cheap on a large bucket; the + // flag marks the value as a floor. + count, atLeast, err = client.CountObjects(prefix, 1) + require.Nil(t, err) + assert.Equal(t, 1, count) + assert.True(t, atLeast) +} + +func TestListSubPrefixes_TrailingSlashHandling(t *testing.T) { + svc := &prefixMockS3{commonPrefixes: map[string][]string{ + "a/b/": {"x", "y"}, + }} + client := &KiroS3Client{S3: svc, Bucket: "bkt"} + + // A caller-supplied prefix may or may not end in a slash; both must resolve + // to the same listing, and the child names come back without the slash. + withSlash, err := client.ListSubPrefixes("a/b/") + require.Nil(t, err) + withoutSlash, err := client.ListSubPrefixes("a/b") + require.Nil(t, err) + + assert.Equal(t, []string{"x", "y"}, withSlash) + assert.Equal(t, withSlash, withoutSlash) +} diff --git a/backend/plugins/kiro/tasks/extractor.go b/backend/plugins/kiro/tasks/extractor.go new file mode 100644 index 00000000000..10867515005 --- /dev/null +++ b/backend/plugins/kiro/tasks/extractor.go @@ -0,0 +1,204 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// parseFunc turns one downloaded object into batches of rows ready for +// insertion. It is a pure function so it can run on a worker goroutine without +// touching the database. +// +// The result is a slice of batches rather than one flat slice because GORM +// derives the target table from the element type: a mixed []interface{} fails +// with "Table not set". A report CSV yields two different models, so each model +// gets its own homogeneous batch. +type parseFunc func(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) + +// rowBatch is a set of rows that all belong to the same table. +type rowBatch struct { + // rows must be a typed slice (e.g. []*models.KiroChatLog), not + // []interface{}, so GORM can resolve the table. + rows interface{} + count int +} + +// parsedFile is a worker's output, handed to the main goroutine for persistence. +type parsedFile struct { + meta *models.KiroS3FileMeta + batches []rowBatch + err errors.Error +} + +// rowCount totals the rows across every batch, for the cursor's record count. +func (p parsedFile) rowCount() int { + total := 0 + for _, batch := range p.batches { + total += batch.count + } + return total +} + +// pendingFiles returns the files still awaiting extraction for a scope. +// +// Files that have already failed MaxAttempts times are excluded. Without that +// bound, one permanently malformed object is retried on every run: the log fills +// with the same error and the scope never reaches a finished state. +func pendingFiles(db dal.Dal, connectionId uint64, scopeId string, fileType string) ([]models.KiroS3FileMeta, errors.Error) { + var files []models.KiroS3FileMeta + err := db.All(&files, + dal.From(&models.KiroS3FileMeta{}), + dal.Where("connection_id = ? AND scope_id = ? AND file_type = ? AND processed = ? AND attempt_count < ?", + connectionId, scopeId, fileType, false, models.MaxAttempts), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to query pending kiro files") + } + return files, nil +} + +// extractFiles downloads and parses every pending file of one type, then stores +// the results. +// +// Downloads run concurrently but all database writes happen on the calling +// goroutine. That split is a correctness requirement, not an optimization: +// twenty workers inserting into the same table concurrently deadlock on MySQL +// gap locks, and DevLake's retry layer would turn those deadlocks into +// intermittent failures that only appear under load. +func extractFiles(taskCtx plugin.SubTaskContext, fileType string, parse parseFunc) errors.Error { + data := taskCtx.GetData().(*KiroTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + + files, err := pendingFiles(db, data.Options.ConnectionId, data.Options.ScopeId, fileType) + if err != nil { + return err + } + if len(files) == 0 { + logger.Info("no pending %s files for scope %s", fileType, data.Options.ScopeId) + return nil + } + + logger.Info("extracting %d %s files", len(files), fileType) + taskCtx.SetProgress(0, len(files)) + + client := data.S3Clients.ForFileType(fileType) + results := make(chan parsedFile, len(files)) + + scheduler, err := helper.NewWorkerScheduler( + taskCtx.GetContext(), + data.WorkerCount(), + // The tick is a global rate limiter, not a per-worker one: every task + // waits for one tick before running, so this value caps total + // throughput regardless of pool size. A one-second tick would pin the + // whole run to one file per second and leave the pool idle - measured + // at 13 seconds for 13 files. Keep it well below the per-object fetch + // time so the workers, not the ticker, set the pace. + time.Millisecond, + logger, + ) + if err != nil { + return err + } + defer scheduler.Release() + + for i := range files { + meta := files[i] + scheduler.SubmitBlocking(func() errors.Error { + // Workers only fetch and parse. Anything touching the database + // happens after WaitAsync below. + body, getErr := client.GetObjectBytes(meta.S3Path) + if getErr != nil { + results <- parsedFile{meta: &meta, err: getErr} + return nil + } + batches, parseErr := parse(body, data.Options.ConnectionId, data.Options.ScopeId) + results <- parsedFile{meta: &meta, batches: batches, err: parseErr} + return nil + }) + } + + if waitErr := scheduler.WaitAsync(); waitErr != nil { + return waitErr + } + close(results) + + for result := range results { + if saveErr := persistFile(db, result); saveErr != nil { + return saveErr + } + taskCtx.IncProgress(1) + } + + return nil +} + +// persistFile stores one file's rows and updates its cursor entry. +// +// A parse or download failure is recorded rather than aborting the run: one bad +// object should not block the rest of the scope. The reason is written to +// error_message so a short month can be explained with a query instead of +// guesswork. +func persistFile(db dal.Dal, result parsedFile) errors.Error { + meta := result.meta + + if result.err != nil { + meta.AttemptCount++ + meta.ErrorMessage = result.err.Error() + // Processed stays false so the file is retried, up to MaxAttempts. + if err := db.Update(meta); err != nil { + return errors.Default.Wrap(err, "failed to record kiro extraction failure") + } + return nil + } + + // Each batch is written separately so GORM sees a typed slice and can + // resolve the target table. + for _, batch := range result.batches { + if batch.count == 0 { + continue + } + if err := db.CreateOrUpdate(batch.rows); err != nil { + // A write failure counts as an attempt too, otherwise a row that + // cannot be stored would be fetched forever. + meta.AttemptCount++ + meta.ErrorMessage = err.Error() + if updateErr := db.Update(meta); updateErr != nil { + return errors.Default.Wrap(updateErr, "failed to record kiro write failure") + } + return nil + } + } + + now := time.Now() + meta.Processed = true + meta.ProcessedTime = &now + meta.RecordCount = result.rowCount() + meta.ErrorMessage = "" + if err := db.Update(meta); err != nil { + return errors.Default.Wrap(err, "failed to mark kiro file processed") + } + return nil +} diff --git a/backend/plugins/kiro/tasks/extractor_test.go b/backend/plugins/kiro/tasks/extractor_test.go new file mode 100644 index 00000000000..5eec6051e3b --- /dev/null +++ b/backend/plugins/kiro/tasks/extractor_test.go @@ -0,0 +1,184 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "os" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// GORM resolves the destination table from a slice's element type, so every +// batch must be a typed slice. Passing []interface{} fails at runtime with +// "Table not set" - a real bug caught only by running against a database, since +// it type-checks fine. These assertions pin the requirement so a future +// refactor back to a flat interface slice fails here instead of in production. +func assertTypedBatch[T any](t *testing.T, batch rowBatch, wantCount int) { + t.Helper() + rows, ok := batch.rows.([]T) + require.True(t, ok, "batch must carry a typed slice, got %T", batch.rows) + assert.Len(t, rows, wantCount) + assert.Equal(t, wantCount, batch.count) +} + +// The adapters exist so one parse produces everything a file yields; these +// verify the adaptation rather than the parsing, which is covered directly. +func TestParseUserReportRows_ReturnsTwoTypedBatches(t *testing.T) { + data := loadReportFixture(t, "02_standard_cli.csv") + + batches, err := parseUserReportRows(data, testConnectionId, testScopeId) + require.Nil(t, err) + + // Two batches, not one mixed slice: a report CSV yields two different + // models, and GORM cannot infer a table from a heterogeneous slice. + require.Len(t, batches, 2) + assertTypedBatch[*models.KiroUserReport](t, batches[0], 1) + assertTypedBatch[*models.KiroUserModelMessage](t, batches[1], 1) +} + +func TestParseLogRows(t *testing.T) { + t.Run("chat log", func(t *testing.T) { + batches, err := parseChatLogRows(loadLogFixture(t, "chat_03_two_records.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, batches, 1) + assertTypedBatch[*models.KiroChatLog](t, batches[0], 2) + }) + + t.Run("completion log", func(t *testing.T) { + batches, err := parseCompletionLogRows(loadLogFixture(t, "completion_01_non_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, batches, 1) + assertTypedBatch[*models.KiroCompletionLog](t, batches[0], 1) + }) + + // A record with no completions still produces a row: those records are the + // denominator for what Kiro offered versus what was taken. + t.Run("empty completions still produce a row", func(t *testing.T) { + batches, err := parseCompletionLogRows(loadLogFixture(t, "completion_02_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, batches, 1) + assertTypedBatch[*models.KiroCompletionLog](t, batches[0], 1) + }) + + t.Run("parse failure propagates", func(t *testing.T) { + _, err := parseChatLogRows([]byte("not gzipped"), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) +} + +// The scheduler's tick is a global rate limiter rather than a per-worker one: +// every submitted task waits one tick before running, so the interval caps total +// throughput no matter how large the pool is. A one-second tick pinned a real run +// to one file per second - 13 seconds for 13 files, with 20 workers idle. +func TestSchedulerTickDoesNotThrottleThePool(t *testing.T) { + source, err := os.ReadFile("extractor.go") + require.NoError(t, err) + + tick := regexp.MustCompile(`NewWorkerScheduler\((?s:.*?)time\.(\w+),`).FindStringSubmatch(string(source)) + require.NotNil(t, tick, "expected a tick interval passed to NewWorkerScheduler") + assert.Equal(t, "Millisecond", tick[1], + "a coarser tick throttles the whole pool to one object per tick") +} + +// Twenty workers inserting into one table concurrently deadlock on MySQL gap +// locks, and DevLake's retry layer turns those deadlocks into failures that only +// appear under load - the hardest kind to reproduce. The worker closure must +// therefore stay free of database access, which is asserted structurally because +// no unit test can observe the absence of a write. +func TestExtractorWorkersDoNotTouchTheDatabase(t *testing.T) { + source, err := os.ReadFile("extractor.go") + require.NoError(t, err) + + body := string(source) + workerStart := regexp.MustCompile(`scheduler\.SubmitBlocking\(func\(\) errors\.Error \{`).FindStringIndex(body) + require.NotNil(t, workerStart, "expected a worker closure submitted to the scheduler") + + // Take the closure body up to the WaitAsync call, which is where the main + // goroutine resumes and persistence begins. + waitIdx := regexp.MustCompile(`scheduler\.WaitAsync\(\)`).FindStringIndex(body) + require.NotNil(t, waitIdx) + require.Less(t, workerStart[1], waitIdx[0]) + workerRegion := body[workerStart[1]:waitIdx[0]] + + for _, forbidden := range []string{"db.Create", "db.Update", "db.CreateOrUpdate", "db.All", "db.First", "db.Exec"} { + assert.NotContains(t, workerRegion, forbidden, + "worker goroutines must not access the database; %s belongs on the main goroutine", forbidden) + } +} + +// A permanently malformed object would otherwise be retried on every run, +// filling the log with the same error while the scope never reaches a finished +// state. +func TestPendingFilesQueryBoundsRetries(t *testing.T) { + source, err := os.ReadFile("extractor.go") + require.NoError(t, err) + + clauses := regexp.MustCompile(`dal\.Where\(\s*"([^"]+)"`).FindAllStringSubmatch(string(source), -1) + require.NotEmpty(t, clauses) + + var found bool + for _, clause := range clauses { + if regexp.MustCompile(`attempt_count\s*<`).MatchString(clause[1]) { + found = true + // The same query must also scope to the connection and scope, or one + // scope's run would pick up another's files. + assert.Contains(t, clause[1], "connection_id") + assert.Contains(t, clause[1], "scope_id") + assert.Contains(t, clause[1], "file_type") + } + } + assert.True(t, found, "the pending-files query must bound attempt_count") +} + +func TestMaxAttemptsIsBounded(t *testing.T) { + // A cap that is zero or negative would stop all extraction; an unbounded one + // would never converge. + assert.Greater(t, models.MaxAttempts, 0) + assert.LessOrEqual(t, models.MaxAttempts, 10) +} + +// Every extractor must declare the collector as a dependency: without the file +// metadata rows there is nothing to extract, and DevLake would otherwise be free +// to schedule extraction before discovery. +func TestExtractorSubTaskMetas(t *testing.T) { + for _, meta := range []plugin.SubTaskMeta{ + ExtractKiroUserReportMeta, + ExtractKiroChatLogMeta, + ExtractKiroCompletionLogMeta, + } { + t.Run(meta.Name, func(t *testing.T) { + assert.NotEmpty(t, meta.Name) + assert.NotNil(t, meta.EntryPoint) + assert.True(t, meta.EnabledByDefault) + + var dependsOnCollector bool + for _, dep := range meta.Dependencies { + if dep.Name == CollectKiroS3FilesMeta.Name { + dependsOnCollector = true + } + } + assert.True(t, dependsOnCollector, "extraction depends on file discovery having run") + }) + } +} diff --git a/backend/plugins/kiro/tasks/identity_client.go b/backend/plugins/kiro/tasks/identity_client.go new file mode 100644 index 00000000000..8e6ed34c2fd --- /dev/null +++ b/backend/plugins/kiro/tasks/identity_client.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/identitystore" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// IdentityStoreAPI is the subset of the Identity Store API used here. +type IdentityStoreAPI interface { + DescribeUser(input *identitystore.DescribeUserInput) (*identitystore.DescribeUserOutput, error) +} + +// KiroIdentityClient resolves user ids to human-readable display names. +// +// This is entirely optional. Identity for joining to git history comes from the +// User_Email column of the report CSV, so a missing or misconfigured Identity +// Store degrades only the display name, never the data pipeline. +type KiroIdentityClient struct { + IdentityStore IdentityStoreAPI + StoreId string +} + +// NewKiroIdentityClient returns nil when Identity Store is not configured, +// which callers treat as "no display names" rather than as an error. +func NewKiroIdentityClient(connection *models.KiroConnection) (*KiroIdentityClient, error) { + if connection.IdentityStoreId == "" || connection.IdentityStoreRegion == "" { + return nil, nil + } + + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(connection.IdentityStoreRegion), + Credentials: credentials.NewStaticCredentials( + connection.AccessKeyId, + connection.SecretAccessKey, + "", + ), + }) + if err != nil { + return nil, err + } + + return &KiroIdentityClient{ + IdentityStore: identitystore.New(sess), + StoreId: connection.IdentityStoreId, + }, nil +} + +// ResolveDisplayName looks up a display name, returning nil when it cannot be +// determined. +// +// nil rather than the raw user id: the column exists for human readability, and +// storing an id there would make it look like a resolved name. +func (client *KiroIdentityClient) ResolveDisplayName(userId string) (*string, error) { + if client == nil || client.IdentityStore == nil || userId == "" { + return nil, nil + } + + result, err := client.IdentityStore.DescribeUser(&identitystore.DescribeUserInput{ + IdentityStoreId: aws.String(client.StoreId), + UserId: aws.String(userId), + }) + if err != nil { + // Surfaced for logging, but callers proceed without a display name. + return nil, err + } + + if result.DisplayName != nil && *result.DisplayName != "" { + name := *result.DisplayName + return &name, nil + } + return nil, nil +} diff --git a/backend/plugins/kiro/tasks/log_extractor.go b/backend/plugins/kiro/tasks/log_extractor.go new file mode 100644 index 00000000000..476b06add9c --- /dev/null +++ b/backend/plugins/kiro/tasks/log_extractor.go @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var ( + _ plugin.SubTaskEntryPoint = ExtractKiroChatLog + _ plugin.SubTaskEntryPoint = ExtractKiroCompletionLog +) + +var ExtractKiroChatLogMeta = plugin.SubTaskMeta{ + Name: "extractKiroChatLog", + EntryPoint: ExtractKiroChatLog, + EnabledByDefault: true, + Description: "Extract Kiro chat interactions from GenerateAssistantResponse logs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectKiroS3FilesMeta}, +} + +var ExtractKiroCompletionLogMeta = plugin.SubTaskMeta{ + Name: "extractKiroCompletionLog", + EntryPoint: ExtractKiroCompletionLog, + EnabledByDefault: true, + Description: "Extract Kiro inline suggestions from GenerateCompletions logs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectKiroS3FilesMeta}, +} + +// ExtractKiroChatLog loads the chat interaction logs for this scope. +// +// This is the high-volume stream: one object per interaction, roughly 700 bytes +// each, which for a single active user reaches several hundred objects a day. +// The cost is request count rather than bytes, which is what the worker pool +// addresses. +func ExtractKiroChatLog(taskCtx plugin.SubTaskContext) errors.Error { + return extractFiles(taskCtx, models.FileTypeChatLog, parseChatLogRows) +} + +// ExtractKiroCompletionLog loads the inline suggestion logs for this scope. +// +// This stream is dormant under agentic usage - the sampled history stops in +// March - but the objects remain in S3, so a backfill picks them up and a team +// that enables IDE inline completion starts producing them again. +func ExtractKiroCompletionLog(taskCtx plugin.SubTaskContext) errors.Error { + return extractFiles(taskCtx, models.FileTypeCompletionLog, parseCompletionLogRows) +} + +// Both adapters return a single typed batch: the slice keeps its concrete +// element type so GORM can resolve the table. +func parseChatLogRows(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) { + logs, err := ParseChatLog(data, connectionId, scopeId) + if err != nil { + return nil, err + } + return []rowBatch{{rows: logs, count: len(logs)}}, nil +} + +func parseCompletionLogRows(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) { + logs, err := ParseCompletionLog(data, connectionId, scopeId) + if err != nil { + return nil, err + } + return []rowBatch{{rows: logs, count: len(logs)}}, nil +} diff --git a/backend/plugins/kiro/tasks/log_parser.go b/backend/plugins/kiro/tasks/log_parser.go new file mode 100644 index 00000000000..3de9a66fdea --- /dev/null +++ b/backend/plugins/kiro/tasks/log_parser.go @@ -0,0 +1,256 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "path/filepath" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// Markers used by the prompt heuristics. These read a prompt's text for signs +// of how the developer invoked Kiro; they are only meaningful when a prompt is +// actually present. +const ( + steeringMarker = ".kiro/steering" + specMarker = ".kiro/specs" +) + +// Every field below is a pointer or checked for presence because Kiro omits +// empty keys entirely rather than emitting a null. Across 4530 sampled records, +// five documented fields never appeared at all and modelId appeared on only 45%, +// so "absent" and "zero" have to stay distinguishable. +type chatLogFile struct { + Records []struct { + Request *struct { + Prompt *string `json:"prompt"` + ChatTriggerType *string `json:"chatTriggerType"` + UserId *string `json:"userId"` + TimeStamp *string `json:"timeStamp"` + ModelId *string `json:"modelId"` + CustomizationArn *string `json:"customizationArn"` + } `json:"generateAssistantResponseEventRequest"` + Response *struct { + AssistantResponse *string `json:"assistantResponse"` + FollowupPrompts *string `json:"followupPrompts"` + RequestId *string `json:"requestId"` + MessageMetadata *struct { + ConversationId *string `json:"conversationId"` + UtteranceId *string `json:"utteranceId"` + } `json:"messageMetadata"` + } `json:"generateAssistantResponseEventResponse"` + } `json:"records"` +} + +type completionLogFile struct { + Records []struct { + Request *struct { + LeftContext *string `json:"leftContext"` + RightContext *string `json:"rightContext"` + FileName *string `json:"fileName"` + UserId *string `json:"userId"` + TimeStamp *string `json:"timeStamp"` + CustomizationArn *string `json:"customizationArn"` + } `json:"generateCompletionsEventRequest"` + Response *struct { + Completions []string `json:"completions"` + RequestId *string `json:"requestId"` + } `json:"generateCompletionsEventResponse"` + } `json:"records"` +} + +// ParseChatLog parses a gzipped GenerateAssistantResponse log. +// +// Neither the prompt nor the assistant response text is persisted: derived +// features are computed here and the originals are dropped, which keeps +// proprietary code and personal content out of the warehouse. +// +// A file holds one or two records, so the array is always walked. +func ParseChatLog(gzData []byte, connectionId uint64, scopeId string) ([]*models.KiroChatLog, errors.Error) { + raw, err := gunzip(gzData) + if err != nil { + return nil, err + } + + var parsed chatLogFile + if jsonErr := json.Unmarshal(raw, &parsed); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to unmarshal chat log") + } + + var result []*models.KiroChatLog + for _, record := range parsed.Records { + if record.Request == nil || record.Response == nil { + // Without both halves there is no usable interaction. + continue + } + requestId := deref(record.Response.RequestId) + if requestId == "" { + // requestId is the primary key; a record without one cannot be + // stored or deduplicated. + continue + } + + userId, identityStoreId := SplitUserId(deref(record.Request.UserId)) + + timestamp, tsErr := ParseKiroTime(deref(record.Request.TimeStamp)) + if tsErr != nil { + return nil, tsErr + } + + log := &models.KiroChatLog{ + ConnectionId: connectionId, + ScopeId: scopeId, + RequestId: requestId, + UserId: userId, + IdentityStoreId: identityStoreId, + Timestamp: timestamp, + ChatTriggerType: deref(record.Request.ChatTriggerType), + // Only ~45% of records carry a model id, so it stays nil when + // absent. Attributing model usage from this column would skew any + // share-of-usage figure; user_model_messages is authoritative. + ModelId: record.Request.ModelId, + ResponseLength: len(deref(record.Response.AssistantResponse)), + // Presence, not content: the follow-up text itself is not stored. + HasFollowupPrompts: record.Response.FollowupPrompts != nil, + } + + prompt := deref(record.Request.Prompt) + if prompt != "" { + // A non-empty prompt means the user spoke this turn. + sum := sha256.Sum256([]byte(prompt)) + hash := hex.EncodeToString(sum[:]) + hasSteering := strings.Contains(prompt, steeringMarker) + isSpecMode := strings.Contains(prompt, specMarker) + + log.HasPrompt = true + log.PromptLength = len(prompt) + log.PromptSha256 = &hash + log.HasSteering = &hasSteering + log.IsSpecMode = &isSpecMode + } + // When the prompt is empty the agent continued on its own. The hash and + // both heuristics stay nil: hashing the empty string would give roughly + // 71% of rows one identical hash and destroy the rework signal, and a + // false heuristic would be indistinguishable from a real negative. + + if md := record.Response.MessageMetadata; md != nil { + log.ConversationId = md.ConversationId + log.UtteranceId = md.UtteranceId + } + + result = append(result, log) + } + + return result, nil +} + +// ParseCompletionLog parses a gzipped GenerateCompletions log. +// +// The counters are named Returned* rather than Accepted*: a record is written +// when the suggestion is requested, and an empty completions array is common, so +// these measure what Kiro offered rather than what was taken. Records with no +// completions are still stored - they are the denominator. +func ParseCompletionLog(gzData []byte, connectionId uint64, scopeId string) ([]*models.KiroCompletionLog, errors.Error) { + raw, err := gunzip(gzData) + if err != nil { + return nil, err + } + + var parsed completionLogFile + if jsonErr := json.Unmarshal(raw, &parsed); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to unmarshal completion log") + } + + var result []*models.KiroCompletionLog + for _, record := range parsed.Records { + if record.Request == nil || record.Response == nil { + continue + } + requestId := deref(record.Response.RequestId) + if requestId == "" { + continue + } + + userId, identityStoreId := SplitUserId(deref(record.Request.UserId)) + + timestamp, tsErr := ParseKiroTime(deref(record.Request.TimeStamp)) + if tsErr != nil { + return nil, tsErr + } + + // fileName is a bare file name with no directory path, so it cannot be + // resolved to a unique repository file. + fileName := deref(record.Request.FileName) + + charCount := 0 + lineCount := 0 + for _, completion := range record.Response.Completions { + charCount += len(completion) + lineCount += strings.Count(completion, "\n") + 1 + } + + result = append(result, &models.KiroCompletionLog{ + ConnectionId: connectionId, + ScopeId: scopeId, + RequestId: requestId, + UserId: userId, + IdentityStoreId: identityStoreId, + Timestamp: timestamp, + FileName: fileName, + FileExtension: strings.TrimPrefix(filepath.Ext(fileName), "."), + // Present on completion records but never on chat records. + HasCustomization: record.Request.CustomizationArn != nil, + CompletionsCount: len(record.Response.Completions), + ReturnedCharCount: charCount, + ReturnedLineCount: lineCount, + LeftContextLength: len(deref(record.Request.LeftContext)), + RightContextLength: len(deref(record.Request.RightContext)), + }) + } + + return result, nil +} + +func gunzip(gzData []byte) ([]byte, errors.Error) { + reader, err := gzip.NewReader(bytes.NewReader(gzData)) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to open gzip reader") + } + defer reader.Close() + + raw, err := io.ReadAll(reader) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to decompress log file") + } + return raw, nil +} + +func deref(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/backend/plugins/kiro/tasks/log_parser_test.go b/backend/plugins/kiro/tasks/log_parser_test.go new file mode 100644 index 00000000000..f883569fecd --- /dev/null +++ b/backend/plugins/kiro/tasks/log_parser_test.go @@ -0,0 +1,319 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "compress/gzip" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fixtures are real log files whose text content was replaced but whose exact +// string lengths and line structure were preserved, because those are the +// values the parser derives and this test asserts. +func loadLogFixture(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "logs", name)) + require.NoError(t, err) + return data +} + +func gzipBytes(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + _, err := w.Write([]byte(s)) + require.NoError(t, err) + require.NoError(t, w.Close()) + return buf.Bytes() +} + +// An empty prompt means the agent continued on its own after a tool call rather +// than the user speaking. This is the majority case - roughly 71% of sampled +// records - and it must not be confused with a user turn. +func TestParseChatLog_EmptyPrompt(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_01_empty_prompt.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.False(t, l.HasPrompt) + assert.Equal(t, 0, l.PromptLength) + // Hashing the empty string would give ~71% of rows one shared hash and + // destroy the rework signal, so it stays NULL. + assert.Nil(t, l.PromptSha256) + // Both heuristics read prompt text, so without a prompt they are unknown + // rather than false. + assert.Nil(t, l.HasSteering) + assert.Nil(t, l.IsSpecMode) + + assert.Equal(t, "64d13ea7-dff5-4563-9285-6a9e351e87a0", l.RequestId) + assert.Equal(t, "MANUAL", l.ChatTriggerType) + require.NotNil(t, l.ModelId) + assert.Equal(t, "claude-opus-5", *l.ModelId) + assert.Equal(t, 259, l.ResponseLength) + + // Log records always carry the identity-store prefix; stripping it is what + // lets this join against the report table. + assert.Equal(t, "11111111-1111-4111-8111-111111111111", l.UserId) + assert.Equal(t, "d-1234567890", l.IdentityStoreId) + + // Nanosecond input truncated for DATETIME(6). + assert.Equal(t, 2026, l.Timestamp.Year()) + assert.Equal(t, 23, l.Timestamp.Hour()) + assert.Equal(t, 0, l.Timestamp.Nanosecond()%1000) + + // Documented but never observed, so they stay NULL - their absence is why + // the S3 logs cannot group interactions into sessions. + assert.Nil(t, l.ConversationId) + assert.Nil(t, l.UtteranceId) +} + +func TestParseChatLog_WithPrompt(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_02_with_prompt.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.True(t, l.HasPrompt) + assert.Equal(t, 1363, l.PromptLength) + require.NotNil(t, l.PromptSha256) + assert.Len(t, *l.PromptSha256, 64) + // With a prompt present the heuristics carry a real verdict. + require.NotNil(t, l.HasSteering) + require.NotNil(t, l.IsSpecMode) + assert.Equal(t, 64, l.ResponseLength) +} + +// Kiro packs one or two records per file, so the array is always walked. +func TestParseChatLog_TwoRecords(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_03_two_records.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 2) + + assert.NotEqual(t, logs[0].RequestId, logs[1].RequestId) + // A zero-length assistant response is real data, not a parse failure. + assert.Equal(t, 0, logs[0].ResponseLength) + assert.Equal(t, 117, logs[1].ResponseLength) +} + +// modelId is absent on more than half of all records. Storing an empty string +// instead of NULL would invent a phantom model accounting for most of the +// traffic. +func TestParseChatLog_MissingModelId(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_04_no_model_id.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.Nil(t, l.ModelId, "an absent modelId must be NULL, never an empty string") + // This record does carry followupPrompts, which a single-day sample had + // suggested never appears. + assert.True(t, l.HasFollowupPrompts) + assert.True(t, l.HasPrompt) + assert.Equal(t, 175, l.PromptLength) +} + +func TestParseChatLog_PromptHashing(t *testing.T) { + // The same prompt text must hash identically, which is what makes repeated + // submission detectable as a rework signal. + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"fix the retry logic","chatTriggerType":"MANUAL",` + + `"userId":"d-abc.user-1","timeStamp":"2026-07-27T23:03:29.027400929Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"%s"}}]}` + + first, err := ParseChatLog(gzipBytes(t, fmt.Sprintf(body, "req-1")), testConnectionId, testScopeId) + require.Nil(t, err) + second, err := ParseChatLog(gzipBytes(t, fmt.Sprintf(body, "req-2")), testConnectionId, testScopeId) + require.Nil(t, err) + + require.Len(t, first, 1) + require.Len(t, second, 1) + require.NotNil(t, first[0].PromptSha256) + require.NotNil(t, second[0].PromptSha256) + assert.Equal(t, *first[0].PromptSha256, *second[0].PromptSha256) + assert.NotEqual(t, first[0].RequestId, second[0].RequestId) +} + +func TestParseChatLog_Heuristics(t *testing.T) { + tests := []struct { + name string + prompt string + wantHasSteering bool + wantIsSpecMode bool + }{ + {"steering reference", "please follow .kiro/steering/style.md", true, false}, + {"spec reference", "implement .kiro/specs/auth/tasks.md", false, true}, + {"both", "read .kiro/steering and .kiro/specs", true, true}, + {"neither", "just fix this bug", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"` + tt.prompt + `","chatTriggerType":"MANUAL",` + + `"userId":"d-abc.user-1","timeStamp":"2026-07-27T23:03:29Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"r1"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + require.NotNil(t, logs[0].HasSteering) + require.NotNil(t, logs[0].IsSpecMode) + assert.Equal(t, tt.wantHasSteering, *logs[0].HasSteering) + assert.Equal(t, tt.wantIsSpecMode, *logs[0].IsSpecMode) + }) + } +} + +func TestParseCompletionLog_NonEmpty(t *testing.T) { + logs, err := ParseCompletionLog(loadLogFixture(t, "completion_01_non_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.Equal(t, "e0d1e760-a907-47d4-baa5-93df0f38b274", l.RequestId) + // A bare file name with no directory path, so it cannot be resolved to a + // unique repository file. + assert.Equal(t, "mcp.json", l.FileName) + assert.Equal(t, "json", l.FileExtension) + assert.Equal(t, 1, l.CompletionsCount) + assert.Equal(t, 347, l.ReturnedCharCount) + assert.Equal(t, 15, l.ReturnedLineCount) + assert.Equal(t, 5324, l.LeftContextLength) + assert.Equal(t, 12, l.RightContextLength) + assert.False(t, l.HasCustomization) + assert.Equal(t, "11111111-1111-4111-8111-111111111111", l.UserId) +} + +// A completion record is written when the suggestion is requested, not when it +// is accepted, so an empty array is normal. The record is the denominator and +// must still be stored. +func TestParseCompletionLog_EmptyCompletions(t *testing.T) { + logs, err := ParseCompletionLog(loadLogFixture(t, "completion_02_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1, "a record with no completions is still stored") + + l := logs[0] + assert.Equal(t, 0, l.CompletionsCount) + assert.Equal(t, 0, l.ReturnedCharCount) + assert.Equal(t, 0, l.ReturnedLineCount) + // Context was still sent, which is what distinguishes this from a truncated + // record. + assert.Equal(t, 5323, l.LeftContextLength) +} + +func TestParseCompletionLog_LineCounting(t *testing.T) { + tests := []struct { + name string + completions string + wantLines int + wantChars int + }{ + {"single line", `["abc"]`, 1, 3}, + {"two lines", `["a\nb"]`, 2, 3}, + {"trailing newline counts the empty final line", `["a\n"]`, 2, 2}, + {"two completions summed", `["a\nb","c"]`, 3, 4}, + {"empty array", `[]`, 0, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := `{"records":[{"generateCompletionsEventRequest":` + + `{"fileName":"f.ts","leftContext":"","rightContext":"",` + + `"userId":"d-abc.user-1","timeStamp":"2026-03-19T13:49:58Z"},` + + `"generateCompletionsEventResponse":{"completions":` + tt.completions + `,"requestId":"r1"}}]}` + logs, err := ParseCompletionLog(gzipBytes(t, body), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + assert.Equal(t, tt.wantLines, logs[0].ReturnedLineCount) + assert.Equal(t, tt.wantChars, logs[0].ReturnedCharCount) + }) + } +} + +func TestParseCompletionLog_Customization(t *testing.T) { + // customizationArn appears on completion records but never on chat records. + body := `{"records":[{"generateCompletionsEventRequest":` + + `{"fileName":"f.ts","userId":"d-abc.u1","timeStamp":"2026-03-19T13:49:58Z",` + + `"customizationArn":"arn:aws:codewhisperer:us-east-1:1:customization/abc"},` + + `"generateCompletionsEventResponse":{"completions":[],"requestId":"r1"}}]}` + logs, err := ParseCompletionLog(gzipBytes(t, body), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + assert.True(t, logs[0].HasCustomization) +} + +func TestParseLog_EdgeCases(t *testing.T) { + t.Run("empty records array", func(t *testing.T) { + logs, err := ParseChatLog(gzipBytes(t, `{"records":[]}`), testConnectionId, testScopeId) + assert.Nil(t, err) + assert.Empty(t, logs) + }) + + t.Run("record without requestId is skipped", func(t *testing.T) { + // requestId is the primary key, so such a record cannot be stored or + // deduplicated. + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"","userId":"d-abc.u1","timeStamp":"2026-07-27T23:03:29Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + assert.Nil(t, err) + assert.Empty(t, logs) + }) + + t.Run("record missing the response half is skipped", func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"x","userId":"d-abc.u1","timeStamp":"2026-07-27T23:03:29Z"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + assert.Nil(t, err) + assert.Empty(t, logs) + }) + + t.Run("bad timestamp is an error", func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"","userId":"d-abc.u1","timeStamp":"not-a-time"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"r1"}}]}` + _, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) + + t.Run("non-gzip input is an error", func(t *testing.T) { + _, err := ParseChatLog([]byte("not gzipped"), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) + + t.Run("malformed json is an error", func(t *testing.T) { + _, err := ParseChatLog(gzipBytes(t, `{"records":`), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) + + t.Run("connection and scope are propagated", func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"","userId":"d-abc.u1","timeStamp":"2026-07-27T23:03:29Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"r1"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), 42, "scope-x") + require.Nil(t, err) + require.Len(t, logs, 1) + assert.Equal(t, uint64(42), logs[0].ConnectionId) + assert.Equal(t, "scope-x", logs[0].ScopeId) + }) +} diff --git a/backend/plugins/kiro/tasks/s3_client.go b/backend/plugins/kiro/tasks/s3_client.go new file mode 100644 index 00000000000..a0e2d19ba5a --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_client.go @@ -0,0 +1,217 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "io" + "sort" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// S3API is the subset of the S3 API this plugin uses, declared as an interface +// so collectors and extractors can be tested without AWS. +type S3API interface { + ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) + GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) +} + +// KiroS3Client is bound to exactly one bucket. +// +// Kiro recommends keeping interaction logs in a bucket separate from the +// activity reports, and the two may carry different KMS keys and IAM +// conditions. One client per bucket keeps those permission boundaries distinct, +// so an access failure points at a specific bucket instead of an ambiguous +// request. +type KiroS3Client struct { + S3 S3API + Bucket string +} + +// KiroS3Clients holds the report and log clients for a connection. When no +// separate log bucket is configured both fields address the same bucket, so +// single-bucket and dual-bucket setups take the same code path everywhere else. +type KiroS3Clients struct { + Report *KiroS3Client + PromptLog *KiroS3Client +} + +// NewKiroS3Clients builds the client pair for a connection. +func NewKiroS3Clients(connection *models.KiroConnection) (*KiroS3Clients, errors.Error) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(connection.Region), + Credentials: credentials.NewStaticCredentials(connection.AccessKeyId, connection.SecretAccessKey, ""), + }) + if err != nil { + return nil, errors.Convert(err) + } + + // A single S3 service client can address both buckets; the split is at the + // KiroS3Client level, which pins the bucket name. + svc := s3.New(sess) + + return &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: connection.Bucket}, + PromptLog: &KiroS3Client{S3: svc, Bucket: connection.GetPromptLogBucket()}, + }, nil +} + +// ForFileType returns the client that owns a given file type. +func (c *KiroS3Clients) ForFileType(fileType string) *KiroS3Client { + if fileType == models.FileTypeReport { + return c.Report + } + return c.PromptLog +} + +// Buckets returns the distinct buckets in use - one entry when reports and logs +// share a bucket, two when they do not. +func (c *KiroS3Clients) Buckets() []string { + if c.Report.Bucket == c.PromptLog.Bucket { + return []string{c.Report.Bucket} + } + return []string{c.Report.Bucket, c.PromptLog.Bucket} +} + +// ListSubPrefixes returns the immediate child "directories" under a prefix. +// +// Kiro's export layout is fully self-describing - accounts, years and months all +// appear as path segments - so this is what lets a scope be picked from what +// actually exists instead of typed by hand. A mistyped prefix is otherwise +// indistinguishable from a month with no data: collection succeeds and finds +// nothing either way. +// +// Uses a delimiter so S3 returns only the segment names, not every object +// beneath them. +func (c *KiroS3Client) ListSubPrefixes(prefix string) ([]string, errors.Error) { + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + var names []string + var continuationToken *string + for { + output, err := c.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(c.Bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), + ContinuationToken: continuationToken, + }) + if err != nil { + return nil, errors.Convert(err) + } + + for _, common := range output.CommonPrefixes { + if common.Prefix == nil { + continue + } + // Strip the queried prefix and the trailing slash to leave just the + // segment name. + name := strings.TrimSuffix(strings.TrimPrefix(*common.Prefix, prefix), "/") + if name != "" { + names = append(names, name) + } + } + + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + continuationToken = output.NextContinuationToken + } + + sort.Strings(names) + return names, nil +} + +// CountObjects reports how many collectable objects sit under a prefix. +// +// This is what turns "did I get the path right?" into an answerable question: +// the connection test reports these counts per stream, so a wrong prefix shows +// as zero before any scope is created. +// +// Counting stops at limit to keep the check cheap; the returned bool reports +// whether more objects remain. +func (c *KiroS3Client) CountObjects(prefix string, limit int) (int, bool, errors.Error) { + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + count := 0 + var continuationToken *string + for { + output, err := c.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(c.Bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if err != nil { + return 0, false, errors.Convert(err) + } + + for _, object := range output.Contents { + if object.Key == nil { + continue + } + // Same filter the collector applies, so the count reflects what + // would actually be collected rather than every object present. + if !strings.HasSuffix(*object.Key, ".csv") && !strings.HasSuffix(*object.Key, ".json.gz") { + continue + } + count++ + if limit > 0 && count >= limit { + return count, true, nil + } + } + + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + continuationToken = output.NextContinuationToken + } + + return count, false, nil +} + +// GetObjectBytes downloads an object in full. +// +// Objects are small - roughly 700 bytes for a chat log, a few KB for a +// completion log, and under 1 KB for a report CSV - so streaming would add +// complexity without saving memory. +func (c *KiroS3Client) GetObjectBytes(key string) ([]byte, errors.Error) { + output, err := c.S3.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(c.Bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, errors.Convert(err) + } + defer output.Body.Close() + + data, readErr := io.ReadAll(output.Body) + if readErr != nil { + return nil, errors.Convert(readErr) + } + return data, nil +} diff --git a/backend/plugins/kiro/tasks/s3_client_test.go b/backend/plugins/kiro/tasks/s3_client_test.go new file mode 100644 index 00000000000..08c910fcf20 --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_client_test.go @@ -0,0 +1,237 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/aws/aws-sdk-go/service/identitystore" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// mockS3 records the bucket each call addressed, so tests can prove a request +// went to the right one. +type mockS3 struct { + getObjectBody string + getObjectErr error + listOutputs []*s3.ListObjectsV2Output + listCallIdx int + seenGetBuckets []string + seenListBuckets []string + seenGetKeys []string +} + +func (m *mockS3) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + m.seenGetBuckets = append(m.seenGetBuckets, *input.Bucket) + m.seenGetKeys = append(m.seenGetKeys, *input.Key) + if m.getObjectErr != nil { + return nil, m.getObjectErr + } + return &s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader([]byte(m.getObjectBody))), + }, nil +} + +func (m *mockS3) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + m.seenListBuckets = append(m.seenListBuckets, *input.Bucket) + if m.listCallIdx >= len(m.listOutputs) { + return &s3.ListObjectsV2Output{}, nil + } + out := m.listOutputs[m.listCallIdx] + m.listCallIdx++ + return out, nil +} + +// The fallback rules mean a single-bucket deployment (what real exports look +// like today) and the dual-bucket layout Kiro recommends both work without +// branching in collectors or extractors. +func TestKiroS3Clients_BucketRouting(t *testing.T) { + t.Run("single bucket routes both file kinds to the same bucket", func(t *testing.T) { + svc := &mockS3{} + clients := &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: "one-bucket"}, + PromptLog: &KiroS3Client{S3: svc, Bucket: "one-bucket"}, + } + + assert.Equal(t, "one-bucket", clients.ForFileType(models.FileTypeReport).Bucket) + assert.Equal(t, "one-bucket", clients.ForFileType(models.FileTypeChatLog).Bucket) + assert.Equal(t, "one-bucket", clients.ForFileType(models.FileTypeCompletionLog).Bucket) + // Deduplicated, so a connection test checks access once rather than twice. + assert.Equal(t, []string{"one-bucket"}, clients.Buckets()) + }) + + t.Run("separate buckets route by file type", func(t *testing.T) { + svc := &mockS3{} + clients := &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: "reports"}, + PromptLog: &KiroS3Client{S3: svc, Bucket: "logs"}, + } + + assert.Equal(t, "reports", clients.ForFileType(models.FileTypeReport).Bucket) + assert.Equal(t, "logs", clients.ForFileType(models.FileTypeChatLog).Bucket) + assert.Equal(t, "logs", clients.ForFileType(models.FileTypeCompletionLog).Bucket) + assert.Equal(t, []string{"reports", "logs"}, clients.Buckets()) + }) + + // An unrecognized file type must not silently read from the report bucket, + // where it would find nothing; log data is the larger and more likely case. + t.Run("unknown file type falls to the log bucket", func(t *testing.T) { + svc := &mockS3{} + clients := &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: "reports"}, + PromptLog: &KiroS3Client{S3: svc, Bucket: "logs"}, + } + assert.Equal(t, "logs", clients.ForFileType("something-new").Bucket) + }) +} + +func TestNewKiroS3Clients_FallbackFromConnection(t *testing.T) { + t.Run("no prompt log bucket falls back to the report bucket", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "kiro-export-test", + }} + clients, err := NewKiroS3Clients(conn) + require.Nil(t, err) + assert.Equal(t, "kiro-export-test", clients.Report.Bucket) + assert.Equal(t, "kiro-export-test", clients.PromptLog.Bucket) + assert.Len(t, clients.Buckets(), 1) + }) + + t.Run("explicit prompt log bucket is used", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "reports-bucket", + PromptLogBucket: "logs-bucket", + }} + clients, err := NewKiroS3Clients(conn) + require.Nil(t, err) + assert.Equal(t, "reports-bucket", clients.Report.Bucket) + assert.Equal(t, "logs-bucket", clients.PromptLog.Bucket) + assert.Len(t, clients.Buckets(), 2) + }) +} + +func TestKiroS3Client_GetObjectBytes(t *testing.T) { + t.Run("reads the body and addresses the bound bucket", func(t *testing.T) { + svc := &mockS3{getObjectBody: "hello"} + client := &KiroS3Client{S3: svc, Bucket: "my-bucket"} + + data, err := client.GetObjectBytes("some/key.csv") + require.Nil(t, err) + assert.Equal(t, "hello", string(data)) + assert.Equal(t, []string{"my-bucket"}, svc.seenGetBuckets) + assert.Equal(t, []string{"some/key.csv"}, svc.seenGetKeys) + }) + + t.Run("propagates an S3 error", func(t *testing.T) { + svc := &mockS3{getObjectErr: errors.New("access denied")} + client := &KiroS3Client{S3: svc, Bucket: "my-bucket"} + + _, err := client.GetObjectBytes("k") + assert.NotNil(t, err) + }) +} + +// mockIdentityStore lets the optional display-name path be exercised without +// AWS. +type mockIdentityStore struct { + displayName *string + err error +} + +func (m *mockIdentityStore) DescribeUser(*identitystore.DescribeUserInput) (*identitystore.DescribeUserOutput, error) { + if m.err != nil { + return nil, m.err + } + return &identitystore.DescribeUserOutput{DisplayName: m.displayName}, nil +} + +func TestKiroIdentityClient_ResolveDisplayName(t *testing.T) { + name := "Some Developer" + + t.Run("resolves a display name", func(t *testing.T) { + client := &KiroIdentityClient{IdentityStore: &mockIdentityStore{displayName: &name}, StoreId: "d-1"} + got, err := client.ResolveDisplayName("user-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, name, *got) + }) + + // The column exists for human readability; falling back to the raw id would + // make an unresolved value look like a resolved one. + t.Run("empty display name yields nil, not the user id", func(t *testing.T) { + empty := "" + client := &KiroIdentityClient{IdentityStore: &mockIdentityStore{displayName: &empty}, StoreId: "d-1"} + got, err := client.ResolveDisplayName("user-1") + require.NoError(t, err) + assert.Nil(t, got) + }) + + // Identity Store is optional, so an unconfigured client must be safe to + // call rather than something every caller has to nil-check. + t.Run("nil client is safe to call", func(t *testing.T) { + var client *KiroIdentityClient + got, err := client.ResolveDisplayName("user-1") + require.NoError(t, err) + assert.Nil(t, got) + }) + + t.Run("error surfaces but yields no name", func(t *testing.T) { + client := &KiroIdentityClient{IdentityStore: &mockIdentityStore{err: errors.New("throttled")}, StoreId: "d-1"} + got, err := client.ResolveDisplayName("user-1") + assert.Error(t, err) + assert.Nil(t, got) + }) +} + +func TestNewKiroIdentityClient_OptionalConfiguration(t *testing.T) { + // Missing configuration is not an error: collection works fully without + // display names because identity comes from the report's email column. + for _, tt := range []struct { + name string + conn models.KiroConn + }{ + {"neither set", models.KiroConn{}}, + {"only store id", models.KiroConn{IdentityStoreId: "d-1"}}, + {"only region", models.KiroConn{IdentityStoreRegion: "us-east-1"}}, + } { + t.Run(tt.name, func(t *testing.T) { + client, err := NewKiroIdentityClient(&models.KiroConnection{KiroConn: tt.conn}) + require.NoError(t, err) + assert.Nil(t, client) + }) + } + + t.Run("fully configured returns a client", func(t *testing.T) { + client, err := NewKiroIdentityClient(&models.KiroConnection{KiroConn: models.KiroConn{ + IdentityStoreId: "d-1234567890", + IdentityStoreRegion: "us-east-1", + }}) + require.NoError(t, err) + require.NotNil(t, client) + assert.Equal(t, "d-1234567890", client.StoreId) + }) +} diff --git a/backend/plugins/kiro/tasks/s3_file_collector.go b/backend/plugins/kiro/tasks/s3_file_collector.go new file mode 100644 index 00000000000..552292e3737 --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_file_collector.go @@ -0,0 +1,171 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "path" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var _ plugin.SubTaskEntryPoint = CollectKiroS3Files + +// CollectKiroS3FilesMeta discovers which S3 objects exist for a scope. +var CollectKiroS3FilesMeta = plugin.SubTaskMeta{ + Name: "collectKiroS3Files", + EntryPoint: CollectKiroS3Files, + EnabledByDefault: true, + Description: "List Kiro export objects in S3 and record them for extraction", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +// CollectKiroS3Files lists every relevant object under the scope's prefixes and +// records the ones not seen before. +// +// Work is batched per listing page rather than per object. A single S3 page +// holds up to 1000 keys, so one SELECT and one INSERT per page replaces two +// round trips per file - at tens of thousands of objects a day that is the +// difference between dozens of queries and tens of thousands. +func CollectKiroS3Files(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*KiroTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + + taskCtx.SetProgress(0, -1) + + for _, spec := range data.Prefixes { + client := data.S3Clients.ForFileType(spec.FileType) + prefix := spec.Prefix + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + logger.Info("scanning s3://%s/%s for %s files", client.Bucket, prefix, spec.FileType) + + var continuationToken *string + for { + output, listErr := client.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(client.Bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if listErr != nil { + return errors.Convert(listErr) + } + + candidates := collectCandidates(output, client.Bucket, spec, data.Options) + inserted, saveErr := saveNewFileMeta(db, data.Options.ConnectionId, candidates) + if saveErr != nil { + return saveErr + } + taskCtx.IncProgress(inserted) + + // IsTruncated is a pointer; dereferencing it unguarded panics on an + // empty response. + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + continuationToken = output.NextContinuationToken + } + } + + return nil +} + +// collectCandidates turns one listing page into file metadata rows. +// +// Only .csv and .json.gz are kept. That filter also excludes the small +// extension-less objects AWS writes at the KiroLogs root as permission probes. +func collectCandidates(output *s3.ListObjectsV2Output, bucket string, spec PrefixSpec, options *KiroOptions) []*models.KiroS3FileMeta { + candidates := make([]*models.KiroS3FileMeta, 0, len(output.Contents)) + for _, object := range output.Contents { + if object.Key == nil { + continue + } + key := *object.Key + if !strings.HasSuffix(key, ".csv") && !strings.HasSuffix(key, ".json.gz") { + continue + } + candidates = append(candidates, &models.KiroS3FileMeta{ + ConnectionId: options.ConnectionId, + S3Path: key, + // Basename only. The full key lives in S3Path, which is sized for + // it; putting a full key here would eventually overflow the column. + FileName: path.Base(key), + Bucket: bucket, + ScopeId: options.ScopeId, + FileType: spec.FileType, + Processed: false, + }) + } + return candidates +} + +// saveNewFileMeta inserts the rows that are not already recorded, returning how +// many were added. +// +// The existence check queries by (connection_id, s3_path), which is exactly the +// primary key. Querying on an unindexed column here would turn each page into a +// full table scan and the task would never finish at scale. +func saveNewFileMeta(db dal.Dal, connectionId uint64, candidates []*models.KiroS3FileMeta) (int, errors.Error) { + if len(candidates) == 0 { + return 0, nil + } + + paths := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + paths = append(paths, candidate.S3Path) + } + + var existingRows []models.KiroS3FileMeta + err := db.All(&existingRows, + dal.Select("s3_path"), + dal.From(&models.KiroS3FileMeta{}), + dal.Where("connection_id = ? AND s3_path IN ?", connectionId, paths), + ) + if err != nil { + return 0, errors.Default.Wrap(err, "failed to query existing kiro file metadata") + } + + existing := make(map[string]struct{}, len(existingRows)) + for _, row := range existingRows { + existing[row.S3Path] = struct{}{} + } + + fresh := make([]*models.KiroS3FileMeta, 0, len(candidates)) + for _, candidate := range candidates { + if _, seen := existing[candidate.S3Path]; seen { + continue + } + fresh = append(fresh, candidate) + } + if len(fresh) == 0 { + return 0, nil + } + + if err := db.Create(fresh); err != nil { + return 0, errors.Default.Wrap(err, "failed to record kiro file metadata") + } + return len(fresh), nil +} diff --git a/backend/plugins/kiro/tasks/s3_file_collector_test.go b/backend/plugins/kiro/tasks/s3_file_collector_test.go new file mode 100644 index 00000000000..e45b26044fb --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_file_collector_test.go @@ -0,0 +1,215 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "os" + "regexp" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +func listOutput(truncated bool, keys ...string) *s3.ListObjectsV2Output { + contents := make([]*s3.Object, 0, len(keys)) + for _, k := range keys { + key := k + contents = append(contents, &s3.Object{Key: &key}) + } + return &s3.ListObjectsV2Output{ + Contents: contents, + IsTruncated: aws.Bool(truncated), + } +} + +func TestCollectCandidates_FileFiltering(t *testing.T) { + spec := PrefixSpec{FileType: models.FileTypeChatLog} + options := &KiroOptions{ConnectionId: 1, ScopeId: "s1"} + + t.Run("keeps csv and json.gz only", func(t *testing.T) { + output := listOutput(false, + "p/report.csv", + "p/log.json.gz", + // AWS writes small extension-less objects at the KiroLogs root as + // permission probes; they must not enter the work queue. + "p/26404955-bf00-40d3-b713-43d18edf0638", + "p/notes.txt", + "p/archive.zip", + ) + candidates := collectCandidates(output, "bkt", spec, options) + require.Len(t, candidates, 2) + assert.Equal(t, "p/report.csv", candidates[0].S3Path) + assert.Equal(t, "p/log.json.gz", candidates[1].S3Path) + }) + + t.Run("stores basename separately from the full key", func(t *testing.T) { + key := "logging/AWSLogs/123456789012/KiroLogs/GenerateAssistantResponse/us-east-1/2026/07/27/23/" + + "123456789012_GenerateAssistantResponse_202607272303_3tbIeIrGJNDFbfVx.json.gz" + candidates := collectCandidates(listOutput(false, key), "bkt", spec, options) + require.Len(t, candidates, 1) + + // The full key goes in S3Path, which is sized for it. FileName holds + // only the basename - a full key there would eventually overflow. + assert.Equal(t, key, candidates[0].S3Path) + assert.Equal(t, "123456789012_GenerateAssistantResponse_202607272303_3tbIeIrGJNDFbfVx.json.gz", + candidates[0].FileName) + assert.Less(t, len(candidates[0].FileName), 255) + }) + + t.Run("records bucket, scope and file type", func(t *testing.T) { + candidates := collectCandidates(listOutput(false, "p/a.csv"), "my-bucket", spec, options) + require.Len(t, candidates, 1) + assert.Equal(t, "my-bucket", candidates[0].Bucket) + assert.Equal(t, "s1", candidates[0].ScopeId) + assert.Equal(t, models.FileTypeChatLog, candidates[0].FileType) + assert.Equal(t, uint64(1), candidates[0].ConnectionId) + assert.False(t, candidates[0].Processed) + }) + + t.Run("nil key is skipped", func(t *testing.T) { + output := &s3.ListObjectsV2Output{ + Contents: []*s3.Object{{Key: nil}, {Key: aws.String("p/a.csv")}}, + IsTruncated: aws.Bool(false), + } + candidates := collectCandidates(output, "bkt", spec, options) + assert.Len(t, candidates, 1) + }) + + t.Run("empty page yields nothing", func(t *testing.T) { + candidates := collectCandidates(listOutput(false), "bkt", spec, options) + assert.Empty(t, candidates) + }) +} + +func TestBuildPrefixes(t *testing.T) { + // Paths verified against real exports. + t.Run("single bucket layout", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "kiro-export-test", + ReportPrefix: "user-report", + PromptLogPrefix: "logging", + }} + prefixes := BuildPrefixes(conn, "123456789012", "2026/07") + + require.Len(t, prefixes, 3) + assert.Equal(t, + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/2026/07", + prefixes[0].Prefix) + assert.Equal(t, models.FileTypeReport, prefixes[0].FileType) + assert.Equal(t, + "logging/AWSLogs/123456789012/KiroLogs/GenerateAssistantResponse/us-east-1/2026/07", + prefixes[1].Prefix) + assert.Equal(t, models.FileTypeChatLog, prefixes[1].FileType) + assert.Equal(t, + "logging/AWSLogs/123456789012/KiroLogs/GenerateCompletions/us-east-1/2026/07", + prefixes[2].Prefix) + assert.Equal(t, models.FileTypeCompletionLog, prefixes[2].FileType) + + for _, p := range prefixes { + assert.Equal(t, "kiro-export-test", p.Bucket) + } + }) + + t.Run("defaults apply when prefixes are unset", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "b", + }} + prefixes := BuildPrefixes(conn, "acct", "2026/07") + assert.Contains(t, prefixes[0].Prefix, "user-report/AWSLogs/acct/KiroLogs/user_report") + assert.Contains(t, prefixes[1].Prefix, "logging/AWSLogs/acct/KiroLogs/GenerateAssistantResponse") + }) + + t.Run("separate buckets are assigned per file type", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "reports", + PromptLogBucket: "logs", + }} + prefixes := BuildPrefixes(conn, "acct", "2026/07") + assert.Equal(t, "reports", prefixes[0].Bucket) + assert.Equal(t, "logs", prefixes[1].Bucket) + assert.Equal(t, "logs", prefixes[2].Bucket) + }) + + // A nil month widens the scope to the whole year, which is how a year-long + // backfill is expressed. + t.Run("year-only time path", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{Region: "us-east-1", Bucket: "b"}} + prefixes := BuildPrefixes(conn, "acct", "2026") + assert.True(t, regexp.MustCompile(`/us-east-1/2026$`).MatchString(prefixes[0].Prefix)) + }) +} + +func TestWorkerCount(t *testing.T) { + assert.Equal(t, DefaultWorkerCount, (&KiroTaskData{Options: &KiroOptions{}}).WorkerCount()) + assert.Equal(t, 5, (&KiroTaskData{Options: &KiroOptions{WorkerCount: 5}}).WorkerCount()) + // A zero or negative override is ignored rather than disabling concurrency. + assert.Equal(t, DefaultWorkerCount, (&KiroTaskData{Options: &KiroOptions{WorkerCount: -1}}).WorkerCount()) + assert.Equal(t, DefaultWorkerCount, (&KiroTaskData{}).WorkerCount()) +} + +// This guards the predecessor defect that motivated the primary key choice: it queried +// its cursor table by s3_path while keying it on file_name, so the lookup falls +// back to scanning every row for the connection and collection never finishes at +// scale. The failure mode is a task that hangs rather than an error, so it is +// worth asserting structurally instead of hoping a reviewer notices. +func TestFileMetaQueriesMatchPrimaryKey(t *testing.T) { + pkColumns := primaryKeyColumns(t, "s3_file_meta.go") + require.Equal(t, []string{"ConnectionId", "S3Path"}, pkColumns, + "the cursor table must be keyed on the connection and the full object path") + + source, err := os.ReadFile("s3_file_collector.go") + require.NoError(t, err) + + whereClauses := regexp.MustCompile(`dal\.Where\(\s*"([^"]+)"`).FindAllStringSubmatch(string(source), -1) + require.NotEmpty(t, whereClauses, "expected at least one filtered query") + + for _, clause := range whereClauses { + condition := clause[1] + assert.Contains(t, condition, "connection_id", + "every cursor query must filter on connection_id, the first key column") + assert.Contains(t, condition, "s3_path", + "every cursor query must filter on s3_path, the second key column") + assert.NotContains(t, condition, "file_name", + "file_name is not indexed and must never appear in a lookup") + } +} + +// primaryKeyColumns extracts the fields tagged as primary keys from a model +// file, in declaration order - which is also the order of the composite index. +func primaryKeyColumns(t *testing.T, modelFile string) []string { + t.Helper() + source, err := os.ReadFile("../models/" + modelFile) + require.NoError(t, err) + + fieldRe := regexp.MustCompile(`(?m)^\s*([A-Z][A-Za-z0-9]*)\s+\S+\s+` + "`" + `[^` + "`" + `]*primaryKey[^` + "`" + `]*` + "`") + matches := fieldRe.FindAllStringSubmatch(string(source), -1) + + columns := make([]string, 0, len(matches)) + for _, m := range matches { + columns = append(columns, m[1]) + } + return columns +} diff --git a/backend/plugins/kiro/tasks/task_data.go b/backend/plugins/kiro/tasks/task_data.go new file mode 100644 index 00000000000..c8b504f1d18 --- /dev/null +++ b/backend/plugins/kiro/tasks/task_data.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// DefaultWorkerCount is how many objects are fetched concurrently. +// +// Collection is bound by request count, not bandwidth: a single user can +// produce ~600 log objects on a busy day, each under a kilobyte. S3 sustains +// thousands of GETs per second per prefix, so 20 is well within limits while +// cutting a peak day from tens of minutes to a couple. +const DefaultWorkerCount = 20 + +// KiroOptions are the blueprint-supplied task options. +type KiroOptions struct { + ConnectionId uint64 `json:"connectionId"` + ScopeId string `json:"scopeId"` + AccountId string `json:"accountId"` + Year int `json:"year"` + Month *int `json:"month"` + // WorkerCount overrides DefaultWorkerCount when set above zero. + WorkerCount int `json:"workerCount"` +} + +// PrefixSpec is one S3 location to scan, along with the kind of file found +// there and which bucket holds it. +type PrefixSpec struct { + Bucket string + Prefix string + FileType string +} + +// KiroTaskData is shared by every subtask in a run. +type KiroTaskData struct { + Options *KiroOptions + Connection *models.KiroConnection + S3Clients *KiroS3Clients + IdentityClient *KiroIdentityClient + // Prefixes are the locations to scan, precomputed so the collector does not + // re-derive paths. + Prefixes []PrefixSpec +} + +// WorkerCount returns the effective concurrency for this run. +func (d *KiroTaskData) WorkerCount() int { + if d.Options != nil && d.Options.WorkerCount > 0 { + return d.Options.WorkerCount + } + return DefaultWorkerCount +} + +// BuildPrefixes derives the three S3 locations a scope covers. +// +// Layout confirmed against real exports: +// +// {bucket}/{reportPrefix}/AWSLogs/{acct}/KiroLogs/user_report/{region}/{y}/{m} +// {bucket}/{logPrefix}/AWSLogs/{acct}/KiroLogs/GenerateAssistantResponse/{region}/{y}/{m} +// {bucket}/{logPrefix}/AWSLogs/{acct}/KiroLogs/GenerateCompletions/{region}/{y}/{m} +// +// The report path's hour segment is always 00 (reports are written at 02:00 +// UTC) while log paths carry a real hour, but neither is included here: the +// prefix stops at the month so a scope lists the whole period in one sweep. +func BuildPrefixes(connection *models.KiroConnection, accountId string, timePath string) []PrefixSpec { + region := connection.Region + reportBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", connection.GetReportPrefix(), accountId) + logBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", connection.GetPromptLogPrefix(), accountId) + + return []PrefixSpec{ + { + Bucket: connection.Bucket, + Prefix: fmt.Sprintf("%s/user_report/%s/%s", reportBase, region, timePath), + FileType: models.FileTypeReport, + }, + { + Bucket: connection.GetPromptLogBucket(), + Prefix: fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", logBase, region, timePath), + FileType: models.FileTypeChatLog, + }, + { + Bucket: connection.GetPromptLogBucket(), + Prefix: fmt.Sprintf("%s/GenerateCompletions/%s/%s", logBase, region, timePath), + FileType: models.FileTypeCompletionLog, + }, + } +} diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_01_empty_prompt.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_01_empty_prompt.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a713fdaf95d50916286ecbd51876db42b1a8f3bb GIT binary patch literal 274 zcmV+t0qy=DiwFP!00002|K(CmZ-gKa{VN;JWrLt1-aPcsr0rpAFWq<`GFp>WC}7jZ z_}>e(@vmgttw#uRMa9&Uk6Ikh?a&B;5PW}YW=*hwTCq(iwgYzME zLB{IQ#3w>`2h_exU)!PY!PegfqWh_Oeytv3aCTrHRP-sFP?lynF9bOzSK!U0(G)j& zLoD~h2rpiaCU&5N=9q9yc}-bKSV_f^P@WPZsCbL%qgHU)ziTgYR)Dpc9nR1if`84o zlWjoxJ0*5J+bbza^YuTm$sKllf(|T9?Y_G0ewq-n`;F`Z2`G`nSnHL}MUqcIXKAA0?h}mi zQGm`%(~Xs>G|tS_n$^<~^RKqNC41052G7dDi|`}p08)P5(dCyHIasp}!JTFZdNH(f zg!{9cqO&%_<{ zKoXDt$GiJ2Z8-Y_?Bo|=zz5}X@E@<4(>fK_?Tr<^i+rk9%O$AkcLwGmejGo&j2{eG zXi)dk{FBHg4s|`|2Y6)f1XTkygTGhzV%+mp49~fUGs8eI@epQ+uqkmuTz+iCd`f(m z+Vs_!$Ek#My^xv4GR{s2)IeS$Uz002AM!wvud literal 0 HcmV?d00001 diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_03_two_records.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_03_two_records.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..37faba6e2e819e16b754275e367813a6c3fcb5ca GIT binary patch literal 315 zcmV-B0mS|viwFP!00002|J6}FPXjRw{VP>wQstcFlDn+~0}?QlvQTx%CB9Or_d$|F zpsN3!ORr^MK@1?_!OF9r?fCgXJlf8?K+4(rK(^STHxU;h>_RkcTw>_EHsJLuCh#&~ zh$HntdfzpDdJ=LqBlg+EmEUc*=vVJOrtsb3{&8`el0!g$<5JAg47iZF(gmAmXQL-4 zlG7J>u~Z&+4L(HE^r-_d$SG44)CzdXLh?Z(DIW3%URyf0~&c|DC Njvt_Pmxgcy000cknU(+m literal 0 HcmV?d00001 diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_04_no_model_id.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_04_no_model_id.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..21551bd3f681b3fe9d4e21259b933bbe00cc740f GIT binary patch literal 368 zcmV-$0gwJ4iwFP!00002|AkUbPs1<_{VOPEIwk#B)82aEfV3erhL9LNWUW^!+N31u zU`+k*H0jFFrfEJ{vEQ@f_+^hIWJ2Z&#W$-xDu9Dz8WN>Sr5V=?P>tXU?zUip&lZ&K zse9B&Q8(s@R>LvI>G!F<@ew5F;3Yd2x#<~n1pV}KVQ}p>bbEucccz!JEDDgzT?6JR z^W@<s&G}dJe zPny+@>EL2_>R{qve~D>KaE$3BoMKD^9Qv=Q>-pCd83HQ_JEd54B=Ac>5G_#E}!M7ll-GzPnxC*WN*?0zjNi%UKa#+9yB=m@#(tfVA zhg91%71Vu-$=wUAXC?nomNGKr@WqA&Q_uAx8bwa-h0O8U+I1);%*kD{@tB*DfO)9v O((V@=W2}$H0ssJYPP&}{ literal 0 HcmV?d00001 diff --git a/backend/plugins/kiro/tasks/testdata/logs/completion_01_non_empty.json.gz b/backend/plugins/kiro/tasks/testdata/logs/completion_01_non_empty.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..c25e2a154b18edad81d87cb8a053348537761bd4 GIT binary patch literal 956 zcmV;t14H~DiwFP!00002|Fv05Z`&{o{wsygOCehC+=gO@U3S=ZTE|1$s9lCUy2Krd z75(qClcse#&_Q;bOD8TZU$_S?@ml z4(sM~_;nw4O)KM7pMpO^YIyC=5Tr$&wiU~TccoZPS?+&vp^hJPbk!R-rxmfUZ&1F%2 zj6h6Rn8ZoK5evlWHd&eG;jdnT*e~%xU( zxPKph(&g;bKv6k%PmBe7=^V2xalOmf8RITN#Lb9@awTl5;}y%%WN~L#3-9n>x7C9l z&w3LCaRdm!T;*h_BotNFUkQ!nvkon9L?|c60R*PFK%50Np%8+iqcpzBzd(~8z-6GP zZ61*TEKDmnG--q?u{cSI4vpiGEEbr`eJ((+jU&B2tqob`B{H1UM^2AM#_+2n$edb` z9!uh^1ioe20Yw^FBEjVf5o8xsXnQ?EGH+hNlp_?6Wh#0NA&*@1RYF7+m%F{rkgyE# ziY5eyCAj)cAK~R-HZt!E*6_hLG}y*qE&xYYPvbKTw=nvt2bTRjgSO2hBtpr ztN3Vg(9_ksvueAz{n0CmiJzkySX0Zxin%U~E)6mA> zOoW_({|a)d{8YPhC5;wBDEL-AwetMo^zN>G4*S#T>sMxXq}>61B~2l_OX&@Y6SK#X zNJt6`l%EbY7%Q__uEiq{1y1+pJ_j)61K^zS@>pS)W{3X?4oMNj9D+4sGV{Vtl%1=| e)oi9GUe0gCjdWMFRn^lIKK%tEg}4TV7ytlO$lWCX literal 0 HcmV?d00001 diff --git a/backend/plugins/kiro/tasks/testdata/logs/completion_02_empty.json.gz b/backend/plugins/kiro/tasks/testdata/logs/completion_02_empty.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a914b79e55feb627b15fe84fb49d08b06afe30b3 GIT binary patch literal 909 zcmV;819JQyiwFP!00002|Fu|4lhZH^{wq5<(;dri_r~mS;KBhovFK3JbScxOrA?U~ zcKY8*+AW*4qQogf4tYqjtxpeI(Nnf7^386(&!#u`PuW9J7Q0zh+-=s|RZ%TB<^J8T zqO3j^KM%#es)e4iRk5hjHs9I(*YdtangTvPMkI+5LDe$nD`MzaGWxWC2ez&k znMm4W_bLIq$6sDzM6zquuJzC2idhzPOfdZu4+4bv+hdZ3VH6-K($xGv7YklWE{f#E z12J7=FHYi)SRf9+p~#UPhK^Gq&A9O>z)mu262+KA>7%eu#PmMlWQ?l>5f>u?#!teQIzh7>LpFDIw+J5p^;kX6 z6Ig44Acg<|m^wKbst84otFMH?@vC-iZ$v01#sLKOae){MB%u(3rlSmAWnZAd58xBf z(=v}p0FDeRI2UPzs&I6Wk~s_xLvlDmRql5IT4fxm^=WRXaaJP3Nq(f%Xk-k(JAz8d z1r>0NpS8eiwjI!61dGpX9P#P!gW$UB)8EahSL`8)x!VU&h2&K)`%>YtK!3KT{P@vuR;!wG*cZF^^G1F?sY^O-tn)$K!t*sb-E?#Fm(R=9Ok6G3#iwev-ZnGT z?za=6CJ;V@n(8n${?==wWIf)0$&R%=-<#fV>-XZ+oX%gEO-CCW?C0L~S<_$zEKIH@ j_0vvtsF(EfSw6|X8R?x3I$x;lc*Nr$TRZ#W85aNm^hUs! literal 0 HcmV?d00001 diff --git a/backend/plugins/kiro/tasks/user_report_extractor.go b/backend/plugins/kiro/tasks/user_report_extractor.go new file mode 100644 index 00000000000..ddef74a1cd5 --- /dev/null +++ b/backend/plugins/kiro/tasks/user_report_extractor.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractKiroUserReport + +var ExtractKiroUserReportMeta = plugin.SubTaskMeta{ + Name: "extractKiroUserReport", + EntryPoint: ExtractKiroUserReport, + EnabledByDefault: true, + Description: "Extract daily per-user activity from Kiro report CSVs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectKiroS3FilesMeta}, +} + +// ExtractKiroUserReport loads the report CSVs discovered for this scope. +// +// Reports are written once per day per client type, so there are only a few +// hundred per year - the concurrency that matters for logs is irrelevant here, +// but reusing extractFiles keeps the retry and bookkeeping behaviour identical +// across all three streams. +func ExtractKiroUserReport(taskCtx plugin.SubTaskContext) errors.Error { + return extractFiles(taskCtx, models.FileTypeReport, parseUserReportRows) +} + +// parseUserReportRows adapts ParseUserReport to the extractor's batch interface. +// +// Both tables come from one parse because the per-model counts are columns of the +// same CSV row; splitting them into two passes would mean reading every file +// twice. They are returned as two batches rather than one mixed slice because +// GORM resolves the target table from the slice's element type. +func parseUserReportRows(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) { + reports, modelMessages, err := ParseUserReport(data, connectionId, scopeId) + if err != nil { + return nil, err + } + + return []rowBatch{ + {rows: reports, count: len(reports)}, + {rows: modelMessages, count: len(modelMessages)}, + }, nil +} diff --git a/backend/plugins/schema_e2e/migration_schema_test.go b/backend/plugins/schema_e2e/migration_schema_test.go index 0fbc02893bc..5cdf115f960 100644 --- a/backend/plugins/schema_e2e/migration_schema_test.go +++ b/backend/plugins/schema_e2e/migration_schema_test.go @@ -70,6 +70,7 @@ import ( issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + kiro "github.com/apache/incubator-devlake/plugins/kiro/impl" linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" @@ -121,6 +122,7 @@ func allGoPlugins() []plugin.PluginMeta { issueTrace.IssueTrace{}, jenkins.Jenkins{}, jira.Jira{}, + kiro.Kiro{}, linear.Linear{}, linker.Linker{}, opsgenie.Opsgenie{}, diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index c3262153dfc..f4182fe9d89 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -46,12 +46,12 @@ import ( issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + kiro "github.com/apache/incubator-devlake/plugins/kiro/impl" linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" org "github.com/apache/incubator-devlake/plugins/org/impl" pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" - q_dev "github.com/apache/incubator-devlake/plugins/q_dev/impl" refdiff "github.com/apache/incubator-devlake/plugins/refdiff/impl" rootly "github.com/apache/incubator-devlake/plugins/rootly/impl" slack "github.com/apache/incubator-devlake/plugins/slack/impl" @@ -114,7 +114,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("opsgenie/models", opsgenie.Opsgenie{}.GetTablesInfo) checker.FeedIn("linker/models", linker.Linker{}.GetTablesInfo) checker.FeedIn("issue_trace/models", issueTrace.IssueTrace{}.GetTablesInfo) - checker.FeedIn("q_dev/models", q_dev.QDev{}.GetTablesInfo) + checker.FeedIn("kiro/models", kiro.Kiro{}.GetTablesInfo) checker.FeedIn("gh-copilot/models", copilot.GhCopilot{}.GetTablesInfo) err := checker.Verify() if err != nil { From 0c61692294beaee675ea29c122bd9ac8d241d58c Mon Sep 17 00:00:00 2001 From: warren Date: Thu, 27 Aug 2026 10:37:56 +0800 Subject: [PATCH 2/6] feat(config-ui): add Kiro connection support Signed-off-by: warren --- config-ui/src/plugins/register/index.ts | 2 + .../src/plugins/register/kiro/assets/icon.svg | 62 +++++ .../src/plugins/register/kiro/config.tsx | 104 ++++++++ config-ui/src/plugins/register/kiro/index.ts | 19 ++ e2e/kiro-full-flow.spec.ts | 252 ++++++++++++++++++ 5 files changed, 439 insertions(+) create mode 100644 config-ui/src/plugins/register/kiro/assets/icon.svg create mode 100644 config-ui/src/plugins/register/kiro/config.tsx create mode 100644 config-ui/src/plugins/register/kiro/index.ts create mode 100644 e2e/kiro-full-flow.spec.ts diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index 259d01bb0f1..441cda5cb2f 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -33,6 +33,7 @@ import { GitLabConfig } from './gitlab'; import { IncidentioConfig } from './incidentio'; import { JenkinsConfig } from './jenkins'; import { JiraConfig } from './jira'; +import { KiroConfig } from './kiro'; import { LinearConfig } from './linear'; import { PagerDutyConfig } from './pagerduty'; import { RootlyConfig } from './rootly'; @@ -64,6 +65,7 @@ export const pluginConfigs: IPluginConfig[] = [ IncidentioConfig, JenkinsConfig, JiraConfig, + KiroConfig, LinearConfig, PagerDutyConfig, RootlyConfig, diff --git a/config-ui/src/plugins/register/kiro/assets/icon.svg b/config-ui/src/plugins/register/kiro/assets/icon.svg new file mode 100644 index 00000000000..503114f140a --- /dev/null +++ b/config-ui/src/plugins/register/kiro/assets/icon.svg @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config-ui/src/plugins/register/kiro/config.tsx b/config-ui/src/plugins/register/kiro/config.tsx new file mode 100644 index 00000000000..d848e8a852e --- /dev/null +++ b/config-ui/src/plugins/register/kiro/config.tsx @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const KiroConfig: IPluginConfig = { + plugin: 'kiro', + name: 'Kiro', + icon: ({ color }) => , + sort: 12, + connection: { + docLink: 'https://kiro.dev/docs/enterprise/monitor-and-track/user-activity/', + initialValues: { + name: '', + region: 'us-east-1', + bucket: '', + reportPrefix: 'user-report', + promptLogBucket: '', + promptLogPrefix: 'logging', + identityStoreId: '', + identityStoreRegion: '', + }, + fields: [ + 'name', + { + key: 'region', + label: 'AWS Region', + subLabel: + 'The region where your Kiro profile was installed. The exports live under this region in the S3 path, so it must match exactly.', + }, + { + key: 'bucket', + label: 'S3 Bucket', + subLabel: 'Bucket holding the daily user activity report CSVs.', + }, + { + key: 'reportPrefix', + label: 'Report Prefix', + subLabel: 'Prefix within the bucket, before AWSLogs/. Leave as user-report unless you configured another.', + defaultValue: 'user-report', + }, + { + key: 'promptLogBucket', + label: 'Prompt Log Bucket (optional)', + subLabel: + 'Only needed if interaction logs go to a different bucket, which Kiro recommends. Leave empty to reuse the bucket above.', + }, + { + key: 'promptLogPrefix', + label: 'Prompt Log Prefix', + subLabel: 'Prefix for the interaction logs. Leave as logging unless you configured another.', + defaultValue: 'logging', + }, + { + key: 'accessKeyId', + label: 'AWS Access Key ID', + }, + { + key: 'secretAccessKey', + label: 'AWS Secret Access Key', + }, + { + key: 'identityStoreId', + label: 'IAM Identity Center Store ID (optional)', + subLabel: + 'Only resolves display names. User identity comes from the report’s User_Email column, so collection works without this. If set, the region below is required too.', + }, + { + key: 'identityStoreRegion', + label: 'IAM Identity Center Region (optional)', + subLabel: 'May differ from the S3 region. Required if a store ID is set.', + }, + ], + }, + dataScope: { + // No custom render: the default picker calls the plugin's remote-scopes + // endpoint, which browses the export layout in S3 as accounts -> years -> + // months and lists only periods that actually hold data. Hand-entering a + // prefix cannot be verified from the outcome, because a typo and an empty + // month both produce a run that succeeds and collects nothing. + title: 'Accounts & Periods', + }, + scopeConfig: { + entities: ['CROSS'], + transformation: {}, + }, +}; diff --git a/config-ui/src/plugins/register/kiro/index.ts b/config-ui/src/plugins/register/kiro/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/kiro/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/e2e/kiro-full-flow.spec.ts b/e2e/kiro-full-flow.spec.ts new file mode 100644 index 00000000000..88234a18844 --- /dev/null +++ b/e2e/kiro-full-flow.spec.ts @@ -0,0 +1,252 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { test, expect, request, Page } from '@playwright/test'; +import * as path from 'path'; +import * as fs from 'fs'; + +const API = 'http://localhost:8080'; +const UI = 'http://localhost:4000'; +const GRAFANA = 'http://localhost:3002'; +const SCREENSHOT_DIR = path.join(__dirname, 'screenshots'); + +// The full-flow test needs an existing Kiro connection with valid credentials. +// Keep environment-specific ids out of the repository. +const EXISTING_CONNECTION_ID = Number(process.env.KIRO_CONNECTION_ID || 0); +const KIRO_ACCOUNT_ID = process.env.KIRO_ACCOUNT_ID || ''; + +const state: { + connectionId: number; + scopeId: string; + blueprintId: number; + pipelineId: number; +} = { connectionId: EXISTING_CONNECTION_ID, scopeId: '', blueprintId: 0, pipelineId: 0 }; + +fs.mkdirSync(SCREENSHOT_DIR, { recursive: true }); + +async function grafanaLogin(page: Page) { + await page.goto(`${GRAFANA}/grafana/login`); + await page.waitForLoadState('networkidle'); + if (page.url().includes('/login')) { + await page.locator('input[name="user"]').fill('admin'); + await page.locator('input[name="password"]').fill('admin'); + await page.locator('button[type="submit"]').click(); + await page.waitForTimeout(2000); + // Handle "change password" prompt if shown + const skipBtn = page.locator('a:has-text("Skip")'); + if (await skipBtn.isVisible({ timeout: 2000 }).catch(() => false)) { + await skipBtn.click(); + } + await page.waitForTimeout(1000); + } +} + +async function openGrafanaDashboard(page: Page, uid: string, screenshotPath: string) { + await grafanaLogin(page); + await page.goto(`${GRAFANA}/grafana/d/${uid}?orgId=1&from=now-90d&to=now`); + + // Wait for first panel data to load + try { + await page.waitForResponse( + (resp) => resp.url().includes('/api/ds/query') && resp.status() === 200, + { timeout: 30000 } + ); + } catch { /* some dashboards may not fire queries immediately */ } + + // Wait for rendering to settle + await page.waitForTimeout(5000); + + // Take viewport screenshot (top section) + await page.screenshot({ path: screenshotPath.replace('.png', '-top.png') }); + + // Scroll down and take more sections + const scrollHeight = await page.evaluate(() => document.body.scrollHeight); + let section = 1; + for (let y = 900; y < scrollHeight; y += 900) { + await page.evaluate((scrollY) => window.scrollTo(0, scrollY), y); + await page.waitForTimeout(3000); + section++; + await page.screenshot({ path: screenshotPath.replace('.png', `-section${section}.png`) }); + } + + // Also take full page screenshot + await page.evaluate(() => window.scrollTo(0, 0)); + await page.waitForTimeout(2000); + await page.screenshot({ path: screenshotPath, fullPage: true }); +} + +test.describe.serial('Kiro Plugin Full Flow', () => { + + test('Step 1: Verify Existing Connection via API', async () => { + expect(EXISTING_CONNECTION_ID, 'set KIRO_CONNECTION_ID').toBeGreaterThan(0); + expect(KIRO_ACCOUNT_ID, 'set KIRO_ACCOUNT_ID').not.toBe(''); + const api = await request.newContext({ baseURL: API }); + + const resp = await api.get(`/plugins/kiro/connections/${state.connectionId}`); + expect(resp.ok()).toBeTruthy(); + const conn = await resp.json(); + console.log(`Using connection: id=${conn.id}, name=${conn.name}, bucket=${conn.bucket}`); + + const testResp = await api.post(`/plugins/kiro/connections/${state.connectionId}/test`); + const testBody = await testResp.json(); + console.log('Test connection:', { accounts: testBody.accounts, streams: testBody.streams, hint: testBody.hint }); + expect(testResp.ok()).toBeTruthy(); + }); + + test('Step 2: View Config-UI Home', async ({ page }) => { + await page.goto(UI); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(1000); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '01-config-ui-home.png'), fullPage: true }); + console.log('Screenshot: Config-UI home'); + }); + + test('Step 3: Create Scope (S3 Slice) via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.put(`/plugins/kiro/connections/${state.connectionId}/scopes`, { + data: { + data: [ + { + accountId: KIRO_ACCOUNT_ID, + basePath: '', + year: 2026, + month: 3, + }, + ], + }, + }); + + const body = await resp.json(); + console.log('Scope created:', resp.status()); + expect(resp.ok()).toBeTruthy(); + state.scopeId = body[0]?.id; + expect(state.scopeId).toBeTruthy(); + console.log(`Scope id: ${state.scopeId}`); + }); + + test('Step 4: Create Blueprint via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post('/blueprints', { + data: { + name: `e2e-blueprint-${Date.now()}`, + mode: 'NORMAL', + enable: true, + cronConfig: '0 0 * * *', + isManual: true, + connections: [ + { + pluginName: 'kiro', + connectionId: state.connectionId, + scopes: [{ scopeId: state.scopeId }], + }, + ], + }, + }); + + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.blueprintId = body.id; + console.log(`Blueprint created: id=${state.blueprintId}`); + }); + + test('Step 5: Trigger Pipeline via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post(`/blueprints/${state.blueprintId}/trigger`, { data: {} }); + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.pipelineId = body.id; + console.log(`Pipeline triggered: id=${state.pipelineId}`); + }); + + test('Step 6: Wait for Pipeline to Complete', async () => { + const api = await request.newContext({ baseURL: API }); + const maxWait = 120000; + const start = Date.now(); + let status = ''; + + while (Date.now() - start < maxWait) { + const resp = await api.get(`/pipelines/${state.pipelineId}`); + const pipeline = await resp.json(); + status = pipeline.status; + console.log(`Pipeline status: ${status} (${Math.round((Date.now() - start) / 1000)}s)`); + if (['TASK_COMPLETED', 'TASK_FAILED', 'TASK_PARTIAL'].includes(status)) break; + await new Promise((r) => setTimeout(r, 3000)); + } + + // Print task details + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + if (tasksResp.ok()) { + const { tasks } = await tasksResp.json(); + for (const t of tasks || []) { + console.log(` Task ${t.id}: ${t.status}${t.failedSubTask ? ` (failed: ${t.failedSubTask})` : ''}`); + if (t.message) console.log(` Error: ${t.message.substring(0, 300)}`); + } + } + + expect(status).toBe('TASK_COMPLETED'); + }); + + test('Step 7: Verify Data via MySQL', async () => { + const api = await request.newContext({ baseURL: API }); + + // Use pipeline tasks to confirm data was processed + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + const { tasks } = await tasksResp.json(); + expect(tasks[0].status).toBe('TASK_COMPLETED'); + console.log(`Pipeline completed in ${tasks[0].spentSeconds}s`); + }); + + test('Step 8: Grafana - Kiro Usage Dashboard (new format)', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_user_report', path.join(SCREENSHOT_DIR, '02-dashboard-user-report.png')); + console.log('Screenshot: Kiro Usage Dashboard'); + }); + + test('Step 9: Grafana - Kiro Feature Metrics', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_feature_metrics', path.join(SCREENSHOT_DIR, '03-dashboard-feature-metrics.png')); + console.log('Screenshot: Kiro Feature Metrics'); + }); + + test('Step 10: Grafana - Kiro AI Activity Insights (logging)', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_logging', path.join(SCREENSHOT_DIR, '04-dashboard-logging.png')); + console.log('Screenshot: Kiro AI Activity Insights'); + }); + + test('Step 11: Grafana - Kiro Executive Dashboard', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_executive', path.join(SCREENSHOT_DIR, '05-dashboard-executive.png')); + console.log('Screenshot: Kiro Executive Dashboard'); + }); + + test('Step 12: View Pipeline in Config-UI', async ({ page }) => { + // Navigate to the API proxy route for pipelines + await page.goto(`${UI}/api/pipelines?pageSize=5`); + await page.waitForLoadState('networkidle'); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '06-config-ui-pipelines.png'), fullPage: true }); + console.log('Screenshot: Pipelines API response'); + }); + + test('Step 13: Cleanup', async () => { + const api = await request.newContext({ baseURL: API }); + if (state.blueprintId) { + await api.delete(`/blueprints/${state.blueprintId}`); + console.log(`Deleted blueprint ${state.blueprintId}`); + } + console.log('Cleanup complete'); + }); +}); From 9d04b3963ab949cd5fc1c70279a8f4845bc589db Mon Sep 17 00:00:00 2001 From: warren Date: Thu, 27 Aug 2026 10:37:58 +0800 Subject: [PATCH 3/6] feat(grafana): add core Kiro dashboards Signed-off-by: warren --- grafana/dashboards/mysql/ai-model-roi.json | 25 +- .../mysql/{q-dev-dora.json => kiro-dora.json} | 386 ++++++++---------- .../kiro_activity_detail.json} | 196 ++++----- ...dev_executive.json => kiro_executive.json} | 147 ++++--- ...metrics.json => kiro_feature_metrics.json} | 168 +++++--- .../{qdev_logging.json => kiro_logging.json} | 201 ++++++--- ...user_report.json => kiro_user_report.json} | 71 +++- .../dashboards/postgresql/ai-model-roi.json | 25 +- .../{q-dev-dora.json => kiro-dora.json} | 330 ++++++--------- .../kiro_activity_detail.json} | 196 ++++----- ...dev_executive.json => kiro_executive.json} | 149 ++++--- ...metrics.json => kiro_feature_metrics.json} | 170 +++++--- .../{qdev_logging.json => kiro_logging.json} | 203 ++++++--- ...user_report.json => kiro_user_report.json} | 71 +++- 14 files changed, 1348 insertions(+), 990 deletions(-) rename grafana/dashboards/mysql/{q-dev-dora.json => kiro-dora.json} (61%) rename grafana/dashboards/{postgresql/qdev_user_data.json => mysql/kiro_activity_detail.json} (69%) rename grafana/dashboards/mysql/{qdev_executive.json => kiro_executive.json} (74%) rename grafana/dashboards/mysql/{qdev_feature_metrics.json => kiro_feature_metrics.json} (71%) rename grafana/dashboards/mysql/{qdev_logging.json => kiro_logging.json} (74%) rename grafana/dashboards/mysql/{qdev_user_report.json => kiro_user_report.json} (82%) rename grafana/dashboards/postgresql/{q-dev-dora.json => kiro-dora.json} (62%) rename grafana/dashboards/{mysql/qdev_user_data.json => postgresql/kiro_activity_detail.json} (70%) rename grafana/dashboards/postgresql/{qdev_executive.json => kiro_executive.json} (71%) rename grafana/dashboards/postgresql/{qdev_feature_metrics.json => kiro_feature_metrics.json} (70%) rename grafana/dashboards/postgresql/{qdev_logging.json => kiro_logging.json} (72%) rename grafana/dashboards/postgresql/{qdev_user_report.json => kiro_user_report.json} (82%) diff --git a/grafana/dashboards/mysql/ai-model-roi.json b/grafana/dashboards/mysql/ai-model-roi.json index 49980a1fadb..275fcf8949a 100644 --- a/grafana/dashboards/mysql/ai-model-roi.json +++ b/grafana/dashboards/mysql/ai-model-roi.json @@ -26,7 +26,7 @@ "targetBlank": true, "title": "Kiro Usage Dashboard", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" } ], "panels": [ @@ -87,7 +87,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used)) AS 'Total Credits'\nFROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], @@ -138,7 +138,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0), 1) AS \"Credits / PR\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT pr.id), 0), 1) AS 'Credits / PR'\nFROM _tool_kiro_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n) pr\nWHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -189,7 +189,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0), 1) AS \"Credits / Deploy\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 1) AS 'Credits / Deploy'\nFROM _tool_kiro_user_report r\nCROSS JOIN (\n SELECT DISTINCT cicd_deployment_id\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n) cdc\nWHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -240,7 +240,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0), 1) AS \"Credits / Issue\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT i.id), 0), 1) AS 'Credits / Issue'\nFROM _tool_kiro_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n) i\nWHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -316,7 +316,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_prs AS (\n SELECT DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY) AS week_start,\n COUNT(*) AS prs\n FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n GROUP BY DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(p.prs, 0), 1) AS 'Credits per PR'\nFROM _credits c\nLEFT JOIN _prs p ON c.week_start = p.week_start\nORDER BY time", "refId": "A" } ], @@ -379,7 +379,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_deploys AS (\n SELECT DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(DISTINCT cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(d.deploys, 0), 1) AS 'Credits per Deploy'\nFROM _credits c\nLEFT JOIN _deploys d ON c.week_start = d.week_start\nORDER BY time", "refId": "A" } ], @@ -442,7 +442,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_issues AS (\n SELECT DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n COUNT(*) AS resolved\n FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(i.resolved, 0), 1) AS 'Credits per Issue'\nFROM _credits c\nLEFT JOIN _issues i ON c.week_start = i.week_start\nORDER BY time", "refId": "A" } ], @@ -505,7 +505,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS 'Credits',\n SUM(total_messages) AS 'Messages',\n SUM(chat_conversations) AS 'Conversations'\nFROM _tool_kiro_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", "refId": "A" } ], @@ -517,7 +517,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", "kiro", "cost", "efficiency" @@ -531,7 +530,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "AI Cost-Efficiency (PostgreSQL)", - "uid": "ai_cost_efficiency-pg", + "title": "AI Model ROI", + "uid": "ai_model_roi", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/mysql/q-dev-dora.json b/grafana/dashboards/mysql/kiro-dora.json similarity index 61% rename from grafana/dashboards/mysql/q-dev-dora.json rename to grafana/dashboards/mysql/kiro-dora.json index b9fd48f3afb..ce2a54b7f37 100644 --- a/grafana/dashboards/mysql/q-dev-dora.json +++ b/grafana/dashboards/mysql/kiro-dora.json @@ -36,10 +36,10 @@ "keepTime": true, "tags": [], "targetBlank": true, - "title": "Q Dev Dashboard", + "title": "Kiro Dashboard", "tooltip": "", "type": "link", - "url": "/d/qdev_user_data/q-dev-user-data-dashboard" + "url": "/d/kiro_user_data/kiro-user-data-dashboard" } ], "panels": [ @@ -61,7 +61,7 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "## AI-Powered DORA Dashboard\nThis dashboard correlates **Q Dev (AI coding assistant)** usage metrics with **DORA** performance indicators to help understand the impact of AI-assisted development on engineering efficiency.\n\n- **Left side**: Q Dev AI usage metrics (code generation, acceptance rate)\n- **Right side**: DORA metrics (Lead Time, Deployment Frequency, Change Failure Rate)\n- **Correlation charts**: Show trends over time to identify potential relationships", + "content": "## Kiro + DORA Dashboard\nThis dashboard compares **Kiro usage** with **DORA** performance indicators at monthly aggregates.\n\n- **Kiro metrics**: active users, credits, messages, conversations, and credits per message\n- **DORA metrics**: lead time, deployment frequency, and change failure rate\n- **Semantic boundary**: Kiro does not export accepted LOC, suggestion acceptance, generated tests, or code-review findings; no proxy is presented as an equivalent measure", "mode": "markdown" }, "pluginVersion": "13.0.2", @@ -82,8 +82,11 @@ "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Number of unique users who used Q Dev AI features", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Unique Kiro users in the selected period, from the daily usage report.", "fieldConfig": { "defaults": { "color": { @@ -115,7 +118,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -126,20 +131,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) as 'Active Q Dev Users'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS 'Active Kiro Users' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Q Dev Active Users", + "title": "Kiro Active Users", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Total AI-generated code lines accepted (Inline + Chat)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro exports credits, not accepted lines of code. Accepted LOC has no equivalent field.", "fieldConfig": { "defaults": { "color": { @@ -172,7 +183,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -183,20 +196,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(inline_ai_code_lines + chat_ai_code_lines) as 'AI Accepted Lines'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used), 1) AS 'Kiro Credits Used' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Total AI Code Lines", + "title": "Total Kiro Credits", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Acceptance rate of inline AI suggestions", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro does not export suggestion acceptance. Credits per message is a usage-efficiency ratio, not an acceptance rate.", "fieldConfig": { "defaults": { "color": { @@ -220,7 +239,7 @@ } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [] }, @@ -237,7 +256,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -248,19 +269,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT \n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Acceptance Rate'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT SUM(credits_used) / NULLIF(SUM(total_messages), 0) AS 'Credits per Message' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "AI Acceptance Rate", + "title": "Credits per Message", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Number of production deployments in selected period", "fieldConfig": { "defaults": { @@ -293,7 +320,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -304,7 +333,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -316,7 +348,10 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Median lead time for changes in hours", "fieldConfig": { "defaults": { @@ -358,7 +393,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -369,7 +406,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -381,7 +421,10 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Percentage of deployments that caused incidents", "fieldConfig": { "defaults": { @@ -423,7 +466,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -434,7 +479,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -455,12 +503,15 @@ }, "id": 9, "panels": [], - "title": "AI Usage vs DORA Metrics Correlation", + "title": "Kiro Usage vs DORA Metrics", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Compare AI code generation trends with Lead Time for Changes. A negative correlation (AI lines up, Lead Time down) suggests AI is helping accelerate delivery.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro credits are compared with Lead Time. Kiro does not export accepted code lines.", "fieldConfig": { "defaults": { "color": { @@ -514,7 +565,7 @@ { "matcher": { "id": "byName", - "options": "AI Accepted Lines" + "options": "Kiro Credits" }, "properties": [ { @@ -533,29 +584,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Median Lead Time (hours)" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Lead Time (hours)" - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] } ] }, @@ -568,7 +596,10 @@ "id": 10, "options": { "legend": { - "calcs": ["mean", "max"], + "calcs": [ + "mean", + "max" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -582,20 +613,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_monthly AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines\n FROM _tool_q_dev_user_data\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\nlead_time_monthly AS (\n SELECT\n DATE_FORMAT(cdc.finished_date, '%Y-%m-01') AS month,\n AVG(ppm.pr_cycle_time) / 60 AS avg_lead_time\n FROM pull_requests pr\n JOIN project_pr_metrics ppm ON ppm.id = pr.id\n JOIN project_mapping pm ON pr.base_repo_id = pm.row_id AND pm.`table` = 'repos'\n JOIN cicd_deployment_commits cdc ON ppm.deployment_commit_id = cdc.id\n WHERE pm.project_name IN (${project})\n AND pr.merged_date IS NOT NULL\n AND ppm.pr_cycle_time IS NOT NULL\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, lt.month), '%Y-%m-%d') AS time,\n ai.ai_lines AS 'AI Accepted Lines',\n ROUND(lt.avg_lead_time, 1) AS 'Median Lead Time (hours)'\nFROM ai_monthly ai\nLEFT JOIN lead_time_monthly lt ON ai.month = lt.month\nWHERE ai.month IS NOT NULL OR lt.month IS NOT NULL\nORDER BY time", + "rawSql": "WITH ai_monthly AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(credits_used) AS kiro_credits\n FROM _tool_kiro_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\nlead_time_monthly AS (\n SELECT\n DATE_FORMAT(cdc.finished_date, '%Y-%m-01') AS month,\n AVG(ppm.pr_cycle_time) / 60 AS avg_lead_time\n FROM pull_requests pr\n JOIN project_pr_metrics ppm ON ppm.id = pr.id\n JOIN project_mapping pm ON pr.base_repo_id = pm.row_id AND pm.`table` = 'repos'\n JOIN cicd_deployment_commits cdc ON ppm.deployment_commit_id = cdc.id\n WHERE pm.project_name IN (${project})\n AND pr.merged_date IS NOT NULL\n AND ppm.pr_cycle_time IS NOT NULL\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, lt.month), '%Y-%m-%d') AS time,\n ai.kiro_credits AS 'Kiro Credits',\n ROUND(lt.avg_lead_time, 1) AS 'Median Lead Time (hours)'\nFROM ai_monthly ai\nLEFT JOIN lead_time_monthly lt ON ai.month = lt.month\nWHERE ai.month IS NOT NULL OR lt.month IS NOT NULL\nORDER BY time", "refId": "A" } ], - "title": "AI Code Generation vs Lead Time Trend", + "title": "Kiro Credits vs Lead Time Trend", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Compare AI suggestion acceptance rate with deployment frequency. Higher acceptance rate may indicate better AI integration and potentially more deployments.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Usage-efficiency ratio versus deployment frequency. This is not suggestion acceptance.", "fieldConfig": { "defaults": { "color": { @@ -649,7 +686,7 @@ { "matcher": { "id": "byName", - "options": "AI Acceptance Rate" + "options": "Credits per Message" }, "properties": [ { @@ -658,7 +695,7 @@ }, { "id": "custom.axisLabel", - "value": "Acceptance Rate" + "value": "Credits per Message" }, { "id": "unit", @@ -672,29 +709,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Deployment Count" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Deployments" - }, - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] } ] }, @@ -707,7 +721,10 @@ "id": 11, "options": { "legend": { - "calcs": ["mean", "max"], + "calcs": [ + "mean", + "max" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -721,20 +738,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_acceptance AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) AS acceptance_rate\n FROM _tool_q_dev_user_data\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\ndeployment_count AS (\n SELECT \n DATE_FORMAT(MAX(cdc.finished_date), '%Y-%m-01') AS month,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deploy_count\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, dc.month), '%Y-%m-%d') AS time,\n ai.acceptance_rate AS 'AI Acceptance Rate',\n dc.deploy_count AS 'Deployment Count'\nFROM ai_acceptance ai\nLEFT JOIN deployment_count dc ON ai.month = dc.month\nWHERE ai.month IS NOT NULL OR dc.month IS NOT NULL\nORDER BY time", + "rawSql": "WITH ai_acceptance AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(credits_used) / NULLIF(SUM(total_messages), 0) AS credits_per_message\n FROM _tool_kiro_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\ndeployment_count AS (\n SELECT \n DATE_FORMAT(MAX(cdc.finished_date), '%Y-%m-01') AS month,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deploy_count\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, dc.month), '%Y-%m-%d') AS time,\n ai.credits_per_message AS 'Credits per Message',\n dc.deploy_count AS 'Deployment Count'\nFROM ai_acceptance ai\nLEFT JOIN deployment_count dc ON ai.month = dc.month\nWHERE ai.month IS NOT NULL OR dc.month IS NOT NULL\nORDER BY time", "refId": "A" } ], - "title": "AI Acceptance Rate vs Deployment Frequency", + "title": "Credits per Message vs Deployment Frequency", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Compare AI-generated tests with Change Failure Rate. More AI-generated tests might correlate with lower failure rates.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro does not export generated-test counts; total messages is the available activity-volume measure.", "fieldConfig": { "defaults": { "color": { @@ -788,7 +811,7 @@ { "matcher": { "id": "byName", - "options": "AI Generated Tests" + "options": "Kiro Messages" }, "properties": [ { @@ -807,33 +830,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Change Failure Rate" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Failure Rate" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] } ] }, @@ -846,7 +842,10 @@ "id": 12, "options": { "legend": { - "calcs": ["mean", "max"], + "calcs": [ + "mean", + "max" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -860,20 +859,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_tests AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(test_generation_generated_tests) AS generated_tests\n FROM _tool_q_dev_user_data\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\ncfr_monthly AS (\n SELECT\n DATE_FORMAT(deployment_finished_date, '%Y-%m-01') AS month,\n SUM(has_incident) / NULLIF(COUNT(deployment_id), 0) AS cfr\n FROM (\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN d.deployment_id ELSE NULL END) AS has_incident\n FROM (\n SELECT\n cdc.cicd_deployment_id AS deployment_id,\n MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n GROUP BY 1\n HAVING $__timeFilter(MAX(cdc.finished_date))\n ) d\n LEFT JOIN project_incident_deployment_relationships pim ON d.deployment_id = pim.deployment_id\n LEFT JOIN incidents i ON pim.id = i.id\n GROUP BY 1, 2\n ) failure_data\n GROUP BY DATE_FORMAT(deployment_finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, cfr.month), '%Y-%m-%d') AS time,\n ai.generated_tests AS 'AI Generated Tests',\n cfr.cfr AS 'Change Failure Rate'\nFROM ai_tests ai\nLEFT JOIN cfr_monthly cfr ON ai.month = cfr.month\nWHERE ai.month IS NOT NULL OR cfr.month IS NOT NULL\nORDER BY time", + "rawSql": "WITH ai_tests AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(total_messages) AS kiro_messages\n FROM _tool_kiro_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\ncfr_monthly AS (\n SELECT\n DATE_FORMAT(deployment_finished_date, '%Y-%m-01') AS month,\n SUM(has_incident) / NULLIF(COUNT(deployment_id), 0) AS cfr\n FROM (\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN d.deployment_id ELSE NULL END) AS has_incident\n FROM (\n SELECT\n cdc.cicd_deployment_id AS deployment_id,\n MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n GROUP BY 1\n HAVING $__timeFilter(MAX(cdc.finished_date))\n ) d\n LEFT JOIN project_incident_deployment_relationships pim ON d.deployment_id = pim.deployment_id\n LEFT JOIN incidents i ON pim.id = i.id\n GROUP BY 1, 2\n ) failure_data\n GROUP BY DATE_FORMAT(deployment_finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, cfr.month), '%Y-%m-%d') AS time,\n ai.kiro_messages AS 'Kiro Messages',\n cfr.cfr AS 'Change Failure Rate'\nFROM ai_tests ai\nLEFT JOIN cfr_monthly cfr ON ai.month = cfr.month\nWHERE ai.month IS NOT NULL OR cfr.month IS NOT NULL\nORDER BY time", "refId": "A" } ], - "title": "AI Test Generation vs Change Failure Rate", + "title": "Kiro Messages vs Change Failure Rate", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Compare active Q Dev users with Code Review findings. More AI-assisted code review might catch issues earlier.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro does not export code-review findings. Shows active users and credits by month.", "fieldConfig": { "defaults": { "color": { @@ -927,30 +932,7 @@ { "matcher": { "id": "byName", - "options": "Active Users" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "left" - }, - { - "id": "custom.axisLabel", - "value": "Users" - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Code Review Findings" + "options": "Kiro Credits" }, "properties": [ { @@ -981,7 +963,10 @@ "id": 13, "options": { "legend": { - "calcs": ["mean", "max"], + "calcs": [ + "mean", + "max" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -995,15 +980,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT \n STR_TO_DATE(DATE_FORMAT(date, '%Y-%m-01'), '%Y-%m-%d') AS time,\n COUNT(DISTINCT user_id) AS 'Active Users',\n SUM(code_review_findings_count) AS 'Code Review Findings'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY DATE_FORMAT(date, '%Y-%m-01')\nORDER BY time", + "rawSql": "SELECT STR_TO_DATE(month_start, '%Y-%m-%d') AS time, active_users AS 'Active Users', kiro_credits AS 'Kiro Credits' FROM (SELECT DATE_FORMAT(date, '%Y-%m-01') AS month_start, COUNT(DISTINCT user_id) AS active_users, SUM(credits_used) AS kiro_credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE_FORMAT(date, '%Y-%m-01')) monthly ORDER BY time", "refId": "A" } ], - "title": "Q Dev Users vs Code Review Findings", + "title": "Kiro Users vs Credits", "type": "timeseries" }, { @@ -1020,8 +1008,11 @@ "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Monthly summary comparing Q Dev AI metrics with DORA metrics side by side", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Monthly Kiro usage metrics beside DORA metrics. LOC, acceptance, generated tests, and review findings are not exported by Kiro.", "fieldConfig": { "defaults": { "color": { @@ -1050,7 +1041,7 @@ { "matcher": { "id": "byName", - "options": "AI Acceptance Rate" + "options": "Credits per Message" }, "properties": [ { @@ -1075,47 +1066,6 @@ "value": 1 } ] - }, - { - "matcher": { - "id": "byName", - "options": "Change Failure Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-RdYlGr" - } - }, - { - "id": "max", - "value": 0.3 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Lead Time (hours)" - }, - "properties": [ - { - "id": "unit", - "value": "h" - } - ] } ] }, @@ -1131,7 +1081,9 @@ "footer": { "countRows": false, "fields": "", - "reducer": ["sum"], + "reducer": [ + "sum" + ], "show": false }, "showHeader": true, @@ -1145,15 +1097,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_metrics AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m') AS month,\n COUNT(DISTINCT user_id) AS active_users,\n SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) AS acceptance_rate,\n SUM(test_generation_generated_tests) AS generated_tests,\n SUM(code_review_findings_count) AS review_findings\n FROM _tool_q_dev_user_data\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m')\n),\ndora_metrics AS (\n SELECT\n DATE_FORMAT(cdc.finished_date, '%Y-%m') AS month,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deployments,\n AVG(ppm.pr_cycle_time) / 60 AS avg_lead_time\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n LEFT JOIN cicd_deployment_commits cdc2 ON cdc.cicd_deployment_id = cdc2.cicd_deployment_id\n LEFT JOIN project_pr_metrics ppm ON ppm.deployment_commit_id = cdc2.id\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m')\n),\ncfr_metrics AS (\n SELECT\n DATE_FORMAT(deployment_finished_date, '%Y-%m') AS month,\n SUM(has_incident) / NULLIF(COUNT(deployment_id), 0) AS cfr\n FROM (\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN d.deployment_id ELSE NULL END) AS has_incident\n FROM (\n SELECT\n cdc.cicd_deployment_id AS deployment_id,\n MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n GROUP BY 1\n HAVING $__timeFilter(MAX(cdc.finished_date))\n ) d\n LEFT JOIN project_incident_deployment_relationships pim ON d.deployment_id = pim.deployment_id\n LEFT JOIN incidents i ON pim.id = i.id\n GROUP BY 1, 2\n ) failure_data\n GROUP BY DATE_FORMAT(deployment_finished_date, '%Y-%m')\n)\nSELECT \n COALESCE(ai.month, dm.month, cfr.month) AS 'Month',\n COALESCE(ai.active_users, 0) AS 'Q Dev Users',\n COALESCE(ai.ai_lines, 0) AS 'AI Code Lines',\n ai.acceptance_rate AS 'AI Acceptance Rate',\n COALESCE(ai.generated_tests, 0) AS 'AI Tests',\n COALESCE(ai.review_findings, 0) AS 'Review Findings',\n COALESCE(dm.deployments, 0) AS 'Deployments',\n ROUND(dm.avg_lead_time, 1) AS 'Lead Time (hours)',\n cfr.cfr AS 'Change Failure Rate'\nFROM ai_metrics ai\nLEFT JOIN dora_metrics dm ON ai.month = dm.month\nLEFT JOIN cfr_metrics cfr ON ai.month = cfr.month\nORDER BY ai.month DESC", + "rawSql": "WITH ai_metrics AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m') AS month,\n COUNT(DISTINCT user_id) AS active_users,\n SUM(credits_used) AS credits,\n SUM(credits_used) / NULLIF(SUM(total_messages), 0) AS credits_per_message,\n SUM(total_messages) AS messages,\n SUM(chat_conversations) AS conversations\n FROM _tool_kiro_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m')\n),\ndora_metrics AS (\n SELECT\n DATE_FORMAT(cdc.finished_date, '%Y-%m') AS month,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deployments,\n AVG(ppm.pr_cycle_time) / 60 AS avg_lead_time\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n LEFT JOIN cicd_deployment_commits cdc2 ON cdc.cicd_deployment_id = cdc2.cicd_deployment_id\n LEFT JOIN project_pr_metrics ppm ON ppm.deployment_commit_id = cdc2.id\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m')\n),\ncfr_metrics AS (\n SELECT\n DATE_FORMAT(deployment_finished_date, '%Y-%m') AS month,\n SUM(has_incident) / NULLIF(COUNT(deployment_id), 0) AS cfr\n FROM (\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN d.deployment_id ELSE NULL END) AS has_incident\n FROM (\n SELECT\n cdc.cicd_deployment_id AS deployment_id,\n MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n GROUP BY 1\n HAVING $__timeFilter(MAX(cdc.finished_date))\n ) d\n LEFT JOIN project_incident_deployment_relationships pim ON d.deployment_id = pim.deployment_id\n LEFT JOIN incidents i ON pim.id = i.id\n GROUP BY 1, 2\n ) failure_data\n GROUP BY DATE_FORMAT(deployment_finished_date, '%Y-%m')\n)\nSELECT \n COALESCE(ai.month, dm.month, cfr.month) AS 'Month',\n COALESCE(ai.active_users, 0) AS 'Kiro Users',\n COALESCE(ai.credits, 0) AS 'Credits Used',\n ai.credits_per_message AS 'Credits per Message',\n COALESCE(ai.messages, 0) AS 'Messages',\n COALESCE(ai.conversations, 0) AS 'Conversations',\n COALESCE(dm.deployments, 0) AS 'Deployments',\n ROUND(dm.avg_lead_time, 1) AS 'Lead Time (hours)',\n cfr.cfr AS 'Change Failure Rate'\nFROM ai_metrics ai\nLEFT JOIN dora_metrics dm ON ai.month = dm.month\nLEFT JOIN cfr_metrics cfr ON ai.month = cfr.month\nORDER BY ai.month DESC", "refId": "A" } ], - "title": "Monthly Q Dev vs DORA Metrics Comparison", + "title": "Monthly Kiro + DORA Metrics", "type": "table" } ], @@ -1161,7 +1116,7 @@ "refresh": "5m", "schemaVersion": 39, "tags": [ - "q_dev", + "kiro", "DORA", "AI", "correlation" @@ -1171,10 +1126,17 @@ { "current": { "selected": true, - "text": ["All"], - "value": ["$__all"] + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" }, - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "SELECT DISTINCT name FROM projects", "hide": 0, "includeAll": true, @@ -1197,7 +1159,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Q Dev + DORA Correlation", - "uid": "qdev_dora_correlation", + "title": "Kiro + DORA Correlation", + "uid": "kiro_dora_correlation", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_user_data.json b/grafana/dashboards/mysql/kiro_activity_detail.json similarity index 69% rename from grafana/dashboards/postgresql/qdev_user_data.json rename to grafana/dashboards/mysql/kiro_activity_detail.json index d446fca81db..ed5c615c4db 100644 --- a/grafana/dashboards/postgresql/qdev_user_data.json +++ b/grafana/dashboards/mysql/kiro_activity_detail.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Overview of key user metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Usage-report totals. Accepted LOC and acceptance rates are not exported by Kiro.", "fieldConfig": { "defaults": { "color": { @@ -66,13 +69,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", SUM(chat_ai_code_lines) AS \"Accepted Lines (Chat)\", SUM(inline_ai_code_lines) AS \"Accepted Lines (Inline Suggestion)\", CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Acceptance Rate (Inline Suggestion)\", SUM(code_review_findings_count) AS \"Findings (Code Review)\", SUM(code_fix_accepted_lines) AS \"Accepted Lines (Code Fix)\", CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) AS \"Acceptance Rate (Code Fix)\", SUM(transformation_lines_ingested) AS \"Ingested Lines (Java Transform)\", SUM(transformation_lines_generated) AS \"Generated Lines (Java Transform)\", SUM(inline_chat_accepted_line_additions) AS \"Accepted Lines (Inline Chat)\", CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) AS \"Acceptance Rate (Inline Chat)\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS 'Active Users', ROUND(SUM(credits_used), 1) AS 'Credits Used', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A", "select": [ [ @@ -115,8 +121,11 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily AI code line changes across all users", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Daily Kiro usage volume from the user report.", "fieldConfig": { "defaults": { "color": { @@ -195,13 +204,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(chat_ai_code_lines) AS \"Chat Accepted Lines\", SUM(code_fix_accepted_lines) AS \"Code Fix Accepted Lines\", SUM(code_fix_generated_lines) AS \"Code Fix Generated Lines\", SUM(transformation_lines_ingested) AS \"Java Transform Ingested Lines\", SUM(transformation_lines_generated) AS \"Java Transform Generated Lines\", SUM(inline_ai_code_lines) AS \"Inline Suggestion Accepted Lines\", SUM(inline_chat_accepted_line_additions) AS \"Inline Chat Accepted Line Additions\", SUM(inline_chat_accepted_line_deletions) AS \"Inline Chat Accepted Line Deletions\", SUM(inline_chat_dismissed_line_additions) AS \"Inline Chat Dismissed Line Additions\", SUM(inline_chat_dismissed_line_deletions) AS \"Inline Chat Dismissed Line Deletions\", SUM(inline_chat_rejected_line_additions) AS \"Inline Chat Rejected Line Additions\", SUM(inline_chat_rejected_line_deletions) AS \"Inline Chat Rejected Line Deletions\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(date) AS time, ROUND(SUM(credits_used), 1) AS 'Credits Used', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE(date) ORDER BY time", "refId": "A", "select": [ [ @@ -240,12 +252,15 @@ ] } ], - "title": "Daily AI Code Line Changes", + "title": "Daily Credits, Messages & Conversations", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily AI interaction trends across all users", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Chat events split into user turns and agent continuations.", "fieldConfig": { "defaults": { "color": { @@ -324,13 +339,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(chat_messages_sent) AS \"Chat Messages Sent\", SUM(code_fix_acceptance_event_count) AS \"Code Fix Accepted Event Count\", SUM(code_fix_generation_event_count) AS \"Code Fix Generated Event Count\", SUM(transformation_event_count) AS \"Java Transform Event Count\", SUM(inline_acceptance_count) AS \"Inline Suggestion Accepted Suggestions\", SUM(inline_suggestions_count) AS \"Inline Suggestion Count\", SUM(inline_chat_total_event_count) AS \"Inline Chat Total Suggestions\", SUM(inline_chat_acceptance_event_count) AS \"Inline Chat Accepted Suggestions\", SUM(inline_chat_dismissal_event_count) AS \"Inline Chat Dismissed Suggestions\", SUM(inline_chat_rejection_event_count) AS \"Inline Chat Rejected Suggestions\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(timestamp) AS time, COUNT(*) AS 'Chat Events', SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS 'User Turns', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A", "select": [ [ @@ -369,12 +387,15 @@ ] } ], - "title": "Daily AI Interactions", + "title": "Daily Chat Activity", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Code review metrics over time", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Authoritative per-model and routing-mode message counts.", "fieldConfig": { "defaults": { "color": { @@ -453,12 +474,15 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(code_fix_acceptance_event_count) AS \"Code Fix Accepted Event Count\", SUM(code_fix_generation_event_count) AS \"Code Fix Generated Event Count\", SUM(code_review_findings_count) AS \"Total Findings\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(date) AS time, model_name AS metric, SUM(message_count) AS value FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY DATE(date), model_name ORDER BY time", "refId": "A", "select": [ [ @@ -497,12 +521,15 @@ ] } ], - "title": "Code Review Metrics", + "title": "Daily Model / Route Messages", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily acceptance rate of AI suggestions", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Agent continuations divided by all chat events. This is not suggestion acceptance.", "fieldConfig": { "defaults": { "color": { @@ -581,13 +608,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) AS \"Code Fix\", CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Inline Suggestions\", CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) AS \"Inline Chat\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) AS 'Agent Continuation Rate' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A", "select": [ [ @@ -626,12 +656,15 @@ ] } ], - "title": "Daily AI Suggestion Acceptance Rate", + "title": "Daily Agent Continuation Rate", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "User AI interaction metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Per-user credits, messages, conversations, overage, and last activity.", "fieldConfig": { "defaults": { "color": { @@ -655,51 +688,7 @@ ] } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Acceptance Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inline Chat Accepted Events" - }, - "properties": [ - { - "id": "custom.width", - "value": 239 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Acceptance Rate (Inline Suggestion)" - }, - "properties": [ - { - "id": "custom.width", - "value": 172 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -724,13 +713,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", SUM(chat_ai_code_lines) AS \"Accepted Lines (Chat)\", SUM(transformation_lines_ingested) AS \"Lines Ingested (Java Transform)\", SUM(transformation_lines_generated) AS \"Lines Generated (Java Transform)\", SUM(transformation_event_count) AS \"Event Count (Java Transform)\", SUM(code_review_findings_count) AS \"Findings (Code Review)\", SUM(code_fix_accepted_lines) AS \"Accepted Lines (Code Fix)\", SUM(code_fix_generated_lines) AS \"Generated Lines (Code Fix)\", SUM(code_fix_acceptance_event_count) AS \"Accepted Count (Code Fix)\", SUM(code_fix_generation_event_count) AS \"Generated Count (Code Fix)\", ROUND(CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) * 100, 2) || '%' AS \"Acceptance Rate (Code Fix)\", SUM(inline_ai_code_lines) AS \"Accepted Lines (Inline Suggestion)\", SUM(inline_acceptance_count) AS \"Accepted Count (Inline Suggestion)\", SUM(inline_suggestions_count) AS \"Total Count (Inline Suggestion)\", ROUND(CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) * 100, 2) || '%' AS \"Acceptance Rate (Inline Suggestion)\", SUM(inline_chat_accepted_line_additions) AS \"Accepted Line Additions (Inline Chat)\", SUM(inline_chat_accepted_line_deletions) AS \"Accepted Line Deletions (Inline Chat)\", SUM(inline_chat_acceptance_event_count) AS \"Accepted Events (Inline Chat)\", SUM(inline_chat_total_event_count) AS \"Total Events (Inline Chat)\", ROUND(CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) * 100, 2) || '%' AS \"Acceptance Rate (Inline Chat)\", SUM(doc_generation_event_count) AS \"Doc Gen Events\", SUM(test_generation_event_count) AS \"Test Gen Events\", SUM(dev_accepted_lines) AS \"Dev Accepted Lines\", MIN(date) AS \"First Activity\", MAX(date) AS \"Last Activity\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"user_id\" ORDER BY SUM(inline_ai_code_lines) DESC NULLS LAST", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS 'User', MAX(subscription_tier) AS 'Tier', ROUND(SUM(credits_used), 1) AS 'Credits Used', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations', SUM(overage_credits_used) AS 'Overage Credits', MAX(date) AS 'Last Activity' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id ORDER BY SUM(credits_used) DESC", "refId": "A", "select": [ [ @@ -769,12 +761,15 @@ ] } ], - "title": "User Interactions", + "title": "Per-User Usage", "type": "table" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily doc generation events and accepted/rejected lines", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Messages by Kiro client type. Doc-generation metrics are not exported.", "fieldConfig": { "defaults": { "color": { @@ -853,13 +848,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(doc_generation_event_count) AS \"Doc Generation Events\", SUM(doc_generation_accepted_line_additions) AS \"Accepted Line Additions\", SUM(doc_generation_accepted_line_updates) AS \"Accepted Line Updates\", SUM(doc_generation_rejected_line_additions) AS \"Rejected Line Additions\", SUM(doc_generation_rejected_line_updates) AS \"Rejected Line Updates\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(date) AS time, client_type AS metric, SUM(total_messages) AS value FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE(date), client_type ORDER BY time", "refId": "A", "select": [ [ @@ -898,12 +896,15 @@ ] } ], - "title": "Doc Generation Metrics", + "title": "Daily Client Type Messages", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily test generation events and lines", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Completion requests, requests with returned suggestions, and returned lines. These are not acceptance events.", "fieldConfig": { "defaults": { "color": { @@ -982,13 +983,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(test_generation_event_count) AS \"Test Generation Events\", SUM(test_generation_accepted_tests) AS \"Accepted Tests\", SUM(test_generation_generated_tests) AS \"Generated Tests\", SUM(test_generation_accepted_lines) AS \"Accepted Lines\", SUM(test_generation_generated_lines) AS \"Generated Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(timestamp) AS time, COUNT(*) AS 'Completion Requests', SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS 'Requests with Results', SUM(returned_line_count) AS 'Returned Lines' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A", "select": [ [ @@ -1027,12 +1031,15 @@ ] } ], - "title": "Test Generation Metrics", + "title": "Daily Completion Activity", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily agentic dev events and lines", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Steering, spec-mode, and agent-continuation events. Kiro does not export accepted agentic LOC.", "fieldConfig": { "defaults": { "color": { @@ -1111,13 +1118,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(dev_generation_event_count) AS \"Dev Generation Events\", SUM(dev_acceptance_event_count) AS \"Dev Acceptance Events\", SUM(dev_generated_lines) AS \"Dev Generated Lines\", SUM(dev_accepted_lines) AS \"Dev Accepted Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS 'Steering', SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS 'Spec Mode', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A", "select": [ [ @@ -1156,7 +1166,7 @@ ] } ], - "title": "Dev (Agentic) Metrics", + "title": "Daily Agentic Signals", "type": "timeseries" } ], @@ -1164,7 +1174,7 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", + "kiro", "user_data" ], "templating": { @@ -1176,7 +1186,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Kiro Code Metrics Dashboard", - "uid": "qdev_user_data-pg", + "title": "Kiro Activity Detail Dashboard", + "uid": "kiro_user_data", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/mysql/qdev_executive.json b/grafana/dashboards/mysql/kiro_executive.json similarity index 74% rename from grafana/dashboards/mysql/qdev_executive.json rename to grafana/dashboards/mysql/kiro_executive.json index fffbc99ba60..32b2c45a377 100644 --- a/grafana/dashboards/mysql/qdev_executive.json +++ b/grafana/dashboards/mysql/kiro_executive.json @@ -27,7 +27,7 @@ "title": "Usage (New)", "tooltip": "Kiro Usage Dashboard - Credits & Messages (new format)", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" }, { "asDropdown": false, @@ -36,10 +36,10 @@ "keepTime": true, "tags": [], "targetBlank": true, - "title": "Feature Metrics (Legacy)", - "tooltip": "Kiro Legacy Feature Metrics (old format)", + "title": "Feature Metrics", + "tooltip": "Kiro Feature Metrics", "type": "link", - "url": "/d/qdev_feature_metrics" + "url": "/d/kiro_feature_metrics" }, { "asDropdown": false, @@ -51,7 +51,7 @@ "title": "Prompt Logging", "tooltip": "Kiro AI Activity Insights - Prompt Logging", "type": "link", - "url": "/d/qdev_logging" + "url": "/d/kiro_logging" } ], "panels": [ @@ -69,7 +69,10 @@ "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Distinct users with chat activity in the last 7 days (from prompt logging)", "fieldConfig": { "defaults": { @@ -116,11 +119,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) as 'WAU'\nFROM lake._tool_q_dev_chat_log\nWHERE timestamp >= DATE_SUB(NOW(), INTERVAL 7 DAY)", + "rawSql": "SELECT COUNT(DISTINCT user_id) as 'WAU'\nFROM lake._tool_kiro_chat_log\nWHERE timestamp >= DATE_SUB(NOW(), INTERVAL 7 DAY)", "refId": "A" } ], @@ -128,8 +134,11 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Average credits spent per accepted line of code (new report + legacy metrics)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Usage efficiency from Kiro reports. Accepted LOC is not exported.", "fieldConfig": { "defaults": { "color": { @@ -175,20 +184,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(SUM(d.total_accepted), 0), 2) as 'Credits per Accepted Line'\nFROM (\n SELECT user_id, date, SUM(credits_used) as credits_used\n FROM lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id, date\n) r\nJOIN (\n SELECT user_id, date,\n (inline_ai_code_lines + chat_ai_code_lines + code_fix_accepted_lines + dev_accepted_lines) as total_accepted\n FROM lake._tool_q_dev_user_data\n WHERE $__timeFilter(date)\n) d ON r.user_id = d.user_id AND r.date = d.date", + "rawSql": "SELECT ROUND(SUM(credits_used) / NULLIF(SUM(total_messages), 0), 2) AS 'Credits per Message' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Credits Efficiency (new + legacy)", + "title": "Credits per Message", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Percentage of inline suggestions accepted (from legacy feature metrics)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Share of chat events generated by agent self-continuation. Suggestion acceptance is not exported.", "fieldConfig": { "defaults": { "color": { @@ -234,19 +249,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 1) as 'Acceptance %'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0), 1) AS 'Agent Continuation %' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], - "title": "Inline Acceptance Rate (legacy)", + "title": "Agent Continuation Rate", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Percentage of users who used steering rules (from prompt logging)", "fieldConfig": { "defaults": { @@ -294,11 +315,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0) as 'Steering %'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0) as 'Steering %'\nFROM lake._tool_kiro_chat_log\nWHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -315,11 +339,14 @@ }, "id": 101, "panels": [], - "title": "User Engagement (logging data: _tool_q_dev_chat_log)", + "title": "User Engagement", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly active user count over time (from prompt logging)", "fieldConfig": { "defaults": { @@ -399,11 +426,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM (\n SELECT user_id, YEARWEEK(timestamp, 1) as yw\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n) t\nGROUP BY yw\nORDER BY time", + "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM (\n SELECT user_id, YEARWEEK(timestamp, 1) as yw\n FROM lake._tool_kiro_chat_log\n WHERE $__timeFilter(timestamp)\n) t\nGROUP BY yw\nORDER BY time", "refId": "A" } ], @@ -411,8 +441,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "New vs returning users by week (from prompt logging)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "New users use the native is_new_user flag; early rows without that field are reported as existing/unknown.", "fieldConfig": { "defaults": { "color": { @@ -491,15 +524,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n SUM(CASE WHEN yw = first_yw THEN 1 ELSE 0 END) as 'New Users',\n SUM(CASE WHEN yw != first_yw THEN 1 ELSE 0 END) as 'Returning Users'\nFROM (\n SELECT DISTINCT u.user_id, YEARWEEK(u.timestamp, 1) as yw, f.first_yw\n FROM lake._tool_q_dev_chat_log u\n JOIN (SELECT user_id, YEARWEEK(MIN(timestamp), 1) as first_yw FROM lake._tool_q_dev_chat_log GROUP BY user_id) f\n ON u.user_id = f.user_id\n WHERE $__timeFilter(u.timestamp)\n) weekly\nGROUP BY yw\nORDER BY time", + "rawSql": "SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time, COUNT(DISTINCT CASE WHEN is_new_user = 1 THEN user_id END) AS 'New Users', COUNT(DISTINCT CASE WHEN is_new_user = 0 OR is_new_user IS NULL THEN user_id END) AS 'Existing/Unknown Users' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) ORDER BY time", "refId": "A" } ], - "title": "New vs Returning Users (Weekly)", + "title": "New vs Existing/Unknown Users (Weekly)", "type": "timeseries" }, { @@ -512,11 +548,14 @@ }, "id": 102, "panels": [], - "title": "Credits & Subscription (new format: _tool_q_dev_user_report)", + "title": "Credits & Subscription", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Cumulative credits this month vs projected total (from new user_report)", "fieldConfig": { "defaults": { @@ -596,11 +635,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(SUM(credits_used)) OVER (ORDER BY date) as 'Cumulative Credits',\n (SELECT SUM(credits_used) / COUNT(DISTINCT date) * DAY(LAST_DAY(CURDATE()))\n FROM lake._tool_q_dev_user_report\n WHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())) as 'Projected Monthly'\nFROM lake._tool_q_dev_user_report\nWHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT\n date as time,\n SUM(SUM(credits_used)) OVER (ORDER BY date) as 'Cumulative Credits',\n (SELECT SUM(credits_used) / COUNT(DISTINCT date) * DAY(LAST_DAY(CURDATE()))\n FROM lake._tool_kiro_user_report\n WHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())) as 'Projected Monthly'\nFROM lake._tool_kiro_user_report\nWHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())\nGROUP BY date\nORDER BY date", "refId": "A" } ], @@ -608,7 +650,10 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Power tier users with no activity in the last 14 days (from new user_report)", "fieldConfig": { "defaults": { @@ -658,11 +703,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as 'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) < DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)", + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as 'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) < DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)", "refId": "A" } ], @@ -679,12 +727,15 @@ }, "id": 103, "panels": [], - "title": "Cross-Source: User Productivity (new report + legacy metrics)", + "title": "Per-User Usage & Efficiency", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Per-user productivity combining credits (new format) with feature metrics (legacy). Only shows users present in both data sources.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Credits, messages, conversations, and credits per message. Accepted LOC is not exported.", "fieldConfig": { "defaults": { "color": { @@ -733,15 +784,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as 'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as 'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 2)\n ELSE NULL END as 'Credits/Line',\n CONCAT(ROUND(SUM(d.inline_acceptance_count) / NULLIF(SUM(d.inline_suggestions_count), 0) * 100, 1), '%') as 'Accept Rate',\n SUM(d.code_review_findings_count) as 'Review Findings',\n SUM(d.test_generation_event_count) as 'Test Gen Events',\n SUM(d.dev_accepted_lines) as 'Agentic Lines',\n MIN(d.date) as 'First Active',\n MAX(d.date) as 'Last Active'\nFROM lake._tool_q_dev_user_data d\nLEFT JOIN (\n SELECT user_id, date, SUM(credits_used) as credits_used, MAX(subscription_tier) as subscription_tier\n FROM lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id, date\n) r ON d.user_id = r.user_id AND d.date = r.date\nWHERE $__timeFilter(d.date)\nGROUP BY d.user_id\nORDER BY SUM(r.credits_used) DESC", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS 'User', MAX(subscription_tier) AS 'Tier', ROUND(SUM(credits_used), 1) AS 'Credits Used', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations', ROUND(SUM(credits_used) / NULLIF(SUM(total_messages), 0), 3) AS 'Credits per Message', MAX(date) AS 'Last Activity' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id ORDER BY SUM(credits_used) DESC", "refId": "A" } ], - "title": "User Productivity & Efficiency", + "title": "Per-User Usage & Efficiency", "type": "table" } ], @@ -749,9 +803,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "executive", - "kiro" + "kiro", + "executive" ], "templating": { "list": [] @@ -763,6 +816,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro Executive Dashboard", - "uid": "qdev_executive", + "uid": "kiro_executive", "version": 1 } diff --git a/grafana/dashboards/mysql/qdev_feature_metrics.json b/grafana/dashboards/mysql/kiro_feature_metrics.json similarity index 71% rename from grafana/dashboards/mysql/qdev_feature_metrics.json rename to grafana/dashboards/mysql/kiro_feature_metrics.json index 5156e652a34..58604d0234c 100644 --- a/grafana/dashboards/mysql/qdev_feature_metrics.json +++ b/grafana/dashboards/mysql/kiro_feature_metrics.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "High-level summary of legacy feature-level activity metrics (from by_user_analytic CSV reports)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro usage-report metrics. Legacy accepted LOC and feature counters are not exported.", "fieldConfig": { "defaults": { "color": { @@ -66,15 +69,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(inline_suggestions_count) as 'Inline Suggestions',\n SUM(inline_acceptance_count) as 'Inline Accepted',\n SUM(chat_messages_sent) as 'Chat Messages',\n SUM(chat_ai_code_lines) as 'Chat AI Lines',\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Agentic Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS 'Active Users', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations', ROUND(SUM(credits_used), 1) AS 'Credits Used' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Legacy Feature Metrics Overview", + "title": "Kiro Usage Overview", "type": "stat" }, { @@ -87,12 +93,15 @@ }, "id": 20, "panels": [], - "title": "Inline Suggestions", + "title": "Completions", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily inline suggestion and acceptance counts", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Returned suggestions are offers, not acceptance events.", "fieldConfig": { "defaults": { "color": { @@ -170,20 +179,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(inline_suggestions_count) as 'Suggestions',\n SUM(inline_acceptance_count) as 'Accepted',\n SUM(inline_ai_code_lines) as 'AI Code Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, COUNT(*) AS 'Completion Requests', SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS 'Requests with Results', SUM(returned_line_count) AS 'Returned Lines' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Inline Suggestions & Acceptance", + "title": "Completion Requests & Results", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Percentage of completion requests that returned at least one suggestion. Kiro does not export acceptance.", "fieldConfig": { "defaults": { "color": { @@ -261,15 +276,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) AS 'Suggestion Return Rate' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Acceptance Rate Trends", + "title": "Suggestion Return Rate", "type": "timeseries" }, { @@ -282,12 +300,15 @@ }, "id": 21, "panels": [], - "title": "Chat & Agentic (Dev)", + "title": "Chat & Agent Continuations", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily chat messages sent and AI-generated code lines from chat", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "User turns and agent self-continuations from chat logs.", "fieldConfig": { "defaults": { "color": { @@ -365,11 +386,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(chat_messages_sent) as 'Messages Sent',\n SUM(chat_messages_interacted) as 'Messages Interacted',\n SUM(chat_ai_code_lines) as 'AI Code Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, COUNT(*) AS 'Chat Events', SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS 'User Turns', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], @@ -377,8 +401,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Agentic (Dev) code generation and acceptance metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Agent continuations are empty-prompt records after tool calls; this measures autonomy, not accepted code.", "fieldConfig": { "defaults": { "color": { @@ -456,15 +483,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(dev_generation_event_count) as 'Generation Events',\n SUM(dev_generated_lines) as 'Generated Lines',\n SUM(dev_accepted_lines) as 'Accepted Lines',\n SUM(dev_acceptance_event_count) as 'Acceptance Events'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS 'User Turns', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) AS 'Agent Continuation Rate' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Agentic (Dev) Activity", + "title": "Agent Continuation Activity", "type": "timeseries" }, { @@ -477,12 +507,15 @@ }, "id": 22, "panels": [], - "title": "Code Review, Test Gen & Transformations", + "title": "Models, Prompts & Clients", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Code review findings and test generation metrics over time", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Authoritative model and routing-mode message counts from daily reports.", "fieldConfig": { "defaults": { "color": { @@ -560,20 +593,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(code_review_succeeded_event_count) as 'Reviews Succeeded',\n SUM(code_review_failed_event_count) as 'Reviews Failed'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(date) AS time, model_name AS metric, SUM(message_count) AS value FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY DATE(date), model_name ORDER BY time", "refId": "A" } ], - "title": "Code Review Activity", + "title": "Model / Route Message Mix", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Test generation events and acceptance over time", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Daily average prompt and response lengths. Test-generation counts are not exported.", "fieldConfig": { "defaults": { "color": { @@ -651,20 +690,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(test_generation_generated_tests) as 'Tests Generated',\n SUM(test_generation_accepted_tests) as 'Tests Accepted',\n SUM(test_generation_generated_lines) as 'Lines Generated',\n SUM(test_generation_accepted_lines) as 'Lines Accepted'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, ROUND(AVG(prompt_length)) AS 'Avg Prompt Length', ROUND(AVG(response_length)) AS 'Avg Response Length' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Test Generation Activity", + "title": "Prompt & Response Length", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Doc generation and code transformation events", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Messages by KIRO_IDE, KIRO_CLI, KIRO_WEB, and PLUGIN. Doc-generation metrics are not exported.", "fieldConfig": { "defaults": { "color": { @@ -742,20 +787,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(doc_generation_accepted_line_additions) as 'Doc Lines Accepted',\n SUM(transformation_event_count) as 'Transformation Events',\n SUM(transformation_lines_generated) as 'Transform Lines Generated'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(date) AS time, client_type AS metric, SUM(total_messages) AS value FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE(date), client_type ORDER BY time", "refId": "A" } ], - "title": "Doc Generation & Transformations", + "title": "Client Type Message Mix", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Number of users who used each feature in the selected period", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Distinct users observed in chat, completion, steering, and spec-mode signals.", "fieldConfig": { "defaults": { "color": { @@ -838,11 +889,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT 'Chat' as Feature, COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) as Users FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Review', COUNT(DISTINCT CASE WHEN code_review_succeeded_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Doc Generation', COUNT(DISTINCT CASE WHEN doc_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Test Generation', COUNT(DISTINCT CASE WHEN test_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Dev (Agentic)', COUNT(DISTINCT CASE WHEN dev_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Transformation', COUNT(DISTINCT CASE WHEN transformation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT 'Chat' AS 'Feature', COUNT(DISTINCT user_id) AS 'Users' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Completion' AS 'Feature', COUNT(DISTINCT user_id) AS 'Users' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Steering' AS 'Feature', COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) AS 'Users' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Spec Mode' AS 'Feature', COUNT(DISTINCT CASE WHEN is_spec_mode = 1 THEN user_id END) AS 'Users' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -863,8 +917,11 @@ "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Per-user breakdown of legacy feature-level metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Per-user chat, completion, autonomy, steering, and spec-mode activity.", "fieldConfig": { "defaults": { "color": { @@ -913,15 +970,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n SUM(inline_suggestions_count) as 'Suggestions',\n SUM(inline_acceptance_count) as 'Accepted',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 1), '%') as 'Accept %',\n SUM(chat_messages_sent) as 'Chat Msgs',\n SUM(chat_ai_code_lines) as 'Chat Lines',\n SUM(dev_accepted_lines) as 'Agentic Lines',\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_accepted_tests) as 'Tests Accepted',\n SUM(doc_generation_event_count) as 'Doc Gen',\n SUM(transformation_event_count) as 'Transforms',\n MIN(date) as 'First Active',\n MAX(date) as 'Last Active'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id\nORDER BY SUM(inline_suggestions_count) DESC", + "rawSql": "WITH users AS (SELECT user_id FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION SELECT user_id FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp)), chat AS (SELECT user_id, COUNT(*) AS chat_events, SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS user_turns, SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS agent_continuations, SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS steering, SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS spec_mode FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id), completion AS (SELECT user_id, COUNT(*) AS completion_requests, SUM(returned_line_count) AS returned_lines FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id), names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, u.user_id) AS 'User', COALESCE(c.chat_events, 0) AS 'Chat Events', COALESCE(c.user_turns, 0) AS 'User Turns', COALESCE(c.agent_continuations, 0) AS 'Agent Continuations', COALESCE(x.completion_requests, 0) AS 'Completion Requests', COALESCE(x.returned_lines, 0) AS 'Returned Lines', COALESCE(c.steering, 0) AS 'Steering', COALESCE(c.spec_mode, 0) AS 'Spec Mode' FROM users u LEFT JOIN chat c ON u.user_id = c.user_id LEFT JOIN completion x ON u.user_id = x.user_id LEFT JOIN names n ON u.user_id = n.user_id ORDER BY COALESCE(c.chat_events, 0) DESC", "refId": "A" } ], - "title": "Per-User Feature Metrics", + "title": "Per-User Kiro Activity", "type": "table" } ], @@ -929,8 +989,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "legacy", "kiro" ], "templating": { @@ -942,7 +1000,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Kiro Legacy Feature Metrics", - "uid": "qdev_feature_metrics", + "title": "Kiro Feature Metrics", + "uid": "kiro_feature_metrics", "version": 1 } diff --git a/grafana/dashboards/mysql/qdev_logging.json b/grafana/dashboards/mysql/kiro_logging.json similarity index 74% rename from grafana/dashboards/mysql/qdev_logging.json rename to grafana/dashboards/mysql/kiro_logging.json index aa7826ae243..538b907aab9 100644 --- a/grafana/dashboards/mysql/qdev_logging.json +++ b/grafana/dashboards/mysql/kiro_logging.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Overview of logging event metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Chat users, user turns, agent continuations, and completion events.", "fieldConfig": { "defaults": { "color": { @@ -66,11 +69,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Users',\n (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id != '') as 'Conversations',\n (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Completion Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Completion Users',\n (SELECT SUM(code_reference_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Code References',\n (SELECT SUM(web_link_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Web Links Cited'", + "rawSql": "SELECT (SELECT COUNT(*) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS 'Chat Events', (SELECT COUNT(DISTINCT user_id) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS 'Chat Users', (SELECT SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS 'User Turns', (SELECT SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS 'Agent Continuations', (SELECT COUNT(*) FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp)) AS 'Completion Events'", "refId": "A" } ], @@ -78,7 +84,10 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Hourly distribution of AI usage activity (chat + completions)", "fieldConfig": { "defaults": { @@ -155,11 +164,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n LPAD(CAST(hour_of_day AS CHAR), 2, '0') as 'Hour',\n SUM(chat_count) as 'Chat Events',\n SUM(completion_count) as 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) as hour_of_day, COUNT(*) as chat_count, 0 as completion_count\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) as hour_of_day, 0 as chat_count, COUNT(*) as completion_count\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) combined\nGROUP BY hour_of_day\nORDER BY hour_of_day", + "rawSql": "SELECT\n LPAD(CAST(hour_of_day AS CHAR), 2, '0') as 'Hour',\n SUM(chat_count) as 'Chat Events',\n SUM(completion_count) as 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) as hour_of_day, COUNT(*) as chat_count, 0 as completion_count\n FROM lake._tool_kiro_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) as hour_of_day, 0 as chat_count, COUNT(*) as completion_count\n FROM lake._tool_kiro_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) combined\nGROUP BY hour_of_day\nORDER BY hour_of_day", "refId": "A" } ], @@ -167,7 +179,10 @@ "type": "barchart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Distribution of chat trigger types: MANUAL (chat window) vs INLINE_CHAT", "fieldConfig": { "defaults": { @@ -223,11 +238,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)'\n ELSE chat_trigger_type\n END as 'Trigger Type',\n COUNT(*) as 'Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY chat_trigger_type\nORDER BY COUNT(*) DESC", + "rawSql": "SELECT\n CASE\n WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)'\n ELSE chat_trigger_type\n END as 'Trigger Type',\n COUNT(*) as 'Events'\nFROM lake._tool_kiro_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY chat_trigger_type\nORDER BY COUNT(*) DESC", "refId": "A" } ], @@ -235,8 +253,11 @@ "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Distribution of model usage across chat events", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Authoritative model and routing-mode counts from daily reports; chat model_id is only partially populated.", "fieldConfig": { "defaults": { "color": { @@ -280,7 +301,7 @@ "calcs": [ "lastNotNull" ], - "fields": "/^Requests$/", + "fields": "/^Messages$/", "values": true }, "tooltip": { @@ -291,19 +312,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN model_id = '' OR model_id IS NULL THEN '(unknown)'\n ELSE model_id\n END as 'Model',\n COUNT(*) as 'Requests'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY model_id\nORDER BY COUNT(*) DESC", + "rawSql": "SELECT model_name AS 'Model / Route', SUM(message_count) AS 'Messages' FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY model_name ORDER BY SUM(message_count) DESC", "refId": "A" } ], - "title": "Model Usage Distribution", + "title": "Model / Route Message Distribution", "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Top file extensions used with inline completions", "fieldConfig": { "defaults": { @@ -359,11 +386,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN file_extension = '' THEN '(unknown)'\n ELSE file_extension\n END as 'File Type',\n COUNT(*) as 'Completions'\nFROM lake._tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "rawSql": "SELECT\n CASE\n WHEN file_extension = '' THEN '(unknown)'\n ELSE file_extension\n END as 'File Type',\n COUNT(*) as 'Completions'\nFROM lake._tool_kiro_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", "refId": "A" } ], @@ -371,8 +401,11 @@ "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Average number of chat events per conversation", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Conversation IDs are absent in sampled Kiro logs, so conversation depth cannot be computed.", "fieldConfig": { "defaults": { "color": { @@ -451,19 +484,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(timestamp) as time,\n COUNT(*) / NULLIF(COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END), 0) as 'Avg Turns per Conversation',\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as 'Unique Conversations',\n COUNT(*) as 'Total Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS 'User Turns', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) AS 'Agent Continuation Rate' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Conversation Depth Analysis", + "title": "User Turns vs Agent Continuations", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Daily chat and completion events over time", "fieldConfig": { "defaults": { @@ -543,11 +582,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(chat) as 'Chat Events', SUM(completions) as 'Completion Events'\nFROM (\n SELECT DATE(timestamp) as time, COUNT(*) as chat, 0 as completions\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n UNION ALL\n SELECT DATE(timestamp) as time, 0 as chat, COUNT(*) as completions\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n) combined\nGROUP BY time\nORDER BY time", + "rawSql": "SELECT time, SUM(chat) as 'Chat Events', SUM(completions) as 'Completion Events'\nFROM (\n SELECT DATE(timestamp) as time, COUNT(*) as chat, 0 as completions\n FROM lake._tool_kiro_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n UNION ALL\n SELECT DATE(timestamp) as time, 0 as chat, COUNT(*) as completions\n FROM lake._tool_kiro_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n) combined\nGROUP BY time\nORDER BY time", "refId": "A" } ], @@ -555,8 +597,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Per-user logging activity summary", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Per-user chat and completion activity with display names resolved from usage reports.", "fieldConfig": { "defaults": { "color": { @@ -605,11 +650,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as 'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n u.conversations as 'Conversations',\n ROUND(u.chat_events / NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events, 0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as 'Avg Response Len',\n u.steering_count as 'Steering Uses',\n u.spec_count as 'Spec Mode Uses',\n u.code_ref_count as 'Code Refs',\n u.web_link_count as 'Web Links',\n u.models_used as 'Models Used',\n u.first_seen as 'First Seen',\n GREATEST(u.last_seen, COALESCE(c.last_seen, u.last_seen)) as 'Last Seen'\nFROM (\n SELECT\n user_id,\n MAX(display_name) as display_name,\n COUNT(*) as chat_events,\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as conversations,\n AVG(prompt_length) as avg_prompt_len,\n AVG(response_length) as avg_response_len,\n GROUP_CONCAT(DISTINCT CASE WHEN model_id != '' AND model_id IS NOT NULL THEN model_id END ORDER BY model_id SEPARATOR ', ') as models_used,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as steering_count,\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as spec_count,\n SUM(code_reference_count) as code_ref_count,\n SUM(web_link_count) as web_link_count,\n MIN(timestamp) as first_seen,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) u\nLEFT JOIN (\n SELECT\n user_id,\n COUNT(*) as completion_events,\n COUNT(DISTINCT file_name) as files_count,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) c ON u.user_id = c.user_id\nORDER BY u.user_id", + "rawSql": "WITH chat AS (SELECT user_id, COUNT(*) AS chat_events, SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS user_turns, SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS agent_continuations FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id), completion AS (SELECT user_id, COUNT(*) AS completion_events, SUM(returned_line_count) AS returned_lines FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id), names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, c.user_id) AS 'User', c.user_id AS 'User ID', c.chat_events AS 'Chat Events', c.user_turns AS 'User Turns', c.agent_continuations AS 'Agent Continuations', COALESCE(x.completion_events, 0) AS 'Completion Events', COALESCE(x.returned_lines, 0) AS 'Returned Lines' FROM chat c LEFT JOIN completion x ON c.user_id = x.user_id LEFT JOIN names n ON c.user_id = n.user_id ORDER BY c.chat_events DESC", "refId": "A" } ], @@ -617,7 +665,10 @@ "type": "table" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Distribution of Kiro feature adoption: Steering, Spec Mode, and Plain Chat", "fieldConfig": { "defaults": { @@ -673,11 +724,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as 'Using Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as 'Using Spec Mode',\n SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) as 'Plain Chat'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "rawSql": "SELECT\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as 'Using Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as 'Using Spec Mode',\n SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) as 'Plain Chat'\nFROM lake._tool_kiro_chat_log\nWHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -685,8 +739,11 @@ "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Top file extensions active during chat events", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Chat logs do not contain active file extensions; completion logs provide file types and returned lines.", "fieldConfig": { "defaults": { "color": { @@ -741,20 +798,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file active)'\n ELSE active_file_extension\n END as 'File Type',\n COUNT(*) as 'Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY active_file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS 'File Type', SUM(returned_line_count) AS 'Returned Lines' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY file_extension ORDER BY SUM(returned_line_count) DESC", "refId": "A" } ], - "title": "Active File Types in Chat", + "title": "Returned Lines by File Type", "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "How often Kiro responses include code references and web links", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro logs expose follow-up prompts but not code-reference or web-link counters.", "fieldConfig": { "defaults": { "color": { @@ -809,19 +872,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n SUM(CASE WHEN code_reference_count > 0 THEN 1 ELSE 0 END) as 'With Code References',\n SUM(CASE WHEN web_link_count > 0 THEN 1 ELSE 0 END) as 'With Web Links',\n SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) as 'With Followup Prompts',\n SUM(CASE WHEN code_reference_count = 0 AND web_link_count = 0 AND has_followup_prompts = 0 THEN 1 ELSE 0 END) as 'Plain Response'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "rawSql": "SELECT SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) AS 'With Follow-up Prompts', SUM(CASE WHEN has_followup_prompts = 0 THEN 1 ELSE 0 END) AS 'Without Follow-up Prompts' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], - "title": "Response Enrichment Breakdown", + "title": "Follow-up Prompt Breakdown", "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Average and maximum prompt/response lengths over time", "fieldConfig": { "defaults": { @@ -901,11 +970,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(timestamp) as time,\n AVG(prompt_length) as 'Avg Prompt Length',\n AVG(response_length) as 'Avg Response Length',\n MAX(prompt_length) as 'Max Prompt Length',\n MAX(response_length) as 'Max Response Length'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "rawSql": "SELECT\n DATE(timestamp) as time,\n AVG(prompt_length) as 'Avg Prompt Length',\n AVG(response_length) as 'Avg Response Length',\n MAX(prompt_length) as 'Max Prompt Length',\n MAX(response_length) as 'Max Response Length'\nFROM lake._tool_kiro_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", "refId": "A" } ], @@ -913,7 +985,10 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Average code context size provided to inline completions over time", "fieldConfig": { "defaults": { @@ -992,11 +1067,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(timestamp) as time,\n ROUND(AVG(left_context_length)) as 'Avg Left Context',\n ROUND(AVG(right_context_length)) as 'Avg Right Context',\n ROUND(AVG(left_context_length + right_context_length)) as 'Avg Total Context'\nFROM lake._tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "rawSql": "SELECT\n DATE(timestamp) as time,\n ROUND(AVG(left_context_length)) as 'Avg Left Context',\n ROUND(AVG(right_context_length)) as 'Avg Right Context',\n ROUND(AVG(left_context_length + right_context_length)) as 'Avg Total Context'\nFROM lake._tool_kiro_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", "refId": "A" } ], @@ -1004,8 +1082,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily trend of code references and web links in chat responses", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Daily responses with and without follow-up prompts. Code-reference and web-link counters are not exported.", "fieldConfig": { "defaults": { "color": { @@ -1083,15 +1164,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(timestamp) as time,\n SUM(code_reference_count) as 'Code References',\n SUM(web_link_count) as 'Web Links',\n SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) as 'Followup Prompts'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) AS 'With Follow-up Prompts', SUM(CASE WHEN has_followup_prompts = 0 THEN 1 ELSE 0 END) AS 'Without Follow-up Prompts' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Response Enrichment Trends", + "title": "Follow-up Prompt Trends", "type": "timeseries" } ], @@ -1099,9 +1183,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "logging", - "kiro" + "kiro", + "logging" ], "templating": { "list": [] @@ -1113,6 +1196,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro AI Activity Insights", - "uid": "qdev_logging", + "uid": "kiro_logging", "version": 1 } diff --git a/grafana/dashboards/mysql/qdev_user_report.json b/grafana/dashboards/mysql/kiro_user_report.json similarity index 82% rename from grafana/dashboards/mysql/qdev_user_report.json rename to grafana/dashboards/mysql/kiro_user_report.json index acd0f14b3b6..3941859756f 100644 --- a/grafana/dashboards/mysql/qdev_user_report.json +++ b/grafana/dashboards/mysql/kiro_user_report.json @@ -19,7 +19,10 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Overview of credits and usage metrics", "fieldConfig": { "defaults": { @@ -66,11 +69,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n SUM(credits_used) as 'Total Credits Used',\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(total_messages) as 'Total Messages',\n SUM(chat_conversations) as 'Total Conversations'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)", + "rawSql": "SELECT\n SUM(credits_used) as 'Total Credits Used',\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(total_messages) as 'Total Messages',\n SUM(chat_conversations) as 'Total Conversations'\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)", "refId": "A" } ], @@ -78,7 +84,10 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Daily credits consumed broken down by subscription tier", "fieldConfig": { "defaults": { @@ -158,11 +167,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n subscription_tier as metric,\n SUM(credits_used) as value\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY date, subscription_tier\nORDER BY date", + "rawSql": "SELECT\n date as time,\n subscription_tier as metric,\n SUM(credits_used) as value\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)\nGROUP BY date, subscription_tier\nORDER BY date", "refId": "A" } ], @@ -170,8 +182,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily messages and conversations broken down by client type", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Daily messages by Kiro IDE, CLI, Web, and Plugin clients, plus conversations.", "fieldConfig": { "defaults": { "color": { @@ -250,11 +265,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) as 'Messages (IDE)',\n SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) as 'Messages (CLI)',\n SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) as 'Messages (Plugin)',\n SUM(chat_conversations) as 'Conversations'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT date AS time, SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) AS 'Messages (IDE)', SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) AS 'Messages (CLI)', SUM(CASE WHEN client_type = 'KIRO_WEB' THEN total_messages ELSE 0 END) AS 'Messages (Web)', SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) AS 'Messages (Plugin)', SUM(chat_conversations) AS 'Conversations' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY date ORDER BY date", "refId": "A" } ], @@ -262,7 +280,10 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Distribution of users across subscription tiers", "fieldConfig": { "defaults": { @@ -318,11 +339,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n subscription_tier as 'Tier',\n COUNT(DISTINCT user_id) as 'Users'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier IS NOT NULL\n AND subscription_tier != ''\nGROUP BY subscription_tier\nORDER BY COUNT(DISTINCT user_id) DESC", + "rawSql": "SELECT\n subscription_tier as 'Tier',\n COUNT(DISTINCT user_id) as 'Users'\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier IS NOT NULL\n AND subscription_tier != ''\nGROUP BY subscription_tier\nORDER BY COUNT(DISTINCT user_id) DESC", "refId": "A" } ], @@ -330,7 +354,10 @@ "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Per-user credits, messages, and subscription details", "fieldConfig": { "defaults": { @@ -429,11 +456,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, subscription_tier, client_type\nORDER BY user_id DESC", + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, subscription_tier, client_type\nORDER BY user_id DESC", "refId": "A" } ], @@ -445,9 +475,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "user_report", - "kiro" + "kiro", + "user_report" ], "templating": { "list": [] @@ -459,6 +488,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro Usage Dashboard", - "uid": "qdev_user_report", + "uid": "kiro_user_report", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/ai-model-roi.json b/grafana/dashboards/postgresql/ai-model-roi.json index ffbda08b227..1a1c1205f79 100644 --- a/grafana/dashboards/postgresql/ai-model-roi.json +++ b/grafana/dashboards/postgresql/ai-model-roi.json @@ -26,7 +26,7 @@ "targetBlank": true, "title": "Kiro Usage Dashboard", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" } ], "panels": [ @@ -87,7 +87,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], @@ -138,7 +138,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0), 0), 1) AS \"Credits / PR\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0), 0), 1) AS \"Credits / PR\" FROM _tool_kiro_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -189,7 +189,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0), 0), 1) AS \"Credits / Deploy\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0), 0), 1) AS \"Credits / Deploy\" FROM _tool_kiro_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -240,7 +240,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0), 0), 1) AS \"Credits / Issue\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0), 0), 1) AS \"Credits / Issue\" FROM _tool_kiro_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -316,7 +316,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -379,7 +379,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -442,7 +442,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -505,7 +505,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -517,7 +517,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", "kiro", "cost", "efficiency" @@ -531,7 +530,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "AI Cost-Efficiency (PostgreSQL)", - "uid": "ai_cost_efficiency-pg", + "title": "AI Model ROI", + "uid": "ai_model_roi-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/q-dev-dora.json b/grafana/dashboards/postgresql/kiro-dora.json similarity index 62% rename from grafana/dashboards/postgresql/q-dev-dora.json rename to grafana/dashboards/postgresql/kiro-dora.json index 2a5368236d1..fecbc0cb490 100644 --- a/grafana/dashboards/postgresql/q-dev-dora.json +++ b/grafana/dashboards/postgresql/kiro-dora.json @@ -36,10 +36,10 @@ "keepTime": true, "tags": [], "targetBlank": true, - "title": "Q Dev Dashboard", + "title": "Kiro Dashboard", "tooltip": "", "type": "link", - "url": "/d/qdev_user_data/q-dev-user-data-dashboard" + "url": "/d/kiro_user_data/kiro-user-data-dashboard" } ], "panels": [ @@ -61,7 +61,7 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "## AI-Powered DORA Dashboard\nThis dashboard correlates **Q Dev (AI coding assistant)** usage metrics with **DORA** performance indicators to help understand the impact of AI-assisted development on engineering efficiency.\n\n- **Left side**: Q Dev AI usage metrics (code generation, acceptance rate)\n- **Right side**: DORA metrics (Lead Time, Deployment Frequency, Change Failure Rate)\n- **Correlation charts**: Show trends over time to identify potential relationships", + "content": "## Kiro + DORA Dashboard\nThis dashboard compares **Kiro usage** with **DORA** performance indicators at monthly aggregates.\n\n- **Kiro metrics**: active users, credits, messages, conversations, and credits per message\n- **DORA metrics**: lead time, deployment frequency, and change failure rate\n- **Semantic boundary**: Kiro does not export accepted LOC, suggestion acceptance, generated tests, or code-review findings; no proxy is presented as an equivalent measure", "mode": "markdown" }, "pluginVersion": "13.0.2", @@ -82,8 +82,11 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Number of unique users who used Q Dev AI features", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Unique Kiro users in the selected period, from the daily usage report.", "fieldConfig": { "defaults": { "color": { @@ -128,20 +131,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Q Dev Users\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Kiro Users\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Q Dev Active Users", + "title": "Kiro Active Users", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Total AI-generated code lines accepted (Inline + Chat)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro exports credits, not accepted lines of code. Accepted LOC has no equivalent field.", "fieldConfig": { "defaults": { "color": { @@ -187,20 +196,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(inline_ai_code_lines + chat_ai_code_lines) AS \"AI Accepted Lines\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used), 1) AS \"Kiro Credits Used\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Total AI Code Lines", + "title": "Total Kiro Credits", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Acceptance rate of inline AI suggestions", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro does not export suggestion acceptance. Credits per message is a usage-efficiency ratio, not an acceptance rate.", "fieldConfig": { "defaults": { "color": { @@ -224,7 +239,7 @@ } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [] }, @@ -254,19 +269,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Acceptance Rate\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT CAST(SUM(credits_used) AS NUMERIC) / NULLIF(SUM(total_messages), 0) AS \"Credits per Message\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "AI Acceptance Rate", + "title": "Credits per Message", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Number of production deployments in selected period", "fieldConfig": { "defaults": { @@ -312,7 +333,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -324,7 +348,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Median lead time for changes in hours", "fieldConfig": { "defaults": { @@ -379,7 +406,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -391,7 +421,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Percentage of deployments that caused incidents", "fieldConfig": { "defaults": { @@ -446,7 +479,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -467,12 +503,15 @@ }, "id": 9, "panels": [], - "title": "AI Usage vs DORA Metrics Correlation", + "title": "Kiro Usage vs DORA Metrics", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Compare AI code generation trends with Lead Time for Changes. A negative correlation (AI lines up, Lead Time down) suggests AI is helping accelerate delivery.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro credits are compared with Lead Time. Kiro does not export accepted code lines.", "fieldConfig": { "defaults": { "color": { @@ -526,7 +565,7 @@ { "matcher": { "id": "byName", - "options": "AI Accepted Lines" + "options": "Kiro Credits" }, "properties": [ { @@ -545,29 +584,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Median Lead Time (hours)" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Lead Time (hours)" - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] } ] }, @@ -597,20 +613,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_monthly AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), lead_time_monthly AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, lt.month), '%Y-%m-%d') AS time, ai.ai_lines AS \"AI Accepted Lines\", ROUND(lt.avg_lead_time, 1) AS \"Median Lead Time (hours)\" FROM ai_monthly AS ai LEFT JOIN lead_time_monthly AS lt ON ai.month = lt.month WHERE NOT ai.month IS NULL OR NOT lt.month IS NULL ORDER BY time NULLS FIRST", + "rawSql": "WITH ai_monthly AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(credits_used) AS kiro_credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), lead_time_monthly AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, lt.month), '%Y-%m-%d') AS time, ai.kiro_credits AS \"Kiro Credits\", ROUND(lt.avg_lead_time, 1) AS \"Median Lead Time (hours)\" FROM ai_monthly AS ai LEFT JOIN lead_time_monthly AS lt ON ai.month = lt.month WHERE NOT ai.month IS NULL OR NOT lt.month IS NULL ORDER BY time NULLS FIRST", "refId": "A" } ], - "title": "AI Code Generation vs Lead Time Trend", + "title": "Kiro Credits vs Lead Time Trend", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Compare AI suggestion acceptance rate with deployment frequency. Higher acceptance rate may indicate better AI integration and potentially more deployments.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Usage-efficiency ratio versus deployment frequency. This is not suggestion acceptance.", "fieldConfig": { "defaults": { "color": { @@ -664,7 +686,7 @@ { "matcher": { "id": "byName", - "options": "AI Acceptance Rate" + "options": "Credits per Message" }, "properties": [ { @@ -673,7 +695,7 @@ }, { "id": "custom.axisLabel", - "value": "Acceptance Rate" + "value": "Credits per Message" }, { "id": "unit", @@ -687,29 +709,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Deployment Count" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Deployments" - }, - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] } ] }, @@ -739,20 +738,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_acceptance AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS acceptance_rate FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), deployment_count AS (SELECT TO_CHAR(CAST(MAX(cdc.finished_date) AS TIMESTAMP), 'YYYY-MM-01') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploy_count FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, dc.month), '%Y-%m-%d') AS time, ai.acceptance_rate AS \"AI Acceptance Rate\", dc.deploy_count AS \"Deployment Count\" FROM ai_acceptance AS ai LEFT JOIN deployment_count AS dc ON ai.month = dc.month WHERE NOT ai.month IS NULL OR NOT dc.month IS NULL ORDER BY time NULLS FIRST", + "rawSql": "WITH ai_acceptance AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(credits_used) AS NUMERIC) / NULLIF(NULLIF(SUM(total_messages), 0), 0) AS credits_per_message FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), deployment_count AS (SELECT TO_CHAR(CAST(MAX(cdc.finished_date) AS TIMESTAMP), 'YYYY-MM-01') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploy_count FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, dc.month), '%Y-%m-%d') AS time, ai.credits_per_message AS \"Credits per Message\", dc.deploy_count AS \"Deployment Count\" FROM ai_acceptance AS ai LEFT JOIN deployment_count AS dc ON ai.month = dc.month WHERE NOT ai.month IS NULL OR NOT dc.month IS NULL ORDER BY time NULLS FIRST", "refId": "A" } ], - "title": "AI Acceptance Rate vs Deployment Frequency", + "title": "Credits per Message vs Deployment Frequency", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Compare AI-generated tests with Change Failure Rate. More AI-generated tests might correlate with lower failure rates.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro does not export generated-test counts; total messages is the available activity-volume measure.", "fieldConfig": { "defaults": { "color": { @@ -806,7 +811,7 @@ { "matcher": { "id": "byName", - "options": "AI Generated Tests" + "options": "Kiro Messages" }, "properties": [ { @@ -825,33 +830,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Change Failure Rate" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Failure Rate" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] } ] }, @@ -881,20 +859,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_tests AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(test_generation_generated_tests) AS generated_tests FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), cfr_monthly AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, cfr.month), '%Y-%m-%d') AS time, ai.generated_tests AS \"AI Generated Tests\", cfr.cfr AS \"Change Failure Rate\" FROM ai_tests AS ai LEFT JOIN cfr_monthly AS cfr ON ai.month = cfr.month WHERE NOT ai.month IS NULL OR NOT cfr.month IS NULL ORDER BY time NULLS FIRST", + "rawSql": "WITH ai_tests AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(total_messages) AS kiro_messages FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), cfr_monthly AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, cfr.month), '%Y-%m-%d') AS time, ai.kiro_messages AS \"Kiro Messages\", cfr.cfr AS \"Change Failure Rate\" FROM ai_tests AS ai LEFT JOIN cfr_monthly AS cfr ON ai.month = cfr.month WHERE NOT ai.month IS NULL OR NOT cfr.month IS NULL ORDER BY time NULLS FIRST", "refId": "A" } ], - "title": "AI Test Generation vs Change Failure Rate", + "title": "Kiro Messages vs Change Failure Rate", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Compare active Q Dev users with Code Review findings. More AI-assisted code review might catch issues earlier.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro does not export code-review findings. Shows active users and credits by month.", "fieldConfig": { "defaults": { "color": { @@ -948,30 +932,7 @@ { "matcher": { "id": "byName", - "options": "Active Users" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "left" - }, - { - "id": "custom.axisLabel", - "value": "Users" - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Code Review Findings" + "options": "Kiro Credits" }, "properties": [ { @@ -1019,15 +980,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT TO_DATE(TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01'), '%Y-%m-%d') AS time, COUNT(DISTINCT user_id) AS \"Active Users\", SUM(code_review_findings_count) AS \"Code Review Findings\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') ORDER BY time NULLS FIRST", + "rawSql": "SELECT TO_DATE(TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01'), '%Y-%m-%d') AS time, COUNT(DISTINCT user_id) AS \"Active Users\", SUM(credits_used) AS \"Kiro Credits\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') ORDER BY time NULLS FIRST", "refId": "A" } ], - "title": "Q Dev Users vs Code Review Findings", + "title": "Kiro Users vs Credits", "type": "timeseries" }, { @@ -1044,8 +1008,11 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Monthly summary comparing Q Dev AI metrics with DORA metrics side by side", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Monthly Kiro usage metrics beside DORA metrics. LOC, acceptance, generated tests, and review findings are not exported by Kiro.", "fieldConfig": { "defaults": { "color": { @@ -1074,7 +1041,7 @@ { "matcher": { "id": "byName", - "options": "AI Acceptance Rate" + "options": "Credits per Message" }, "properties": [ { @@ -1099,47 +1066,6 @@ "value": 1 } ] - }, - { - "matcher": { - "id": "byName", - "options": "Change Failure Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-RdYlGr" - } - }, - { - "id": "max", - "value": 0.3 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Lead Time (hours)" - }, - "properties": [ - { - "id": "unit", - "value": "h" - } - ] } ] }, @@ -1171,15 +1097,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_metrics AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT user_id) AS active_users, SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines, CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS acceptance_rate, SUM(test_generation_generated_tests) AS generated_tests, SUM(code_review_findings_count) AS review_findings FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM')), dora_metrics AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deployments, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' LEFT JOIN cicd_deployment_commits AS cdc2 ON cdc.cicd_deployment_id = cdc2.cicd_deployment_id LEFT JOIN project_pr_metrics AS ppm ON ppm.deployment_commit_id = cdc2.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM')), cfr_metrics AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM')) SELECT COALESCE(ai.month, dm.month, cfr.month) AS \"Month\", COALESCE(ai.active_users, 0) AS \"Q Dev Users\", COALESCE(ai.ai_lines, 0) AS \"AI Code Lines\", ai.acceptance_rate AS \"AI Acceptance Rate\", COALESCE(ai.generated_tests, 0) AS \"AI Tests\", COALESCE(ai.review_findings, 0) AS \"Review Findings\", COALESCE(dm.deployments, 0) AS \"Deployments\", ROUND(dm.avg_lead_time, 1) AS \"Lead Time (hours)\", cfr.cfr AS \"Change Failure Rate\" FROM ai_metrics AS ai LEFT JOIN dora_metrics AS dm ON ai.month = dm.month LEFT JOIN cfr_metrics AS cfr ON ai.month = cfr.month ORDER BY ai.month DESC NULLS LAST", + "rawSql": "WITH ai_metrics AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT user_id) AS active_users, SUM(credits_used) AS credits, CAST(SUM(credits_used) AS NUMERIC) / NULLIF(NULLIF(SUM(total_messages), 0), 0) AS credits_per_message, SUM(total_messages) AS messages, SUM(chat_conversations) AS conversations FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM')), dora_metrics AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deployments, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' LEFT JOIN cicd_deployment_commits AS cdc2 ON cdc.cicd_deployment_id = cdc2.cicd_deployment_id LEFT JOIN project_pr_metrics AS ppm ON ppm.deployment_commit_id = cdc2.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM')), cfr_metrics AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM')) SELECT COALESCE(ai.month, dm.month, cfr.month) AS \"Month\", COALESCE(ai.active_users, 0) AS \"Kiro Users\", COALESCE(ai.credits, 0) AS \"Credits Used\", ai.credits_per_message AS \"Credits per Message\", COALESCE(ai.messages, 0) AS \"Messages\", COALESCE(ai.conversations, 0) AS \"Conversations\", COALESCE(dm.deployments, 0) AS \"Deployments\", ROUND(dm.avg_lead_time, 1) AS \"Lead Time (hours)\", cfr.cfr AS \"Change Failure Rate\" FROM ai_metrics AS ai LEFT JOIN dora_metrics AS dm ON ai.month = dm.month LEFT JOIN cfr_metrics AS cfr ON ai.month = cfr.month ORDER BY ai.month DESC NULLS LAST", "refId": "A" } ], - "title": "Monthly Q Dev vs DORA Metrics Comparison", + "title": "Monthly Kiro + DORA Metrics", "type": "table" } ], @@ -1187,7 +1116,7 @@ "refresh": "5m", "schemaVersion": 39, "tags": [ - "q_dev", + "kiro", "DORA", "AI", "correlation" @@ -1204,7 +1133,10 @@ "$__all" ] }, - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "definition": "SELECT DISTINCT name FROM projects", "hide": 0, "includeAll": true, @@ -1227,7 +1159,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Q Dev + DORA Correlation", - "uid": "qdev_dora_correlation-pg", + "title": "Kiro + DORA Correlation", + "uid": "kiro_dora_correlation-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/mysql/qdev_user_data.json b/grafana/dashboards/postgresql/kiro_activity_detail.json similarity index 70% rename from grafana/dashboards/mysql/qdev_user_data.json rename to grafana/dashboards/postgresql/kiro_activity_detail.json index affaf57244c..f67b94fd9a2 100644 --- a/grafana/dashboards/mysql/qdev_user_data.json +++ b/grafana/dashboards/postgresql/kiro_activity_detail.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Overview of key user metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Usage-report totals. Accepted LOC and acceptance rates are not exported by Kiro.", "fieldConfig": { "defaults": { "color": { @@ -66,13 +69,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Acceptance Rate (Inline Suggestion)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Acceptance Rate (Code Fix)',\n SUM(transformation_lines_ingested) as 'Ingested Lines (Java Transform)',\n SUM(transformation_lines_generated) as 'Generated Lines (Java Transform)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Lines (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Acceptance Rate (Inline Chat)'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", ROUND(SUM(credits_used), 1) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A", "select": [ [ @@ -115,8 +121,11 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily AI code line changes across all users", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Daily Kiro usage volume from the user report.", "fieldConfig": { "defaults": { "color": { @@ -195,13 +204,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(chat_ai_code_lines) as 'Chat Accepted Lines',\n SUM(code_fix_accepted_lines) as 'Code Fix Accepted Lines',\n SUM(code_fix_generated_lines) as 'Code Fix Generated Lines',\n SUM(transformation_lines_ingested) as 'Java Transform Ingested Lines',\n SUM(transformation_lines_generated) as 'Java Transform Generated Lines',\n SUM(inline_ai_code_lines) as 'Inline Suggestion Accepted Lines',\n SUM(inline_chat_accepted_line_additions) as 'Inline Chat Accepted Line Additions',\n SUM(inline_chat_accepted_line_deletions) as 'Inline Chat Accepted Line Deletions',\n SUM(inline_chat_dismissed_line_additions) as 'Inline Chat Dismissed Line Additions',\n SUM(inline_chat_dismissed_line_deletions) as 'Inline Chat Dismissed Line Deletions',\n SUM(inline_chat_rejected_line_additions) as 'Inline Chat Rejected Line Additions',\n SUM(inline_chat_rejected_line_deletions) as 'Inline Chat Rejected Line Deletions'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(date AS DATE) AS time, ROUND(SUM(credits_used), 1) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -240,12 +252,15 @@ ] } ], - "title": "Daily AI Code Line Changes", + "title": "Daily Credits, Messages & Conversations", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily AI interaction trends across all users", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Chat events split into user turns and agent continuations.", "fieldConfig": { "defaults": { "color": { @@ -324,13 +339,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(chat_messages_sent) as 'Chat Messages Sent',\n SUM(code_fix_acceptance_event_count) as 'Code Fix Accepted Event Count',\n SUM(code_fix_generation_event_count) as 'Code Fix Generated Event Count',\n SUM(transformation_event_count) as 'Java Transform Event Count',\n SUM(inline_acceptance_count) as 'Inline Suggestion Accepted Suggestions',\n SUM(inline_suggestions_count) as 'Inline Suggestion Count',\n SUM(inline_chat_total_event_count) as 'Inline Chat Total Suggestions',\n SUM(inline_chat_acceptance_event_count) as 'Inline Chat Accepted Suggestions',\n SUM(inline_chat_dismissal_event_count) as 'Inline Chat Dismissed Suggestions',\n SUM(inline_chat_rejection_event_count) as 'Inline Chat Rejected Suggestions'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS \"Chat Events\", SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS \"User Turns\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -369,12 +387,15 @@ ] } ], - "title": "Daily AI Interactions", + "title": "Daily Chat Activity", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Code review metrics over time", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Authoritative per-model and routing-mode message counts.", "fieldConfig": { "defaults": { "color": { @@ -453,12 +474,15 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(code_fix_acceptance_event_count) as 'Code Fix Accepted Event Count',\n SUM(code_fix_generation_event_count) as 'Code Fix Generated Event Count',\n SUM(code_review_findings_count) as 'Total Findings'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(date AS DATE) AS time, model_name AS metric, SUM(message_count) AS value FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE), model_name ORDER BY time", "refId": "A", "select": [ [ @@ -497,12 +521,15 @@ ] } ], - "title": "Code Review Metrics", + "title": "Daily Model / Route Messages", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily acceptance rate of AI suggestions", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Agent continuations divided by all chat events. This is not suggestion acceptance.", "fieldConfig": { "defaults": { "color": { @@ -581,13 +608,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CAST(SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0) AS \"Agent Continuation Rate\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -626,12 +656,15 @@ ] } ], - "title": "Daily AI Suggestion Acceptance Rate", + "title": "Daily Agent Continuation Rate", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "User AI interaction metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Per-user credits, messages, conversations, overage, and last activity.", "fieldConfig": { "defaults": { "color": { @@ -655,51 +688,7 @@ ] } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Acceptance Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inline Chat Accepted Events" - }, - "properties": [ - { - "id": "custom.width", - "value": 239 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Acceptance Rate (Inline Suggestion)" - }, - "properties": [ - { - "id": "custom.width", - "value": 172 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -724,13 +713,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_generated_lines) as 'Generated Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) as 'Accepted Count (Code Fix)',\n SUM(code_fix_generation_event_count) as 'Generated Count (Code Fix)',\n CONCAT(ROUND(SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Code Fix)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) as 'Accepted Count (Inline Suggestion)',\n SUM(inline_suggestions_count) as 'Total Count (Inline Suggestion)',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Suggestion)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Line Additions (Inline Chat)',\n SUM(inline_chat_accepted_line_deletions) as 'Accepted Line Deletions (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) as 'Accepted Events (Inline Chat)',\n SUM(inline_chat_total_event_count) as 'Total Events (Inline Chat)',\n CONCAT(ROUND(SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Chat)',\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id\nORDER BY SUM(inline_ai_code_lines) DESC", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", MAX(subscription_tier) AS \"Tier\", ROUND(SUM(credits_used), 1) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", SUM(overage_credits_used) AS \"Overage Credits\", MAX(date) AS \"Last Activity\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id ORDER BY SUM(credits_used) DESC", "refId": "A", "select": [ [ @@ -769,12 +761,15 @@ ] } ], - "title": "User Interactions", + "title": "Per-User Usage", "type": "table" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily doc generation events and accepted/rejected lines", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Messages by Kiro client type. Doc-generation metrics are not exported.", "fieldConfig": { "defaults": { "color": { @@ -853,13 +848,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(doc_generation_event_count) as 'Doc Generation Events',\n SUM(doc_generation_accepted_line_additions) as 'Accepted Line Additions',\n SUM(doc_generation_accepted_line_updates) as 'Accepted Line Updates',\n SUM(doc_generation_rejected_line_additions) as 'Rejected Line Additions',\n SUM(doc_generation_rejected_line_updates) as 'Rejected Line Updates'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(date AS DATE) AS time, client_type AS metric, SUM(total_messages) AS value FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE), client_type ORDER BY time", "refId": "A", "select": [ [ @@ -898,12 +896,15 @@ ] } ], - "title": "Doc Generation Metrics", + "title": "Daily Client Type Messages", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily test generation events and lines", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Completion requests, requests with returned suggestions, and returned lines. These are not acceptance events.", "fieldConfig": { "defaults": { "color": { @@ -982,13 +983,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(test_generation_event_count) as 'Test Generation Events',\n SUM(test_generation_accepted_tests) as 'Accepted Tests',\n SUM(test_generation_generated_tests) as 'Generated Tests',\n SUM(test_generation_accepted_lines) as 'Accepted Lines',\n SUM(test_generation_generated_lines) as 'Generated Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS \"Completion Requests\", SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS \"Requests with Results\", SUM(returned_line_count) AS \"Returned Lines\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -1027,12 +1031,15 @@ ] } ], - "title": "Test Generation Metrics", + "title": "Daily Completion Activity", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily agentic dev events and lines", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Steering, spec-mode, and agent-continuation events. Kiro does not export accepted agentic LOC.", "fieldConfig": { "defaults": { "color": { @@ -1111,13 +1118,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(dev_generation_event_count) as 'Dev Generation Events',\n SUM(dev_acceptance_event_count) as 'Dev Acceptance Events',\n SUM(dev_generated_lines) as 'Dev Generated Lines',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Spec Mode\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -1156,7 +1166,7 @@ ] } ], - "title": "Dev (Agentic) Metrics", + "title": "Daily Agentic Signals", "type": "timeseries" } ], @@ -1164,7 +1174,7 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", + "kiro", "user_data" ], "templating": { @@ -1176,7 +1186,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Kiro Code Metrics Dashboard", - "uid": "qdev_user_data", + "title": "Kiro Activity Detail Dashboard", + "uid": "kiro_user_data-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_executive.json b/grafana/dashboards/postgresql/kiro_executive.json similarity index 71% rename from grafana/dashboards/postgresql/qdev_executive.json rename to grafana/dashboards/postgresql/kiro_executive.json index 8ea7bbb8c8f..9a13ba80109 100644 --- a/grafana/dashboards/postgresql/qdev_executive.json +++ b/grafana/dashboards/postgresql/kiro_executive.json @@ -27,7 +27,7 @@ "title": "Usage (New)", "tooltip": "Kiro Usage Dashboard - Credits & Messages (new format)", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" }, { "asDropdown": false, @@ -36,10 +36,10 @@ "keepTime": true, "tags": [], "targetBlank": true, - "title": "Feature Metrics (Legacy)", - "tooltip": "Kiro Legacy Feature Metrics (old format)", + "title": "Feature Metrics", + "tooltip": "Kiro Feature Metrics", "type": "link", - "url": "/d/qdev_feature_metrics" + "url": "/d/kiro_feature_metrics" }, { "asDropdown": false, @@ -51,7 +51,7 @@ "title": "Prompt Logging", "tooltip": "Kiro AI Activity Insights - Prompt Logging", "type": "link", - "url": "/d/qdev_logging" + "url": "/d/kiro_logging" } ], "panels": [ @@ -69,7 +69,10 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Distinct users with chat activity in the last 7 days (from prompt logging)", "fieldConfig": { "defaults": { @@ -116,11 +119,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"WAU\" FROM lake._tool_q_dev_chat_log WHERE timestamp >= NOW() - INTERVAL '7 DAY'", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"WAU\" FROM lake._tool_kiro_chat_log WHERE timestamp >= NOW() - INTERVAL '7 DAY'", "refId": "A" } ], @@ -128,8 +134,11 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Average credits spent per accepted line of code (new report + legacy metrics)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Usage efficiency from Kiro reports. Accepted LOC is not exported.", "fieldConfig": { "defaults": { "color": { @@ -175,20 +184,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(SUM(d.total_accepted), 0), 0), 2) AS \"Credits per Accepted Line\" FROM (SELECT user_id, date, SUM(credits_used) AS credits_used FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", date) AS r JOIN (SELECT user_id, \"date\", (inline_ai_code_lines + chat_ai_code_lines + code_fix_accepted_lines + dev_accepted_lines) AS total_accepted FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)) AS d ON r.user_id = d.user_id AND r.date = d.date", + "rawSql": "SELECT ROUND(CAST(SUM(credits_used) AS NUMERIC) / NULLIF(SUM(total_messages), 0), 2) AS \"Credits per Message\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Credits Efficiency (new + legacy)", + "title": "Credits per Message", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Percentage of inline suggestions accepted (from legacy feature metrics)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Share of chat events generated by agent self-continuation. Suggestion acceptance is not exported.", "fieldConfig": { "defaults": { "color": { @@ -234,19 +249,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) * 100, 1) AS \"Acceptance %\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(CAST(SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0), 1) AS \"Agent Continuation %\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], - "title": "Inline Acceptance Rate (legacy)", + "title": "Agent Continuation Rate", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Percentage of users who used steering rules (from prompt logging)", "fieldConfig": { "defaults": { @@ -294,11 +315,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) * 100 AS DECIMAL), 0) AS \"Steering %\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) * 100 AS DECIMAL), 0) AS \"Steering %\" FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -315,11 +339,14 @@ }, "id": 101, "panels": [], - "title": "User Engagement (logging data: _tool_q_dev_chat_log)", + "title": "User Engagement", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly active user count over time (from prompt logging)", "fieldConfig": { "defaults": { @@ -399,11 +426,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT TO_DATE(yw || ' Monday', 'IYYYIW FMDay') AS time, COUNT(DISTINCT user_id) AS \"Active Users\" FROM (SELECT user_id, (EXTRACT(ISOYEAR FROM timestamp) * 100 + EXTRACT(WEEK FROM timestamp))::int AS yw FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS t GROUP BY \"yw\" ORDER BY time NULLS FIRST", + "rawSql": "SELECT TO_DATE(yw || ' Monday', 'IYYYIW FMDay') AS time, COUNT(DISTINCT user_id) AS \"Active Users\" FROM (SELECT user_id, (EXTRACT(ISOYEAR FROM timestamp) * 100 + EXTRACT(WEEK FROM timestamp))::int AS yw FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS t GROUP BY \"yw\" ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -411,8 +441,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "New vs returning users by week (from prompt logging)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "New users use the native is_new_user flag; early rows without that field are reported as existing/unknown.", "fieldConfig": { "defaults": { "color": { @@ -491,15 +524,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT TO_DATE(yw || ' Monday', 'IYYYIW FMDay') AS time, SUM(CASE WHEN yw = first_yw THEN 1 ELSE 0 END) AS \"New Users\", SUM(CASE WHEN yw <> first_yw THEN 1 ELSE 0 END) AS \"Returning Users\" FROM (SELECT DISTINCT u.user_id, (EXTRACT(ISOYEAR FROM u.timestamp) * 100 + EXTRACT(WEEK FROM u.timestamp))::int AS yw, f.first_yw FROM lake._tool_q_dev_chat_log AS u JOIN (SELECT user_id, (EXTRACT(ISOYEAR FROM MIN(timestamp)) * 100 + EXTRACT(WEEK FROM MIN(timestamp)))::int AS first_yw FROM lake._tool_q_dev_chat_log GROUP BY user_id) AS f ON u.user_id = f.user_id WHERE $__timeFilter(u.timestamp)) AS weekly GROUP BY yw ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(DISTINCT CASE WHEN is_new_user IS TRUE THEN user_id END) AS \"New Users\", COUNT(DISTINCT CASE WHEN is_new_user IS NOT TRUE THEN user_id END) AS \"Existing/Unknown Users\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time", "refId": "A" } ], - "title": "New vs Returning Users (Weekly)", + "title": "New vs Existing/Unknown Users (Weekly)", "type": "timeseries" }, { @@ -512,11 +548,14 @@ }, "id": 102, "panels": [], - "title": "Credits & Subscription (new format: _tool_q_dev_user_report)", + "title": "Credits & Subscription", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Cumulative credits this month vs projected total (from new user_report)", "fieldConfig": { "defaults": { @@ -596,11 +635,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(SUM(credits_used)) OVER (ORDER BY date NULLS FIRST) AS \"Cumulative Credits\", (SELECT CAST(SUM(credits_used) AS NUMERIC) / NULLIF(COUNT(DISTINCT date), 0) * EXTRACT(DAY FROM CAST(CAST(DATE_TRUNC('MONTH', CURRENT_DATE) + INTERVAL '1 MONTH' - INTERVAL '1 DAY' AS DATE) AS DATE)) FROM lake._tool_q_dev_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE))) AS \"Projected Monthly\" FROM lake._tool_q_dev_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE)) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT date AS time, SUM(SUM(credits_used)) OVER (ORDER BY date NULLS FIRST) AS \"Cumulative Credits\", (SELECT CAST(SUM(credits_used) AS NUMERIC) / NULLIF(COUNT(DISTINCT date), 0) * EXTRACT(DAY FROM CAST(CAST(DATE_TRUNC('MONTH', CURRENT_DATE) + INTERVAL '1 MONTH' - INTERVAL '1 DAY' AS DATE) AS DATE)) FROM lake._tool_kiro_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE))) AS \"Projected Monthly\" FROM lake._tool_kiro_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE)) GROUP BY \"date\" ORDER BY date NULLS FIRST", "refId": "A" } ], @@ -608,7 +650,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Power tier users with no activity in the last 14 days (from new user_report)", "fieldConfig": { "defaults": { @@ -658,11 +703,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", MAX(subscription_tier) AS \"Tier\", ROUND(SUM(credits_used), 1) AS \"Total Credits Used\", MAX(date) AS \"Last Activity\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) AND subscription_tier = 'POWER' GROUP BY \"user_id\" HAVING MAX(date) < NOW() - INTERVAL '14 DAY' ORDER BY MAX(date) NULLS FIRST", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", MAX(subscription_tier) AS \"Tier\", ROUND(SUM(credits_used), 1) AS \"Total Credits Used\", MAX(date) AS \"Last Activity\" FROM lake._tool_kiro_user_report WHERE $__timeFilter(date) AND subscription_tier = 'POWER' GROUP BY \"user_id\" HAVING MAX(date) < NOW() - INTERVAL '14 DAY' ORDER BY MAX(date) NULLS FIRST", "refId": "A" } ], @@ -679,12 +727,15 @@ }, "id": 103, "panels": [], - "title": "Cross-Source: User Productivity (new report + legacy metrics)", + "title": "Per-User Usage & Efficiency", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Per-user productivity combining credits (new format) with feature metrics (legacy). Only shows users present in both data sources.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Credits, messages, conversations, and credits per message. Accepted LOC is not exported.", "fieldConfig": { "defaults": { "color": { @@ -733,15 +784,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(d.display_name), d.user_id) AS \"User\", COALESCE(MAX(r.subscription_tier), '') AS \"Tier\", ROUND(SUM(r.credits_used), 1) AS \"Credits Used\", SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) AS \"Total Accepted Lines\", CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0 THEN ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 0), 2) ELSE NULL END AS \"Credits/Line\", ROUND(CAST(SUM(d.inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(d.inline_suggestions_count), 0), 0) * 100, 1) || '%' AS \"Accept Rate\", SUM(d.code_review_findings_count) AS \"Review Findings\", SUM(d.test_generation_event_count) AS \"Test Gen Events\", SUM(d.dev_accepted_lines) AS \"Agentic Lines\", MIN(d.date) AS \"First Active\", MAX(d.date) AS \"Last Active\" FROM lake._tool_q_dev_user_data AS d LEFT JOIN (SELECT user_id, date, SUM(credits_used) AS credits_used, MAX(subscription_tier) AS subscription_tier FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", date) AS r ON d.user_id = r.user_id AND d.date = r.date WHERE $__timeFilter(d.date) GROUP BY d.user_id ORDER BY SUM(r.credits_used) DESC NULLS LAST", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", MAX(subscription_tier) AS \"Tier\", ROUND(SUM(credits_used), 1) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", ROUND(SUM(credits_used) / NULLIF(SUM(total_messages), 0), 3) AS \"Credits per Message\", MAX(date) AS \"Last Activity\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id ORDER BY SUM(credits_used) DESC", "refId": "A" } ], - "title": "User Productivity & Efficiency", + "title": "Per-User Usage & Efficiency", "type": "table" } ], @@ -749,9 +803,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "executive", - "kiro" + "kiro", + "executive" ], "templating": { "list": [] @@ -763,6 +816,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro Executive Dashboard", - "uid": "qdev_executive-pg", + "uid": "kiro_executive-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_feature_metrics.json b/grafana/dashboards/postgresql/kiro_feature_metrics.json similarity index 70% rename from grafana/dashboards/postgresql/qdev_feature_metrics.json rename to grafana/dashboards/postgresql/kiro_feature_metrics.json index 3ea698aa550..ac8b3a4872d 100644 --- a/grafana/dashboards/postgresql/qdev_feature_metrics.json +++ b/grafana/dashboards/postgresql/kiro_feature_metrics.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "High-level summary of legacy feature-level activity metrics (from by_user_analytic CSV reports)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro usage-report metrics. Legacy accepted LOC and feature counters are not exported.", "fieldConfig": { "defaults": { "color": { @@ -66,15 +69,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", SUM(inline_suggestions_count) AS \"Inline Suggestions\", SUM(inline_acceptance_count) AS \"Inline Accepted\", SUM(chat_messages_sent) AS \"Chat Messages\", SUM(chat_ai_code_lines) AS \"Chat AI Lines\", SUM(code_review_findings_count) AS \"Review Findings\", SUM(test_generation_event_count) AS \"Test Gen Events\", SUM(dev_accepted_lines) AS \"Agentic Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", ROUND(SUM(credits_used), 1) AS \"Credits Used\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Legacy Feature Metrics Overview", + "title": "Kiro Usage Overview", "type": "stat" }, { @@ -87,12 +93,15 @@ }, "id": 20, "panels": [], - "title": "Inline Suggestions", + "title": "Completions", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily inline suggestion and acceptance counts", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Returned suggestions are offers, not acceptance events.", "fieldConfig": { "defaults": { "color": { @@ -170,20 +179,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(inline_suggestions_count) AS \"Suggestions\", SUM(inline_acceptance_count) AS \"Accepted\", SUM(inline_ai_code_lines) AS \"AI Code Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS \"Completion Requests\", SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS \"Requests with Results\", SUM(returned_line_count) AS \"Returned Lines\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Inline Suggestions & Acceptance", + "title": "Completion Requests & Results", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Percentage of completion requests that returned at least one suggestion. Kiro does not export acceptance.", "fieldConfig": { "defaults": { "color": { @@ -261,15 +276,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Inline Suggestions\", CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) AS \"Code Fix\", CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) AS \"Inline Chat\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CAST(SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0) AS \"Suggestion Return Rate\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Acceptance Rate Trends", + "title": "Suggestion Return Rate", "type": "timeseries" }, { @@ -282,12 +300,15 @@ }, "id": 21, "panels": [], - "title": "Chat & Agentic (Dev)", + "title": "Chat & Agent Continuations", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily chat messages sent and AI-generated code lines from chat", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "User turns and agent self-continuations from chat logs.", "fieldConfig": { "defaults": { "color": { @@ -365,11 +386,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(chat_messages_sent) AS \"Messages Sent\", SUM(chat_messages_interacted) AS \"Messages Interacted\", SUM(chat_ai_code_lines) AS \"AI Code Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS \"Chat Events\", SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS \"User Turns\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], @@ -377,8 +401,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Agentic (Dev) code generation and acceptance metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Agent continuations are empty-prompt records after tool calls; this measures autonomy, not accepted code.", "fieldConfig": { "defaults": { "color": { @@ -456,15 +483,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(dev_generation_event_count) AS \"Generation Events\", SUM(dev_generated_lines) AS \"Generated Lines\", SUM(dev_accepted_lines) AS \"Accepted Lines\", SUM(dev_acceptance_event_count) AS \"Acceptance Events\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS \"User Turns\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\", CAST(SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0) AS \"Agent Continuation Rate\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Agentic (Dev) Activity", + "title": "Agent Continuation Activity", "type": "timeseries" }, { @@ -477,12 +507,15 @@ }, "id": 22, "panels": [], - "title": "Code Review, Test Gen & Transformations", + "title": "Models, Prompts & Clients", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Code review findings and test generation metrics over time", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Authoritative model and routing-mode message counts from daily reports.", "fieldConfig": { "defaults": { "color": { @@ -560,20 +593,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(code_review_findings_count) AS \"Review Findings\", SUM(code_review_succeeded_event_count) AS \"Reviews Succeeded\", SUM(code_review_failed_event_count) AS \"Reviews Failed\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) AS time, model_name AS metric, SUM(message_count) AS value FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE), model_name ORDER BY time", "refId": "A" } ], - "title": "Code Review Activity", + "title": "Model / Route Message Mix", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Test generation events and acceptance over time", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Daily average prompt and response lengths. Test-generation counts are not exported.", "fieldConfig": { "defaults": { "color": { @@ -651,20 +690,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(test_generation_event_count) AS \"Test Gen Events\", SUM(test_generation_generated_tests) AS \"Tests Generated\", SUM(test_generation_accepted_tests) AS \"Tests Accepted\", SUM(test_generation_generated_lines) AS \"Lines Generated\", SUM(test_generation_accepted_lines) AS \"Lines Accepted\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, ROUND(AVG(prompt_length)) AS \"Avg Prompt Length\", ROUND(AVG(response_length)) AS \"Avg Response Length\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Test Generation Activity", + "title": "Prompt & Response Length", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Doc generation and code transformation events", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Messages by KIRO_IDE, KIRO_CLI, KIRO_WEB, and PLUGIN. Doc-generation metrics are not exported.", "fieldConfig": { "defaults": { "color": { @@ -742,20 +787,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(doc_generation_event_count) AS \"Doc Gen Events\", SUM(doc_generation_accepted_line_additions) AS \"Doc Lines Accepted\", SUM(transformation_event_count) AS \"Transformation Events\", SUM(transformation_lines_generated) AS \"Transform Lines Generated\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) AS time, client_type AS metric, SUM(total_messages) AS value FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE), client_type ORDER BY time", "refId": "A" } ], - "title": "Doc Generation & Transformations", + "title": "Client Type Message Mix", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Number of users who used each feature in the selected period", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Distinct users observed in chat, completion, steering, and spec-mode signals.", "fieldConfig": { "defaults": { "color": { @@ -838,11 +889,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT 'Chat' AS \"Feature\", COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) AS \"Users\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Code Review', COUNT(DISTINCT CASE WHEN code_review_succeeded_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Doc Generation', COUNT(DISTINCT CASE WHEN doc_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Test Generation', COUNT(DISTINCT CASE WHEN test_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Dev (Agentic)', COUNT(DISTINCT CASE WHEN dev_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Transformation', COUNT(DISTINCT CASE WHEN transformation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT 'Chat' AS \"Feature\", COUNT(DISTINCT user_id) AS \"Users\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Completion' AS \"Feature\", COUNT(DISTINCT user_id) AS \"Users\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Steering' AS \"Feature\", COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) AS \"Users\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Spec Mode' AS \"Feature\", COUNT(DISTINCT CASE WHEN is_spec_mode = TRUE THEN user_id END) AS \"Users\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -863,8 +917,11 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Per-user breakdown of legacy feature-level metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Per-user chat, completion, autonomy, steering, and spec-mode activity.", "fieldConfig": { "defaults": { "color": { @@ -913,15 +970,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", SUM(inline_suggestions_count) AS \"Suggestions\", SUM(inline_acceptance_count) AS \"Accepted\", ROUND(CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) * 100, 1) || '%' AS \"Accept %\", SUM(chat_messages_sent) AS \"Chat Msgs\", SUM(chat_ai_code_lines) AS \"Chat Lines\", SUM(dev_accepted_lines) AS \"Agentic Lines\", SUM(code_review_findings_count) AS \"Review Findings\", SUM(test_generation_accepted_tests) AS \"Tests Accepted\", SUM(doc_generation_event_count) AS \"Doc Gen\", SUM(transformation_event_count) AS \"Transforms\", MIN(date) AS \"First Active\", MAX(date) AS \"Last Active\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"user_id\" ORDER BY SUM(inline_suggestions_count) DESC NULLS LAST", + "rawSql": "WITH users AS (SELECT user_id FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION SELECT user_id FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp)), chat AS (SELECT user_id, COUNT(*) AS chat_events, SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS user_turns, SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS agent_continuations, SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS steering, SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS spec_mode FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id), completion AS (SELECT user_id, COUNT(*) AS completion_requests, SUM(returned_line_count) AS returned_lines FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id), names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, u.user_id) AS \"User\", COALESCE(c.chat_events, 0) AS \"Chat Events\", COALESCE(c.user_turns, 0) AS \"User Turns\", COALESCE(c.agent_continuations, 0) AS \"Agent Continuations\", COALESCE(x.completion_requests, 0) AS \"Completion Requests\", COALESCE(x.returned_lines, 0) AS \"Returned Lines\", COALESCE(c.steering, 0) AS \"Steering\", COALESCE(c.spec_mode, 0) AS \"Spec Mode\" FROM users u LEFT JOIN chat c ON u.user_id = c.user_id LEFT JOIN completion x ON u.user_id = x.user_id LEFT JOIN names n ON u.user_id = n.user_id ORDER BY COALESCE(c.chat_events, 0) DESC", "refId": "A" } ], - "title": "Per-User Feature Metrics", + "title": "Per-User Kiro Activity", "type": "table" } ], @@ -929,8 +989,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "legacy", "kiro" ], "templating": { @@ -942,7 +1000,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Kiro Legacy Feature Metrics", - "uid": "qdev_feature_metrics-pg", + "title": "Kiro Feature Metrics", + "uid": "kiro_feature_metrics-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_logging.json b/grafana/dashboards/postgresql/kiro_logging.json similarity index 72% rename from grafana/dashboards/postgresql/qdev_logging.json rename to grafana/dashboards/postgresql/kiro_logging.json index 032b4fcfa16..c0b4396cf33 100644 --- a/grafana/dashboards/postgresql/qdev_logging.json +++ b/grafana/dashboards/postgresql/kiro_logging.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Overview of logging event metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Chat users, user turns, agent continuations, and completion events.", "fieldConfig": { "defaults": { "color": { @@ -66,11 +69,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Events\", (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Users\", (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id <> '') AS \"Conversations\", (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) AS \"Completion Events\", (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) AS \"Completion Users\", (SELECT SUM(code_reference_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Code References\", (SELECT SUM(web_link_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Web Links Cited\"", + "rawSql": "SELECT (SELECT COUNT(*) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Events\", (SELECT COUNT(DISTINCT user_id) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Users\", (SELECT SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS \"User Turns\", (SELECT SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS \"Agent Continuations\", (SELECT COUNT(*) FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp)) AS \"Completion Events\"", "refId": "A" } ], @@ -78,7 +84,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Hourly distribution of AI usage activity (chat + completions)", "fieldConfig": { "defaults": { @@ -155,11 +164,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT LPAD(CAST(hour_of_day AS TEXT), 2, '0') AS \"Hour\", SUM(chat_count) AS \"Chat Events\", SUM(completion_count) AS \"Completion Events\" FROM (SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, COUNT(*) AS chat_count, 0 AS completion_count FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) UNION ALL SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, 0 AS chat_count, COUNT(*) AS completion_count FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp)) AS combined GROUP BY hour_of_day ORDER BY hour_of_day NULLS FIRST", + "rawSql": "SELECT LPAD(CAST(hour_of_day AS TEXT), 2, '0') AS \"Hour\", SUM(chat_count) AS \"Chat Events\", SUM(completion_count) AS \"Completion Events\" FROM (SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, COUNT(*) AS chat_count, 0 AS completion_count FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) UNION ALL SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, 0 AS chat_count, COUNT(*) AS completion_count FROM lake._tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp)) AS combined GROUP BY hour_of_day ORDER BY hour_of_day NULLS FIRST", "refId": "A" } ], @@ -167,7 +179,10 @@ "type": "barchart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Distribution of chat trigger types: MANUAL (chat window) vs INLINE_CHAT", "fieldConfig": { "defaults": { @@ -223,11 +238,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)' ELSE chat_trigger_type END AS \"Trigger Type\", COUNT(*) AS \"Events\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"chat_trigger_type\" ORDER BY COUNT(*) DESC NULLS LAST", + "rawSql": "SELECT CASE WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)' ELSE chat_trigger_type END AS \"Trigger Type\", COUNT(*) AS \"Events\" FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"chat_trigger_type\" ORDER BY COUNT(*) DESC NULLS LAST", "refId": "A" } ], @@ -235,8 +253,11 @@ "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Distribution of model usage across chat events", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Authoritative model and routing-mode counts from daily reports; chat model_id is only partially populated.", "fieldConfig": { "defaults": { "color": { @@ -280,7 +301,7 @@ "calcs": [ "lastNotNull" ], - "fields": "/^Requests$/", + "fields": "/^Messages$/", "values": true }, "tooltip": { @@ -291,19 +312,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN model_id = '' OR model_id IS NULL THEN '(unknown)' ELSE model_id END AS \"Model\", COUNT(*) AS \"Requests\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"model_id\" ORDER BY COUNT(*) DESC NULLS LAST", + "rawSql": "SELECT model_name AS \"Model / Route\", SUM(message_count) AS \"Messages\" FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY model_name ORDER BY SUM(message_count) DESC", "refId": "A" } ], - "title": "Model Usage Distribution", + "title": "Model / Route Message Distribution", "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Top file extensions used with inline completions", "fieldConfig": { "defaults": { @@ -359,11 +386,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"File Type\", COUNT(*) AS \"Completions\" FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 15", + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"File Type\", COUNT(*) AS \"Completions\" FROM lake._tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 15", "refId": "A" } ], @@ -371,8 +401,11 @@ "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Average number of chat events per conversation", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Conversation IDs are absent in sampled Kiro logs, so conversation depth cannot be computed.", "fieldConfig": { "defaults": { "color": { @@ -451,19 +484,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CAST(COUNT(*) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT CASE WHEN conversation_id <> '' THEN conversation_id END), 0), 0) AS \"Avg Turns per Conversation\", COUNT(DISTINCT CASE WHEN conversation_id <> '' THEN conversation_id END) AS \"Unique Conversations\", COUNT(*) AS \"Total Chat Events\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS \"User Turns\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\", CAST(SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0) AS \"Agent Continuation Rate\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Conversation Depth Analysis", + "title": "User Turns vs Agent Continuations", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Daily chat and completion events over time", "fieldConfig": { "defaults": { @@ -543,11 +582,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(chat) AS \"Chat Events\", SUM(completions) AS \"Completion Events\" FROM (SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS chat, 0 AS completions FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) UNION ALL SELECT CAST(timestamp AS DATE) AS time, 0 AS chat, COUNT(*) AS completions FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE)) AS combined GROUP BY time ORDER BY time NULLS FIRST", + "rawSql": "SELECT time, SUM(chat) AS \"Chat Events\", SUM(completions) AS \"Completion Events\" FROM (SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS chat, 0 AS completions FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) UNION ALL SELECT CAST(timestamp AS DATE) AS time, 0 AS chat, COUNT(*) AS completions FROM lake._tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE)) AS combined GROUP BY time ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -555,8 +597,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Per-user logging activity summary", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Per-user chat and completion activity with display names resolved from usage reports.", "fieldConfig": { "defaults": { "color": { @@ -605,11 +650,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(u.display_name, u.user_id) AS \"User\", u.user_id AS \"User ID\", u.chat_events AS \"Chat Events\", u.conversations AS \"Conversations\", ROUND(CAST(u.chat_events AS NUMERIC) / NULLIF(NULLIF(u.conversations, 0), 0), 1) AS \"Avg Turns\", COALESCE(c.completion_events, 0) AS \"Completion Events\", COALESCE(c.files_count, 0) AS \"Distinct Files\", ROUND(u.avg_prompt_len) AS \"Avg Prompt Len\", ROUND(u.avg_response_len) AS \"Avg Response Len\", u.steering_count AS \"Steering Uses\", u.spec_count AS \"Spec Mode Uses\", u.code_ref_count AS \"Code Refs\", u.web_link_count AS \"Web Links\", u.models_used AS \"Models Used\", u.first_seen AS \"First Seen\", GREATEST(u.last_seen, COALESCE(c.last_seen, u.last_seen)) AS \"Last Seen\" FROM (SELECT user_id, MAX(display_name) AS display_name, COUNT(*) AS chat_events, COUNT(DISTINCT CASE WHEN conversation_id <> '' THEN conversation_id END) AS conversations, AVG(prompt_length) AS avg_prompt_len, AVG(response_length) AS avg_response_len, STRING_AGG(DISTINCT CASE WHEN model_id <> '' AND NOT model_id IS NULL THEN model_id END, ', ' ORDER BY model_id NULLS FIRST) AS models_used, SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS steering_count, SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS spec_count, SUM(code_reference_count) AS code_ref_count, SUM(web_link_count) AS web_link_count, MIN(timestamp) AS first_seen, MAX(timestamp) AS last_seen FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id) AS u LEFT JOIN (SELECT user_id, COUNT(*) AS completion_events, COUNT(DISTINCT file_name) AS files_count, MAX(timestamp) AS last_seen FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id) AS c ON u.user_id = c.user_id ORDER BY u.user_id NULLS FIRST", + "rawSql": "WITH chat AS (SELECT user_id, COUNT(*) AS chat_events, SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS user_turns, SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS agent_continuations FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id), completion AS (SELECT user_id, COUNT(*) AS completion_events, SUM(returned_line_count) AS returned_lines FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id), names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, c.user_id) AS \"User\", c.user_id AS \"User ID\", c.chat_events AS \"Chat Events\", c.user_turns AS \"User Turns\", c.agent_continuations AS \"Agent Continuations\", COALESCE(x.completion_events, 0) AS \"Completion Events\", COALESCE(x.returned_lines, 0) AS \"Returned Lines\" FROM chat c LEFT JOIN completion x ON c.user_id = x.user_id LEFT JOIN names n ON c.user_id = n.user_id ORDER BY c.chat_events DESC", "refId": "A" } ], @@ -617,7 +665,10 @@ "type": "table" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Distribution of Kiro feature adoption: Steering, Spec Mode, and Plain Chat", "fieldConfig": { "defaults": { @@ -673,11 +724,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Using Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Using Spec Mode\", SUM(CASE WHEN has_steering = FALSE AND is_spec_mode = FALSE THEN 1 ELSE 0 END) AS \"Plain Chat\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Using Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Using Spec Mode\", SUM(CASE WHEN has_steering = FALSE AND is_spec_mode = FALSE THEN 1 ELSE 0 END) AS \"Plain Chat\" FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -685,8 +739,11 @@ "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Top file extensions active during chat events", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Chat logs do not contain active file extensions; completion logs provide file types and returned lines.", "fieldConfig": { "defaults": { "color": { @@ -741,20 +798,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file active)' ELSE active_file_extension END AS \"File Type\", COUNT(*) AS \"Chat Events\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"active_file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 15", + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"File Type\", SUM(returned_line_count) AS \"Returned Lines\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY file_extension ORDER BY SUM(returned_line_count) DESC", "refId": "A" } ], - "title": "Active File Types in Chat", + "title": "Returned Lines by File Type", "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "How often Kiro responses include code references and web links", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro logs expose follow-up prompts but not code-reference or web-link counters.", "fieldConfig": { "defaults": { "color": { @@ -809,19 +872,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(CASE WHEN code_reference_count > 0 THEN 1 ELSE 0 END) AS \"With Code References\", SUM(CASE WHEN web_link_count > 0 THEN 1 ELSE 0 END) AS \"With Web Links\", SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"With Followup Prompts\", SUM(CASE WHEN code_reference_count = '0' AND web_link_count = '0' AND has_followup_prompts = FALSE THEN 1 ELSE 0 END) AS \"Plain Response\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"With Follow-up Prompts\", SUM(CASE WHEN has_followup_prompts = FALSE THEN 1 ELSE 0 END) AS \"Without Follow-up Prompts\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], - "title": "Response Enrichment Breakdown", + "title": "Follow-up Prompt Breakdown", "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Average and maximum prompt/response lengths over time", "fieldConfig": { "defaults": { @@ -901,11 +970,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, AVG(prompt_length) AS \"Avg Prompt Length\", AVG(response_length) AS \"Avg Response Length\", MAX(prompt_length) AS \"Max Prompt Length\", MAX(response_length) AS \"Max Response Length\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, AVG(prompt_length) AS \"Avg Prompt Length\", AVG(response_length) AS \"Avg Response Length\", MAX(prompt_length) AS \"Max Prompt Length\", MAX(response_length) AS \"Max Response Length\" FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", "refId": "A" } ], @@ -913,7 +985,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Average code context size provided to inline completions over time", "fieldConfig": { "defaults": { @@ -992,11 +1067,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, ROUND(AVG(left_context_length)) AS \"Avg Left Context\", ROUND(AVG(right_context_length)) AS \"Avg Right Context\", ROUND(AVG(left_context_length + right_context_length)) AS \"Avg Total Context\" FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, ROUND(AVG(left_context_length)) AS \"Avg Left Context\", ROUND(AVG(right_context_length)) AS \"Avg Right Context\", ROUND(AVG(left_context_length + right_context_length)) AS \"Avg Total Context\" FROM lake._tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", "refId": "A" } ], @@ -1004,8 +1082,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily trend of code references and web links in chat responses", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Daily responses with and without follow-up prompts. Code-reference and web-link counters are not exported.", "fieldConfig": { "defaults": { "color": { @@ -1083,15 +1164,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(code_reference_count) AS \"Code References\", SUM(web_link_count) AS \"Web Links\", SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"Followup Prompts\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"With Follow-up Prompts\", SUM(CASE WHEN has_followup_prompts = FALSE THEN 1 ELSE 0 END) AS \"Without Follow-up Prompts\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Response Enrichment Trends", + "title": "Follow-up Prompt Trends", "type": "timeseries" } ], @@ -1099,9 +1183,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "logging", - "kiro" + "kiro", + "logging" ], "templating": { "list": [] @@ -1113,6 +1196,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro AI Activity Insights", - "uid": "qdev_logging-pg", + "uid": "kiro_logging-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_user_report.json b/grafana/dashboards/postgresql/kiro_user_report.json similarity index 82% rename from grafana/dashboards/postgresql/qdev_user_report.json rename to grafana/dashboards/postgresql/kiro_user_report.json index 592d8a75daf..4f720f1aaf4 100644 --- a/grafana/dashboards/postgresql/qdev_user_report.json +++ b/grafana/dashboards/postgresql/kiro_user_report.json @@ -19,7 +19,10 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Overview of credits and usage metrics", "fieldConfig": { "defaults": { @@ -66,11 +69,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(credits_used) AS \"Total Credits Used\", COUNT(DISTINCT user_id) AS \"Active Users\", SUM(total_messages) AS \"Total Messages\", SUM(chat_conversations) AS \"Total Conversations\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date)", + "rawSql": "SELECT SUM(credits_used) AS \"Total Credits Used\", COUNT(DISTINCT user_id) AS \"Active Users\", SUM(total_messages) AS \"Total Messages\", SUM(chat_conversations) AS \"Total Conversations\" FROM lake._tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], @@ -78,7 +84,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Daily credits consumed broken down by subscription tier", "fieldConfig": { "defaults": { @@ -158,11 +167,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, subscription_tier AS metric, SUM(credits_used) AS value FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"date\", \"subscription_tier\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT date AS time, subscription_tier AS metric, SUM(credits_used) AS value FROM lake._tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY \"date\", \"subscription_tier\" ORDER BY date NULLS FIRST", "refId": "A" } ], @@ -170,8 +182,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily messages and conversations broken down by client type", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Daily messages by Kiro IDE, CLI, Web, and Plugin clients, plus conversations.", "fieldConfig": { "defaults": { "color": { @@ -250,11 +265,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) AS \"Messages (IDE)\", SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) AS \"Messages (CLI)\", SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) AS \"Messages (Plugin)\", SUM(chat_conversations) AS \"Conversations\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT date AS time, SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) AS \"Messages (IDE)\", SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) AS \"Messages (CLI)\", SUM(CASE WHEN client_type = 'KIRO_WEB' THEN total_messages ELSE 0 END) AS \"Messages (Web)\", SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) AS \"Messages (Plugin)\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY date ORDER BY date", "refId": "A" } ], @@ -262,7 +280,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Distribution of users across subscription tiers", "fieldConfig": { "defaults": { @@ -318,11 +339,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT subscription_tier AS \"Tier\", COUNT(DISTINCT user_id) AS \"Users\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) AND NOT subscription_tier IS NULL AND subscription_tier <> '' GROUP BY \"subscription_tier\" ORDER BY COUNT(DISTINCT user_id) DESC NULLS LAST", + "rawSql": "SELECT subscription_tier AS \"Tier\", COUNT(DISTINCT user_id) AS \"Users\" FROM lake._tool_kiro_user_report WHERE $__timeFilter(date) AND NOT subscription_tier IS NULL AND subscription_tier <> '' GROUP BY \"subscription_tier\" ORDER BY COUNT(DISTINCT user_id) DESC NULLS LAST", "refId": "A" } ], @@ -330,7 +354,10 @@ "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Per-user credits, messages, and subscription details", "fieldConfig": { "defaults": { @@ -429,11 +456,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", subscription_tier AS \"Tier\", client_type AS \"Client\", SUM(credits_used) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", SUM(overage_credits_used) AS \"Overage Credits\", CASE WHEN MAX(CAST(overage_enabled AS BIGINT)) = 1 THEN 'Yes' ELSE 'No' END AS \"Overage\", MIN(date) AS \"First Activity\", MAX(date) AS \"Last Activity\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", \"subscription_tier\", \"client_type\" ORDER BY user_id DESC NULLS LAST", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", subscription_tier AS \"Tier\", client_type AS \"Client\", SUM(credits_used) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", SUM(overage_credits_used) AS \"Overage Credits\", CASE WHEN MAX(CAST(overage_enabled AS BIGINT)) = 1 THEN 'Yes' ELSE 'No' END AS \"Overage\", MIN(date) AS \"First Activity\", MAX(date) AS \"Last Activity\" FROM lake._tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", \"subscription_tier\", \"client_type\" ORDER BY user_id DESC NULLS LAST", "refId": "A" } ], @@ -445,9 +475,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "user_report", - "kiro" + "kiro", + "user_report" ], "templating": { "list": [] @@ -459,6 +488,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro Usage Dashboard", - "uid": "qdev_user_report-pg", + "uid": "kiro_user_report-pg", "version": 1 -} \ No newline at end of file +} From e89a5a8eba8cfd47bb4a1eaf6bdfc88843add444 Mon Sep 17 00:00:00 2001 From: warren Date: Thu, 27 Aug 2026 10:38:00 +0800 Subject: [PATCH 4/6] feat(grafana): migrate Kiro engineering dashboards Signed-off-by: warren --- .../dashboards/mysql/ai-cost-efficiency.json | 423 +++++++++++--- .../mysql/developer-productivity-hours.json | 208 +++++-- .../dashboards/mysql/kiro-credits-dora.json | 523 +++++++++++++++--- .../dashboards/mysql/language-ai-heatmap.json | 227 ++++++-- .../dashboards/mysql/multi-ai-comparison.json | 343 +++++++++--- .../mysql/steering-adoption-tracker.json | 286 ++++++++-- .../postgresql/ai-cost-efficiency.json | 101 +++- .../developer-productivity-hours.json | 51 +- .../postgresql/kiro-credits-dora.json | 116 +++- .../postgresql/language-ai-heatmap.json | 55 +- .../postgresql/multi-ai-comparison.json | 98 +++- .../postgresql/steering-adoption-tracker.json | 77 ++- 12 files changed, 1993 insertions(+), 515 deletions(-) diff --git a/grafana/dashboards/mysql/ai-cost-efficiency.json b/grafana/dashboards/mysql/ai-cost-efficiency.json index d52905d0cca..68cfe802779 100644 --- a/grafana/dashboards/mysql/ai-cost-efficiency.json +++ b/grafana/dashboards/mysql/ai-cost-efficiency.json @@ -26,42 +26,74 @@ "targetBlank": true, "title": "Kiro Usage Dashboard", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" } ], "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "panels": [], "title": "Summary", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(credits_used)) AS 'Total Credits'\nFROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used)) AS 'Total Credits'\nFROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], @@ -69,29 +101,56 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT pr.id), 0), 1) AS 'Credits / PR'\nFROM _tool_q_dev_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n) pr\nWHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT pr.id), 0), 1) AS 'Credits / PR'\nFROM _tool_kiro_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n) pr\nWHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -99,29 +158,56 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 12, "y": 1 }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 1 + }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 1) AS 'Credits / Deploy'\nFROM _tool_q_dev_user_report r\nCROSS JOIN (\n SELECT DISTINCT cicd_deployment_id\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n) cdc\nWHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 1) AS 'Credits / Deploy'\nFROM _tool_kiro_user_report r\nCROSS JOIN (\n SELECT DISTINCT cicd_deployment_id\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n) cdc\nWHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -129,29 +215,56 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 18, "y": 1 }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 1 + }, "id": 5, "options": { "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT i.id), 0), 1) AS 'Credits / Issue'\nFROM _tool_q_dev_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n) i\nWHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT i.id), 0), 1) AS 'Credits / Issue'\nFROM _tool_kiro_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n) i\nWHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -160,40 +273,80 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, "id": 10, "panels": [], "title": "Weekly Trends", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly cost per merged PR", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, "id": 11, "options": { - "legend": { "calcs": ["mean", "min"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_prs AS (\n SELECT DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY) AS week_start,\n COUNT(*) AS prs\n FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n GROUP BY DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(p.prs, 0), 1) AS 'Credits per PR'\nFROM _credits c\nLEFT JOIN _prs p ON c.week_start = p.week_start\nORDER BY time", + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_prs AS (\n SELECT DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY) AS week_start,\n COUNT(*) AS prs\n FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n GROUP BY DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(p.prs, 0), 1) AS 'Credits per PR'\nFROM _credits c\nLEFT JOIN _prs p ON c.week_start = p.week_start\nORDER BY time", "refId": "A" } ], @@ -201,33 +354,68 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly cost per production deployment", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, "id": 12, "options": { - "legend": { "calcs": ["mean", "min"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_deploys AS (\n SELECT DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(DISTINCT cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(d.deploys, 0), 1) AS 'Credits per Deploy'\nFROM _credits c\nLEFT JOIN _deploys d ON c.week_start = d.week_start\nORDER BY time", + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_deploys AS (\n SELECT DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(DISTINCT cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(d.deploys, 0), 1) AS 'Credits per Deploy'\nFROM _credits c\nLEFT JOIN _deploys d ON c.week_start = d.week_start\nORDER BY time", "refId": "A" } ], @@ -235,33 +423,68 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly cost per resolved issue", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, "id": 13, "options": { - "legend": { "calcs": ["mean", "min"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_issues AS (\n SELECT DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n COUNT(*) AS resolved\n FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(i.resolved, 0), 1) AS 'Credits per Issue'\nFROM _credits c\nLEFT JOIN _issues i ON c.week_start = i.week_start\nORDER BY time", + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_issues AS (\n SELECT DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n COUNT(*) AS resolved\n FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(i.resolved, 0), 1) AS 'Credits per Issue'\nFROM _credits c\nLEFT JOIN _issues i ON c.week_start = i.week_start\nORDER BY time", "refId": "A" } ], @@ -269,33 +492,68 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Is cost efficiency improving over time?", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, "id": 14, "options": { - "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS 'Credits',\n SUM(total_messages) AS 'Messages',\n SUM(chat_conversations) AS 'Conversations'\nFROM _tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", + "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS 'Credits',\n SUM(total_messages) AS 'Messages',\n SUM(chat_conversations) AS 'Conversations'\nFROM _tool_kiro_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", "refId": "A" } ], @@ -306,9 +564,18 @@ "preload": false, "refresh": "5m", "schemaVersion": 41, - "tags": ["q_dev", "kiro", "cost", "efficiency"], - "templating": { "list": [] }, - "time": { "from": "now-90d", "to": "now" }, + "tags": [ + "kiro", + "cost", + "efficiency" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, "timepicker": {}, "timezone": "utc", "title": "AI Cost-Efficiency", diff --git a/grafana/dashboards/mysql/developer-productivity-hours.json b/grafana/dashboards/mysql/developer-productivity-hours.json index 2259f6d5bdd..c5b4504aae5 100644 --- a/grafana/dashboards/mysql/developer-productivity-hours.json +++ b/grafana/dashboards/mysql/developer-productivity-hours.json @@ -19,31 +19,60 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Chat and completion events by hour of day", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, - "stacking": { "mode": "normal" }, "thresholdsStyle": { "mode": "off" } + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "stacking": { + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "options": { - "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT LPAD(CAST(hr AS CHAR), 2, '0') AS 'Hour',\n SUM(chat) AS 'Chat Events', SUM(comp) AS 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) AS hr, COUNT(*) AS chat, 0 AS comp\n FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) AS hr, 0 AS chat, COUNT(*) AS comp\n FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) t GROUP BY hr ORDER BY hr", + "rawSql": "SELECT LPAD(CAST(hr AS CHAR), 2, '0') AS 'Hour',\n SUM(chat) AS 'Chat Events', SUM(comp) AS 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) AS hr, COUNT(*) AS chat, 0 AS comp\n FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) AS hr, 0 AS chat, COUNT(*) AS comp\n FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) t GROUP BY hr ORDER BY hr", "refId": "A" } ], @@ -51,32 +80,64 @@ "type": "barchart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Average prompt complexity and response richness by hour", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "showPoints": "auto", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, "id": 2, "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT LPAD(CAST(HOUR(timestamp) AS CHAR), 2, '0') AS 'Hour',\n ROUND(AVG(prompt_length)) AS 'Avg Prompt Length',\n ROUND(AVG(response_length)) AS 'Avg Response Length'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY HOUR(timestamp)\nORDER BY HOUR(timestamp)", + "rawSql": "SELECT LPAD(CAST(hr AS CHAR), 2, '0') AS 'Hour', ROUND(avg_prompt) AS 'Avg Prompt Length', ROUND(avg_response) AS 'Avg Response Length' FROM (SELECT HOUR(timestamp) AS hr, AVG(prompt_length) AS avg_prompt, AVG(response_length) AS avg_response FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY HOUR(timestamp)) hourly ORDER BY hr", "refId": "A" } ], @@ -84,31 +145,60 @@ "type": "barchart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Steering and spec mode usage concentration by hour", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, - "stacking": { "mode": "normal" }, "thresholdsStyle": { "mode": "off" } + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "stacking": { + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, "id": 3, "options": { - "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT LPAD(CAST(HOUR(timestamp) AS CHAR), 2, '0') AS 'Hour',\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS 'Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS 'Spec Mode',\n SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) AS 'Plain Chat'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY HOUR(timestamp)\nORDER BY HOUR(timestamp)", + "rawSql": "SELECT LPAD(CAST(hr AS CHAR), 2, '0') AS 'Hour', steering AS 'Steering', spec_mode AS 'Spec Mode', plain_chat AS 'Plain Chat' FROM (SELECT HOUR(timestamp) AS hr, SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS steering, SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS spec_mode, SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) AS plain_chat FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY HOUR(timestamp)) hourly ORDER BY hr", "refId": "A" } ], @@ -116,31 +206,60 @@ "type": "barchart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Day-of-week activity pattern", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, - "stacking": { "mode": "normal" }, "thresholdsStyle": { "mode": "off" } + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "stacking": { + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, "id": 4, "options": { - "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE DAYOFWEEK(timestamp)\n WHEN 1 THEN 'Sun' WHEN 2 THEN 'Mon' WHEN 3 THEN 'Tue'\n WHEN 4 THEN 'Wed' WHEN 5 THEN 'Thu' WHEN 6 THEN 'Fri' WHEN 7 THEN 'Sat'\n END AS 'Day',\n SUM(chat) AS 'Chat Events', SUM(comp) AS 'Completions'\nFROM (\n SELECT timestamp, COUNT(*) AS chat, 0 AS comp FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY timestamp\n UNION ALL\n SELECT timestamp, 0, COUNT(*) FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY timestamp\n) t\nGROUP BY DAYOFWEEK(timestamp)\nORDER BY DAYOFWEEK(timestamp)", + "rawSql": "SELECT CASE weekday_num WHEN 1 THEN 'Sun' WHEN 2 THEN 'Mon' WHEN 3 THEN 'Tue' WHEN 4 THEN 'Wed' WHEN 5 THEN 'Thu' WHEN 6 THEN 'Fri' WHEN 7 THEN 'Sat' END AS 'Day', SUM(chat) AS 'Chat Events', SUM(comp) AS 'Completions' FROM (SELECT DAYOFWEEK(timestamp) AS weekday_num, COUNT(*) AS chat, 0 AS comp FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DAYOFWEEK(timestamp) UNION ALL SELECT DAYOFWEEK(timestamp) AS weekday_num, 0 AS chat, COUNT(*) AS comp FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DAYOFWEEK(timestamp)) activity GROUP BY weekday_num ORDER BY weekday_num", "refId": "A" } ], @@ -151,9 +270,18 @@ "preload": false, "refresh": "5m", "schemaVersion": 41, - "tags": ["q_dev", "kiro", "productivity", "hours"], - "templating": { "list": [] }, - "time": { "from": "now-90d", "to": "now" }, + "tags": [ + "kiro", + "productivity", + "hours" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, "timepicker": {}, "timezone": "utc", "title": "Developer AI Productivity Hours", diff --git a/grafana/dashboards/mysql/kiro-credits-dora.json b/grafana/dashboards/mysql/kiro-credits-dora.json index 46614cc0834..e587f1002fa 100644 --- a/grafana/dashboards/mysql/kiro-credits-dora.json +++ b/grafana/dashboards/mysql/kiro-credits-dora.json @@ -37,7 +37,7 @@ "targetBlank": true, "title": "Kiro Usage Dashboard", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" } ], "panels": [ @@ -46,10 +46,19 @@ "type": "datasource", "uid": "grafana" }, - "gridPos": { "h": 3, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "options": { - "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, "content": "## Kiro Credits + DORA Correlation\nThis dashboard correlates **Kiro AI usage (credits and messages)** with **DORA** performance indicators at a weekly aggregate level.\n\n- **Pearson's r** measures linear correlation: negative r suggests higher AI usage may correlate with shorter cycle times.\n- Data is aggregated by **week** and joined on `week_start`.", "mode": "markdown" }, @@ -58,48 +67,91 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, "id": 2, "panels": [], "title": "Overview", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Pearson correlation coefficient between weekly Kiro credits and PR cycle time", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [{ "options": { "match": "null", "result": { "text": "Need more data" } }, "type": "special" }], + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "Need more data" + } + }, + "type": "special" + } + ], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": -0.3 }, - { "color": "orange", "value": 0.3 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": -0.3 + }, + { + "color": "orange", + "value": 0.3 + } ] } }, "overrides": [] }, - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 4 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 4 + }, "id": 3, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showPercentChange": false, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH _kiro_weekly AS (\n SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS weekly_credits\n FROM _tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_pr_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_cycle_hours\n FROM project_pr_metrics\n WHERE pr_merged_date IS NOT NULL\n AND pr_cycle_time IS NOT NULL\n AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_joined AS (\n SELECT k.weekly_credits AS x, p.avg_cycle_hours AS y\n FROM _kiro_weekly k\n INNER JOIN _pr_weekly p ON k.week_start = p.week_start\n),\n_stats AS (\n SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy\n FROM _joined\n)\nSELECT CASE\n WHEN s.n < 4 THEN NULL\n WHEN s.sx = 0 OR s.sy = 0 THEN 0\n ELSE ROUND((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined j) / (s.n * s.sx * s.sy), 2)\nEND AS 'r'\nFROM _stats s", + "rawSql": "WITH _kiro_weekly AS (\n SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS weekly_credits\n FROM _tool_kiro_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_pr_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_cycle_hours\n FROM project_pr_metrics\n WHERE pr_merged_date IS NOT NULL\n AND pr_cycle_time IS NOT NULL\n AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_joined AS (\n SELECT k.weekly_credits AS x, p.avg_cycle_hours AS y\n FROM _kiro_weekly k\n INNER JOIN _pr_weekly p ON k.week_start = p.week_start\n),\n_stats AS (\n SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy\n FROM _joined\n)\nSELECT CASE\n WHEN s.n < 4 THEN NULL\n WHEN s.sx = 0 OR s.sy = 0 THEN 0\n ELSE ROUND((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined j) / (s.n * s.sx * s.sy), 2)\nEND AS 'r'\nFROM _stats s", "refId": "A" } ], @@ -107,30 +159,57 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Total Kiro credits consumed in period", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 6, "w": 4, "x": 8, "y": 4 }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 4 + }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(credits_used) AS 'Credits Used'\nFROM _tool_q_dev_user_report\nWHERE $__timeFilter(date)", + "rawSql": "SELECT SUM(credits_used) AS 'Credits Used'\nFROM _tool_kiro_user_report\nWHERE $__timeFilter(date)", "refId": "A" } ], @@ -138,28 +217,55 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Median PR cycle time in hours", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "h" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 4, "x": 12, "y": 4 }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 4 + }, "id": 5, "options": { "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, "rawSql": "WITH _ranked AS (\n SELECT pr_cycle_time / 60.0 AS ct,\n PERCENT_RANK() OVER (ORDER BY pr_cycle_time) AS prank\n FROM project_pr_metrics\n WHERE pr_merged_date IS NOT NULL AND pr_cycle_time IS NOT NULL\n AND $__timeFilter(pr_merged_date)\n)\nSELECT ROUND(MAX(ct), 1) AS 'Median Cycle Time'\nFROM _ranked WHERE prank <= 0.5", @@ -170,18 +276,36 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Cycle time comparison: weeks with above-median vs below-median AI credits usage", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "h" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 4 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 4 + }, "id": 6, "options": { "barRadius": 0.1, @@ -189,14 +313,19 @@ "orientation": "horizontal", "showValue": "auto", "stacking": "none", - "tooltip": { "mode": "single" } + "tooltip": { + "mode": "single" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "WITH _kiro_weekly AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS weekly_credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_pr_weekly AS (\n SELECT DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_ct\n FROM project_pr_metrics\n WHERE pr_merged_date IS NOT NULL AND pr_cycle_time IS NOT NULL AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_joined AS (\n SELECT k.weekly_credits, p.avg_ct FROM _kiro_weekly k INNER JOIN _pr_weekly p ON k.week_start = p.week_start\n),\n_med AS (\n SELECT weekly_credits AS med FROM _joined ORDER BY weekly_credits LIMIT 1 OFFSET (SELECT FLOOR(COUNT(*)/2) FROM _joined)\n)\nSELECT\n CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END AS 'Tier',\n ROUND(AVG(j.avg_ct), 1) AS 'Avg Cycle Time'\nFROM _joined j, _med m\nGROUP BY CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END", + "rawSql": "WITH _kiro_weekly AS (SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)), _pr_weekly AS (SELECT DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start, AVG(pr_cycle_time) / 60.0 AS avg_ct FROM project_pr_metrics WHERE pr_merged_date IS NOT NULL AND pr_cycle_time IS NOT NULL AND $__timeFilter(pr_merged_date) GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)), _joined AS (SELECT k.weekly_credits, p.avg_ct FROM _kiro_weekly k INNER JOIN _pr_weekly p ON k.week_start = p.week_start), _ranked AS (SELECT weekly_credits, ROW_NUMBER() OVER (ORDER BY weekly_credits) AS rn, COUNT(*) OVER () AS cnt FROM _joined), _med AS (SELECT AVG(weekly_credits) AS med FROM _ranked WHERE rn IN (FLOOR((cnt + 1) / 2), FLOOR((cnt + 2) / 2))) SELECT CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END AS 'Tier', ROUND(AVG(j.avg_ct), 1) AS 'Avg Cycle Time' FROM _joined j CROSS JOIN _med m GROUP BY CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END", "refId": "A" } ], @@ -205,40 +334,80 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, "id": 10, "panels": [], "title": "Weekly Trends", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly Kiro credits consumed", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, "id": 11, "options": { - "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS 'Credits Used',\n COUNT(DISTINCT user_id) AS 'Active Users'\nFROM _tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", + "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS 'Credits Used',\n COUNT(DISTINCT user_id) AS 'Active Users'\nFROM _tool_kiro_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", "refId": "A" } ], @@ -246,30 +415,65 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly average PR cycle time in hours", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "h" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, "id": 12, "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, "rawSql": "SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS time,\n ROUND(AVG(pr_cycle_time) / 60.0, 1) AS 'Avg Cycle Time (hrs)',\n COUNT(*) AS 'PRs Merged'\nFROM project_pr_metrics\nWHERE pr_merged_date IS NOT NULL AND pr_cycle_time IS NOT NULL\n AND $__timeFilter(pr_merged_date)\nGROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\nORDER BY time", @@ -281,37 +485,79 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 19 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, "id": 20, "panels": [], "title": "Deployment Frequency", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Pearson correlation between weekly credits and deployment count", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [{ "options": { "match": "null", "result": { "text": "Need more data" } }, "type": "special" }], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "Need more data" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 0, "y": 20 }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 20 + }, "id": 21, "options": { "colorMode": "value", "graphMode": "none", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "WITH _kiro_weekly AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS weekly_credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_deploy_weekly AS (\n SELECT DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY) AS week_start,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY)\n),\n_joined AS (\n SELECT k.weekly_credits AS x, d.deploys AS y\n FROM _kiro_weekly k INNER JOIN _deploy_weekly d ON k.week_start = d.week_start\n),\n_stats AS (\n SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy FROM _joined\n)\nSELECT CASE\n WHEN s.n < 4 THEN NULL\n WHEN s.sx = 0 OR s.sy = 0 THEN 0\n ELSE ROUND((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined j) / (s.n * s.sx * s.sy), 2)\nEND AS 'r'\nFROM _stats s", + "rawSql": "WITH _kiro_weekly AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS weekly_credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_deploy_weekly AS (\n SELECT DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY) AS week_start,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY)\n),\n_joined AS (\n SELECT k.weekly_credits AS x, d.deploys AS y\n FROM _kiro_weekly k INNER JOIN _deploy_weekly d ON k.week_start = d.week_start\n),\n_stats AS (\n SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy FROM _joined\n)\nSELECT CASE\n WHEN s.n < 4 THEN NULL\n WHEN s.sx = 0 OR s.sy = 0 THEN 0\n ELSE ROUND((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined j) / (s.n * s.sx * s.sy), 2)\nEND AS 'r'\nFROM _stats s", "refId": "A" } ], @@ -319,33 +565,68 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly credits overlaid with deployment count", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 18, "x": 6, "y": 20 }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 20 + }, "id": 22, "options": { - "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(credits) AS 'Credits Used', SUM(deploys) AS 'Deployments'\nFROM (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS credits, 0 AS deploys\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n UNION ALL\n SELECT DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY) AS time,\n 0 AS credits, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY)\n) combined\nGROUP BY time ORDER BY time", + "rawSql": "SELECT time, SUM(credits) AS 'Credits Used', SUM(deploys) AS 'Deployments'\nFROM (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS credits, 0 AS deploys\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n UNION ALL\n SELECT DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY) AS time,\n 0 AS credits, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY)\n) combined\nGROUP BY time ORDER BY time", "refId": "A" } ], @@ -354,40 +635,80 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, "id": 30, "panels": [], "title": "Change Failure Rate", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Change failure rate: % of deployments that caused incidents", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 29 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 29 + }, "id": 31, "options": { - "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(credits) AS 'Credits Used', SUM(cfr) AS 'Change Failure Rate'\nFROM (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS credits, 0 AS cfr\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n UNION ALL\n SELECT DATE_SUB(DATE(d.deployment_finished_date), INTERVAL WEEKDAY(DATE(d.deployment_finished_date)) DAY) AS time,\n 0 AS credits,\n SUM(CASE WHEN i.id IS NOT NULL THEN 1 ELSE 0 END) / COUNT(DISTINCT d.deployment_id) AS cfr\n FROM (\n SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY cdc.cicd_deployment_id\n ) d\n LEFT JOIN project_incident_deployment_relationships pidr ON d.deployment_id = pidr.deployment_id\n LEFT JOIN incidents i ON pidr.id = i.id\n GROUP BY DATE_SUB(DATE(d.deployment_finished_date), INTERVAL WEEKDAY(DATE(d.deployment_finished_date)) DAY)\n) combined\nGROUP BY time ORDER BY time", + "rawSql": "SELECT time, SUM(credits) AS 'Credits Used', SUM(cfr) AS 'Change Failure Rate'\nFROM (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS credits, 0 AS cfr\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n UNION ALL\n SELECT DATE_SUB(DATE(d.deployment_finished_date), INTERVAL WEEKDAY(DATE(d.deployment_finished_date)) DAY) AS time,\n 0 AS credits,\n SUM(CASE WHEN i.id IS NOT NULL THEN 1 ELSE 0 END) / COUNT(DISTINCT d.deployment_id) AS cfr\n FROM (\n SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY cdc.cicd_deployment_id\n ) d\n LEFT JOIN project_incident_deployment_relationships pidr ON d.deployment_id = pidr.deployment_id\n LEFT JOIN incidents i ON pidr.id = i.id\n GROUP BY DATE_SUB(DATE(d.deployment_finished_date), INTERVAL WEEKDAY(DATE(d.deployment_finished_date)) DAY)\n) combined\nGROUP BY time ORDER BY time", "refId": "A" } ], @@ -398,12 +719,27 @@ "preload": false, "refresh": "5m", "schemaVersion": 41, - "tags": ["q_dev", "dora", "kiro", "correlation"], + "tags": [ + "kiro", + "dora", + "correlation" + ], "templating": { "list": [ { - "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "definition": "SELECT DISTINCT name FROM projects", "hide": 0, "includeAll": true, @@ -417,7 +753,10 @@ } ] }, - "time": { "from": "now-90d", "to": "now" }, + "time": { + "from": "now-90d", + "to": "now" + }, "timepicker": {}, "timezone": "utc", "title": "Kiro Credits + DORA Correlation", diff --git a/grafana/dashboards/mysql/language-ai-heatmap.json b/grafana/dashboards/mysql/language-ai-heatmap.json index 789b291e5c1..46927ea0683 100644 --- a/grafana/dashboards/mysql/language-ai-heatmap.json +++ b/grafana/dashboards/mysql/language-ai-heatmap.json @@ -19,25 +19,60 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Completion requests, avg context size, and completion rate per language", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, - "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] } + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, - "options": { "cellHeight": "sm", "showHeader": true, "sortBy": [{ "desc": true, "displayName": "Requests" }] }, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Requests" + } + ] + }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS 'Language',\n COUNT(*) AS 'Requests',\n ROUND(AVG(completions_count), 2) AS 'Avg Completions',\n ROUND(AVG(left_context_length)) AS 'Avg Left Context',\n ROUND(AVG(right_context_length)) AS 'Avg Right Context',\n ROUND(AVG(left_context_length + right_context_length)) AS 'Avg Total Context',\n COUNT(DISTINCT user_id) AS 'Users'\nFROM _tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC", + "rawSql": "SELECT\n CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS 'Language',\n COUNT(*) AS 'Requests',\n ROUND(AVG(completions_count), 2) AS 'Avg Completions',\n ROUND(AVG(left_context_length)) AS 'Avg Left Context',\n ROUND(AVG(right_context_length)) AS 'Avg Right Context',\n ROUND(AVG(left_context_length + right_context_length)) AS 'Avg Total Context',\n COUNT(DISTINCT user_id) AS 'Users'\nFROM _tool_kiro_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC", "refId": "A" } ], @@ -45,31 +80,60 @@ "type": "table" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Daily completion requests by language", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, - "stacking": { "mode": "normal" }, "thresholdsStyle": { "mode": "off" } + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "stacking": { + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 9 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, "id": 2, "options": { - "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT DATE(timestamp) AS time,\n CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS metric,\n COUNT(*) AS value\nFROM _tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp), file_extension\nORDER BY time", + "rawSql": "SELECT DATE(timestamp) AS time,\n CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS metric,\n COUNT(*) AS value\nFROM _tool_kiro_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp), file_extension\nORDER BY time", "refId": "A" } ], @@ -77,64 +141,134 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Which file types are most active during chat sessions", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Chat logs do not contain active file types. This uses completion-log file extensions and returned lines.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, "id": 3, "options": { - "displayLabels": ["name", "percent"], - "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": ["value", "percent"] }, + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, "pieType": "donut", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": true }, - "tooltip": { "mode": "single" } + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file)' ELSE active_file_extension END AS 'File Type',\n COUNT(*) AS 'Chat Events'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY active_file_extension\nORDER BY COUNT(*) DESC\nLIMIT 10", + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS 'File Type', SUM(returned_line_count) AS 'Returned Lines' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY file_extension ORDER BY SUM(returned_line_count) DESC", "refId": "A" } ], - "title": "Active File Types During Chat", + "title": "Returned Lines by File Type", "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Context size trends for top languages", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, "id": 4, "options": { - "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT DATE(timestamp) AS time,\n file_extension AS metric,\n ROUND(AVG(left_context_length + right_context_length)) AS value\nFROM _tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\n AND file_extension IN (SELECT file_extension FROM _tool_q_dev_completion_log GROUP BY file_extension ORDER BY COUNT(*) DESC LIMIT 5)\nGROUP BY DATE(timestamp), file_extension\nORDER BY time", + "rawSql": "WITH top_extensions AS (SELECT file_extension FROM _tool_kiro_completion_log GROUP BY file_extension ORDER BY COUNT(*) DESC LIMIT 5) SELECT DATE(c.timestamp) AS time, c.file_extension AS metric, ROUND(AVG(c.left_context_length + c.right_context_length)) AS value FROM _tool_kiro_completion_log c INNER JOIN top_extensions t ON c.file_extension = t.file_extension WHERE $__timeFilter(c.timestamp) GROUP BY DATE(c.timestamp), c.file_extension ORDER BY time", "refId": "A" } ], @@ -145,9 +279,18 @@ "preload": false, "refresh": "5m", "schemaVersion": 41, - "tags": ["q_dev", "kiro", "language", "completions"], - "templating": { "list": [] }, - "time": { "from": "now-90d", "to": "now" }, + "tags": [ + "kiro", + "language", + "completions" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, "timepicker": {}, "timezone": "utc", "title": "Kiro Language AI Heatmap", diff --git a/grafana/dashboards/mysql/multi-ai-comparison.json b/grafana/dashboards/mysql/multi-ai-comparison.json index 617cda77637..15084441420 100644 --- a/grafana/dashboards/mysql/multi-ai-comparison.json +++ b/grafana/dashboards/mysql/multi-ai-comparison.json @@ -26,7 +26,7 @@ "targetBlank": true, "title": "Kiro Usage", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" }, { "asDropdown": false, @@ -46,11 +46,20 @@ "type": "datasource", "uid": "grafana" }, - "gridPos": { "h": 3, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "options": { - "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "## Multi-AI Tool Comparison: Copilot vs Kiro\nCompare GitHub Copilot and Kiro (Amazon Q Developer) adoption and usage side by side.\n\n**Note:** Copilot metrics come from `_tool_copilot_enterprise_daily_metrics` (enterprise-level aggregates). Kiro metrics come from `_tool_q_dev_user_report` and `_tool_q_dev_user_data`. Select the Copilot connection and scope using the dropdowns above.", + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## Multi-AI Tool Comparison: Copilot vs Kiro\nCompare GitHub Copilot and Kiro adoption and activity side by side.\n\n**Semantic boundary:** Copilot exports code-generation and acceptance events. Kiro exports usage reports and completion responses, but no acceptance event. Returned Kiro lines and Copilot LOC added are displayed with explicit labels and must not be treated as equivalent productivity measures.", "mode": "markdown" }, "title": "About", @@ -58,44 +67,87 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, "id": 2, "panels": [], "title": "Active Users", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly active users for both tools", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, "id": 3, "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n COUNT(DISTINCT user_id) AS 'Kiro Active Users'\nFROM _tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", + "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n COUNT(DISTINCT user_id) AS 'Kiro Active Users'\nFROM _tool_kiro_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", "refId": "A" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, "rawSql": "SELECT\n DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n MAX(daily_active_users) AS 'Copilot Active Users'\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\nGROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\nORDER BY time", @@ -107,71 +159,146 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, "id": 10, "panels": [], - "title": "Code Generation", + "title": "Assistant Activity", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Weekly code generation and acceptance events", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro completion requests and requests returning suggestions. Returned does not mean accepted.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, "id": 11, "options": { - "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(inline_suggestions_count) AS 'Kiro: Suggestions',\n SUM(inline_acceptance_count) AS 'Kiro: Accepted'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", + "rawSql": "SELECT DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) AS time, COUNT(*) AS 'Kiro: Completion Requests', SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS 'Kiro: Requests with Results' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) ORDER BY time", "refId": "A" } ], - "title": "Kiro: Code Suggestions & Acceptance", + "title": "Kiro Completion Requests & Results", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly Copilot code generation and acceptance events", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, "id": 12, "options": { - "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, "rawSql": "SELECT\n DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n SUM(code_generation_activity_count) AS 'Copilot: Suggestions',\n SUM(code_acceptance_activity_count) AS 'Copilot: Accepted'\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\nGROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\nORDER BY time", @@ -183,59 +310,117 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, "id": 20, "panels": [], - "title": "Lines of Code & Acceptance Rate", + "title": "Returned Lines & Acceptance Availability", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Weekly AI-generated lines of code accepted", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Different semantics: Kiro counts lines returned to the IDE; Copilot counts LOC added. Do not interpret as an acceptance comparison.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, "id": 21, "options": { - "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(kiro_loc) AS 'Kiro LOC Accepted', SUM(copilot_loc) AS 'Copilot LOC Added'\nFROM (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(inline_ai_code_lines + chat_ai_code_lines) AS kiro_loc, 0 AS copilot_loc\n FROM _tool_q_dev_user_data WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n UNION ALL\n SELECT DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n 0 AS kiro_loc, SUM(loc_added_sum) AS copilot_loc\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\n GROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\n) combined\nGROUP BY time ORDER BY time", + "rawSql": "SELECT time, SUM(kiro_lines) AS 'Kiro Returned Lines', SUM(copilot_loc) AS 'Copilot LOC Added' FROM (SELECT DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) AS time, SUM(returned_line_count) AS kiro_lines, 0 AS copilot_loc FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) UNION ALL SELECT DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time, 0 AS kiro_lines, SUM(loc_added_sum) AS copilot_loc FROM _tool_copilot_enterprise_daily_metrics WHERE connection_id = ${copilot_connection_id} AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)) combined GROUP BY time ORDER BY time", "refId": "A" } ], - "title": "LOC Accepted: Kiro vs Copilot", + "title": "Kiro Returned Lines vs Copilot LOC Added", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Overall acceptance rates for both tools", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro exports no suggestion-acceptance event, so its value is NULL. Copilot acceptance remains available.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "percent" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, "id": 22, "options": { "barRadius": 0.1, @@ -243,30 +428,42 @@ "orientation": "horizontal", "showValue": "auto", "stacking": "none", - "tooltip": { "mode": "single" } + "tooltip": { + "mode": "single" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT 'Kiro' AS Tool,\n ROUND(SUM(inline_acceptance_count) * 100.0 / NULLIF(SUM(inline_suggestions_count), 0), 1) AS 'Acceptance Rate'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)\nUNION ALL\nSELECT 'Copilot' AS Tool,\n ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1)\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)", + "rawSql": "SELECT 'Kiro (not exported)' AS 'Tool', CAST(NULL AS DECIMAL(10,2)) AS 'Acceptance Rate' UNION ALL SELECT 'Copilot' AS 'Tool', ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) AS 'Acceptance Rate' FROM _tool_copilot_enterprise_daily_metrics WHERE connection_id = ${copilot_connection_id} AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day)", "refId": "A" } ], - "title": "Acceptance Rate Comparison", + "title": "Acceptance Rate Availability", "type": "bargauge" } ], "preload": false, "refresh": "5m", "schemaVersion": 41, - "tags": ["q_dev", "copilot", "kiro", "comparison"], + "tags": [ + "kiro", + "copilot", + "comparison" + ], "templating": { "list": [ { "current": {}, - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC", "hide": 0, "label": "Copilot Connection", @@ -278,7 +475,10 @@ }, { "current": {}, - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "definition": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${copilot_connection_id}' AS UNSIGNED)", "hide": 0, "label": "Copilot Scope", @@ -290,7 +490,10 @@ } ] }, - "time": { "from": "now-90d", "to": "now" }, + "time": { + "from": "now-90d", + "to": "now" + }, "timepicker": {}, "timezone": "utc", "title": "Multi-AI Tool Comparison", diff --git a/grafana/dashboards/mysql/steering-adoption-tracker.json b/grafana/dashboards/mysql/steering-adoption-tracker.json index 14711cfd609..fa17c3d6fa3 100644 --- a/grafana/dashboards/mysql/steering-adoption-tracker.json +++ b/grafana/dashboards/mysql/steering-adoption-tracker.json @@ -19,29 +19,56 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue" }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, "unit": "percent" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, "id": 1, "options": { - "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -49,29 +76,56 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "purple" }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple" + } + ] + }, "unit": "percent" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 0 + }, "id": 2, "options": { - "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN is_spec_mode = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN is_spec_mode = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -79,29 +133,56 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, "unit": "percent" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 0 + }, "id": 3, "options": { - "colorMode": "value", "graphMode": "area", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS 'value'\nFROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -109,32 +190,64 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly trend of steering and spec mode adoption rate", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { - "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, - "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "unit": "percentunit" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, "id": 4, "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi" } + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) AS time,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Steering Rate',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Spec Mode Rate'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY)\nORDER BY time", + "rawSql": "SELECT\n DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) AS time,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Steering Rate',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Spec Mode Rate'\nFROM _tool_kiro_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY)\nORDER BY time", "refId": "A" } ], @@ -142,29 +255,54 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Compare prompt and response quality between steering and non-steering sessions", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, "id": 5, "options": { - "barRadius": 0.1, "barWidth": 0.5, - "orientation": "horizontal", "showValue": "auto", "stacking": "none", - "tooltip": { "mode": "single" } + "barRadius": 0.1, + "barWidth": 0.5, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single" + } }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT 'With Steering' AS 'Mode', ROUND(AVG(prompt_length)) AS 'Avg Prompt', ROUND(AVG(response_length)) AS 'Avg Response'\nFROM _tool_q_dev_chat_log WHERE has_steering = 1 AND $__timeFilter(timestamp)\nUNION ALL\nSELECT 'Without Steering', ROUND(AVG(prompt_length)), ROUND(AVG(response_length))\nFROM _tool_q_dev_chat_log WHERE has_steering = 0 AND $__timeFilter(timestamp)", + "rawSql": "SELECT 'With Steering' AS 'Mode', ROUND(AVG(prompt_length)) AS 'Avg Prompt', ROUND(AVG(response_length)) AS 'Avg Response'\nFROM _tool_kiro_chat_log WHERE has_steering = 1 AND $__timeFilter(timestamp)\nUNION ALL\nSELECT 'Without Steering', ROUND(AVG(prompt_length)), ROUND(AVG(response_length))\nFROM _tool_kiro_chat_log WHERE has_steering = 0 AND $__timeFilter(timestamp)", "refId": "A" } ], @@ -172,25 +310,54 @@ "type": "bargauge" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Per-user steering and spec mode usage", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Per-user steering and spec-mode use, with names resolved from usage reports.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, - "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] } + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, "id": 6, - "options": { "cellHeight": "sm", "showHeader": true }, + "options": { + "cellHeight": "sm", + "showHeader": true + }, "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) AS 'User',\n COUNT(*) AS 'Total Chats',\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS 'Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS 'Spec Mode',\n ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS 'Steering %'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY user_id\nORDER BY COUNT(*) DESC", + "rawSql": "WITH names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, c.user_id) AS 'User', COUNT(*) AS 'Total Chats', SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS 'Steering', SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS 'Spec Mode', ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0), 1) AS 'Steering %' FROM _tool_kiro_chat_log c LEFT JOIN names n ON c.user_id = n.user_id WHERE $__timeFilter(c.timestamp) GROUP BY c.user_id, n.display_name ORDER BY COUNT(*) DESC", "refId": "A" } ], @@ -201,9 +368,18 @@ "preload": false, "refresh": "5m", "schemaVersion": 41, - "tags": ["q_dev", "kiro", "steering", "adoption"], - "templating": { "list": [] }, - "time": { "from": "now-90d", "to": "now" }, + "tags": [ + "kiro", + "steering", + "adoption" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, "timepicker": {}, "timezone": "utc", "title": "Kiro Steering & Spec Mode Adoption", diff --git a/grafana/dashboards/postgresql/ai-cost-efficiency.json b/grafana/dashboards/postgresql/ai-cost-efficiency.json index 497a05b0f34..2b7723c7081 100644 --- a/grafana/dashboards/postgresql/ai-cost-efficiency.json +++ b/grafana/dashboards/postgresql/ai-cost-efficiency.json @@ -26,7 +26,7 @@ "targetBlank": true, "title": "Kiro Usage Dashboard", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" } ], "panels": [ @@ -44,7 +44,10 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "fieldConfig": { "defaults": { "color": { @@ -84,10 +87,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], @@ -95,7 +101,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "fieldConfig": { "defaults": { "color": { @@ -135,10 +144,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0) AS DECIMAL), 1) AS \"Credits / PR\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(CAST(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0) AS DECIMAL), 1) AS \"Credits / PR\" FROM _tool_kiro_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -146,7 +158,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "fieldConfig": { "defaults": { "color": { @@ -186,10 +201,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0) AS DECIMAL), 1) AS \"Credits / Deploy\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(CAST(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0) AS DECIMAL), 1) AS \"Credits / Deploy\" FROM _tool_kiro_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -197,7 +215,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "fieldConfig": { "defaults": { "color": { @@ -237,10 +258,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0) AS DECIMAL), 1) AS \"Credits / Issue\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(CAST(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0) AS DECIMAL), 1) AS \"Credits / Issue\" FROM _tool_kiro_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -261,7 +285,10 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly cost per merged PR", "fieldConfig": { "defaults": { @@ -313,10 +340,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(merged_date AS DATE)) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(merged_date AS DATE)) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -324,7 +354,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly cost per production deployment", "fieldConfig": { "defaults": { @@ -376,10 +409,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -387,7 +423,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly cost per resolved issue", "fieldConfig": { "defaults": { @@ -439,10 +478,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -450,7 +492,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Is cost efficiency improving over time?", "fieldConfig": { "defaults": { @@ -502,10 +547,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -517,7 +565,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", "kiro", "cost", "efficiency" @@ -534,4 +581,4 @@ "title": "AI Cost-Efficiency", "uid": "ai_cost_efficiency-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/developer-productivity-hours.json b/grafana/dashboards/postgresql/developer-productivity-hours.json index ba9848d1c43..05fb09212cc 100644 --- a/grafana/dashboards/postgresql/developer-productivity-hours.json +++ b/grafana/dashboards/postgresql/developer-productivity-hours.json @@ -19,7 +19,10 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Chat and completion events by hour of day", "fieldConfig": { "defaults": { @@ -63,10 +66,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT LPAD(CAST(hr AS TEXT), 2, '0') AS \"Hour\", SUM(chat) AS \"Chat Events\", SUM(comp) AS \"Completion Events\" FROM (SELECT EXTRACT(HOUR FROM timestamp) AS hr, COUNT(*) AS chat, 0 AS comp FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) UNION ALL SELECT EXTRACT(HOUR FROM timestamp) AS hr, 0 AS chat, COUNT(*) AS comp FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp)) AS t GROUP BY hr ORDER BY hr NULLS FIRST", + "rawSql": "SELECT LPAD(CAST(hr AS TEXT), 2, '0') AS \"Hour\", SUM(chat) AS \"Chat Events\", SUM(comp) AS \"Completion Events\" FROM (SELECT EXTRACT(HOUR FROM timestamp) AS hr, COUNT(*) AS chat, 0 AS comp FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) UNION ALL SELECT EXTRACT(HOUR FROM timestamp) AS hr, 0 AS chat, COUNT(*) AS comp FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp)) AS t GROUP BY hr ORDER BY hr NULLS FIRST", "refId": "A" } ], @@ -74,7 +80,10 @@ "type": "barchart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Average prompt complexity and response richness by hour", "fieldConfig": { "defaults": { @@ -122,10 +131,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT LPAD(CAST(EXTRACT(HOUR FROM timestamp) AS TEXT), 2, '0') AS \"Hour\", ROUND(AVG(prompt_length)) AS \"Avg Prompt Length\", ROUND(AVG(response_length)) AS \"Avg Response Length\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) ORDER BY EXTRACT(HOUR FROM timestamp) NULLS FIRST", + "rawSql": "SELECT LPAD(CAST(EXTRACT(HOUR FROM timestamp) AS TEXT), 2, '0') AS \"Hour\", ROUND(AVG(prompt_length)) AS \"Avg Prompt Length\", ROUND(AVG(response_length)) AS \"Avg Response Length\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) ORDER BY EXTRACT(HOUR FROM timestamp) NULLS FIRST", "refId": "A" } ], @@ -133,7 +145,10 @@ "type": "barchart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Steering and spec mode usage concentration by hour", "fieldConfig": { "defaults": { @@ -177,10 +192,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT LPAD(CAST(EXTRACT(HOUR FROM timestamp) AS TEXT), 2, '0') AS \"Hour\", SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Spec Mode\", SUM(CASE WHEN has_steering = FALSE AND is_spec_mode = FALSE THEN 1 ELSE 0 END) AS \"Plain Chat\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) ORDER BY EXTRACT(HOUR FROM timestamp) NULLS FIRST", + "rawSql": "SELECT LPAD(CAST(EXTRACT(HOUR FROM timestamp) AS TEXT), 2, '0') AS \"Hour\", SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Spec Mode\", SUM(CASE WHEN has_steering = FALSE AND is_spec_mode = FALSE THEN 1 ELSE 0 END) AS \"Plain Chat\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) ORDER BY EXTRACT(HOUR FROM timestamp) NULLS FIRST", "refId": "A" } ], @@ -188,7 +206,10 @@ "type": "barchart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Day-of-week activity pattern", "fieldConfig": { "defaults": { @@ -232,10 +253,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) WHEN 1 THEN 'Sun' WHEN 2 THEN 'Mon' WHEN 3 THEN 'Tue' WHEN 4 THEN 'Wed' WHEN 5 THEN 'Thu' WHEN 6 THEN 'Fri' WHEN 7 THEN 'Sat' END AS \"Day\", SUM(chat) AS \"Chat Events\", SUM(comp) AS \"Completions\" FROM (SELECT timestamp, COUNT(*) AS chat, 0 AS comp FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY timestamp UNION ALL SELECT timestamp, 0, COUNT(*) FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY timestamp) AS t GROUP BY EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) ORDER BY EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) NULLS FIRST", + "rawSql": "SELECT CASE EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) WHEN 1 THEN 'Sun' WHEN 2 THEN 'Mon' WHEN 3 THEN 'Tue' WHEN 4 THEN 'Wed' WHEN 5 THEN 'Thu' WHEN 6 THEN 'Fri' WHEN 7 THEN 'Sat' END AS \"Day\", SUM(chat) AS \"Chat Events\", SUM(comp) AS \"Completions\" FROM (SELECT timestamp, COUNT(*) AS chat, 0 AS comp FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY timestamp UNION ALL SELECT timestamp, 0, COUNT(*) FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY timestamp) AS t GROUP BY EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) ORDER BY EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) NULLS FIRST", "refId": "A" } ], @@ -247,7 +271,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", "kiro", "productivity", "hours" @@ -264,4 +287,4 @@ "title": "Developer AI Productivity Hours", "uid": "kiro_productivity_hours-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/kiro-credits-dora.json b/grafana/dashboards/postgresql/kiro-credits-dora.json index 551529fd01a..6f6ff79fc73 100644 --- a/grafana/dashboards/postgresql/kiro-credits-dora.json +++ b/grafana/dashboards/postgresql/kiro-credits-dora.json @@ -37,7 +37,7 @@ "targetBlank": true, "title": "Kiro Usage Dashboard", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" } ], "panels": [ @@ -79,7 +79,10 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Pearson correlation coefficient between weekly Kiro credits and PR cycle time", "fieldConfig": { "defaults": { @@ -141,11 +144,14 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH _kiro_weekly AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _pr_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_cycle_hours FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT k.weekly_credits AS x, p.avg_cycle_hours AS y FROM _kiro_weekly AS k INNER JOIN _pr_weekly AS p ON k.week_start = p.week_start), _stats AS (SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy FROM _joined) SELECT CASE WHEN s.n < 4 THEN NULL WHEN s.sx = '0' OR s.sy = '0' THEN 0 ELSE ROUND(CAST((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined AS j) AS NUMERIC) / NULLIF((s.n * s.sx * s.sy), 0), 2) END AS \"r\" FROM _stats AS s", + "rawSql": "WITH _kiro_weekly AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _pr_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_cycle_hours FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT k.weekly_credits AS x, p.avg_cycle_hours AS y FROM _kiro_weekly AS k INNER JOIN _pr_weekly AS p ON k.week_start = p.week_start), _stats AS (SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy FROM _joined) SELECT CASE WHEN s.n < 4 THEN NULL WHEN s.sx = '0' OR s.sy = '0' THEN 0 ELSE ROUND(CAST((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined AS j) AS NUMERIC) / NULLIF((s.n * s.sx * s.sy), 0), 2) END AS \"r\" FROM _stats AS s", "refId": "A" } ], @@ -153,7 +159,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Total Kiro credits consumed in period", "fieldConfig": { "defaults": { @@ -194,10 +203,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(credits_used) AS \"Credits Used\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "rawSql": "SELECT SUM(credits_used) AS \"Credits Used\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], @@ -205,7 +217,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Median PR cycle time in hours", "fieldConfig": { "defaults": { @@ -247,7 +262,10 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, "rawSql": "WITH _ranked AS (SELECT CAST(pr_cycle_time AS NUMERIC) / NULLIF(60.0, 0) AS ct, PERCENT_RANK() OVER (ORDER BY pr_cycle_time NULLS FIRST) AS prank FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date)) SELECT ROUND(MAX(ct), 1) AS \"Median Cycle Time\" FROM _ranked WHERE prank <= 0.5", @@ -258,7 +276,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Cycle time comparison: weeks with above-median vs below-median AI credits usage", "fieldConfig": { "defaults": { @@ -298,10 +319,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "WITH _kiro_weekly AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _pr_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_ct FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT k.weekly_credits, p.avg_ct FROM _kiro_weekly AS k INNER JOIN _pr_weekly AS p ON k.week_start = p.week_start), _med AS (SELECT weekly_credits AS med FROM _joined ORDER BY weekly_credits NULLS FIRST LIMIT 1 OFFSET (SELECT FLOOR(CAST(COUNT(*) AS NUMERIC) / NULLIF(2, 0)) FROM _joined)) SELECT CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END AS \"Tier\", ROUND(CAST(AVG(j.avg_ct) AS DECIMAL), 1) AS \"Avg Cycle Time\" FROM _joined AS j, _med AS m GROUP BY CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END", + "rawSql": "WITH _kiro_weekly AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _pr_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_ct FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT k.weekly_credits, p.avg_ct FROM _kiro_weekly AS k INNER JOIN _pr_weekly AS p ON k.week_start = p.week_start), _med AS (SELECT weekly_credits AS med FROM _joined ORDER BY weekly_credits NULLS FIRST LIMIT 1 OFFSET (SELECT FLOOR(CAST(COUNT(*) AS NUMERIC) / NULLIF(2, 0)) FROM _joined)) SELECT CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END AS \"Tier\", ROUND(CAST(AVG(j.avg_ct) AS DECIMAL), 1) AS \"Avg Cycle Time\" FROM _joined AS j, _med AS m GROUP BY CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END", "refId": "A" } ], @@ -322,7 +346,10 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly Kiro credits consumed", "fieldConfig": { "defaults": { @@ -374,10 +401,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits Used\", COUNT(DISTINCT user_id) AS \"Active Users\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits Used\", COUNT(DISTINCT user_id) AS \"Active Users\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -385,7 +415,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly average PR cycle time in hours", "fieldConfig": { "defaults": { @@ -437,7 +470,10 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, "rawSql": "SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS time, ROUND(CAST(CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS DECIMAL), 1) AS \"Avg Cycle Time (hrs)\", COUNT(*) AS \"PRs Merged\" FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", @@ -461,7 +497,10 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Pearson correlation between weekly credits and deployment count", "fieldConfig": { "defaults": { @@ -512,10 +551,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "WITH _kiro_weekly AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _deploy_weekly AS (SELECT CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT k.weekly_credits AS x, d.deploys AS y FROM _kiro_weekly AS k INNER JOIN _deploy_weekly AS d ON k.week_start = d.week_start), _stats AS (SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy FROM _joined) SELECT CASE WHEN s.n < 4 THEN NULL WHEN s.sx = '0' OR s.sy = '0' THEN 0 ELSE ROUND(CAST((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined AS j) AS NUMERIC) / NULLIF((s.n * s.sx * s.sy), 0), 2) END AS \"r\" FROM _stats AS s", + "rawSql": "WITH _kiro_weekly AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _deploy_weekly AS (SELECT CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT k.weekly_credits AS x, d.deploys AS y FROM _kiro_weekly AS k INNER JOIN _deploy_weekly AS d ON k.week_start = d.week_start), _stats AS (SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy FROM _joined) SELECT CASE WHEN s.n < 4 THEN NULL WHEN s.sx = '0' OR s.sy = '0' THEN 0 ELSE ROUND(CAST((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined AS j) AS NUMERIC) / NULLIF((s.n * s.sx * s.sy), 0), 2) END AS \"r\" FROM _stats AS s", "refId": "A" } ], @@ -523,7 +565,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly credits overlaid with deployment count", "fieldConfig": { "defaults": { @@ -575,10 +620,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(credits) AS \"Credits Used\", SUM(deploys) AS \"Deployments\" FROM (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS credits, 0 AS deploys FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' UNION ALL SELECT CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day' AS time, 0 AS credits, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day') AS combined GROUP BY time ORDER BY time NULLS FIRST", + "rawSql": "SELECT time, SUM(credits) AS \"Credits Used\", SUM(deploys) AS \"Deployments\" FROM (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS credits, 0 AS deploys FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' UNION ALL SELECT CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day' AS time, 0 AS credits, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day') AS combined GROUP BY time ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -599,7 +647,10 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Change failure rate: % of deployments that caused incidents", "fieldConfig": { "defaults": { @@ -651,10 +702,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(credits) AS \"Credits Used\", SUM(cfr) AS \"Change Failure Rate\" FROM (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS credits, 0 AS cfr FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' UNION ALL SELECT CAST(d.deployment_finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(d.deployment_finished_date AS DATE)) - 1) * INTERVAL '1 day' AS time, 0 AS credits, CAST(SUM(CASE WHEN NOT i.id IS NULL THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(DISTINCT d.deployment_id), 0) AS cfr FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY cdc.cicd_deployment_id) AS d LEFT JOIN project_incident_deployment_relationships AS pidr ON d.deployment_id = pidr.deployment_id LEFT JOIN incidents AS i ON pidr.id = i.id GROUP BY CAST(d.deployment_finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(d.deployment_finished_date AS DATE)) - 1) * INTERVAL '1 day') AS combined GROUP BY time ORDER BY time NULLS FIRST", + "rawSql": "SELECT time, SUM(credits) AS \"Credits Used\", SUM(cfr) AS \"Change Failure Rate\" FROM (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS credits, 0 AS cfr FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' UNION ALL SELECT CAST(d.deployment_finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(d.deployment_finished_date AS DATE)) - 1) * INTERVAL '1 day' AS time, 0 AS credits, CAST(SUM(CASE WHEN NOT i.id IS NULL THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(DISTINCT d.deployment_id), 0) AS cfr FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY cdc.cicd_deployment_id) AS d LEFT JOIN project_incident_deployment_relationships AS pidr ON d.deployment_id = pidr.deployment_id LEFT JOIN incidents AS i ON pidr.id = i.id GROUP BY CAST(d.deployment_finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(d.deployment_finished_date AS DATE)) - 1) * INTERVAL '1 day') AS combined GROUP BY time ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -666,9 +720,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "dora", "kiro", + "dora", "correlation" ], "templating": { @@ -683,7 +736,10 @@ "$__all" ] }, - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "definition": "SELECT DISTINCT name FROM projects", "hide": 0, "includeAll": true, @@ -706,4 +762,4 @@ "title": "Kiro Credits + DORA Correlation", "uid": "kiro_credits_dora-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/language-ai-heatmap.json b/grafana/dashboards/postgresql/language-ai-heatmap.json index 63aa23d65da..75b45818b57 100644 --- a/grafana/dashboards/postgresql/language-ai-heatmap.json +++ b/grafana/dashboards/postgresql/language-ai-heatmap.json @@ -19,7 +19,10 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Completion requests, avg context size, and completion rate per language", "fieldConfig": { "defaults": { @@ -63,10 +66,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"Language\", COUNT(*) AS \"Requests\", ROUND(CAST(AVG(completions_count) AS DECIMAL), 2) AS \"Avg Completions\", ROUND(AVG(left_context_length)) AS \"Avg Left Context\", ROUND(AVG(right_context_length)) AS \"Avg Right Context\", ROUND(AVG(left_context_length + right_context_length)) AS \"Avg Total Context\", COUNT(DISTINCT user_id) AS \"Users\" FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST", + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"Language\", COUNT(*) AS \"Requests\", ROUND(CAST(AVG(completions_count) AS DECIMAL), 2) AS \"Avg Completions\", ROUND(AVG(left_context_length)) AS \"Avg Left Context\", ROUND(AVG(right_context_length)) AS \"Avg Right Context\", ROUND(AVG(left_context_length + right_context_length)) AS \"Avg Total Context\", COUNT(DISTINCT user_id) AS \"Users\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST", "refId": "A" } ], @@ -74,7 +80,10 @@ "type": "table" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Daily completion requests by language", "fieldConfig": { "defaults": { @@ -118,10 +127,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS metric, COUNT(*) AS value FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE), \"file_extension\" ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS metric, COUNT(*) AS value FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE), \"file_extension\" ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -129,8 +141,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Which file types are most active during chat sessions", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Chat logs do not contain active file types. This uses completion-log file extensions and returned lines.", "fieldConfig": { "defaults": { "color": { @@ -183,18 +198,24 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file)' ELSE active_file_extension END AS \"File Type\", COUNT(*) AS \"Chat Events\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"active_file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 10", + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"File Type\", SUM(returned_line_count) AS \"Returned Lines\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY file_extension ORDER BY SUM(returned_line_count) DESC", "refId": "A" } ], - "title": "Active File Types During Chat", + "title": "Returned Lines by File Type", "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Context size trends for top languages", "fieldConfig": { "defaults": { @@ -241,10 +262,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, file_extension AS metric, ROUND(AVG(left_context_length + right_context_length)) AS value FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) AND file_extension IN (SELECT file_extension FROM _tool_q_dev_completion_log GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 5) GROUP BY CAST(timestamp AS DATE), \"file_extension\" ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, file_extension AS metric, ROUND(AVG(left_context_length + right_context_length)) AS value FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) AND file_extension IN (SELECT file_extension FROM _tool_kiro_completion_log GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 5) GROUP BY CAST(timestamp AS DATE), \"file_extension\" ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -256,7 +280,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", "kiro", "language", "completions" @@ -273,4 +296,4 @@ "title": "Kiro Language AI Heatmap", "uid": "kiro_language_heatmap-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/multi-ai-comparison.json b/grafana/dashboards/postgresql/multi-ai-comparison.json index bca47881221..3787072c130 100644 --- a/grafana/dashboards/postgresql/multi-ai-comparison.json +++ b/grafana/dashboards/postgresql/multi-ai-comparison.json @@ -26,7 +26,7 @@ "targetBlank": true, "title": "Kiro Usage", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" }, { "asDropdown": false, @@ -59,7 +59,7 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "## Multi-AI Tool Comparison: Copilot vs Kiro\nCompare GitHub Copilot and Kiro (Amazon Q Developer) adoption and usage side by side.\n\n**Note:** Copilot metrics come from `_tool_copilot_enterprise_daily_metrics` (enterprise-level aggregates). Kiro metrics come from `_tool_q_dev_user_report` and `_tool_q_dev_user_data`. Select the Copilot connection and scope using the dropdowns above.", + "content": "## Multi-AI Tool Comparison: Copilot vs Kiro\nCompare GitHub Copilot and Kiro adoption and activity side by side.\n\n**Semantic boundary:** Copilot exports code-generation and acceptance events. Kiro exports usage reports and completion responses, but no acceptance event. Returned Kiro lines and Copilot LOC added are displayed with explicit labels and must not be treated as equivalent productivity measures.", "mode": "markdown" }, "title": "About", @@ -79,7 +79,10 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly active users for both tools", "fieldConfig": { "defaults": { @@ -131,14 +134,20 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(DISTINCT user_id) AS \"Kiro Active Users\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(DISTINCT user_id) AS \"Kiro Active Users\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", "refId": "A" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, "rawSql": "SELECT CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' AS time, MAX(daily_active_users) AS \"Copilot Active Users\" FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", @@ -158,12 +167,15 @@ }, "id": 10, "panels": [], - "title": "Code Generation", + "title": "Assistant Activity", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Weekly code generation and acceptance events", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro completion requests and requests returning suggestions. Returned does not mean accepted.", "fieldConfig": { "defaults": { "color": { @@ -214,18 +226,24 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(inline_suggestions_count) AS \"Kiro: Suggestions\", SUM(inline_acceptance_count) AS \"Kiro: Accepted\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(*) AS \"Kiro: Completion Requests\", SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS \"Kiro: Requests with Results\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time", "refId": "A" } ], - "title": "Kiro: Code Suggestions & Acceptance", + "title": "Kiro Completion Requests & Results", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly Copilot code generation and acceptance events", "fieldConfig": { "defaults": { @@ -277,7 +295,10 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, "rawSql": "SELECT CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(code_generation_activity_count) AS \"Copilot: Suggestions\", SUM(code_acceptance_activity_count) AS \"Copilot: Accepted\" FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", @@ -297,12 +318,15 @@ }, "id": 20, "panels": [], - "title": "Lines of Code & Acceptance Rate", + "title": "Returned Lines & Acceptance Availability", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Weekly AI-generated lines of code accepted", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Different semantics: Kiro counts lines returned to the IDE; Copilot counts LOC added. Do not interpret as an acceptance comparison.", "fieldConfig": { "defaults": { "color": { @@ -353,19 +377,25 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(kiro_loc) AS \"Kiro LOC Accepted\", SUM(copilot_loc) AS \"Copilot LOC Added\" FROM (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(inline_ai_code_lines + chat_ai_code_lines) AS kiro_loc, 0 AS copilot_loc FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' UNION ALL SELECT CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' AS time, 0 AS kiro_loc, SUM(loc_added_sum) AS copilot_loc FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day') AS combined GROUP BY time ORDER BY time NULLS FIRST", + "rawSql": "SELECT time, SUM(kiro_lines) AS \"Kiro Returned Lines\", SUM(copilot_loc) AS \"Copilot LOC Added\" FROM (SELECT CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(returned_line_count) AS kiro_lines, 0 AS copilot_loc FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' UNION ALL SELECT CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' AS time, 0 AS kiro_lines, SUM(loc_added_sum) AS copilot_loc FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day') combined GROUP BY time ORDER BY time", "refId": "A" } ], - "title": "LOC Accepted: Kiro vs Copilot", + "title": "Kiro Returned Lines vs Copilot LOC Added", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Overall acceptance rates for both tools", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro exports no suggestion-acceptance event, so its value is NULL. Copilot acceptance remains available.", "fieldConfig": { "defaults": { "color": { @@ -404,14 +434,17 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT 'Kiro' AS \"Tool\", ROUND(CAST(SUM(inline_acceptance_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0), 1) AS \"Acceptance Rate\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Copilot' AS \"Tool\", ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day)", + "rawSql": "SELECT 'Kiro (not exported)' AS \"Tool\", CAST(NULL AS NUMERIC) AS \"Acceptance Rate\" UNION ALL SELECT 'Copilot' AS \"Tool\", ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(SUM(code_generation_activity_count), 0), 1) AS \"Acceptance Rate\" FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day)", "refId": "A" } ], - "title": "Acceptance Rate Comparison", + "title": "Acceptance Rate Availability", "type": "bargauge" } ], @@ -419,16 +452,18 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "copilot", "kiro", + "copilot", "comparison" ], "templating": { "list": [ { "current": {}, - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC NULLS LAST", "hide": 0, "label": "Copilot Connection", @@ -440,7 +475,10 @@ }, { "current": {}, - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "definition": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${copilot_connection_id}' AS BIGINT)", "hide": 0, "label": "Copilot Scope", @@ -461,4 +499,4 @@ "title": "Multi-AI Tool Comparison", "uid": "multi_ai_comparison-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/steering-adoption-tracker.json b/grafana/dashboards/postgresql/steering-adoption-tracker.json index f332a314483..31403278474 100644 --- a/grafana/dashboards/postgresql/steering-adoption-tracker.json +++ b/grafana/dashboards/postgresql/steering-adoption-tracker.json @@ -19,7 +19,10 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "fieldConfig": { "defaults": { "color": { @@ -59,10 +62,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) * 100.0 AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) AS DECIMAL), 0) AS \"value\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) * 100.0 AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) AS DECIMAL), 0) AS \"value\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -70,7 +76,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "fieldConfig": { "defaults": { "color": { @@ -110,10 +119,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN is_spec_mode = TRUE THEN user_id END) * 100.0 AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) AS DECIMAL), 0) AS \"value\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN is_spec_mode = TRUE THEN user_id END) * 100.0 AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) AS DECIMAL), 0) AS \"value\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -121,7 +133,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "fieldConfig": { "defaults": { "color": { @@ -161,10 +176,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(CAST(SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) * 100.0 AS NUMERIC) / NULLIF(COUNT(*), 0) AS DECIMAL), 1) AS \"value\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(CAST(CAST(SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) * 100.0 AS NUMERIC) / NULLIF(COUNT(*), 0) AS DECIMAL), 1) AS \"value\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -172,7 +190,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly trend of steering and spec mode adoption rate", "fieldConfig": { "defaults": { @@ -220,10 +241,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' AS time, CAST(SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Steering Rate\", CAST(SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Spec Mode Rate\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' AS time, CAST(SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Steering Rate\", CAST(SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Spec Mode Rate\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -231,7 +255,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Compare prompt and response quality between steering and non-steering sessions", "fieldConfig": { "defaults": { @@ -269,10 +296,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT 'With Steering' AS \"Mode\", ROUND(AVG(prompt_length)) AS \"Avg Prompt\", ROUND(AVG(response_length)) AS \"Avg Response\" FROM _tool_q_dev_chat_log WHERE has_steering = TRUE AND $__timeFilter(timestamp) UNION ALL SELECT 'Without Steering', ROUND(AVG(prompt_length)), ROUND(AVG(response_length)) FROM _tool_q_dev_chat_log WHERE has_steering = FALSE AND $__timeFilter(timestamp)", + "rawSql": "SELECT 'With Steering' AS \"Mode\", ROUND(AVG(prompt_length)) AS \"Avg Prompt\", ROUND(AVG(response_length)) AS \"Avg Response\" FROM _tool_kiro_chat_log WHERE has_steering = TRUE AND $__timeFilter(timestamp) UNION ALL SELECT 'Without Steering', ROUND(AVG(prompt_length)), ROUND(AVG(response_length)) FROM _tool_kiro_chat_log WHERE has_steering = FALSE AND $__timeFilter(timestamp)", "refId": "A" } ], @@ -280,8 +310,11 @@ "type": "bargauge" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Per-user steering and spec mode usage", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Per-user steering and spec-mode use, with names resolved from usage reports.", "fieldConfig": { "defaults": { "color": { @@ -318,10 +351,13 @@ }, "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", COUNT(*) AS \"Total Chats\", SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Spec Mode\", ROUND(CAST(CAST(SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) * 100.0 AS NUMERIC) / NULLIF(COUNT(*), 0) AS DECIMAL), 1) AS \"Steering %\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"user_id\" ORDER BY COUNT(*) DESC NULLS LAST", + "rawSql": "WITH names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, c.user_id) AS \"User\", COUNT(*) AS \"Total Chats\", SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Spec Mode\", ROUND(CAST(SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0), 1) AS \"Steering %\" FROM _tool_kiro_chat_log c LEFT JOIN names n ON c.user_id = n.user_id WHERE $__timeFilter(c.timestamp) GROUP BY c.user_id, n.display_name ORDER BY COUNT(*) DESC", "refId": "A" } ], @@ -333,7 +369,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", "kiro", "steering", "adoption" @@ -350,4 +385,4 @@ "title": "Kiro Steering & Spec Mode Adoption", "uid": "kiro_steering_adoption-pg", "version": 1 -} \ No newline at end of file +} From 16ece9c11713e5a80ed49a15cc738a09f2f1a582 Mon Sep 17 00:00:00 2001 From: warren Date: Fri, 28 Aug 2026 14:32:51 +0800 Subject: [PATCH 5/6] fix(kiro): use portable timestamp precision Signed-off-by: warren --- backend/plugins/kiro/models/chat_log.go | 2 +- backend/plugins/kiro/models/completion_log.go | 2 +- .../models/migrationscripts/archived/init.go | 4 +- .../kiro/models/timestamp_schema_test.go | 55 +++++++++++++++++++ 4 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 backend/plugins/kiro/models/timestamp_schema_test.go diff --git a/backend/plugins/kiro/models/chat_log.go b/backend/plugins/kiro/models/chat_log.go index ecd943696fe..cde329f517d 100644 --- a/backend/plugins/kiro/models/chat_log.go +++ b/backend/plugins/kiro/models/chat_log.go @@ -44,7 +44,7 @@ type KiroChatLog struct { UserId string `gorm:"type:varchar(64);index" json:"userId"` // IdentityStoreId is the stripped prefix, retained for auditability. IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` // ChatTriggerType is MANUAL or INLINE_CHAT per the docs; only MANUAL has // been observed. Not validated against a fixed set. ChatTriggerType string `gorm:"type:varchar(20)" json:"chatTriggerType"` diff --git a/backend/plugins/kiro/models/completion_log.go b/backend/plugins/kiro/models/completion_log.go index 0d09c28e0ad..cfd633e5335 100644 --- a/backend/plugins/kiro/models/completion_log.go +++ b/backend/plugins/kiro/models/completion_log.go @@ -40,7 +40,7 @@ type KiroCompletionLog struct { UserId string `gorm:"type:varchar(64);index" json:"userId"` IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` // FileName has no path component; see the type comment. FileName string `gorm:"type:varchar(255);index" json:"fileName"` diff --git a/backend/plugins/kiro/models/migrationscripts/archived/init.go b/backend/plugins/kiro/models/migrationscripts/archived/init.go index 5c1c9a3224f..53422db0499 100644 --- a/backend/plugins/kiro/models/migrationscripts/archived/init.go +++ b/backend/plugins/kiro/models/migrationscripts/archived/init.go @@ -127,7 +127,7 @@ type KiroChatLog struct { RequestId string `gorm:"primaryKey;type:varchar(64)"` UserId string `gorm:"type:varchar(64);index" json:"userId"` IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` ChatTriggerType string `gorm:"type:varchar(20)" json:"chatTriggerType"` ModelId *string `gorm:"type:varchar(100)" json:"modelId"` HasPrompt bool `gorm:"index" json:"hasPrompt"` @@ -152,7 +152,7 @@ type KiroCompletionLog struct { RequestId string `gorm:"primaryKey;type:varchar(64)"` UserId string `gorm:"type:varchar(64);index" json:"userId"` IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` FileName string `gorm:"type:varchar(255);index" json:"fileName"` FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` HasCustomization bool `json:"hasCustomization"` diff --git a/backend/plugins/kiro/models/timestamp_schema_test.go b/backend/plugins/kiro/models/timestamp_schema_test.go new file mode 100644 index 00000000000..4bc479b1c62 --- /dev/null +++ b/backend/plugins/kiro/models/timestamp_schema_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "sync" + "testing" + + "github.com/apache/incubator-devlake/plugins/kiro/models/migrationscripts/archived" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + mysqlDriver "gorm.io/driver/mysql" + postgresDriver "gorm.io/driver/postgres" + "gorm.io/gorm/schema" +) + +func TestLogTimestampUsesPortableMicrosecondPrecision(t *testing.T) { + logModels := []struct { + name string + model any + }{ + {name: "chat runtime model", model: &KiroChatLog{}}, + {name: "completion runtime model", model: &KiroCompletionLog{}}, + {name: "chat migration model", model: &archived.KiroChatLog{}}, + {name: "completion migration model", model: &archived.KiroCompletionLog{}}, + } + + for _, logModel := range logModels { + t.Run(logModel.name, func(t *testing.T) { + sch, err := schema.Parse(logModel.model, &sync.Map{}, schema.NamingStrategy{}) + require.NoError(t, err) + + field := sch.LookUpField("Timestamp") + require.NotNil(t, field) + assert.Equal(t, 6, field.Precision) + assert.Equal(t, "datetime(6) NULL", mysqlDriver.New(mysqlDriver.Config{}).DataTypeOf(field)) + assert.Equal(t, "timestamptz(6)", postgresDriver.New(postgresDriver.Config{}).DataTypeOf(field)) + }) + } +} From c0f832bad05b2002960f9ea799a289eeebb02307 Mon Sep 17 00:00:00 2001 From: warren Date: Fri, 28 Aug 2026 15:00:01 +0800 Subject: [PATCH 6/6] fix(kiro): keep q_dev in table info checks Signed-off-by: warren --- backend/plugins/table_info_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index f4182fe9d89..afe5fa1a31d 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -52,6 +52,7 @@ import ( opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" org "github.com/apache/incubator-devlake/plugins/org/impl" pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" + q_dev "github.com/apache/incubator-devlake/plugins/q_dev/impl" refdiff "github.com/apache/incubator-devlake/plugins/refdiff/impl" rootly "github.com/apache/incubator-devlake/plugins/rootly/impl" slack "github.com/apache/incubator-devlake/plugins/slack/impl" @@ -114,6 +115,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("opsgenie/models", opsgenie.Opsgenie{}.GetTablesInfo) checker.FeedIn("linker/models", linker.Linker{}.GetTablesInfo) checker.FeedIn("issue_trace/models", issueTrace.IssueTrace{}.GetTablesInfo) + checker.FeedIn("q_dev/models", q_dev.QDev{}.GetTablesInfo) checker.FeedIn("kiro/models", kiro.Kiro{}.GetTablesInfo) checker.FeedIn("gh-copilot/models", copilot.GhCopilot{}.GetTablesInfo) err := checker.Verify()