diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local index 2c8b7ef1dcb..d4fab0a1480 100644 --- a/backend/Dockerfile.local +++ b/backend/Dockerfile.local @@ -73,6 +73,12 @@ RUN apt-get update && apt-get install -y \ libssh2-1 \ libssl3 \ ca-certificates \ + # gitextractor shells out to the git CLI unless + # USE_GO_GIT_IN_GIT_EXTRACTOR is set, so the binary must be present or every + # clone fails with "git: executable file not found in $PATH". + # debian:bookworm-slim does not ship it; the official image inherits it from + # its python base. + git \ && rm -rf /var/lib/apt/lists/* # Copy libgit2 diff --git a/backend/plugins/kiro/api/blueprint_v200.go b/backend/plugins/kiro/api/blueprint_v200.go new file mode 100644 index 00000000000..ceea9cbaee5 --- /dev/null +++ b/backend/plugins/kiro/api/blueprint_v200.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + + // No domain layer scopes: this plugin writes only to _tool_kiro_* tables. + // Cross-tool AI modelling in the domain layer is separate work. + return plan, []plugin.Scope{}, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.KiroS3Slice, srvhelper.NoScopeConfig], + connection *models.KiroConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + slice := scopeDetail.Scope + + op := &tasks.KiroOptions{ + ConnectionId: slice.ConnectionId, + ScopeId: slice.Id, + AccountId: slice.AccountId, + Year: slice.Year, + Month: slice.Month, + } + + // An empty entity list enables every subtask; the three streams are + // always collected together because they describe the same activity. + task, err := helper.MakePipelinePlanTask("kiro", subtaskMetas, []string{}, op) + if err != nil { + return nil, err + } + + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + plan[i] = append(stage, task) + } + return plan, nil +} diff --git a/backend/plugins/kiro/api/connection.go b/backend/plugins/kiro/api/connection.go new file mode 100644 index 00000000000..075274b1d56 --- /dev/null +++ b/backend/plugins/kiro/api/connection.go @@ -0,0 +1,163 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// PostConnections creates a new connection. +// @Summary create kiro connection +// @Description Create kiro connection +// @Tags plugins/kiro +// @Param body body models.KiroConnection true "json body" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + // Wrapped as BadInput so a struct-tag validation failure reports 400 rather + // than 500 - the difference between "fix your input" and "the server broke". + if err := api.Decode(input.Body, connection, vld); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid connection payload") + } + if err := validateConnection(&connection.KiroConn); err != nil { + return nil, errors.BadInput.Wrap(err, "connection validation failed") + } + if err := connectionHelper.Create(connection, input); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil +} + +// PatchConnection updates an existing connection. +// @Summary patch kiro connection +// @Description Patch kiro connection +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Param body body models.KiroConnection true "json body" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + if err := (&models.KiroConnection{}).MergeFromRequest(connection, input.Body); err != nil { + return nil, errors.Convert(err) + } + if err := validateConnection(&connection.KiroConn); err != nil { + return nil, errors.BadInput.Wrap(err, "connection validation failed") + } + if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil +} + +// DeleteConnection removes a connection. +// @Summary delete a kiro connection +// @Description Delete a kiro connection +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} srvhelper.DsRefs "References exist to this connection" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + conn := &models.KiroConnection{} + output, err := connectionHelper.Delete(conn, input) + if err != nil { + return output, err + } + output.Body = conn.Sanitize() + return output, nil +} + +// ListConnections lists all connections. +// @Summary get all kiro connections +// @Description Get all kiro connections +// @Tags plugins/kiro +// @Success 200 {object} []models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connections []models.KiroConnection + if err := connectionHelper.List(&connections); err != nil { + return nil, err + } + for i := range connections { + connections[i] = connections[i].Sanitize() + } + return &plugin.ApiResourceOutput{Body: connections}, nil +} + +// GetConnection returns one connection. +// @Summary get kiro connection detail +// @Description Get kiro connection detail +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +// validateConnection checks the fields collection cannot proceed without. +// +// Identity Store fields are deliberately not required: they only resolve display +// names, and identity for joining to git history comes from the report's +// User_Email column. Requiring them would block a working setup. +func validateConnection(conn *models.KiroConn) error { + if conn.AccessKeyId == "" { + return errors.BadInput.New("AccessKeyId is required") + } + if conn.SecretAccessKey == "" { + return errors.BadInput.New("SecretAccessKey is required") + } + if conn.Region == "" { + return errors.BadInput.New("Region is required") + } + if conn.Bucket == "" { + return errors.BadInput.New("Bucket is required") + } + // A partial Identity Store configuration is a mistake worth reporting: it + // silently yields no display names, which looks like a data problem rather + // than a configuration one. + if (conn.IdentityStoreId == "") != (conn.IdentityStoreRegion == "") { + return errors.BadInput.New("IdentityStoreId and IdentityStoreRegion must be set together") + } + return nil +} diff --git a/backend/plugins/kiro/api/init.go b/backend/plugins/kiro/api/init.go new file mode 100644 index 00000000000..e4b614a743d --- /dev/null +++ b/backend/plugins/kiro/api/init.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/go-playground/validator/v10" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var ( + vld *validator.Validate + connectionHelper *api.ConnectionApiHelper + basicRes context.BasicRes + // Scope config is NoScopeConfig: the report CSV's meaning is fixed by AWS + // and uniform across an organization, so there is nothing per-scope to + // configure. + dsHelper *api.DsHelper[models.KiroConnection, models.KiroS3Slice, srvhelper.NoScopeConfig] +) + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + connectionHelper = api.NewConnectionHelper( + basicRes, + vld, + p.Name(), + ) + + dsHelper = api.NewDataSourceHelper[ + models.KiroConnection, models.KiroS3Slice, srvhelper.NoScopeConfig, + ]( + basicRes, + p.Name(), + // Searchable scope fields. + []string{"accountId", "name"}, + func(c models.KiroConnection) models.KiroConnection { return c.Sanitize() }, + func(s models.KiroS3Slice) models.KiroS3Slice { return s.Sanitize() }, + nil, + ) + + // Scope browsing and search are implemented directly in remote_api.go rather + // than through the shared DsRemoteApiScopeList/Search helpers. Those build an + // HTTP client from the connection first, and that constructor runs a DNS + // check on the endpoint - a bucket name is not a hostname, so it fails. The + // helpers assume an HTTP data source; this one is S3. +} diff --git a/backend/plugins/kiro/api/remote_api.go b/backend/plugins/kiro/api/remote_api.go new file mode 100644 index 00000000000..1be7635558f --- /dev/null +++ b/backend/plugins/kiro/api/remote_api.go @@ -0,0 +1,338 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +// listKiroRemoteScopes browses the export layout as a tree. +// +// Three levels, mirroring Kiro's own S3 partitioning: +// +// (root) -> one group per AWS account with exported data +// {account} -> one group per year, plus a whole-year scope +// {account}/{y} -> one selectable scope per month +// +// Everything comes from S3 rather than user input. That is the point: a +// hand-typed prefix cannot be validated from the outcome, because a typo and a +// month with no data both produce a successful run that collects nothing. +func listKiroRemoteScopes(connection *models.KiroConnection, groupId string) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], + err errors.Error, +) { + if connection == nil { + return nil, errors.BadInput.New("connection is required") + } + + discovery, err := tasks.NewDiscovery(connection) + if err != nil { + return nil, err + } + + accountId, year, err := parseGroupId(groupId) + if err != nil { + return nil, err + } + + switch { + case accountId == "": + return listAccountGroups(discovery) + case year == 0: + return listYearGroups(discovery, accountId) + default: + return listMonthScopes(discovery, accountId, year) + } +} + +// listAccountGroups is the tree root: the accounts that actually have exports. +func listAccountGroups(discovery *tasks.Discovery) ( + []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], errors.Error, +) { + accounts, err := discovery.ListAccounts() + if err != nil { + return nil, err + } + + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0, len(accounts)) + for _, accountId := range accounts { + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + Id: accountId, + Name: accountId, + FullName: accountId, + }) + } + return entries, nil +} + +// listYearGroups lists the years under an account. +// +// Each year is offered both as a group to expand and as a directly selectable +// scope, because a nil month means "collect the whole year" - which is how a +// year-long backfill is expressed without creating twelve scopes by hand. +func listYearGroups(discovery *tasks.Discovery, accountId string) ( + []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], errors.Error, +) { + years, err := discovery.ListYears(accountId) + if err != nil { + return nil, err + } + + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0, len(years)*2) + for _, year := range years { + groupId := fmt.Sprintf("%s/%04d", accountId, year) + parent := accountId + + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parent, + Id: groupId, + Name: fmt.Sprintf("%04d", year), + FullName: groupId, + }) + + wholeYear := &models.KiroS3Slice{AccountId: accountId, Year: year} + *wholeYear = wholeYear.Sanitize() + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: &parent, + Id: wholeYear.Id, + Name: fmt.Sprintf("%04d (whole year)", year), + FullName: wholeYear.ScopeName(), + Data: wholeYear, + }) + } + return entries, nil +} + +// listMonthScopes lists the months that hold data for an account and year. +func listMonthScopes(discovery *tasks.Discovery, accountId string, year int) ( + []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], errors.Error, +) { + months, err := discovery.ListMonths(accountId, year) + if err != nil { + return nil, err + } + + parent := fmt.Sprintf("%s/%04d", accountId, year) + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0, len(months)) + for _, month := range months { + m := month + slice := &models.KiroS3Slice{AccountId: accountId, Year: year, Month: &m} + *slice = slice.Sanitize() + + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: &parent, + Id: slice.Id, + Name: fmt.Sprintf("%04d-%02d", year, month), + FullName: slice.ScopeName(), + Data: slice, + }) + } + return entries, nil +} + +// searchKiroRemoteScopes filters the discovered months by substring. +// +// Matching is against "{account} {year}-{month}", so "2026-07" or an account +// number both work. The search space is one listing per year, small enough to +// scan without an index. +func searchKiroRemoteScopes( + connection *models.KiroConnection, + query string, + page int, + pageSize int, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], + err errors.Error, +) { + empty := []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{} + if connection == nil { + return empty, nil + } + query = strings.ToLower(strings.TrimSpace(query)) + if query == "" { + return empty, nil + } + + discovery, err := tasks.NewDiscovery(connection) + if err != nil { + return nil, err + } + accounts, err := discovery.ListAccounts() + if err != nil { + return nil, err + } + + matches := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0) + for _, accountId := range accounts { + years, yearErr := discovery.ListYears(accountId) + if yearErr != nil { + return nil, yearErr + } + for _, year := range years { + months, monthErr := discovery.ListMonths(accountId, year) + if monthErr != nil { + return nil, monthErr + } + for _, month := range months { + m := month + slice := &models.KiroS3Slice{AccountId: accountId, Year: year, Month: &m} + *slice = slice.Sanitize() + + if !strings.Contains(strings.ToLower(slice.ScopeName()), query) && + !strings.Contains(strings.ToLower(slice.Id), query) { + continue + } + matches = append(matches, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: slice.Id, + Name: slice.ScopeName(), + FullName: slice.ScopeName(), + Data: slice, + }) + } + } + } + + return paginate(matches, page, pageSize), nil +} + +// paginate applies the requested page window to an in-memory result set. +func paginate( + entries []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], + page int, pageSize int, +) []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice] { + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 50 + } + start := (page - 1) * pageSize + if start >= len(entries) { + return []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{} + } + end := start + pageSize + if end > len(entries) { + end = len(entries) + } + return entries[start:end] +} + +// parseGroupId splits a tree node id into its parts. +// +// "" is the root, "{account}" is an account node, "{account}/{year}" is a year +// node. +func parseGroupId(groupId string) (accountId string, year int, err errors.Error) { + trimmed := strings.Trim(strings.TrimSpace(groupId), "/") + if trimmed == "" { + return "", 0, nil + } + + parts := strings.Split(trimmed, "/") + switch len(parts) { + case 1: + return parts[0], 0, nil + case 2: + parsedYear, convErr := strconv.Atoi(parts[1]) + if convErr != nil { + return "", 0, errors.BadInput.New("invalid year in groupId: " + groupId) + } + return parts[0], parsedYear, nil + default: + return "", 0, errors.BadInput.New("unrecognized groupId: " + groupId) + } +} + +// RemoteScopes browses the Kiro export layout in S3. +// +// Implemented directly rather than through the shared scope-list helper. That +// helper builds an HTTP client from the connection first, and its constructor +// runs a DNS check on the endpoint - which fails here, because a bucket name is +// not a hostname. The helper is built for HTTP data sources; this one is S3. +// @Summary list available kiro scopes discovered from S3 +// @Description Browse accounts, years and months that actually have exported data +// @Tags plugins/kiro +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param groupId query string false "account id, or account/year" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.KiroS3Slice] +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + + children, err := listKiroRemoteScopes(connection, input.Query.Get("groupId")) + if err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: dsmodels.DsRemoteApiScopeList[models.KiroS3Slice]{Children: children}, + }, nil +} + +// SearchRemoteScopes finds discovered scopes by substring. +// +// Implemented directly rather than through the shared search helper: that +// helper's callback receives only an HTTP ApiClient, and discovery here needs +// the connection itself to build an S3 client. +// @Summary search kiro scopes discovered from S3 +// @Description Search the discovered months by account or year-month +// @Tags plugins/kiro +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param search query string false "search" +// @Param page query int false "page number" +// @Param pageSize query int false "page size per page" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.KiroS3Slice] "the parentIds are always null" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/search-remote-scopes [GET] +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + + page, _ := strconv.Atoi(input.Query.Get("page")) + pageSize, _ := strconv.Atoi(input.Query.Get("pageSize")) + + children, err := searchKiroRemoteScopes(connection, input.Query.Get("search"), page, pageSize) + if err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: dsmodels.DsRemoteApiScopeList[models.KiroS3Slice]{Children: children}, + }, nil +} diff --git a/backend/plugins/kiro/api/s3_slice_api.go b/backend/plugins/kiro/api/s3_slice_api.go new file mode 100644 index 00000000000..f8a05fb185f --- /dev/null +++ b/backend/plugins/kiro/api/s3_slice_api.go @@ -0,0 +1,117 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +type PutScopesReqBody = helper.PutScopesReqBody[models.KiroS3Slice] +type ScopeDetail = srvhelper.ScopeDetail[models.KiroS3Slice, srvhelper.NoScopeConfig] + +// PutScopes creates or updates Kiro collection scopes. +// @Summary create or update kiro scopes +// @Description Create or update kiro scopes, each covering one AWS account for one month +// @Tags plugins/kiro +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.KiroS3Slice +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// GetScopeList returns the scopes for a connection. +// @Summary get kiro scopes +// @Description get kiro scopes +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param pageSize query int false "page size" +// @Param page query int false "page number" +// @Param blueprints query bool false "include blueprint references" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope returns a single scope. +// @Summary get one kiro scope +// @Description get one kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// PatchScope updates a scope. +// @Summary patch a kiro scope +// @Description patch a kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param scope body models.KiroS3Slice true "json" +// @Success 200 {object} models.KiroS3Slice +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// DeleteScope removes a scope and optionally its collected data. +// @Summary delete a kiro scope +// @Description delete a kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +// GetScopeLatestSyncState reports the most recent sync for a scope. +// @Summary get the latest sync state of a kiro scope +// @Description get the latest sync state of a kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} []models.LatestSyncState +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/kiro/api/test_connection.go b/backend/plugins/kiro/api/test_connection.go new file mode 100644 index 00000000000..8df055e582f --- /dev/null +++ b/backend/plugins/kiro/api/test_connection.go @@ -0,0 +1,204 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/http" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +// TestConnection validates a connection that has not been saved yet. +// @Summary test kiro connection +// @Description Test kiro connection +// @Tags plugins/kiro +// @Param body body models.KiroConn true "json body" +// @Success 200 {object} ConnectionReport +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.KiroConnection + // Wrapped as BadInput: a struct-tag validation failure is the caller's + // problem, and an unwrapped Decode error surfaces as HTTP 500 - which sends + // the user looking at server logs instead of at their own form. + if err := api.Decode(input.Body, &connection, vld); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid connection payload") + } + if err := validateConnection(&connection.KiroConn); err != nil { + return nil, errors.BadInput.Wrap(err, "connection validation failed") + } + if err := testConnection(&connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: buildConnectionReport(&connection), + Status: http.StatusOK, + }, nil +} + +// TestExistingConnection validates a saved connection, optionally with overrides +// from the request body. +// @Summary test existing kiro connection +// @Description Test existing kiro connection +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Success 200 {object} ConnectionReport +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := api.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid connection payload") + } + if err := testConnection(connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: buildConnectionReport(connection), + Status: http.StatusOK, + }, nil +} + +// ConnectionReport describes what the connection can actually see. +// +// A bare success/failure verdict is not enough to tell whether a configuration +// is right, because a wrong prefix and a genuinely empty period produce the same +// outcome: collection runs, finds nothing, and reports success. Returning the +// discovered accounts and per-stream object counts makes the difference visible +// before any scope is created. +type ConnectionReport struct { + ReportBucket string `json:"reportBucket"` + PromptLogBucket string `json:"promptLogBucket"` + // Accounts are the AWS account ids found under the report prefix. An empty + // list is the clearest sign that the bucket or prefix is wrong. + Accounts []string `json:"accounts"` + // Streams reports object counts for the most recent discovered period. + Streams []tasks.StreamCount `json:"streams"` + // Hint explains an empty result in plain terms. + Hint string `json:"hint,omitempty"` +} + +// connectionReportCountLimit caps counting so the check stays fast on a bucket +// holding hundreds of thousands of objects. The exact number does not matter for +// verifying a path - only whether it is zero. +const connectionReportCountLimit = 500 + +// buildConnectionReport probes the layout and summarizes what was found. +// +// Errors are folded into the report rather than returned: the connection itself +// is already known to work at this point, and a discovery failure is more useful +// shown as an empty result with a hint than as a failed request. +func buildConnectionReport(connection *models.KiroConnection) *ConnectionReport { + report := &ConnectionReport{ + ReportBucket: connection.Bucket, + PromptLogBucket: connection.GetPromptLogBucket(), + Accounts: []string{}, + Streams: []tasks.StreamCount{}, + } + + discovery, err := tasks.NewDiscovery(connection) + if err != nil { + report.Hint = "could not initialise S3 discovery: " + err.Error() + return report + } + + accounts, err := discovery.ListAccounts() + if err != nil { + report.Hint = "could not list accounts under the report prefix: " + err.Error() + return report + } + report.Accounts = accounts + + if len(accounts) == 0 { + report.Hint = fmt.Sprintf( + "no account directories under s3://%s/%s/AWSLogs/ - check the bucket and report prefix", + connection.Bucket, connection.GetReportPrefix()) + return report + } + + // Probe the newest period that exists, since that is where data is most + // likely to be and therefore the most informative check. + accountId := accounts[len(accounts)-1] + years, err := discovery.ListYears(accountId) + if err != nil || len(years) == 0 { + report.Hint = fmt.Sprintf( + "account %s has no year directories - check the region in the connection", accountId) + return report + } + year := years[len(years)-1] + + var month *int + if months, monthErr := discovery.ListMonths(accountId, year); monthErr == nil && len(months) > 0 { + latest := months[len(months)-1] + month = &latest + } + + report.Streams = discovery.CountStreams(accountId, year, month, connectionReportCountLimit) + + period := fmt.Sprintf("%04d", year) + if month != nil { + period = fmt.Sprintf("%04d-%02d", year, *month) + } + report.Hint = fmt.Sprintf("counts are for account %s, period %s", accountId, period) + return report +} + +// testConnection issues a real request against every bucket in use. +// +// Constructing a client proves nothing - the AWS SDK builds one happily from +// invalid credentials, so a test that stops there reports success for a +// connection that cannot read anything. A single-key list is the cheapest call +// that actually exercises credentials and bucket permissions. +// +// Both buckets are checked when reports and logs are separated, since they may +// carry different KMS keys and IAM conditions; a connection that can read +// reports but not logs would otherwise pass and then silently collect nothing. +func testConnection(connection *models.KiroConnection) errors.Error { + clients, err := tasks.NewKiroS3Clients(connection) + if err != nil { + return err + } + + for _, bucket := range clients.Buckets() { + client := clients.Report + if bucket == clients.PromptLog.Bucket { + client = clients.PromptLog + } + if _, listErr := client.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + MaxKeys: aws.Int64(1), + }); listErr != nil { + return errors.BadInput.Wrap(listErr, "cannot access s3 bucket "+bucket) + } + } + + return nil +} diff --git a/backend/plugins/kiro/impl/impl.go b/backend/plugins/kiro/impl/impl.go new file mode 100644 index 00000000000..c6b19708aeb --- /dev/null +++ b/backend/plugins/kiro/impl/impl.go @@ -0,0 +1,208 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.DataSourcePluginBlueprintV200 +} = (*Kiro)(nil) + +// Kiro collects Kiro enterprise usage exports from S3. +// +// This is a separate plugin rather than an evolution of the retired predecessor: +// the old format is frozen, and a plugin that keeps evolving should not depend +// on frozen code. No implementation code is shared, following the same split +// as bitbucket and bitbucket_server. +type Kiro struct{} + +func (p Kiro) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Kiro) Name() string { + return "kiro" +} + +func (p Kiro) Description() string { + return "collect Kiro usage reports and interaction logs from S3" +} + +func (p Kiro) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/kiro" +} + +// GetTablesInfo must list every model or plugins/table_info_test.go fails. +func (p Kiro) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.KiroConnection{}, + &models.KiroS3Slice{}, + &models.KiroS3FileMeta{}, + &models.KiroUserReport{}, + &models.KiroUserModelMessage{}, + &models.KiroChatLog{}, + &models.KiroCompletionLog{}, + } +} + +func (p Kiro) Connection() dal.Tabler { + return &models.KiroConnection{} +} + +func (p Kiro) Scope() plugin.ToolLayerScope { + return &models.KiroS3Slice{} +} + +// ScopeConfig returns nil: the export format is defined by AWS and uniform +// across an organization, so there is nothing per-scope to configure. +func (p Kiro) ScopeConfig() dal.Tabler { + return nil +} + +func (p Kiro) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +// SubTaskMetas lists discovery first, then one extractor per stream. The +// extractors declare their dependency on discovery, so the split is safe and +// gives each stream its own progress reporting - useful when a scope holds tens +// of thousands of log objects and one needs to know which stream is slow. +func (p Kiro) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectKiroS3FilesMeta, + tasks.ExtractKiroUserReportMeta, + tasks.ExtractKiroChatLogMeta, + tasks.ExtractKiroCompletionLogMeta, + } +} + +func (p Kiro) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.KiroOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, err + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("connectionId is required") + } + if op.AccountId == "" { + return nil, errors.BadInput.New("accountId is required") + } + if op.Year <= 0 { + return nil, errors.BadInput.New("year is required") + } + + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + connection := &models.KiroConnection{} + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, err + } + + s3Clients, err := tasks.NewKiroS3Clients(connection) + if err != nil { + return nil, err + } + + // Identity Store is optional and only supplies display names, so a failure + // here degrades presentation rather than collection. + identityClient, identityErr := tasks.NewKiroIdentityClient(connection) + if identityErr != nil { + taskCtx.GetLogger().Warn(identityErr, "identity store unavailable, proceeding without display names") + identityClient = nil + } + + timePath := fmt.Sprintf("%04d", op.Year) + if op.Month != nil { + timePath = fmt.Sprintf("%04d/%02d", op.Year, *op.Month) + } + + return &tasks.KiroTaskData{ + Options: &op, + Connection: connection, + S3Clients: s3Clients, + IdentityClient: identityClient, + Prefixes: tasks.BuildPrefixes(connection, op.AccountId, timePath), + }, nil +} + +func (p Kiro) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Kiro) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + // Scope discovery: lists the accounts, years and months that actually + // have exported data, so a scope is selected instead of hand-entered. + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "GET": api.GetScopeLatestSyncState, + }, + } +} diff --git a/backend/plugins/kiro/impl/table_info_test.go b/backend/plugins/kiro/impl/table_info_test.go new file mode 100644 index 00000000000..cb195602d63 --- /dev/null +++ b/backend/plugins/kiro/impl/table_info_test.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "testing" + + "github.com/apache/incubator-devlake/helpers/unithelper" +) + +// The repo-wide plugins/table_info_test.go performs this same check for every +// plugin, but building that package requires gitextractor and therefore a +// specific libgit2. Running it here as well keeps the feedback local: a model +// added without registering it in GetTablesInfo fails immediately rather than +// only in CI. +func TestKiroTableInfo(t *testing.T) { + checker := unithelper.NewTableInfoChecker(unithelper.TableInfoCheckerConfig{}) + checker.FeedIn("../models", Kiro{}.GetTablesInfo) + if err := checker.Verify(); err != nil { + t.Error(err) + } +} diff --git a/backend/plugins/kiro/kiro.go b/backend/plugins/kiro/kiro.go new file mode 100644 index 00000000000..27c8dff3032 --- /dev/null +++ b/backend/plugins/kiro/kiro.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/kiro/impl" +) + +var PluginEntry impl.Kiro + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "kiro"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "kiro connection id") + accountId := cmd.Flags().StringP("accountId", "a", "", "AWS account id that Kiro exports for") + year := cmd.Flags().IntP("year", "y", 0, "year to collect") + month := cmd.Flags().IntP("month", "m", 0, "month to collect; omit to collect the whole year") + + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("accountId") + _ = cmd.MarkFlagRequired("year") + + cmd.Run = func(cmd *cobra.Command, args []string) { + options := map[string]interface{}{ + "connectionId": *connectionId, + "accountId": *accountId, + "year": *year, + } + // A zero month means the whole year, matching the scope model where a + // nil month widens collection. + if *month > 0 { + options["month"] = *month + } + runner.DirectRun(cmd, args, PluginEntry, options, "") + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/kiro/models/chat_log.go b/backend/plugins/kiro/models/chat_log.go index ecd943696fe..cde329f517d 100644 --- a/backend/plugins/kiro/models/chat_log.go +++ b/backend/plugins/kiro/models/chat_log.go @@ -44,7 +44,7 @@ type KiroChatLog struct { UserId string `gorm:"type:varchar(64);index" json:"userId"` // IdentityStoreId is the stripped prefix, retained for auditability. IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` // ChatTriggerType is MANUAL or INLINE_CHAT per the docs; only MANUAL has // been observed. Not validated against a fixed set. ChatTriggerType string `gorm:"type:varchar(20)" json:"chatTriggerType"` diff --git a/backend/plugins/kiro/models/completion_log.go b/backend/plugins/kiro/models/completion_log.go index 0d09c28e0ad..cfd633e5335 100644 --- a/backend/plugins/kiro/models/completion_log.go +++ b/backend/plugins/kiro/models/completion_log.go @@ -40,7 +40,7 @@ type KiroCompletionLog struct { UserId string `gorm:"type:varchar(64);index" json:"userId"` IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` // FileName has no path component; see the type comment. FileName string `gorm:"type:varchar(255);index" json:"fileName"` diff --git a/backend/plugins/kiro/models/migrationscripts/archived/init.go b/backend/plugins/kiro/models/migrationscripts/archived/init.go index 5c1c9a3224f..53422db0499 100644 --- a/backend/plugins/kiro/models/migrationscripts/archived/init.go +++ b/backend/plugins/kiro/models/migrationscripts/archived/init.go @@ -127,7 +127,7 @@ type KiroChatLog struct { RequestId string `gorm:"primaryKey;type:varchar(64)"` UserId string `gorm:"type:varchar(64);index" json:"userId"` IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` ChatTriggerType string `gorm:"type:varchar(20)" json:"chatTriggerType"` ModelId *string `gorm:"type:varchar(100)" json:"modelId"` HasPrompt bool `gorm:"index" json:"hasPrompt"` @@ -152,7 +152,7 @@ type KiroCompletionLog struct { RequestId string `gorm:"primaryKey;type:varchar(64)"` UserId string `gorm:"type:varchar(64);index" json:"userId"` IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` FileName string `gorm:"type:varchar(255);index" json:"fileName"` FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` HasCustomization bool `json:"hasCustomization"` diff --git a/backend/plugins/kiro/models/timestamp_schema_test.go b/backend/plugins/kiro/models/timestamp_schema_test.go new file mode 100644 index 00000000000..4bc479b1c62 --- /dev/null +++ b/backend/plugins/kiro/models/timestamp_schema_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "sync" + "testing" + + "github.com/apache/incubator-devlake/plugins/kiro/models/migrationscripts/archived" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + mysqlDriver "gorm.io/driver/mysql" + postgresDriver "gorm.io/driver/postgres" + "gorm.io/gorm/schema" +) + +func TestLogTimestampUsesPortableMicrosecondPrecision(t *testing.T) { + logModels := []struct { + name string + model any + }{ + {name: "chat runtime model", model: &KiroChatLog{}}, + {name: "completion runtime model", model: &KiroCompletionLog{}}, + {name: "chat migration model", model: &archived.KiroChatLog{}}, + {name: "completion migration model", model: &archived.KiroCompletionLog{}}, + } + + for _, logModel := range logModels { + t.Run(logModel.name, func(t *testing.T) { + sch, err := schema.Parse(logModel.model, &sync.Map{}, schema.NamingStrategy{}) + require.NoError(t, err) + + field := sch.LookUpField("Timestamp") + require.NotNil(t, field) + assert.Equal(t, 6, field.Precision) + assert.Equal(t, "datetime(6) NULL", mysqlDriver.New(mysqlDriver.Config{}).DataTypeOf(field)) + assert.Equal(t, "timestamptz(6)", postgresDriver.New(postgresDriver.Config{}).DataTypeOf(field)) + }) + } +} diff --git a/backend/plugins/kiro/tasks/discovery.go b/backend/plugins/kiro/tasks/discovery.go new file mode 100644 index 00000000000..1bd6a134759 --- /dev/null +++ b/backend/plugins/kiro/tasks/discovery.go @@ -0,0 +1,158 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "strconv" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// Discovery walks the report prefix to find what data actually exists. +// +// Kiro's S3 layout encodes every scope dimension as a path segment: +// +// {reportPrefix}/AWSLogs/{accountId}/KiroLogs/user_report/{region}/{year}/{month}/ +// +// so a scope never has to be typed by hand. That matters beyond convenience: a +// mistyped prefix produces exactly the same outcome as a month with no data - +// collection succeeds and finds nothing - so hand-entered paths cannot be +// verified from the result. +type Discovery struct { + clients *KiroS3Clients + connection *models.KiroConnection +} + +func NewDiscovery(connection *models.KiroConnection) (*Discovery, errors.Error) { + clients, err := NewKiroS3Clients(connection) + if err != nil { + return nil, err + } + return &Discovery{clients: clients, connection: connection}, nil +} + +// reportRoot is the prefix holding the per-account directories. +func (d *Discovery) reportRoot() string { + return fmt.Sprintf("%s/AWSLogs", d.connection.GetReportPrefix()) +} + +// accountReportPrefix is where one account's report months live. +func (d *Discovery) accountReportPrefix(accountId string) string { + return fmt.Sprintf("%s/AWSLogs/%s/KiroLogs/user_report/%s", + d.connection.GetReportPrefix(), accountId, d.connection.Region) +} + +// ListAccounts returns the AWS account ids that have exported data. +// +// Kiro requires a bucket per account holding subscriptions and does not support +// cross-account buckets, so in practice this is usually one entry - but reading +// it from S3 removes the chance of a typo in a 12-digit number. +func (d *Discovery) ListAccounts() ([]string, errors.Error) { + return d.clients.Report.ListSubPrefixes(d.reportRoot()) +} + +// ListYears returns the years with report data for an account. +func (d *Discovery) ListYears(accountId string) ([]int, errors.Error) { + names, err := d.clients.Report.ListSubPrefixes(d.accountReportPrefix(accountId)) + if err != nil { + return nil, err + } + return parseNumericSegments(names), nil +} + +// ListMonths returns the months with report data for an account and year. +func (d *Discovery) ListMonths(accountId string, year int) ([]int, errors.Error) { + prefix := fmt.Sprintf("%s/%04d", d.accountReportPrefix(accountId), year) + names, err := d.clients.Report.ListSubPrefixes(prefix) + if err != nil { + return nil, err + } + return parseNumericSegments(names), nil +} + +// StreamCount is how many collectable objects one stream holds. +type StreamCount struct { + FileType string `json:"fileType"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + Count int `json:"count"` + // AtLeast is true when counting stopped at the cap, so Count is a floor. + AtLeast bool `json:"atLeast"` + // Error explains why a stream could not be counted, e.g. missing + // permission on the log bucket while the report bucket is readable. + Error string `json:"error,omitempty"` +} + +// CountStreams reports the object count for each of the three streams. +// +// This is the answer to "is my configuration right?". Reporting counts per +// stream distinguishes the three cases that otherwise look identical: a wrong +// prefix (zero everywhere), a genuinely dormant stream (zero for one type - +// inline completions stopped being produced under agentic usage), and a +// permissions gap on one bucket (an error for the log streams only). +// +// countLimit caps the work per stream; pass 0 to count everything. +func (d *Discovery) CountStreams(accountId string, year int, month *int, countLimit int) []StreamCount { + timePath := fmt.Sprintf("%04d", year) + if month != nil { + timePath = fmt.Sprintf("%04d/%02d", year, *month) + } + + specs := BuildPrefixes(d.connection, accountId, timePath) + results := make([]StreamCount, 0, len(specs)) + + for _, spec := range specs { + client := d.clients.ForFileType(spec.FileType) + result := StreamCount{ + FileType: spec.FileType, + Bucket: client.Bucket, + Prefix: spec.Prefix, + } + count, atLeast, err := client.CountObjects(spec.Prefix, countLimit) + if err != nil { + // Recorded rather than returned: one unreadable stream should still + // leave the others' counts visible, since that contrast is what + // identifies a per-bucket permission problem. + result.Error = err.Error() + } else { + result.Count = count + result.AtLeast = atLeast + } + results = append(results, result) + } + + return results +} + +// parseNumericSegments keeps only the segments that are numbers, in order. +// +// S3 prefixes are strings, and a non-numeric directory would otherwise surface +// as a year or month. +func parseNumericSegments(names []string) []int { + values := make([]int, 0, len(names)) + for _, name := range names { + value, err := strconv.Atoi(name) + if err != nil { + continue + } + values = append(values, value) + } + return values +} diff --git a/backend/plugins/kiro/tasks/discovery_test.go b/backend/plugins/kiro/tasks/discovery_test.go new file mode 100644 index 00000000000..785f79cbd58 --- /dev/null +++ b/backend/plugins/kiro/tasks/discovery_test.go @@ -0,0 +1,211 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// prefixMockS3 answers listings from a canned prefix tree, and records the +// requests so tests can assert that a delimiter was used. +type prefixMockS3 struct { + // commonPrefixes maps a queried prefix to the child prefixes returned. + commonPrefixes map[string][]string + // objects maps a queried prefix to the object keys beneath it. + objects map[string][]string + seenDelimiter []string + seenPrefix []string +} + +func (m *prefixMockS3) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + m.seenPrefix = append(m.seenPrefix, prefix) + if input.Delimiter != nil { + m.seenDelimiter = append(m.seenDelimiter, *input.Delimiter) + } else { + m.seenDelimiter = append(m.seenDelimiter, "") + } + + out := &s3.ListObjectsV2Output{IsTruncated: aws.Bool(false)} + for _, child := range m.commonPrefixes[prefix] { + full := prefix + child + "/" + out.CommonPrefixes = append(out.CommonPrefixes, &s3.CommonPrefix{Prefix: aws.String(full)}) + } + for _, key := range m.objects[prefix] { + out.Contents = append(out.Contents, &s3.Object{Key: aws.String(prefix + key)}) + } + return out, nil +} + +func (m *prefixMockS3) GetObject(*s3.GetObjectInput) (*s3.GetObjectOutput, error) { + return nil, nil +} + +func discoveryFixture(svc S3API) *Discovery { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "kiro-export-test", + ReportPrefix: "user-report", + PromptLogPrefix: "logging", + }} + client := &KiroS3Client{S3: svc, Bucket: conn.Bucket} + return &Discovery{ + clients: &KiroS3Clients{Report: client, PromptLog: client}, + connection: conn, + } +} + +// Kiro encodes every scope dimension as a path segment, which is what allows a +// scope to be selected rather than typed. This matters beyond convenience: a +// mistyped prefix and a month with no data produce the same outcome - a +// successful run that collects nothing - so a hand-entered path cannot be +// verified from the result. +func TestDiscovery_ListsLayoutFromS3(t *testing.T) { + // The real layout, as confirmed against the live bucket. + svc := &prefixMockS3{commonPrefixes: map[string][]string{ + "user-report/AWSLogs/": {"123456789012"}, + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/": {"2026"}, + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/2026/": {"02", "03", "07"}, + }} + discovery := discoveryFixture(svc) + + accounts, err := discovery.ListAccounts() + require.Nil(t, err) + assert.Equal(t, []string{"123456789012"}, accounts) + + years, err := discovery.ListYears("123456789012") + require.Nil(t, err) + assert.Equal(t, []int{2026}, years) + + months, err := discovery.ListMonths("123456789012", 2026) + require.Nil(t, err) + // Zero-padded segments must parse to plain integers, and only months that + // actually hold data are offered. + assert.Equal(t, []int{2, 3, 7}, months) + + // A delimiter is required: without it S3 returns every object under the + // prefix instead of just the segment names, which on a log prefix would be + // hundreds of thousands of keys. + for _, delimiter := range svc.seenDelimiter { + assert.Equal(t, "/", delimiter) + } +} + +func TestDiscovery_IgnoresNonNumericSegments(t *testing.T) { + svc := &prefixMockS3{commonPrefixes: map[string][]string{ + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/": {"2026", "unexpected", "2025"}, + }} + years, err := discoveryFixture(svc).ListYears("123456789012") + require.Nil(t, err) + // S3 prefixes are strings; a stray directory must not surface as a year. + assert.Equal(t, []int{2025, 2026}, years) +} + +func TestDiscovery_EmptyLayout(t *testing.T) { + // An empty account list is the clearest signal that the bucket or report + // prefix is wrong, so it must come back as an empty result rather than an + // error. + discovery := discoveryFixture(&prefixMockS3{}) + + accounts, err := discovery.ListAccounts() + require.Nil(t, err) + assert.Empty(t, accounts) +} + +// Per-stream counts separate three situations that a single pass/fail verdict +// cannot: a wrong prefix (zero everywhere), a dormant stream (zero for one type, +// which is what inline completions look like under agentic usage), and a +// permission gap on one bucket. +func TestDiscovery_CountStreams(t *testing.T) { + base := "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/2026/07/" + logBase := "logging/AWSLogs/123456789012/KiroLogs/" + svc := &prefixMockS3{objects: map[string][]string{ + base: {"KIRO_CLI_x_user_report_1.csv", "KIRO_IDE_x_user_report_1.csv"}, + logBase + "GenerateAssistantResponse/us-east-1/2026/07/": {"a.json.gz", "b.json.gz", "c.json.gz"}, + // GenerateCompletions intentionally absent: dormant, not misconfigured. + }} + + month := 7 + counts := discoveryFixture(svc).CountStreams("123456789012", 2026, &month, 0) + require.Len(t, counts, 3) + + byType := map[string]StreamCount{} + for _, c := range counts { + byType[c.FileType] = c + } + assert.Equal(t, 2, byType[models.FileTypeReport].Count) + assert.Equal(t, 3, byType[models.FileTypeChatLog].Count) + assert.Equal(t, 0, byType[models.FileTypeCompletionLog].Count) + + // The prefix is reported alongside the count so a zero can be checked + // against the bucket directly. + assert.Contains(t, byType[models.FileTypeReport].Prefix, "user_report/us-east-1/2026/07") + for _, c := range counts { + assert.NotEmpty(t, c.Bucket) + assert.Empty(t, c.Error) + } +} + +func TestCountObjects(t *testing.T) { + prefix := "p/" + svc := &prefixMockS3{objects: map[string][]string{ + // Only .csv and .json.gz are collectable; the count must match what + // would actually be collected, not every object present. + prefix: {"a.csv", "b.json.gz", "c.txt", "d", "e.zip"}, + }} + client := &KiroS3Client{S3: svc, Bucket: "b"} + + count, atLeast, err := client.CountObjects(prefix, 0) + require.Nil(t, err) + assert.Equal(t, 2, count) + assert.False(t, atLeast) + + // Counting stops at the cap so the check stays cheap on a large bucket; the + // flag marks the value as a floor. + count, atLeast, err = client.CountObjects(prefix, 1) + require.Nil(t, err) + assert.Equal(t, 1, count) + assert.True(t, atLeast) +} + +func TestListSubPrefixes_TrailingSlashHandling(t *testing.T) { + svc := &prefixMockS3{commonPrefixes: map[string][]string{ + "a/b/": {"x", "y"}, + }} + client := &KiroS3Client{S3: svc, Bucket: "bkt"} + + // A caller-supplied prefix may or may not end in a slash; both must resolve + // to the same listing, and the child names come back without the slash. + withSlash, err := client.ListSubPrefixes("a/b/") + require.Nil(t, err) + withoutSlash, err := client.ListSubPrefixes("a/b") + require.Nil(t, err) + + assert.Equal(t, []string{"x", "y"}, withSlash) + assert.Equal(t, withSlash, withoutSlash) +} diff --git a/backend/plugins/kiro/tasks/extractor.go b/backend/plugins/kiro/tasks/extractor.go new file mode 100644 index 00000000000..10867515005 --- /dev/null +++ b/backend/plugins/kiro/tasks/extractor.go @@ -0,0 +1,204 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// parseFunc turns one downloaded object into batches of rows ready for +// insertion. It is a pure function so it can run on a worker goroutine without +// touching the database. +// +// The result is a slice of batches rather than one flat slice because GORM +// derives the target table from the element type: a mixed []interface{} fails +// with "Table not set". A report CSV yields two different models, so each model +// gets its own homogeneous batch. +type parseFunc func(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) + +// rowBatch is a set of rows that all belong to the same table. +type rowBatch struct { + // rows must be a typed slice (e.g. []*models.KiroChatLog), not + // []interface{}, so GORM can resolve the table. + rows interface{} + count int +} + +// parsedFile is a worker's output, handed to the main goroutine for persistence. +type parsedFile struct { + meta *models.KiroS3FileMeta + batches []rowBatch + err errors.Error +} + +// rowCount totals the rows across every batch, for the cursor's record count. +func (p parsedFile) rowCount() int { + total := 0 + for _, batch := range p.batches { + total += batch.count + } + return total +} + +// pendingFiles returns the files still awaiting extraction for a scope. +// +// Files that have already failed MaxAttempts times are excluded. Without that +// bound, one permanently malformed object is retried on every run: the log fills +// with the same error and the scope never reaches a finished state. +func pendingFiles(db dal.Dal, connectionId uint64, scopeId string, fileType string) ([]models.KiroS3FileMeta, errors.Error) { + var files []models.KiroS3FileMeta + err := db.All(&files, + dal.From(&models.KiroS3FileMeta{}), + dal.Where("connection_id = ? AND scope_id = ? AND file_type = ? AND processed = ? AND attempt_count < ?", + connectionId, scopeId, fileType, false, models.MaxAttempts), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to query pending kiro files") + } + return files, nil +} + +// extractFiles downloads and parses every pending file of one type, then stores +// the results. +// +// Downloads run concurrently but all database writes happen on the calling +// goroutine. That split is a correctness requirement, not an optimization: +// twenty workers inserting into the same table concurrently deadlock on MySQL +// gap locks, and DevLake's retry layer would turn those deadlocks into +// intermittent failures that only appear under load. +func extractFiles(taskCtx plugin.SubTaskContext, fileType string, parse parseFunc) errors.Error { + data := taskCtx.GetData().(*KiroTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + + files, err := pendingFiles(db, data.Options.ConnectionId, data.Options.ScopeId, fileType) + if err != nil { + return err + } + if len(files) == 0 { + logger.Info("no pending %s files for scope %s", fileType, data.Options.ScopeId) + return nil + } + + logger.Info("extracting %d %s files", len(files), fileType) + taskCtx.SetProgress(0, len(files)) + + client := data.S3Clients.ForFileType(fileType) + results := make(chan parsedFile, len(files)) + + scheduler, err := helper.NewWorkerScheduler( + taskCtx.GetContext(), + data.WorkerCount(), + // The tick is a global rate limiter, not a per-worker one: every task + // waits for one tick before running, so this value caps total + // throughput regardless of pool size. A one-second tick would pin the + // whole run to one file per second and leave the pool idle - measured + // at 13 seconds for 13 files. Keep it well below the per-object fetch + // time so the workers, not the ticker, set the pace. + time.Millisecond, + logger, + ) + if err != nil { + return err + } + defer scheduler.Release() + + for i := range files { + meta := files[i] + scheduler.SubmitBlocking(func() errors.Error { + // Workers only fetch and parse. Anything touching the database + // happens after WaitAsync below. + body, getErr := client.GetObjectBytes(meta.S3Path) + if getErr != nil { + results <- parsedFile{meta: &meta, err: getErr} + return nil + } + batches, parseErr := parse(body, data.Options.ConnectionId, data.Options.ScopeId) + results <- parsedFile{meta: &meta, batches: batches, err: parseErr} + return nil + }) + } + + if waitErr := scheduler.WaitAsync(); waitErr != nil { + return waitErr + } + close(results) + + for result := range results { + if saveErr := persistFile(db, result); saveErr != nil { + return saveErr + } + taskCtx.IncProgress(1) + } + + return nil +} + +// persistFile stores one file's rows and updates its cursor entry. +// +// A parse or download failure is recorded rather than aborting the run: one bad +// object should not block the rest of the scope. The reason is written to +// error_message so a short month can be explained with a query instead of +// guesswork. +func persistFile(db dal.Dal, result parsedFile) errors.Error { + meta := result.meta + + if result.err != nil { + meta.AttemptCount++ + meta.ErrorMessage = result.err.Error() + // Processed stays false so the file is retried, up to MaxAttempts. + if err := db.Update(meta); err != nil { + return errors.Default.Wrap(err, "failed to record kiro extraction failure") + } + return nil + } + + // Each batch is written separately so GORM sees a typed slice and can + // resolve the target table. + for _, batch := range result.batches { + if batch.count == 0 { + continue + } + if err := db.CreateOrUpdate(batch.rows); err != nil { + // A write failure counts as an attempt too, otherwise a row that + // cannot be stored would be fetched forever. + meta.AttemptCount++ + meta.ErrorMessage = err.Error() + if updateErr := db.Update(meta); updateErr != nil { + return errors.Default.Wrap(updateErr, "failed to record kiro write failure") + } + return nil + } + } + + now := time.Now() + meta.Processed = true + meta.ProcessedTime = &now + meta.RecordCount = result.rowCount() + meta.ErrorMessage = "" + if err := db.Update(meta); err != nil { + return errors.Default.Wrap(err, "failed to mark kiro file processed") + } + return nil +} diff --git a/backend/plugins/kiro/tasks/extractor_test.go b/backend/plugins/kiro/tasks/extractor_test.go new file mode 100644 index 00000000000..5eec6051e3b --- /dev/null +++ b/backend/plugins/kiro/tasks/extractor_test.go @@ -0,0 +1,184 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "os" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// GORM resolves the destination table from a slice's element type, so every +// batch must be a typed slice. Passing []interface{} fails at runtime with +// "Table not set" - a real bug caught only by running against a database, since +// it type-checks fine. These assertions pin the requirement so a future +// refactor back to a flat interface slice fails here instead of in production. +func assertTypedBatch[T any](t *testing.T, batch rowBatch, wantCount int) { + t.Helper() + rows, ok := batch.rows.([]T) + require.True(t, ok, "batch must carry a typed slice, got %T", batch.rows) + assert.Len(t, rows, wantCount) + assert.Equal(t, wantCount, batch.count) +} + +// The adapters exist so one parse produces everything a file yields; these +// verify the adaptation rather than the parsing, which is covered directly. +func TestParseUserReportRows_ReturnsTwoTypedBatches(t *testing.T) { + data := loadReportFixture(t, "02_standard_cli.csv") + + batches, err := parseUserReportRows(data, testConnectionId, testScopeId) + require.Nil(t, err) + + // Two batches, not one mixed slice: a report CSV yields two different + // models, and GORM cannot infer a table from a heterogeneous slice. + require.Len(t, batches, 2) + assertTypedBatch[*models.KiroUserReport](t, batches[0], 1) + assertTypedBatch[*models.KiroUserModelMessage](t, batches[1], 1) +} + +func TestParseLogRows(t *testing.T) { + t.Run("chat log", func(t *testing.T) { + batches, err := parseChatLogRows(loadLogFixture(t, "chat_03_two_records.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, batches, 1) + assertTypedBatch[*models.KiroChatLog](t, batches[0], 2) + }) + + t.Run("completion log", func(t *testing.T) { + batches, err := parseCompletionLogRows(loadLogFixture(t, "completion_01_non_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, batches, 1) + assertTypedBatch[*models.KiroCompletionLog](t, batches[0], 1) + }) + + // A record with no completions still produces a row: those records are the + // denominator for what Kiro offered versus what was taken. + t.Run("empty completions still produce a row", func(t *testing.T) { + batches, err := parseCompletionLogRows(loadLogFixture(t, "completion_02_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, batches, 1) + assertTypedBatch[*models.KiroCompletionLog](t, batches[0], 1) + }) + + t.Run("parse failure propagates", func(t *testing.T) { + _, err := parseChatLogRows([]byte("not gzipped"), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) +} + +// The scheduler's tick is a global rate limiter rather than a per-worker one: +// every submitted task waits one tick before running, so the interval caps total +// throughput no matter how large the pool is. A one-second tick pinned a real run +// to one file per second - 13 seconds for 13 files, with 20 workers idle. +func TestSchedulerTickDoesNotThrottleThePool(t *testing.T) { + source, err := os.ReadFile("extractor.go") + require.NoError(t, err) + + tick := regexp.MustCompile(`NewWorkerScheduler\((?s:.*?)time\.(\w+),`).FindStringSubmatch(string(source)) + require.NotNil(t, tick, "expected a tick interval passed to NewWorkerScheduler") + assert.Equal(t, "Millisecond", tick[1], + "a coarser tick throttles the whole pool to one object per tick") +} + +// Twenty workers inserting into one table concurrently deadlock on MySQL gap +// locks, and DevLake's retry layer turns those deadlocks into failures that only +// appear under load - the hardest kind to reproduce. The worker closure must +// therefore stay free of database access, which is asserted structurally because +// no unit test can observe the absence of a write. +func TestExtractorWorkersDoNotTouchTheDatabase(t *testing.T) { + source, err := os.ReadFile("extractor.go") + require.NoError(t, err) + + body := string(source) + workerStart := regexp.MustCompile(`scheduler\.SubmitBlocking\(func\(\) errors\.Error \{`).FindStringIndex(body) + require.NotNil(t, workerStart, "expected a worker closure submitted to the scheduler") + + // Take the closure body up to the WaitAsync call, which is where the main + // goroutine resumes and persistence begins. + waitIdx := regexp.MustCompile(`scheduler\.WaitAsync\(\)`).FindStringIndex(body) + require.NotNil(t, waitIdx) + require.Less(t, workerStart[1], waitIdx[0]) + workerRegion := body[workerStart[1]:waitIdx[0]] + + for _, forbidden := range []string{"db.Create", "db.Update", "db.CreateOrUpdate", "db.All", "db.First", "db.Exec"} { + assert.NotContains(t, workerRegion, forbidden, + "worker goroutines must not access the database; %s belongs on the main goroutine", forbidden) + } +} + +// A permanently malformed object would otherwise be retried on every run, +// filling the log with the same error while the scope never reaches a finished +// state. +func TestPendingFilesQueryBoundsRetries(t *testing.T) { + source, err := os.ReadFile("extractor.go") + require.NoError(t, err) + + clauses := regexp.MustCompile(`dal\.Where\(\s*"([^"]+)"`).FindAllStringSubmatch(string(source), -1) + require.NotEmpty(t, clauses) + + var found bool + for _, clause := range clauses { + if regexp.MustCompile(`attempt_count\s*<`).MatchString(clause[1]) { + found = true + // The same query must also scope to the connection and scope, or one + // scope's run would pick up another's files. + assert.Contains(t, clause[1], "connection_id") + assert.Contains(t, clause[1], "scope_id") + assert.Contains(t, clause[1], "file_type") + } + } + assert.True(t, found, "the pending-files query must bound attempt_count") +} + +func TestMaxAttemptsIsBounded(t *testing.T) { + // A cap that is zero or negative would stop all extraction; an unbounded one + // would never converge. + assert.Greater(t, models.MaxAttempts, 0) + assert.LessOrEqual(t, models.MaxAttempts, 10) +} + +// Every extractor must declare the collector as a dependency: without the file +// metadata rows there is nothing to extract, and DevLake would otherwise be free +// to schedule extraction before discovery. +func TestExtractorSubTaskMetas(t *testing.T) { + for _, meta := range []plugin.SubTaskMeta{ + ExtractKiroUserReportMeta, + ExtractKiroChatLogMeta, + ExtractKiroCompletionLogMeta, + } { + t.Run(meta.Name, func(t *testing.T) { + assert.NotEmpty(t, meta.Name) + assert.NotNil(t, meta.EntryPoint) + assert.True(t, meta.EnabledByDefault) + + var dependsOnCollector bool + for _, dep := range meta.Dependencies { + if dep.Name == CollectKiroS3FilesMeta.Name { + dependsOnCollector = true + } + } + assert.True(t, dependsOnCollector, "extraction depends on file discovery having run") + }) + } +} diff --git a/backend/plugins/kiro/tasks/identity_client.go b/backend/plugins/kiro/tasks/identity_client.go new file mode 100644 index 00000000000..8e6ed34c2fd --- /dev/null +++ b/backend/plugins/kiro/tasks/identity_client.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/identitystore" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// IdentityStoreAPI is the subset of the Identity Store API used here. +type IdentityStoreAPI interface { + DescribeUser(input *identitystore.DescribeUserInput) (*identitystore.DescribeUserOutput, error) +} + +// KiroIdentityClient resolves user ids to human-readable display names. +// +// This is entirely optional. Identity for joining to git history comes from the +// User_Email column of the report CSV, so a missing or misconfigured Identity +// Store degrades only the display name, never the data pipeline. +type KiroIdentityClient struct { + IdentityStore IdentityStoreAPI + StoreId string +} + +// NewKiroIdentityClient returns nil when Identity Store is not configured, +// which callers treat as "no display names" rather than as an error. +func NewKiroIdentityClient(connection *models.KiroConnection) (*KiroIdentityClient, error) { + if connection.IdentityStoreId == "" || connection.IdentityStoreRegion == "" { + return nil, nil + } + + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(connection.IdentityStoreRegion), + Credentials: credentials.NewStaticCredentials( + connection.AccessKeyId, + connection.SecretAccessKey, + "", + ), + }) + if err != nil { + return nil, err + } + + return &KiroIdentityClient{ + IdentityStore: identitystore.New(sess), + StoreId: connection.IdentityStoreId, + }, nil +} + +// ResolveDisplayName looks up a display name, returning nil when it cannot be +// determined. +// +// nil rather than the raw user id: the column exists for human readability, and +// storing an id there would make it look like a resolved name. +func (client *KiroIdentityClient) ResolveDisplayName(userId string) (*string, error) { + if client == nil || client.IdentityStore == nil || userId == "" { + return nil, nil + } + + result, err := client.IdentityStore.DescribeUser(&identitystore.DescribeUserInput{ + IdentityStoreId: aws.String(client.StoreId), + UserId: aws.String(userId), + }) + if err != nil { + // Surfaced for logging, but callers proceed without a display name. + return nil, err + } + + if result.DisplayName != nil && *result.DisplayName != "" { + name := *result.DisplayName + return &name, nil + } + return nil, nil +} diff --git a/backend/plugins/kiro/tasks/log_extractor.go b/backend/plugins/kiro/tasks/log_extractor.go new file mode 100644 index 00000000000..476b06add9c --- /dev/null +++ b/backend/plugins/kiro/tasks/log_extractor.go @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var ( + _ plugin.SubTaskEntryPoint = ExtractKiroChatLog + _ plugin.SubTaskEntryPoint = ExtractKiroCompletionLog +) + +var ExtractKiroChatLogMeta = plugin.SubTaskMeta{ + Name: "extractKiroChatLog", + EntryPoint: ExtractKiroChatLog, + EnabledByDefault: true, + Description: "Extract Kiro chat interactions from GenerateAssistantResponse logs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectKiroS3FilesMeta}, +} + +var ExtractKiroCompletionLogMeta = plugin.SubTaskMeta{ + Name: "extractKiroCompletionLog", + EntryPoint: ExtractKiroCompletionLog, + EnabledByDefault: true, + Description: "Extract Kiro inline suggestions from GenerateCompletions logs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectKiroS3FilesMeta}, +} + +// ExtractKiroChatLog loads the chat interaction logs for this scope. +// +// This is the high-volume stream: one object per interaction, roughly 700 bytes +// each, which for a single active user reaches several hundred objects a day. +// The cost is request count rather than bytes, which is what the worker pool +// addresses. +func ExtractKiroChatLog(taskCtx plugin.SubTaskContext) errors.Error { + return extractFiles(taskCtx, models.FileTypeChatLog, parseChatLogRows) +} + +// ExtractKiroCompletionLog loads the inline suggestion logs for this scope. +// +// This stream is dormant under agentic usage - the sampled history stops in +// March - but the objects remain in S3, so a backfill picks them up and a team +// that enables IDE inline completion starts producing them again. +func ExtractKiroCompletionLog(taskCtx plugin.SubTaskContext) errors.Error { + return extractFiles(taskCtx, models.FileTypeCompletionLog, parseCompletionLogRows) +} + +// Both adapters return a single typed batch: the slice keeps its concrete +// element type so GORM can resolve the table. +func parseChatLogRows(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) { + logs, err := ParseChatLog(data, connectionId, scopeId) + if err != nil { + return nil, err + } + return []rowBatch{{rows: logs, count: len(logs)}}, nil +} + +func parseCompletionLogRows(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) { + logs, err := ParseCompletionLog(data, connectionId, scopeId) + if err != nil { + return nil, err + } + return []rowBatch{{rows: logs, count: len(logs)}}, nil +} diff --git a/backend/plugins/kiro/tasks/log_parser.go b/backend/plugins/kiro/tasks/log_parser.go new file mode 100644 index 00000000000..3de9a66fdea --- /dev/null +++ b/backend/plugins/kiro/tasks/log_parser.go @@ -0,0 +1,256 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "path/filepath" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// Markers used by the prompt heuristics. These read a prompt's text for signs +// of how the developer invoked Kiro; they are only meaningful when a prompt is +// actually present. +const ( + steeringMarker = ".kiro/steering" + specMarker = ".kiro/specs" +) + +// Every field below is a pointer or checked for presence because Kiro omits +// empty keys entirely rather than emitting a null. Across 4530 sampled records, +// five documented fields never appeared at all and modelId appeared on only 45%, +// so "absent" and "zero" have to stay distinguishable. +type chatLogFile struct { + Records []struct { + Request *struct { + Prompt *string `json:"prompt"` + ChatTriggerType *string `json:"chatTriggerType"` + UserId *string `json:"userId"` + TimeStamp *string `json:"timeStamp"` + ModelId *string `json:"modelId"` + CustomizationArn *string `json:"customizationArn"` + } `json:"generateAssistantResponseEventRequest"` + Response *struct { + AssistantResponse *string `json:"assistantResponse"` + FollowupPrompts *string `json:"followupPrompts"` + RequestId *string `json:"requestId"` + MessageMetadata *struct { + ConversationId *string `json:"conversationId"` + UtteranceId *string `json:"utteranceId"` + } `json:"messageMetadata"` + } `json:"generateAssistantResponseEventResponse"` + } `json:"records"` +} + +type completionLogFile struct { + Records []struct { + Request *struct { + LeftContext *string `json:"leftContext"` + RightContext *string `json:"rightContext"` + FileName *string `json:"fileName"` + UserId *string `json:"userId"` + TimeStamp *string `json:"timeStamp"` + CustomizationArn *string `json:"customizationArn"` + } `json:"generateCompletionsEventRequest"` + Response *struct { + Completions []string `json:"completions"` + RequestId *string `json:"requestId"` + } `json:"generateCompletionsEventResponse"` + } `json:"records"` +} + +// ParseChatLog parses a gzipped GenerateAssistantResponse log. +// +// Neither the prompt nor the assistant response text is persisted: derived +// features are computed here and the originals are dropped, which keeps +// proprietary code and personal content out of the warehouse. +// +// A file holds one or two records, so the array is always walked. +func ParseChatLog(gzData []byte, connectionId uint64, scopeId string) ([]*models.KiroChatLog, errors.Error) { + raw, err := gunzip(gzData) + if err != nil { + return nil, err + } + + var parsed chatLogFile + if jsonErr := json.Unmarshal(raw, &parsed); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to unmarshal chat log") + } + + var result []*models.KiroChatLog + for _, record := range parsed.Records { + if record.Request == nil || record.Response == nil { + // Without both halves there is no usable interaction. + continue + } + requestId := deref(record.Response.RequestId) + if requestId == "" { + // requestId is the primary key; a record without one cannot be + // stored or deduplicated. + continue + } + + userId, identityStoreId := SplitUserId(deref(record.Request.UserId)) + + timestamp, tsErr := ParseKiroTime(deref(record.Request.TimeStamp)) + if tsErr != nil { + return nil, tsErr + } + + log := &models.KiroChatLog{ + ConnectionId: connectionId, + ScopeId: scopeId, + RequestId: requestId, + UserId: userId, + IdentityStoreId: identityStoreId, + Timestamp: timestamp, + ChatTriggerType: deref(record.Request.ChatTriggerType), + // Only ~45% of records carry a model id, so it stays nil when + // absent. Attributing model usage from this column would skew any + // share-of-usage figure; user_model_messages is authoritative. + ModelId: record.Request.ModelId, + ResponseLength: len(deref(record.Response.AssistantResponse)), + // Presence, not content: the follow-up text itself is not stored. + HasFollowupPrompts: record.Response.FollowupPrompts != nil, + } + + prompt := deref(record.Request.Prompt) + if prompt != "" { + // A non-empty prompt means the user spoke this turn. + sum := sha256.Sum256([]byte(prompt)) + hash := hex.EncodeToString(sum[:]) + hasSteering := strings.Contains(prompt, steeringMarker) + isSpecMode := strings.Contains(prompt, specMarker) + + log.HasPrompt = true + log.PromptLength = len(prompt) + log.PromptSha256 = &hash + log.HasSteering = &hasSteering + log.IsSpecMode = &isSpecMode + } + // When the prompt is empty the agent continued on its own. The hash and + // both heuristics stay nil: hashing the empty string would give roughly + // 71% of rows one identical hash and destroy the rework signal, and a + // false heuristic would be indistinguishable from a real negative. + + if md := record.Response.MessageMetadata; md != nil { + log.ConversationId = md.ConversationId + log.UtteranceId = md.UtteranceId + } + + result = append(result, log) + } + + return result, nil +} + +// ParseCompletionLog parses a gzipped GenerateCompletions log. +// +// The counters are named Returned* rather than Accepted*: a record is written +// when the suggestion is requested, and an empty completions array is common, so +// these measure what Kiro offered rather than what was taken. Records with no +// completions are still stored - they are the denominator. +func ParseCompletionLog(gzData []byte, connectionId uint64, scopeId string) ([]*models.KiroCompletionLog, errors.Error) { + raw, err := gunzip(gzData) + if err != nil { + return nil, err + } + + var parsed completionLogFile + if jsonErr := json.Unmarshal(raw, &parsed); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to unmarshal completion log") + } + + var result []*models.KiroCompletionLog + for _, record := range parsed.Records { + if record.Request == nil || record.Response == nil { + continue + } + requestId := deref(record.Response.RequestId) + if requestId == "" { + continue + } + + userId, identityStoreId := SplitUserId(deref(record.Request.UserId)) + + timestamp, tsErr := ParseKiroTime(deref(record.Request.TimeStamp)) + if tsErr != nil { + return nil, tsErr + } + + // fileName is a bare file name with no directory path, so it cannot be + // resolved to a unique repository file. + fileName := deref(record.Request.FileName) + + charCount := 0 + lineCount := 0 + for _, completion := range record.Response.Completions { + charCount += len(completion) + lineCount += strings.Count(completion, "\n") + 1 + } + + result = append(result, &models.KiroCompletionLog{ + ConnectionId: connectionId, + ScopeId: scopeId, + RequestId: requestId, + UserId: userId, + IdentityStoreId: identityStoreId, + Timestamp: timestamp, + FileName: fileName, + FileExtension: strings.TrimPrefix(filepath.Ext(fileName), "."), + // Present on completion records but never on chat records. + HasCustomization: record.Request.CustomizationArn != nil, + CompletionsCount: len(record.Response.Completions), + ReturnedCharCount: charCount, + ReturnedLineCount: lineCount, + LeftContextLength: len(deref(record.Request.LeftContext)), + RightContextLength: len(deref(record.Request.RightContext)), + }) + } + + return result, nil +} + +func gunzip(gzData []byte) ([]byte, errors.Error) { + reader, err := gzip.NewReader(bytes.NewReader(gzData)) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to open gzip reader") + } + defer reader.Close() + + raw, err := io.ReadAll(reader) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to decompress log file") + } + return raw, nil +} + +func deref(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/backend/plugins/kiro/tasks/log_parser_test.go b/backend/plugins/kiro/tasks/log_parser_test.go new file mode 100644 index 00000000000..f883569fecd --- /dev/null +++ b/backend/plugins/kiro/tasks/log_parser_test.go @@ -0,0 +1,319 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "compress/gzip" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fixtures are real log files whose text content was replaced but whose exact +// string lengths and line structure were preserved, because those are the +// values the parser derives and this test asserts. +func loadLogFixture(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "logs", name)) + require.NoError(t, err) + return data +} + +func gzipBytes(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + _, err := w.Write([]byte(s)) + require.NoError(t, err) + require.NoError(t, w.Close()) + return buf.Bytes() +} + +// An empty prompt means the agent continued on its own after a tool call rather +// than the user speaking. This is the majority case - roughly 71% of sampled +// records - and it must not be confused with a user turn. +func TestParseChatLog_EmptyPrompt(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_01_empty_prompt.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.False(t, l.HasPrompt) + assert.Equal(t, 0, l.PromptLength) + // Hashing the empty string would give ~71% of rows one shared hash and + // destroy the rework signal, so it stays NULL. + assert.Nil(t, l.PromptSha256) + // Both heuristics read prompt text, so without a prompt they are unknown + // rather than false. + assert.Nil(t, l.HasSteering) + assert.Nil(t, l.IsSpecMode) + + assert.Equal(t, "64d13ea7-dff5-4563-9285-6a9e351e87a0", l.RequestId) + assert.Equal(t, "MANUAL", l.ChatTriggerType) + require.NotNil(t, l.ModelId) + assert.Equal(t, "claude-opus-5", *l.ModelId) + assert.Equal(t, 259, l.ResponseLength) + + // Log records always carry the identity-store prefix; stripping it is what + // lets this join against the report table. + assert.Equal(t, "11111111-1111-4111-8111-111111111111", l.UserId) + assert.Equal(t, "d-1234567890", l.IdentityStoreId) + + // Nanosecond input truncated for DATETIME(6). + assert.Equal(t, 2026, l.Timestamp.Year()) + assert.Equal(t, 23, l.Timestamp.Hour()) + assert.Equal(t, 0, l.Timestamp.Nanosecond()%1000) + + // Documented but never observed, so they stay NULL - their absence is why + // the S3 logs cannot group interactions into sessions. + assert.Nil(t, l.ConversationId) + assert.Nil(t, l.UtteranceId) +} + +func TestParseChatLog_WithPrompt(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_02_with_prompt.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.True(t, l.HasPrompt) + assert.Equal(t, 1363, l.PromptLength) + require.NotNil(t, l.PromptSha256) + assert.Len(t, *l.PromptSha256, 64) + // With a prompt present the heuristics carry a real verdict. + require.NotNil(t, l.HasSteering) + require.NotNil(t, l.IsSpecMode) + assert.Equal(t, 64, l.ResponseLength) +} + +// Kiro packs one or two records per file, so the array is always walked. +func TestParseChatLog_TwoRecords(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_03_two_records.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 2) + + assert.NotEqual(t, logs[0].RequestId, logs[1].RequestId) + // A zero-length assistant response is real data, not a parse failure. + assert.Equal(t, 0, logs[0].ResponseLength) + assert.Equal(t, 117, logs[1].ResponseLength) +} + +// modelId is absent on more than half of all records. Storing an empty string +// instead of NULL would invent a phantom model accounting for most of the +// traffic. +func TestParseChatLog_MissingModelId(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_04_no_model_id.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.Nil(t, l.ModelId, "an absent modelId must be NULL, never an empty string") + // This record does carry followupPrompts, which a single-day sample had + // suggested never appears. + assert.True(t, l.HasFollowupPrompts) + assert.True(t, l.HasPrompt) + assert.Equal(t, 175, l.PromptLength) +} + +func TestParseChatLog_PromptHashing(t *testing.T) { + // The same prompt text must hash identically, which is what makes repeated + // submission detectable as a rework signal. + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"fix the retry logic","chatTriggerType":"MANUAL",` + + `"userId":"d-abc.user-1","timeStamp":"2026-07-27T23:03:29.027400929Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"%s"}}]}` + + first, err := ParseChatLog(gzipBytes(t, fmt.Sprintf(body, "req-1")), testConnectionId, testScopeId) + require.Nil(t, err) + second, err := ParseChatLog(gzipBytes(t, fmt.Sprintf(body, "req-2")), testConnectionId, testScopeId) + require.Nil(t, err) + + require.Len(t, first, 1) + require.Len(t, second, 1) + require.NotNil(t, first[0].PromptSha256) + require.NotNil(t, second[0].PromptSha256) + assert.Equal(t, *first[0].PromptSha256, *second[0].PromptSha256) + assert.NotEqual(t, first[0].RequestId, second[0].RequestId) +} + +func TestParseChatLog_Heuristics(t *testing.T) { + tests := []struct { + name string + prompt string + wantHasSteering bool + wantIsSpecMode bool + }{ + {"steering reference", "please follow .kiro/steering/style.md", true, false}, + {"spec reference", "implement .kiro/specs/auth/tasks.md", false, true}, + {"both", "read .kiro/steering and .kiro/specs", true, true}, + {"neither", "just fix this bug", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"` + tt.prompt + `","chatTriggerType":"MANUAL",` + + `"userId":"d-abc.user-1","timeStamp":"2026-07-27T23:03:29Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"r1"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + require.NotNil(t, logs[0].HasSteering) + require.NotNil(t, logs[0].IsSpecMode) + assert.Equal(t, tt.wantHasSteering, *logs[0].HasSteering) + assert.Equal(t, tt.wantIsSpecMode, *logs[0].IsSpecMode) + }) + } +} + +func TestParseCompletionLog_NonEmpty(t *testing.T) { + logs, err := ParseCompletionLog(loadLogFixture(t, "completion_01_non_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.Equal(t, "e0d1e760-a907-47d4-baa5-93df0f38b274", l.RequestId) + // A bare file name with no directory path, so it cannot be resolved to a + // unique repository file. + assert.Equal(t, "mcp.json", l.FileName) + assert.Equal(t, "json", l.FileExtension) + assert.Equal(t, 1, l.CompletionsCount) + assert.Equal(t, 347, l.ReturnedCharCount) + assert.Equal(t, 15, l.ReturnedLineCount) + assert.Equal(t, 5324, l.LeftContextLength) + assert.Equal(t, 12, l.RightContextLength) + assert.False(t, l.HasCustomization) + assert.Equal(t, "11111111-1111-4111-8111-111111111111", l.UserId) +} + +// A completion record is written when the suggestion is requested, not when it +// is accepted, so an empty array is normal. The record is the denominator and +// must still be stored. +func TestParseCompletionLog_EmptyCompletions(t *testing.T) { + logs, err := ParseCompletionLog(loadLogFixture(t, "completion_02_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1, "a record with no completions is still stored") + + l := logs[0] + assert.Equal(t, 0, l.CompletionsCount) + assert.Equal(t, 0, l.ReturnedCharCount) + assert.Equal(t, 0, l.ReturnedLineCount) + // Context was still sent, which is what distinguishes this from a truncated + // record. + assert.Equal(t, 5323, l.LeftContextLength) +} + +func TestParseCompletionLog_LineCounting(t *testing.T) { + tests := []struct { + name string + completions string + wantLines int + wantChars int + }{ + {"single line", `["abc"]`, 1, 3}, + {"two lines", `["a\nb"]`, 2, 3}, + {"trailing newline counts the empty final line", `["a\n"]`, 2, 2}, + {"two completions summed", `["a\nb","c"]`, 3, 4}, + {"empty array", `[]`, 0, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := `{"records":[{"generateCompletionsEventRequest":` + + `{"fileName":"f.ts","leftContext":"","rightContext":"",` + + `"userId":"d-abc.user-1","timeStamp":"2026-03-19T13:49:58Z"},` + + `"generateCompletionsEventResponse":{"completions":` + tt.completions + `,"requestId":"r1"}}]}` + logs, err := ParseCompletionLog(gzipBytes(t, body), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + assert.Equal(t, tt.wantLines, logs[0].ReturnedLineCount) + assert.Equal(t, tt.wantChars, logs[0].ReturnedCharCount) + }) + } +} + +func TestParseCompletionLog_Customization(t *testing.T) { + // customizationArn appears on completion records but never on chat records. + body := `{"records":[{"generateCompletionsEventRequest":` + + `{"fileName":"f.ts","userId":"d-abc.u1","timeStamp":"2026-03-19T13:49:58Z",` + + `"customizationArn":"arn:aws:codewhisperer:us-east-1:1:customization/abc"},` + + `"generateCompletionsEventResponse":{"completions":[],"requestId":"r1"}}]}` + logs, err := ParseCompletionLog(gzipBytes(t, body), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + assert.True(t, logs[0].HasCustomization) +} + +func TestParseLog_EdgeCases(t *testing.T) { + t.Run("empty records array", func(t *testing.T) { + logs, err := ParseChatLog(gzipBytes(t, `{"records":[]}`), testConnectionId, testScopeId) + assert.Nil(t, err) + assert.Empty(t, logs) + }) + + t.Run("record without requestId is skipped", func(t *testing.T) { + // requestId is the primary key, so such a record cannot be stored or + // deduplicated. + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"","userId":"d-abc.u1","timeStamp":"2026-07-27T23:03:29Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + assert.Nil(t, err) + assert.Empty(t, logs) + }) + + t.Run("record missing the response half is skipped", func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"x","userId":"d-abc.u1","timeStamp":"2026-07-27T23:03:29Z"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + assert.Nil(t, err) + assert.Empty(t, logs) + }) + + t.Run("bad timestamp is an error", func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"","userId":"d-abc.u1","timeStamp":"not-a-time"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"r1"}}]}` + _, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) + + t.Run("non-gzip input is an error", func(t *testing.T) { + _, err := ParseChatLog([]byte("not gzipped"), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) + + t.Run("malformed json is an error", func(t *testing.T) { + _, err := ParseChatLog(gzipBytes(t, `{"records":`), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) + + t.Run("connection and scope are propagated", func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"","userId":"d-abc.u1","timeStamp":"2026-07-27T23:03:29Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"r1"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), 42, "scope-x") + require.Nil(t, err) + require.Len(t, logs, 1) + assert.Equal(t, uint64(42), logs[0].ConnectionId) + assert.Equal(t, "scope-x", logs[0].ScopeId) + }) +} diff --git a/backend/plugins/kiro/tasks/s3_client.go b/backend/plugins/kiro/tasks/s3_client.go new file mode 100644 index 00000000000..a0e2d19ba5a --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_client.go @@ -0,0 +1,217 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "io" + "sort" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// S3API is the subset of the S3 API this plugin uses, declared as an interface +// so collectors and extractors can be tested without AWS. +type S3API interface { + ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) + GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) +} + +// KiroS3Client is bound to exactly one bucket. +// +// Kiro recommends keeping interaction logs in a bucket separate from the +// activity reports, and the two may carry different KMS keys and IAM +// conditions. One client per bucket keeps those permission boundaries distinct, +// so an access failure points at a specific bucket instead of an ambiguous +// request. +type KiroS3Client struct { + S3 S3API + Bucket string +} + +// KiroS3Clients holds the report and log clients for a connection. When no +// separate log bucket is configured both fields address the same bucket, so +// single-bucket and dual-bucket setups take the same code path everywhere else. +type KiroS3Clients struct { + Report *KiroS3Client + PromptLog *KiroS3Client +} + +// NewKiroS3Clients builds the client pair for a connection. +func NewKiroS3Clients(connection *models.KiroConnection) (*KiroS3Clients, errors.Error) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(connection.Region), + Credentials: credentials.NewStaticCredentials(connection.AccessKeyId, connection.SecretAccessKey, ""), + }) + if err != nil { + return nil, errors.Convert(err) + } + + // A single S3 service client can address both buckets; the split is at the + // KiroS3Client level, which pins the bucket name. + svc := s3.New(sess) + + return &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: connection.Bucket}, + PromptLog: &KiroS3Client{S3: svc, Bucket: connection.GetPromptLogBucket()}, + }, nil +} + +// ForFileType returns the client that owns a given file type. +func (c *KiroS3Clients) ForFileType(fileType string) *KiroS3Client { + if fileType == models.FileTypeReport { + return c.Report + } + return c.PromptLog +} + +// Buckets returns the distinct buckets in use - one entry when reports and logs +// share a bucket, two when they do not. +func (c *KiroS3Clients) Buckets() []string { + if c.Report.Bucket == c.PromptLog.Bucket { + return []string{c.Report.Bucket} + } + return []string{c.Report.Bucket, c.PromptLog.Bucket} +} + +// ListSubPrefixes returns the immediate child "directories" under a prefix. +// +// Kiro's export layout is fully self-describing - accounts, years and months all +// appear as path segments - so this is what lets a scope be picked from what +// actually exists instead of typed by hand. A mistyped prefix is otherwise +// indistinguishable from a month with no data: collection succeeds and finds +// nothing either way. +// +// Uses a delimiter so S3 returns only the segment names, not every object +// beneath them. +func (c *KiroS3Client) ListSubPrefixes(prefix string) ([]string, errors.Error) { + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + var names []string + var continuationToken *string + for { + output, err := c.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(c.Bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), + ContinuationToken: continuationToken, + }) + if err != nil { + return nil, errors.Convert(err) + } + + for _, common := range output.CommonPrefixes { + if common.Prefix == nil { + continue + } + // Strip the queried prefix and the trailing slash to leave just the + // segment name. + name := strings.TrimSuffix(strings.TrimPrefix(*common.Prefix, prefix), "/") + if name != "" { + names = append(names, name) + } + } + + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + continuationToken = output.NextContinuationToken + } + + sort.Strings(names) + return names, nil +} + +// CountObjects reports how many collectable objects sit under a prefix. +// +// This is what turns "did I get the path right?" into an answerable question: +// the connection test reports these counts per stream, so a wrong prefix shows +// as zero before any scope is created. +// +// Counting stops at limit to keep the check cheap; the returned bool reports +// whether more objects remain. +func (c *KiroS3Client) CountObjects(prefix string, limit int) (int, bool, errors.Error) { + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + count := 0 + var continuationToken *string + for { + output, err := c.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(c.Bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if err != nil { + return 0, false, errors.Convert(err) + } + + for _, object := range output.Contents { + if object.Key == nil { + continue + } + // Same filter the collector applies, so the count reflects what + // would actually be collected rather than every object present. + if !strings.HasSuffix(*object.Key, ".csv") && !strings.HasSuffix(*object.Key, ".json.gz") { + continue + } + count++ + if limit > 0 && count >= limit { + return count, true, nil + } + } + + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + continuationToken = output.NextContinuationToken + } + + return count, false, nil +} + +// GetObjectBytes downloads an object in full. +// +// Objects are small - roughly 700 bytes for a chat log, a few KB for a +// completion log, and under 1 KB for a report CSV - so streaming would add +// complexity without saving memory. +func (c *KiroS3Client) GetObjectBytes(key string) ([]byte, errors.Error) { + output, err := c.S3.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(c.Bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, errors.Convert(err) + } + defer output.Body.Close() + + data, readErr := io.ReadAll(output.Body) + if readErr != nil { + return nil, errors.Convert(readErr) + } + return data, nil +} diff --git a/backend/plugins/kiro/tasks/s3_client_test.go b/backend/plugins/kiro/tasks/s3_client_test.go new file mode 100644 index 00000000000..08c910fcf20 --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_client_test.go @@ -0,0 +1,237 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/aws/aws-sdk-go/service/identitystore" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// mockS3 records the bucket each call addressed, so tests can prove a request +// went to the right one. +type mockS3 struct { + getObjectBody string + getObjectErr error + listOutputs []*s3.ListObjectsV2Output + listCallIdx int + seenGetBuckets []string + seenListBuckets []string + seenGetKeys []string +} + +func (m *mockS3) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + m.seenGetBuckets = append(m.seenGetBuckets, *input.Bucket) + m.seenGetKeys = append(m.seenGetKeys, *input.Key) + if m.getObjectErr != nil { + return nil, m.getObjectErr + } + return &s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader([]byte(m.getObjectBody))), + }, nil +} + +func (m *mockS3) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + m.seenListBuckets = append(m.seenListBuckets, *input.Bucket) + if m.listCallIdx >= len(m.listOutputs) { + return &s3.ListObjectsV2Output{}, nil + } + out := m.listOutputs[m.listCallIdx] + m.listCallIdx++ + return out, nil +} + +// The fallback rules mean a single-bucket deployment (what real exports look +// like today) and the dual-bucket layout Kiro recommends both work without +// branching in collectors or extractors. +func TestKiroS3Clients_BucketRouting(t *testing.T) { + t.Run("single bucket routes both file kinds to the same bucket", func(t *testing.T) { + svc := &mockS3{} + clients := &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: "one-bucket"}, + PromptLog: &KiroS3Client{S3: svc, Bucket: "one-bucket"}, + } + + assert.Equal(t, "one-bucket", clients.ForFileType(models.FileTypeReport).Bucket) + assert.Equal(t, "one-bucket", clients.ForFileType(models.FileTypeChatLog).Bucket) + assert.Equal(t, "one-bucket", clients.ForFileType(models.FileTypeCompletionLog).Bucket) + // Deduplicated, so a connection test checks access once rather than twice. + assert.Equal(t, []string{"one-bucket"}, clients.Buckets()) + }) + + t.Run("separate buckets route by file type", func(t *testing.T) { + svc := &mockS3{} + clients := &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: "reports"}, + PromptLog: &KiroS3Client{S3: svc, Bucket: "logs"}, + } + + assert.Equal(t, "reports", clients.ForFileType(models.FileTypeReport).Bucket) + assert.Equal(t, "logs", clients.ForFileType(models.FileTypeChatLog).Bucket) + assert.Equal(t, "logs", clients.ForFileType(models.FileTypeCompletionLog).Bucket) + assert.Equal(t, []string{"reports", "logs"}, clients.Buckets()) + }) + + // An unrecognized file type must not silently read from the report bucket, + // where it would find nothing; log data is the larger and more likely case. + t.Run("unknown file type falls to the log bucket", func(t *testing.T) { + svc := &mockS3{} + clients := &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: "reports"}, + PromptLog: &KiroS3Client{S3: svc, Bucket: "logs"}, + } + assert.Equal(t, "logs", clients.ForFileType("something-new").Bucket) + }) +} + +func TestNewKiroS3Clients_FallbackFromConnection(t *testing.T) { + t.Run("no prompt log bucket falls back to the report bucket", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "kiro-export-test", + }} + clients, err := NewKiroS3Clients(conn) + require.Nil(t, err) + assert.Equal(t, "kiro-export-test", clients.Report.Bucket) + assert.Equal(t, "kiro-export-test", clients.PromptLog.Bucket) + assert.Len(t, clients.Buckets(), 1) + }) + + t.Run("explicit prompt log bucket is used", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "reports-bucket", + PromptLogBucket: "logs-bucket", + }} + clients, err := NewKiroS3Clients(conn) + require.Nil(t, err) + assert.Equal(t, "reports-bucket", clients.Report.Bucket) + assert.Equal(t, "logs-bucket", clients.PromptLog.Bucket) + assert.Len(t, clients.Buckets(), 2) + }) +} + +func TestKiroS3Client_GetObjectBytes(t *testing.T) { + t.Run("reads the body and addresses the bound bucket", func(t *testing.T) { + svc := &mockS3{getObjectBody: "hello"} + client := &KiroS3Client{S3: svc, Bucket: "my-bucket"} + + data, err := client.GetObjectBytes("some/key.csv") + require.Nil(t, err) + assert.Equal(t, "hello", string(data)) + assert.Equal(t, []string{"my-bucket"}, svc.seenGetBuckets) + assert.Equal(t, []string{"some/key.csv"}, svc.seenGetKeys) + }) + + t.Run("propagates an S3 error", func(t *testing.T) { + svc := &mockS3{getObjectErr: errors.New("access denied")} + client := &KiroS3Client{S3: svc, Bucket: "my-bucket"} + + _, err := client.GetObjectBytes("k") + assert.NotNil(t, err) + }) +} + +// mockIdentityStore lets the optional display-name path be exercised without +// AWS. +type mockIdentityStore struct { + displayName *string + err error +} + +func (m *mockIdentityStore) DescribeUser(*identitystore.DescribeUserInput) (*identitystore.DescribeUserOutput, error) { + if m.err != nil { + return nil, m.err + } + return &identitystore.DescribeUserOutput{DisplayName: m.displayName}, nil +} + +func TestKiroIdentityClient_ResolveDisplayName(t *testing.T) { + name := "Some Developer" + + t.Run("resolves a display name", func(t *testing.T) { + client := &KiroIdentityClient{IdentityStore: &mockIdentityStore{displayName: &name}, StoreId: "d-1"} + got, err := client.ResolveDisplayName("user-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, name, *got) + }) + + // The column exists for human readability; falling back to the raw id would + // make an unresolved value look like a resolved one. + t.Run("empty display name yields nil, not the user id", func(t *testing.T) { + empty := "" + client := &KiroIdentityClient{IdentityStore: &mockIdentityStore{displayName: &empty}, StoreId: "d-1"} + got, err := client.ResolveDisplayName("user-1") + require.NoError(t, err) + assert.Nil(t, got) + }) + + // Identity Store is optional, so an unconfigured client must be safe to + // call rather than something every caller has to nil-check. + t.Run("nil client is safe to call", func(t *testing.T) { + var client *KiroIdentityClient + got, err := client.ResolveDisplayName("user-1") + require.NoError(t, err) + assert.Nil(t, got) + }) + + t.Run("error surfaces but yields no name", func(t *testing.T) { + client := &KiroIdentityClient{IdentityStore: &mockIdentityStore{err: errors.New("throttled")}, StoreId: "d-1"} + got, err := client.ResolveDisplayName("user-1") + assert.Error(t, err) + assert.Nil(t, got) + }) +} + +func TestNewKiroIdentityClient_OptionalConfiguration(t *testing.T) { + // Missing configuration is not an error: collection works fully without + // display names because identity comes from the report's email column. + for _, tt := range []struct { + name string + conn models.KiroConn + }{ + {"neither set", models.KiroConn{}}, + {"only store id", models.KiroConn{IdentityStoreId: "d-1"}}, + {"only region", models.KiroConn{IdentityStoreRegion: "us-east-1"}}, + } { + t.Run(tt.name, func(t *testing.T) { + client, err := NewKiroIdentityClient(&models.KiroConnection{KiroConn: tt.conn}) + require.NoError(t, err) + assert.Nil(t, client) + }) + } + + t.Run("fully configured returns a client", func(t *testing.T) { + client, err := NewKiroIdentityClient(&models.KiroConnection{KiroConn: models.KiroConn{ + IdentityStoreId: "d-1234567890", + IdentityStoreRegion: "us-east-1", + }}) + require.NoError(t, err) + require.NotNil(t, client) + assert.Equal(t, "d-1234567890", client.StoreId) + }) +} diff --git a/backend/plugins/kiro/tasks/s3_file_collector.go b/backend/plugins/kiro/tasks/s3_file_collector.go new file mode 100644 index 00000000000..552292e3737 --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_file_collector.go @@ -0,0 +1,171 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "path" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var _ plugin.SubTaskEntryPoint = CollectKiroS3Files + +// CollectKiroS3FilesMeta discovers which S3 objects exist for a scope. +var CollectKiroS3FilesMeta = plugin.SubTaskMeta{ + Name: "collectKiroS3Files", + EntryPoint: CollectKiroS3Files, + EnabledByDefault: true, + Description: "List Kiro export objects in S3 and record them for extraction", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +// CollectKiroS3Files lists every relevant object under the scope's prefixes and +// records the ones not seen before. +// +// Work is batched per listing page rather than per object. A single S3 page +// holds up to 1000 keys, so one SELECT and one INSERT per page replaces two +// round trips per file - at tens of thousands of objects a day that is the +// difference between dozens of queries and tens of thousands. +func CollectKiroS3Files(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*KiroTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + + taskCtx.SetProgress(0, -1) + + for _, spec := range data.Prefixes { + client := data.S3Clients.ForFileType(spec.FileType) + prefix := spec.Prefix + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + logger.Info("scanning s3://%s/%s for %s files", client.Bucket, prefix, spec.FileType) + + var continuationToken *string + for { + output, listErr := client.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(client.Bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if listErr != nil { + return errors.Convert(listErr) + } + + candidates := collectCandidates(output, client.Bucket, spec, data.Options) + inserted, saveErr := saveNewFileMeta(db, data.Options.ConnectionId, candidates) + if saveErr != nil { + return saveErr + } + taskCtx.IncProgress(inserted) + + // IsTruncated is a pointer; dereferencing it unguarded panics on an + // empty response. + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + continuationToken = output.NextContinuationToken + } + } + + return nil +} + +// collectCandidates turns one listing page into file metadata rows. +// +// Only .csv and .json.gz are kept. That filter also excludes the small +// extension-less objects AWS writes at the KiroLogs root as permission probes. +func collectCandidates(output *s3.ListObjectsV2Output, bucket string, spec PrefixSpec, options *KiroOptions) []*models.KiroS3FileMeta { + candidates := make([]*models.KiroS3FileMeta, 0, len(output.Contents)) + for _, object := range output.Contents { + if object.Key == nil { + continue + } + key := *object.Key + if !strings.HasSuffix(key, ".csv") && !strings.HasSuffix(key, ".json.gz") { + continue + } + candidates = append(candidates, &models.KiroS3FileMeta{ + ConnectionId: options.ConnectionId, + S3Path: key, + // Basename only. The full key lives in S3Path, which is sized for + // it; putting a full key here would eventually overflow the column. + FileName: path.Base(key), + Bucket: bucket, + ScopeId: options.ScopeId, + FileType: spec.FileType, + Processed: false, + }) + } + return candidates +} + +// saveNewFileMeta inserts the rows that are not already recorded, returning how +// many were added. +// +// The existence check queries by (connection_id, s3_path), which is exactly the +// primary key. Querying on an unindexed column here would turn each page into a +// full table scan and the task would never finish at scale. +func saveNewFileMeta(db dal.Dal, connectionId uint64, candidates []*models.KiroS3FileMeta) (int, errors.Error) { + if len(candidates) == 0 { + return 0, nil + } + + paths := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + paths = append(paths, candidate.S3Path) + } + + var existingRows []models.KiroS3FileMeta + err := db.All(&existingRows, + dal.Select("s3_path"), + dal.From(&models.KiroS3FileMeta{}), + dal.Where("connection_id = ? AND s3_path IN ?", connectionId, paths), + ) + if err != nil { + return 0, errors.Default.Wrap(err, "failed to query existing kiro file metadata") + } + + existing := make(map[string]struct{}, len(existingRows)) + for _, row := range existingRows { + existing[row.S3Path] = struct{}{} + } + + fresh := make([]*models.KiroS3FileMeta, 0, len(candidates)) + for _, candidate := range candidates { + if _, seen := existing[candidate.S3Path]; seen { + continue + } + fresh = append(fresh, candidate) + } + if len(fresh) == 0 { + return 0, nil + } + + if err := db.Create(fresh); err != nil { + return 0, errors.Default.Wrap(err, "failed to record kiro file metadata") + } + return len(fresh), nil +} diff --git a/backend/plugins/kiro/tasks/s3_file_collector_test.go b/backend/plugins/kiro/tasks/s3_file_collector_test.go new file mode 100644 index 00000000000..e45b26044fb --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_file_collector_test.go @@ -0,0 +1,215 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "os" + "regexp" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +func listOutput(truncated bool, keys ...string) *s3.ListObjectsV2Output { + contents := make([]*s3.Object, 0, len(keys)) + for _, k := range keys { + key := k + contents = append(contents, &s3.Object{Key: &key}) + } + return &s3.ListObjectsV2Output{ + Contents: contents, + IsTruncated: aws.Bool(truncated), + } +} + +func TestCollectCandidates_FileFiltering(t *testing.T) { + spec := PrefixSpec{FileType: models.FileTypeChatLog} + options := &KiroOptions{ConnectionId: 1, ScopeId: "s1"} + + t.Run("keeps csv and json.gz only", func(t *testing.T) { + output := listOutput(false, + "p/report.csv", + "p/log.json.gz", + // AWS writes small extension-less objects at the KiroLogs root as + // permission probes; they must not enter the work queue. + "p/26404955-bf00-40d3-b713-43d18edf0638", + "p/notes.txt", + "p/archive.zip", + ) + candidates := collectCandidates(output, "bkt", spec, options) + require.Len(t, candidates, 2) + assert.Equal(t, "p/report.csv", candidates[0].S3Path) + assert.Equal(t, "p/log.json.gz", candidates[1].S3Path) + }) + + t.Run("stores basename separately from the full key", func(t *testing.T) { + key := "logging/AWSLogs/123456789012/KiroLogs/GenerateAssistantResponse/us-east-1/2026/07/27/23/" + + "123456789012_GenerateAssistantResponse_202607272303_3tbIeIrGJNDFbfVx.json.gz" + candidates := collectCandidates(listOutput(false, key), "bkt", spec, options) + require.Len(t, candidates, 1) + + // The full key goes in S3Path, which is sized for it. FileName holds + // only the basename - a full key there would eventually overflow. + assert.Equal(t, key, candidates[0].S3Path) + assert.Equal(t, "123456789012_GenerateAssistantResponse_202607272303_3tbIeIrGJNDFbfVx.json.gz", + candidates[0].FileName) + assert.Less(t, len(candidates[0].FileName), 255) + }) + + t.Run("records bucket, scope and file type", func(t *testing.T) { + candidates := collectCandidates(listOutput(false, "p/a.csv"), "my-bucket", spec, options) + require.Len(t, candidates, 1) + assert.Equal(t, "my-bucket", candidates[0].Bucket) + assert.Equal(t, "s1", candidates[0].ScopeId) + assert.Equal(t, models.FileTypeChatLog, candidates[0].FileType) + assert.Equal(t, uint64(1), candidates[0].ConnectionId) + assert.False(t, candidates[0].Processed) + }) + + t.Run("nil key is skipped", func(t *testing.T) { + output := &s3.ListObjectsV2Output{ + Contents: []*s3.Object{{Key: nil}, {Key: aws.String("p/a.csv")}}, + IsTruncated: aws.Bool(false), + } + candidates := collectCandidates(output, "bkt", spec, options) + assert.Len(t, candidates, 1) + }) + + t.Run("empty page yields nothing", func(t *testing.T) { + candidates := collectCandidates(listOutput(false), "bkt", spec, options) + assert.Empty(t, candidates) + }) +} + +func TestBuildPrefixes(t *testing.T) { + // Paths verified against real exports. + t.Run("single bucket layout", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "kiro-export-test", + ReportPrefix: "user-report", + PromptLogPrefix: "logging", + }} + prefixes := BuildPrefixes(conn, "123456789012", "2026/07") + + require.Len(t, prefixes, 3) + assert.Equal(t, + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/2026/07", + prefixes[0].Prefix) + assert.Equal(t, models.FileTypeReport, prefixes[0].FileType) + assert.Equal(t, + "logging/AWSLogs/123456789012/KiroLogs/GenerateAssistantResponse/us-east-1/2026/07", + prefixes[1].Prefix) + assert.Equal(t, models.FileTypeChatLog, prefixes[1].FileType) + assert.Equal(t, + "logging/AWSLogs/123456789012/KiroLogs/GenerateCompletions/us-east-1/2026/07", + prefixes[2].Prefix) + assert.Equal(t, models.FileTypeCompletionLog, prefixes[2].FileType) + + for _, p := range prefixes { + assert.Equal(t, "kiro-export-test", p.Bucket) + } + }) + + t.Run("defaults apply when prefixes are unset", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "b", + }} + prefixes := BuildPrefixes(conn, "acct", "2026/07") + assert.Contains(t, prefixes[0].Prefix, "user-report/AWSLogs/acct/KiroLogs/user_report") + assert.Contains(t, prefixes[1].Prefix, "logging/AWSLogs/acct/KiroLogs/GenerateAssistantResponse") + }) + + t.Run("separate buckets are assigned per file type", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "reports", + PromptLogBucket: "logs", + }} + prefixes := BuildPrefixes(conn, "acct", "2026/07") + assert.Equal(t, "reports", prefixes[0].Bucket) + assert.Equal(t, "logs", prefixes[1].Bucket) + assert.Equal(t, "logs", prefixes[2].Bucket) + }) + + // A nil month widens the scope to the whole year, which is how a year-long + // backfill is expressed. + t.Run("year-only time path", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{Region: "us-east-1", Bucket: "b"}} + prefixes := BuildPrefixes(conn, "acct", "2026") + assert.True(t, regexp.MustCompile(`/us-east-1/2026$`).MatchString(prefixes[0].Prefix)) + }) +} + +func TestWorkerCount(t *testing.T) { + assert.Equal(t, DefaultWorkerCount, (&KiroTaskData{Options: &KiroOptions{}}).WorkerCount()) + assert.Equal(t, 5, (&KiroTaskData{Options: &KiroOptions{WorkerCount: 5}}).WorkerCount()) + // A zero or negative override is ignored rather than disabling concurrency. + assert.Equal(t, DefaultWorkerCount, (&KiroTaskData{Options: &KiroOptions{WorkerCount: -1}}).WorkerCount()) + assert.Equal(t, DefaultWorkerCount, (&KiroTaskData{}).WorkerCount()) +} + +// This guards the predecessor defect that motivated the primary key choice: it queried +// its cursor table by s3_path while keying it on file_name, so the lookup falls +// back to scanning every row for the connection and collection never finishes at +// scale. The failure mode is a task that hangs rather than an error, so it is +// worth asserting structurally instead of hoping a reviewer notices. +func TestFileMetaQueriesMatchPrimaryKey(t *testing.T) { + pkColumns := primaryKeyColumns(t, "s3_file_meta.go") + require.Equal(t, []string{"ConnectionId", "S3Path"}, pkColumns, + "the cursor table must be keyed on the connection and the full object path") + + source, err := os.ReadFile("s3_file_collector.go") + require.NoError(t, err) + + whereClauses := regexp.MustCompile(`dal\.Where\(\s*"([^"]+)"`).FindAllStringSubmatch(string(source), -1) + require.NotEmpty(t, whereClauses, "expected at least one filtered query") + + for _, clause := range whereClauses { + condition := clause[1] + assert.Contains(t, condition, "connection_id", + "every cursor query must filter on connection_id, the first key column") + assert.Contains(t, condition, "s3_path", + "every cursor query must filter on s3_path, the second key column") + assert.NotContains(t, condition, "file_name", + "file_name is not indexed and must never appear in a lookup") + } +} + +// primaryKeyColumns extracts the fields tagged as primary keys from a model +// file, in declaration order - which is also the order of the composite index. +func primaryKeyColumns(t *testing.T, modelFile string) []string { + t.Helper() + source, err := os.ReadFile("../models/" + modelFile) + require.NoError(t, err) + + fieldRe := regexp.MustCompile(`(?m)^\s*([A-Z][A-Za-z0-9]*)\s+\S+\s+` + "`" + `[^` + "`" + `]*primaryKey[^` + "`" + `]*` + "`") + matches := fieldRe.FindAllStringSubmatch(string(source), -1) + + columns := make([]string, 0, len(matches)) + for _, m := range matches { + columns = append(columns, m[1]) + } + return columns +} diff --git a/backend/plugins/kiro/tasks/task_data.go b/backend/plugins/kiro/tasks/task_data.go new file mode 100644 index 00000000000..c8b504f1d18 --- /dev/null +++ b/backend/plugins/kiro/tasks/task_data.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// DefaultWorkerCount is how many objects are fetched concurrently. +// +// Collection is bound by request count, not bandwidth: a single user can +// produce ~600 log objects on a busy day, each under a kilobyte. S3 sustains +// thousands of GETs per second per prefix, so 20 is well within limits while +// cutting a peak day from tens of minutes to a couple. +const DefaultWorkerCount = 20 + +// KiroOptions are the blueprint-supplied task options. +type KiroOptions struct { + ConnectionId uint64 `json:"connectionId"` + ScopeId string `json:"scopeId"` + AccountId string `json:"accountId"` + Year int `json:"year"` + Month *int `json:"month"` + // WorkerCount overrides DefaultWorkerCount when set above zero. + WorkerCount int `json:"workerCount"` +} + +// PrefixSpec is one S3 location to scan, along with the kind of file found +// there and which bucket holds it. +type PrefixSpec struct { + Bucket string + Prefix string + FileType string +} + +// KiroTaskData is shared by every subtask in a run. +type KiroTaskData struct { + Options *KiroOptions + Connection *models.KiroConnection + S3Clients *KiroS3Clients + IdentityClient *KiroIdentityClient + // Prefixes are the locations to scan, precomputed so the collector does not + // re-derive paths. + Prefixes []PrefixSpec +} + +// WorkerCount returns the effective concurrency for this run. +func (d *KiroTaskData) WorkerCount() int { + if d.Options != nil && d.Options.WorkerCount > 0 { + return d.Options.WorkerCount + } + return DefaultWorkerCount +} + +// BuildPrefixes derives the three S3 locations a scope covers. +// +// Layout confirmed against real exports: +// +// {bucket}/{reportPrefix}/AWSLogs/{acct}/KiroLogs/user_report/{region}/{y}/{m} +// {bucket}/{logPrefix}/AWSLogs/{acct}/KiroLogs/GenerateAssistantResponse/{region}/{y}/{m} +// {bucket}/{logPrefix}/AWSLogs/{acct}/KiroLogs/GenerateCompletions/{region}/{y}/{m} +// +// The report path's hour segment is always 00 (reports are written at 02:00 +// UTC) while log paths carry a real hour, but neither is included here: the +// prefix stops at the month so a scope lists the whole period in one sweep. +func BuildPrefixes(connection *models.KiroConnection, accountId string, timePath string) []PrefixSpec { + region := connection.Region + reportBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", connection.GetReportPrefix(), accountId) + logBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", connection.GetPromptLogPrefix(), accountId) + + return []PrefixSpec{ + { + Bucket: connection.Bucket, + Prefix: fmt.Sprintf("%s/user_report/%s/%s", reportBase, region, timePath), + FileType: models.FileTypeReport, + }, + { + Bucket: connection.GetPromptLogBucket(), + Prefix: fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", logBase, region, timePath), + FileType: models.FileTypeChatLog, + }, + { + Bucket: connection.GetPromptLogBucket(), + Prefix: fmt.Sprintf("%s/GenerateCompletions/%s/%s", logBase, region, timePath), + FileType: models.FileTypeCompletionLog, + }, + } +} diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_01_empty_prompt.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_01_empty_prompt.json.gz new file mode 100644 index 00000000000..a713fdaf95d Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/chat_01_empty_prompt.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_02_with_prompt.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_02_with_prompt.json.gz new file mode 100644 index 00000000000..36fb2f6eed1 Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/chat_02_with_prompt.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_03_two_records.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_03_two_records.json.gz new file mode 100644 index 00000000000..37faba6e2e8 Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/chat_03_two_records.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_04_no_model_id.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_04_no_model_id.json.gz new file mode 100644 index 00000000000..21551bd3f68 Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/chat_04_no_model_id.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/completion_01_non_empty.json.gz b/backend/plugins/kiro/tasks/testdata/logs/completion_01_non_empty.json.gz new file mode 100644 index 00000000000..c25e2a154b1 Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/completion_01_non_empty.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/completion_02_empty.json.gz b/backend/plugins/kiro/tasks/testdata/logs/completion_02_empty.json.gz new file mode 100644 index 00000000000..a914b79e55f Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/completion_02_empty.json.gz differ diff --git a/backend/plugins/kiro/tasks/user_report_extractor.go b/backend/plugins/kiro/tasks/user_report_extractor.go new file mode 100644 index 00000000000..ddef74a1cd5 --- /dev/null +++ b/backend/plugins/kiro/tasks/user_report_extractor.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractKiroUserReport + +var ExtractKiroUserReportMeta = plugin.SubTaskMeta{ + Name: "extractKiroUserReport", + EntryPoint: ExtractKiroUserReport, + EnabledByDefault: true, + Description: "Extract daily per-user activity from Kiro report CSVs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectKiroS3FilesMeta}, +} + +// ExtractKiroUserReport loads the report CSVs discovered for this scope. +// +// Reports are written once per day per client type, so there are only a few +// hundred per year - the concurrency that matters for logs is irrelevant here, +// but reusing extractFiles keeps the retry and bookkeeping behaviour identical +// across all three streams. +func ExtractKiroUserReport(taskCtx plugin.SubTaskContext) errors.Error { + return extractFiles(taskCtx, models.FileTypeReport, parseUserReportRows) +} + +// parseUserReportRows adapts ParseUserReport to the extractor's batch interface. +// +// Both tables come from one parse because the per-model counts are columns of the +// same CSV row; splitting them into two passes would mean reading every file +// twice. They are returned as two batches rather than one mixed slice because +// GORM resolves the target table from the slice's element type. +func parseUserReportRows(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) { + reports, modelMessages, err := ParseUserReport(data, connectionId, scopeId) + if err != nil { + return nil, err + } + + return []rowBatch{ + {rows: reports, count: len(reports)}, + {rows: modelMessages, count: len(modelMessages)}, + }, nil +} diff --git a/backend/plugins/schema_e2e/migration_schema_test.go b/backend/plugins/schema_e2e/migration_schema_test.go index 0fbc02893bc..5cdf115f960 100644 --- a/backend/plugins/schema_e2e/migration_schema_test.go +++ b/backend/plugins/schema_e2e/migration_schema_test.go @@ -70,6 +70,7 @@ import ( issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + kiro "github.com/apache/incubator-devlake/plugins/kiro/impl" linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" @@ -121,6 +122,7 @@ func allGoPlugins() []plugin.PluginMeta { issueTrace.IssueTrace{}, jenkins.Jenkins{}, jira.Jira{}, + kiro.Kiro{}, linear.Linear{}, linker.Linker{}, opsgenie.Opsgenie{}, diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index c3262153dfc..afe5fa1a31d 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -46,6 +46,7 @@ import ( issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + kiro "github.com/apache/incubator-devlake/plugins/kiro/impl" linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" @@ -115,6 +116,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("linker/models", linker.Linker{}.GetTablesInfo) checker.FeedIn("issue_trace/models", issueTrace.IssueTrace{}.GetTablesInfo) checker.FeedIn("q_dev/models", q_dev.QDev{}.GetTablesInfo) + checker.FeedIn("kiro/models", kiro.Kiro{}.GetTablesInfo) checker.FeedIn("gh-copilot/models", copilot.GhCopilot{}.GetTablesInfo) err := checker.Verify() if err != nil { diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index 259d01bb0f1..441cda5cb2f 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -33,6 +33,7 @@ import { GitLabConfig } from './gitlab'; import { IncidentioConfig } from './incidentio'; import { JenkinsConfig } from './jenkins'; import { JiraConfig } from './jira'; +import { KiroConfig } from './kiro'; import { LinearConfig } from './linear'; import { PagerDutyConfig } from './pagerduty'; import { RootlyConfig } from './rootly'; @@ -64,6 +65,7 @@ export const pluginConfigs: IPluginConfig[] = [ IncidentioConfig, JenkinsConfig, JiraConfig, + KiroConfig, LinearConfig, PagerDutyConfig, RootlyConfig, diff --git a/config-ui/src/plugins/register/kiro/assets/icon.svg b/config-ui/src/plugins/register/kiro/assets/icon.svg new file mode 100644 index 00000000000..503114f140a --- /dev/null +++ b/config-ui/src/plugins/register/kiro/assets/icon.svg @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config-ui/src/plugins/register/kiro/config.tsx b/config-ui/src/plugins/register/kiro/config.tsx new file mode 100644 index 00000000000..d848e8a852e --- /dev/null +++ b/config-ui/src/plugins/register/kiro/config.tsx @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const KiroConfig: IPluginConfig = { + plugin: 'kiro', + name: 'Kiro', + icon: ({ color }) => , + sort: 12, + connection: { + docLink: 'https://kiro.dev/docs/enterprise/monitor-and-track/user-activity/', + initialValues: { + name: '', + region: 'us-east-1', + bucket: '', + reportPrefix: 'user-report', + promptLogBucket: '', + promptLogPrefix: 'logging', + identityStoreId: '', + identityStoreRegion: '', + }, + fields: [ + 'name', + { + key: 'region', + label: 'AWS Region', + subLabel: + 'The region where your Kiro profile was installed. The exports live under this region in the S3 path, so it must match exactly.', + }, + { + key: 'bucket', + label: 'S3 Bucket', + subLabel: 'Bucket holding the daily user activity report CSVs.', + }, + { + key: 'reportPrefix', + label: 'Report Prefix', + subLabel: 'Prefix within the bucket, before AWSLogs/. Leave as user-report unless you configured another.', + defaultValue: 'user-report', + }, + { + key: 'promptLogBucket', + label: 'Prompt Log Bucket (optional)', + subLabel: + 'Only needed if interaction logs go to a different bucket, which Kiro recommends. Leave empty to reuse the bucket above.', + }, + { + key: 'promptLogPrefix', + label: 'Prompt Log Prefix', + subLabel: 'Prefix for the interaction logs. Leave as logging unless you configured another.', + defaultValue: 'logging', + }, + { + key: 'accessKeyId', + label: 'AWS Access Key ID', + }, + { + key: 'secretAccessKey', + label: 'AWS Secret Access Key', + }, + { + key: 'identityStoreId', + label: 'IAM Identity Center Store ID (optional)', + subLabel: + 'Only resolves display names. User identity comes from the report’s User_Email column, so collection works without this. If set, the region below is required too.', + }, + { + key: 'identityStoreRegion', + label: 'IAM Identity Center Region (optional)', + subLabel: 'May differ from the S3 region. Required if a store ID is set.', + }, + ], + }, + dataScope: { + // No custom render: the default picker calls the plugin's remote-scopes + // endpoint, which browses the export layout in S3 as accounts -> years -> + // months and lists only periods that actually hold data. Hand-entering a + // prefix cannot be verified from the outcome, because a typo and an empty + // month both produce a run that succeeds and collects nothing. + title: 'Accounts & Periods', + }, + scopeConfig: { + entities: ['CROSS'], + transformation: {}, + }, +}; diff --git a/config-ui/src/plugins/register/kiro/index.ts b/config-ui/src/plugins/register/kiro/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/kiro/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/e2e/kiro-full-flow.spec.ts b/e2e/kiro-full-flow.spec.ts new file mode 100644 index 00000000000..88234a18844 --- /dev/null +++ b/e2e/kiro-full-flow.spec.ts @@ -0,0 +1,252 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { test, expect, request, Page } from '@playwright/test'; +import * as path from 'path'; +import * as fs from 'fs'; + +const API = 'http://localhost:8080'; +const UI = 'http://localhost:4000'; +const GRAFANA = 'http://localhost:3002'; +const SCREENSHOT_DIR = path.join(__dirname, 'screenshots'); + +// The full-flow test needs an existing Kiro connection with valid credentials. +// Keep environment-specific ids out of the repository. +const EXISTING_CONNECTION_ID = Number(process.env.KIRO_CONNECTION_ID || 0); +const KIRO_ACCOUNT_ID = process.env.KIRO_ACCOUNT_ID || ''; + +const state: { + connectionId: number; + scopeId: string; + blueprintId: number; + pipelineId: number; +} = { connectionId: EXISTING_CONNECTION_ID, scopeId: '', blueprintId: 0, pipelineId: 0 }; + +fs.mkdirSync(SCREENSHOT_DIR, { recursive: true }); + +async function grafanaLogin(page: Page) { + await page.goto(`${GRAFANA}/grafana/login`); + await page.waitForLoadState('networkidle'); + if (page.url().includes('/login')) { + await page.locator('input[name="user"]').fill('admin'); + await page.locator('input[name="password"]').fill('admin'); + await page.locator('button[type="submit"]').click(); + await page.waitForTimeout(2000); + // Handle "change password" prompt if shown + const skipBtn = page.locator('a:has-text("Skip")'); + if (await skipBtn.isVisible({ timeout: 2000 }).catch(() => false)) { + await skipBtn.click(); + } + await page.waitForTimeout(1000); + } +} + +async function openGrafanaDashboard(page: Page, uid: string, screenshotPath: string) { + await grafanaLogin(page); + await page.goto(`${GRAFANA}/grafana/d/${uid}?orgId=1&from=now-90d&to=now`); + + // Wait for first panel data to load + try { + await page.waitForResponse( + (resp) => resp.url().includes('/api/ds/query') && resp.status() === 200, + { timeout: 30000 } + ); + } catch { /* some dashboards may not fire queries immediately */ } + + // Wait for rendering to settle + await page.waitForTimeout(5000); + + // Take viewport screenshot (top section) + await page.screenshot({ path: screenshotPath.replace('.png', '-top.png') }); + + // Scroll down and take more sections + const scrollHeight = await page.evaluate(() => document.body.scrollHeight); + let section = 1; + for (let y = 900; y < scrollHeight; y += 900) { + await page.evaluate((scrollY) => window.scrollTo(0, scrollY), y); + await page.waitForTimeout(3000); + section++; + await page.screenshot({ path: screenshotPath.replace('.png', `-section${section}.png`) }); + } + + // Also take full page screenshot + await page.evaluate(() => window.scrollTo(0, 0)); + await page.waitForTimeout(2000); + await page.screenshot({ path: screenshotPath, fullPage: true }); +} + +test.describe.serial('Kiro Plugin Full Flow', () => { + + test('Step 1: Verify Existing Connection via API', async () => { + expect(EXISTING_CONNECTION_ID, 'set KIRO_CONNECTION_ID').toBeGreaterThan(0); + expect(KIRO_ACCOUNT_ID, 'set KIRO_ACCOUNT_ID').not.toBe(''); + const api = await request.newContext({ baseURL: API }); + + const resp = await api.get(`/plugins/kiro/connections/${state.connectionId}`); + expect(resp.ok()).toBeTruthy(); + const conn = await resp.json(); + console.log(`Using connection: id=${conn.id}, name=${conn.name}, bucket=${conn.bucket}`); + + const testResp = await api.post(`/plugins/kiro/connections/${state.connectionId}/test`); + const testBody = await testResp.json(); + console.log('Test connection:', { accounts: testBody.accounts, streams: testBody.streams, hint: testBody.hint }); + expect(testResp.ok()).toBeTruthy(); + }); + + test('Step 2: View Config-UI Home', async ({ page }) => { + await page.goto(UI); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(1000); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '01-config-ui-home.png'), fullPage: true }); + console.log('Screenshot: Config-UI home'); + }); + + test('Step 3: Create Scope (S3 Slice) via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.put(`/plugins/kiro/connections/${state.connectionId}/scopes`, { + data: { + data: [ + { + accountId: KIRO_ACCOUNT_ID, + basePath: '', + year: 2026, + month: 3, + }, + ], + }, + }); + + const body = await resp.json(); + console.log('Scope created:', resp.status()); + expect(resp.ok()).toBeTruthy(); + state.scopeId = body[0]?.id; + expect(state.scopeId).toBeTruthy(); + console.log(`Scope id: ${state.scopeId}`); + }); + + test('Step 4: Create Blueprint via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post('/blueprints', { + data: { + name: `e2e-blueprint-${Date.now()}`, + mode: 'NORMAL', + enable: true, + cronConfig: '0 0 * * *', + isManual: true, + connections: [ + { + pluginName: 'kiro', + connectionId: state.connectionId, + scopes: [{ scopeId: state.scopeId }], + }, + ], + }, + }); + + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.blueprintId = body.id; + console.log(`Blueprint created: id=${state.blueprintId}`); + }); + + test('Step 5: Trigger Pipeline via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post(`/blueprints/${state.blueprintId}/trigger`, { data: {} }); + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.pipelineId = body.id; + console.log(`Pipeline triggered: id=${state.pipelineId}`); + }); + + test('Step 6: Wait for Pipeline to Complete', async () => { + const api = await request.newContext({ baseURL: API }); + const maxWait = 120000; + const start = Date.now(); + let status = ''; + + while (Date.now() - start < maxWait) { + const resp = await api.get(`/pipelines/${state.pipelineId}`); + const pipeline = await resp.json(); + status = pipeline.status; + console.log(`Pipeline status: ${status} (${Math.round((Date.now() - start) / 1000)}s)`); + if (['TASK_COMPLETED', 'TASK_FAILED', 'TASK_PARTIAL'].includes(status)) break; + await new Promise((r) => setTimeout(r, 3000)); + } + + // Print task details + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + if (tasksResp.ok()) { + const { tasks } = await tasksResp.json(); + for (const t of tasks || []) { + console.log(` Task ${t.id}: ${t.status}${t.failedSubTask ? ` (failed: ${t.failedSubTask})` : ''}`); + if (t.message) console.log(` Error: ${t.message.substring(0, 300)}`); + } + } + + expect(status).toBe('TASK_COMPLETED'); + }); + + test('Step 7: Verify Data via MySQL', async () => { + const api = await request.newContext({ baseURL: API }); + + // Use pipeline tasks to confirm data was processed + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + const { tasks } = await tasksResp.json(); + expect(tasks[0].status).toBe('TASK_COMPLETED'); + console.log(`Pipeline completed in ${tasks[0].spentSeconds}s`); + }); + + test('Step 8: Grafana - Kiro Usage Dashboard (new format)', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_user_report', path.join(SCREENSHOT_DIR, '02-dashboard-user-report.png')); + console.log('Screenshot: Kiro Usage Dashboard'); + }); + + test('Step 9: Grafana - Kiro Feature Metrics', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_feature_metrics', path.join(SCREENSHOT_DIR, '03-dashboard-feature-metrics.png')); + console.log('Screenshot: Kiro Feature Metrics'); + }); + + test('Step 10: Grafana - Kiro AI Activity Insights (logging)', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_logging', path.join(SCREENSHOT_DIR, '04-dashboard-logging.png')); + console.log('Screenshot: Kiro AI Activity Insights'); + }); + + test('Step 11: Grafana - Kiro Executive Dashboard', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_executive', path.join(SCREENSHOT_DIR, '05-dashboard-executive.png')); + console.log('Screenshot: Kiro Executive Dashboard'); + }); + + test('Step 12: View Pipeline in Config-UI', async ({ page }) => { + // Navigate to the API proxy route for pipelines + await page.goto(`${UI}/api/pipelines?pageSize=5`); + await page.waitForLoadState('networkidle'); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '06-config-ui-pipelines.png'), fullPage: true }); + console.log('Screenshot: Pipelines API response'); + }); + + test('Step 13: Cleanup', async () => { + const api = await request.newContext({ baseURL: API }); + if (state.blueprintId) { + await api.delete(`/blueprints/${state.blueprintId}`); + console.log(`Deleted blueprint ${state.blueprintId}`); + } + console.log('Cleanup complete'); + }); +}); diff --git a/grafana/dashboards/mysql/ai-model-roi.json b/grafana/dashboards/mysql/ai-model-roi.json index 49980a1fadb..275fcf8949a 100644 --- a/grafana/dashboards/mysql/ai-model-roi.json +++ b/grafana/dashboards/mysql/ai-model-roi.json @@ -26,7 +26,7 @@ "targetBlank": true, "title": "Kiro Usage Dashboard", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" } ], "panels": [ @@ -87,7 +87,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used)) AS 'Total Credits'\nFROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], @@ -138,7 +138,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0), 1) AS \"Credits / PR\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT pr.id), 0), 1) AS 'Credits / PR'\nFROM _tool_kiro_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n) pr\nWHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -189,7 +189,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0), 1) AS \"Credits / Deploy\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 1) AS 'Credits / Deploy'\nFROM _tool_kiro_user_report r\nCROSS JOIN (\n SELECT DISTINCT cicd_deployment_id\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n) cdc\nWHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -240,7 +240,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0), 1) AS \"Credits / Issue\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT i.id), 0), 1) AS 'Credits / Issue'\nFROM _tool_kiro_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n) i\nWHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -316,7 +316,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_prs AS (\n SELECT DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY) AS week_start,\n COUNT(*) AS prs\n FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n GROUP BY DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(p.prs, 0), 1) AS 'Credits per PR'\nFROM _credits c\nLEFT JOIN _prs p ON c.week_start = p.week_start\nORDER BY time", "refId": "A" } ], @@ -379,7 +379,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_deploys AS (\n SELECT DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(DISTINCT cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(d.deploys, 0), 1) AS 'Credits per Deploy'\nFROM _credits c\nLEFT JOIN _deploys d ON c.week_start = d.week_start\nORDER BY time", "refId": "A" } ], @@ -442,7 +442,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_kiro_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_issues AS (\n SELECT DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n COUNT(*) AS resolved\n FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(i.resolved, 0), 1) AS 'Credits per Issue'\nFROM _credits c\nLEFT JOIN _issues i ON c.week_start = i.week_start\nORDER BY time", "refId": "A" } ], @@ -505,7 +505,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS 'Credits',\n SUM(total_messages) AS 'Messages',\n SUM(chat_conversations) AS 'Conversations'\nFROM _tool_kiro_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", "refId": "A" } ], @@ -517,7 +517,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", "kiro", "cost", "efficiency" @@ -531,7 +530,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "AI Cost-Efficiency (PostgreSQL)", - "uid": "ai_cost_efficiency-pg", + "title": "AI Model ROI", + "uid": "ai_model_roi", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/mysql/q-dev-dora.json b/grafana/dashboards/mysql/kiro-dora.json similarity index 61% rename from grafana/dashboards/mysql/q-dev-dora.json rename to grafana/dashboards/mysql/kiro-dora.json index b9fd48f3afb..ce2a54b7f37 100644 --- a/grafana/dashboards/mysql/q-dev-dora.json +++ b/grafana/dashboards/mysql/kiro-dora.json @@ -36,10 +36,10 @@ "keepTime": true, "tags": [], "targetBlank": true, - "title": "Q Dev Dashboard", + "title": "Kiro Dashboard", "tooltip": "", "type": "link", - "url": "/d/qdev_user_data/q-dev-user-data-dashboard" + "url": "/d/kiro_user_data/kiro-user-data-dashboard" } ], "panels": [ @@ -61,7 +61,7 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "## AI-Powered DORA Dashboard\nThis dashboard correlates **Q Dev (AI coding assistant)** usage metrics with **DORA** performance indicators to help understand the impact of AI-assisted development on engineering efficiency.\n\n- **Left side**: Q Dev AI usage metrics (code generation, acceptance rate)\n- **Right side**: DORA metrics (Lead Time, Deployment Frequency, Change Failure Rate)\n- **Correlation charts**: Show trends over time to identify potential relationships", + "content": "## Kiro + DORA Dashboard\nThis dashboard compares **Kiro usage** with **DORA** performance indicators at monthly aggregates.\n\n- **Kiro metrics**: active users, credits, messages, conversations, and credits per message\n- **DORA metrics**: lead time, deployment frequency, and change failure rate\n- **Semantic boundary**: Kiro does not export accepted LOC, suggestion acceptance, generated tests, or code-review findings; no proxy is presented as an equivalent measure", "mode": "markdown" }, "pluginVersion": "13.0.2", @@ -82,8 +82,11 @@ "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Number of unique users who used Q Dev AI features", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Unique Kiro users in the selected period, from the daily usage report.", "fieldConfig": { "defaults": { "color": { @@ -115,7 +118,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -126,20 +131,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) as 'Active Q Dev Users'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS 'Active Kiro Users' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Q Dev Active Users", + "title": "Kiro Active Users", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Total AI-generated code lines accepted (Inline + Chat)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro exports credits, not accepted lines of code. Accepted LOC has no equivalent field.", "fieldConfig": { "defaults": { "color": { @@ -172,7 +183,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -183,20 +196,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(inline_ai_code_lines + chat_ai_code_lines) as 'AI Accepted Lines'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used), 1) AS 'Kiro Credits Used' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Total AI Code Lines", + "title": "Total Kiro Credits", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Acceptance rate of inline AI suggestions", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro does not export suggestion acceptance. Credits per message is a usage-efficiency ratio, not an acceptance rate.", "fieldConfig": { "defaults": { "color": { @@ -220,7 +239,7 @@ } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [] }, @@ -237,7 +256,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -248,19 +269,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT \n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Acceptance Rate'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT SUM(credits_used) / NULLIF(SUM(total_messages), 0) AS 'Credits per Message' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "AI Acceptance Rate", + "title": "Credits per Message", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Number of production deployments in selected period", "fieldConfig": { "defaults": { @@ -293,7 +320,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -304,7 +333,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -316,7 +348,10 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Median lead time for changes in hours", "fieldConfig": { "defaults": { @@ -358,7 +393,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -369,7 +406,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -381,7 +421,10 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Percentage of deployments that caused incidents", "fieldConfig": { "defaults": { @@ -423,7 +466,9 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -434,7 +479,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -455,12 +503,15 @@ }, "id": 9, "panels": [], - "title": "AI Usage vs DORA Metrics Correlation", + "title": "Kiro Usage vs DORA Metrics", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Compare AI code generation trends with Lead Time for Changes. A negative correlation (AI lines up, Lead Time down) suggests AI is helping accelerate delivery.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro credits are compared with Lead Time. Kiro does not export accepted code lines.", "fieldConfig": { "defaults": { "color": { @@ -514,7 +565,7 @@ { "matcher": { "id": "byName", - "options": "AI Accepted Lines" + "options": "Kiro Credits" }, "properties": [ { @@ -533,29 +584,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Median Lead Time (hours)" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Lead Time (hours)" - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] } ] }, @@ -568,7 +596,10 @@ "id": 10, "options": { "legend": { - "calcs": ["mean", "max"], + "calcs": [ + "mean", + "max" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -582,20 +613,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_monthly AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines\n FROM _tool_q_dev_user_data\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\nlead_time_monthly AS (\n SELECT\n DATE_FORMAT(cdc.finished_date, '%Y-%m-01') AS month,\n AVG(ppm.pr_cycle_time) / 60 AS avg_lead_time\n FROM pull_requests pr\n JOIN project_pr_metrics ppm ON ppm.id = pr.id\n JOIN project_mapping pm ON pr.base_repo_id = pm.row_id AND pm.`table` = 'repos'\n JOIN cicd_deployment_commits cdc ON ppm.deployment_commit_id = cdc.id\n WHERE pm.project_name IN (${project})\n AND pr.merged_date IS NOT NULL\n AND ppm.pr_cycle_time IS NOT NULL\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, lt.month), '%Y-%m-%d') AS time,\n ai.ai_lines AS 'AI Accepted Lines',\n ROUND(lt.avg_lead_time, 1) AS 'Median Lead Time (hours)'\nFROM ai_monthly ai\nLEFT JOIN lead_time_monthly lt ON ai.month = lt.month\nWHERE ai.month IS NOT NULL OR lt.month IS NOT NULL\nORDER BY time", + "rawSql": "WITH ai_monthly AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(credits_used) AS kiro_credits\n FROM _tool_kiro_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\nlead_time_monthly AS (\n SELECT\n DATE_FORMAT(cdc.finished_date, '%Y-%m-01') AS month,\n AVG(ppm.pr_cycle_time) / 60 AS avg_lead_time\n FROM pull_requests pr\n JOIN project_pr_metrics ppm ON ppm.id = pr.id\n JOIN project_mapping pm ON pr.base_repo_id = pm.row_id AND pm.`table` = 'repos'\n JOIN cicd_deployment_commits cdc ON ppm.deployment_commit_id = cdc.id\n WHERE pm.project_name IN (${project})\n AND pr.merged_date IS NOT NULL\n AND ppm.pr_cycle_time IS NOT NULL\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, lt.month), '%Y-%m-%d') AS time,\n ai.kiro_credits AS 'Kiro Credits',\n ROUND(lt.avg_lead_time, 1) AS 'Median Lead Time (hours)'\nFROM ai_monthly ai\nLEFT JOIN lead_time_monthly lt ON ai.month = lt.month\nWHERE ai.month IS NOT NULL OR lt.month IS NOT NULL\nORDER BY time", "refId": "A" } ], - "title": "AI Code Generation vs Lead Time Trend", + "title": "Kiro Credits vs Lead Time Trend", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Compare AI suggestion acceptance rate with deployment frequency. Higher acceptance rate may indicate better AI integration and potentially more deployments.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Usage-efficiency ratio versus deployment frequency. This is not suggestion acceptance.", "fieldConfig": { "defaults": { "color": { @@ -649,7 +686,7 @@ { "matcher": { "id": "byName", - "options": "AI Acceptance Rate" + "options": "Credits per Message" }, "properties": [ { @@ -658,7 +695,7 @@ }, { "id": "custom.axisLabel", - "value": "Acceptance Rate" + "value": "Credits per Message" }, { "id": "unit", @@ -672,29 +709,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Deployment Count" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Deployments" - }, - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] } ] }, @@ -707,7 +721,10 @@ "id": 11, "options": { "legend": { - "calcs": ["mean", "max"], + "calcs": [ + "mean", + "max" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -721,20 +738,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_acceptance AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) AS acceptance_rate\n FROM _tool_q_dev_user_data\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\ndeployment_count AS (\n SELECT \n DATE_FORMAT(MAX(cdc.finished_date), '%Y-%m-01') AS month,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deploy_count\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, dc.month), '%Y-%m-%d') AS time,\n ai.acceptance_rate AS 'AI Acceptance Rate',\n dc.deploy_count AS 'Deployment Count'\nFROM ai_acceptance ai\nLEFT JOIN deployment_count dc ON ai.month = dc.month\nWHERE ai.month IS NOT NULL OR dc.month IS NOT NULL\nORDER BY time", + "rawSql": "WITH ai_acceptance AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(credits_used) / NULLIF(SUM(total_messages), 0) AS credits_per_message\n FROM _tool_kiro_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\ndeployment_count AS (\n SELECT \n DATE_FORMAT(MAX(cdc.finished_date), '%Y-%m-01') AS month,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deploy_count\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, dc.month), '%Y-%m-%d') AS time,\n ai.credits_per_message AS 'Credits per Message',\n dc.deploy_count AS 'Deployment Count'\nFROM ai_acceptance ai\nLEFT JOIN deployment_count dc ON ai.month = dc.month\nWHERE ai.month IS NOT NULL OR dc.month IS NOT NULL\nORDER BY time", "refId": "A" } ], - "title": "AI Acceptance Rate vs Deployment Frequency", + "title": "Credits per Message vs Deployment Frequency", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Compare AI-generated tests with Change Failure Rate. More AI-generated tests might correlate with lower failure rates.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro does not export generated-test counts; total messages is the available activity-volume measure.", "fieldConfig": { "defaults": { "color": { @@ -788,7 +811,7 @@ { "matcher": { "id": "byName", - "options": "AI Generated Tests" + "options": "Kiro Messages" }, "properties": [ { @@ -807,33 +830,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Change Failure Rate" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Failure Rate" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] } ] }, @@ -846,7 +842,10 @@ "id": 12, "options": { "legend": { - "calcs": ["mean", "max"], + "calcs": [ + "mean", + "max" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -860,20 +859,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_tests AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(test_generation_generated_tests) AS generated_tests\n FROM _tool_q_dev_user_data\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\ncfr_monthly AS (\n SELECT\n DATE_FORMAT(deployment_finished_date, '%Y-%m-01') AS month,\n SUM(has_incident) / NULLIF(COUNT(deployment_id), 0) AS cfr\n FROM (\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN d.deployment_id ELSE NULL END) AS has_incident\n FROM (\n SELECT\n cdc.cicd_deployment_id AS deployment_id,\n MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n GROUP BY 1\n HAVING $__timeFilter(MAX(cdc.finished_date))\n ) d\n LEFT JOIN project_incident_deployment_relationships pim ON d.deployment_id = pim.deployment_id\n LEFT JOIN incidents i ON pim.id = i.id\n GROUP BY 1, 2\n ) failure_data\n GROUP BY DATE_FORMAT(deployment_finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, cfr.month), '%Y-%m-%d') AS time,\n ai.generated_tests AS 'AI Generated Tests',\n cfr.cfr AS 'Change Failure Rate'\nFROM ai_tests ai\nLEFT JOIN cfr_monthly cfr ON ai.month = cfr.month\nWHERE ai.month IS NOT NULL OR cfr.month IS NOT NULL\nORDER BY time", + "rawSql": "WITH ai_tests AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m-01') AS month,\n SUM(total_messages) AS kiro_messages\n FROM _tool_kiro_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m-01')\n),\ncfr_monthly AS (\n SELECT\n DATE_FORMAT(deployment_finished_date, '%Y-%m-01') AS month,\n SUM(has_incident) / NULLIF(COUNT(deployment_id), 0) AS cfr\n FROM (\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN d.deployment_id ELSE NULL END) AS has_incident\n FROM (\n SELECT\n cdc.cicd_deployment_id AS deployment_id,\n MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n GROUP BY 1\n HAVING $__timeFilter(MAX(cdc.finished_date))\n ) d\n LEFT JOIN project_incident_deployment_relationships pim ON d.deployment_id = pim.deployment_id\n LEFT JOIN incidents i ON pim.id = i.id\n GROUP BY 1, 2\n ) failure_data\n GROUP BY DATE_FORMAT(deployment_finished_date, '%Y-%m-01')\n)\nSELECT \n STR_TO_DATE(COALESCE(ai.month, cfr.month), '%Y-%m-%d') AS time,\n ai.kiro_messages AS 'Kiro Messages',\n cfr.cfr AS 'Change Failure Rate'\nFROM ai_tests ai\nLEFT JOIN cfr_monthly cfr ON ai.month = cfr.month\nWHERE ai.month IS NOT NULL OR cfr.month IS NOT NULL\nORDER BY time", "refId": "A" } ], - "title": "AI Test Generation vs Change Failure Rate", + "title": "Kiro Messages vs Change Failure Rate", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Compare active Q Dev users with Code Review findings. More AI-assisted code review might catch issues earlier.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro does not export code-review findings. Shows active users and credits by month.", "fieldConfig": { "defaults": { "color": { @@ -927,30 +932,7 @@ { "matcher": { "id": "byName", - "options": "Active Users" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "left" - }, - { - "id": "custom.axisLabel", - "value": "Users" - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Code Review Findings" + "options": "Kiro Credits" }, "properties": [ { @@ -981,7 +963,10 @@ "id": 13, "options": { "legend": { - "calcs": ["mean", "max"], + "calcs": [ + "mean", + "max" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -995,15 +980,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT \n STR_TO_DATE(DATE_FORMAT(date, '%Y-%m-01'), '%Y-%m-%d') AS time,\n COUNT(DISTINCT user_id) AS 'Active Users',\n SUM(code_review_findings_count) AS 'Code Review Findings'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY DATE_FORMAT(date, '%Y-%m-01')\nORDER BY time", + "rawSql": "SELECT STR_TO_DATE(month_start, '%Y-%m-%d') AS time, active_users AS 'Active Users', kiro_credits AS 'Kiro Credits' FROM (SELECT DATE_FORMAT(date, '%Y-%m-01') AS month_start, COUNT(DISTINCT user_id) AS active_users, SUM(credits_used) AS kiro_credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE_FORMAT(date, '%Y-%m-01')) monthly ORDER BY time", "refId": "A" } ], - "title": "Q Dev Users vs Code Review Findings", + "title": "Kiro Users vs Credits", "type": "timeseries" }, { @@ -1020,8 +1008,11 @@ "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Monthly summary comparing Q Dev AI metrics with DORA metrics side by side", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Monthly Kiro usage metrics beside DORA metrics. LOC, acceptance, generated tests, and review findings are not exported by Kiro.", "fieldConfig": { "defaults": { "color": { @@ -1050,7 +1041,7 @@ { "matcher": { "id": "byName", - "options": "AI Acceptance Rate" + "options": "Credits per Message" }, "properties": [ { @@ -1075,47 +1066,6 @@ "value": 1 } ] - }, - { - "matcher": { - "id": "byName", - "options": "Change Failure Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-RdYlGr" - } - }, - { - "id": "max", - "value": 0.3 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Lead Time (hours)" - }, - "properties": [ - { - "id": "unit", - "value": "h" - } - ] } ] }, @@ -1131,7 +1081,9 @@ "footer": { "countRows": false, "fields": "", - "reducer": ["sum"], + "reducer": [ + "sum" + ], "show": false }, "showHeader": true, @@ -1145,15 +1097,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_metrics AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m') AS month,\n COUNT(DISTINCT user_id) AS active_users,\n SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) AS acceptance_rate,\n SUM(test_generation_generated_tests) AS generated_tests,\n SUM(code_review_findings_count) AS review_findings\n FROM _tool_q_dev_user_data\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m')\n),\ndora_metrics AS (\n SELECT\n DATE_FORMAT(cdc.finished_date, '%Y-%m') AS month,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deployments,\n AVG(ppm.pr_cycle_time) / 60 AS avg_lead_time\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n LEFT JOIN cicd_deployment_commits cdc2 ON cdc.cicd_deployment_id = cdc2.cicd_deployment_id\n LEFT JOIN project_pr_metrics ppm ON ppm.deployment_commit_id = cdc2.id\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m')\n),\ncfr_metrics AS (\n SELECT\n DATE_FORMAT(deployment_finished_date, '%Y-%m') AS month,\n SUM(has_incident) / NULLIF(COUNT(deployment_id), 0) AS cfr\n FROM (\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN d.deployment_id ELSE NULL END) AS has_incident\n FROM (\n SELECT\n cdc.cicd_deployment_id AS deployment_id,\n MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n GROUP BY 1\n HAVING $__timeFilter(MAX(cdc.finished_date))\n ) d\n LEFT JOIN project_incident_deployment_relationships pim ON d.deployment_id = pim.deployment_id\n LEFT JOIN incidents i ON pim.id = i.id\n GROUP BY 1, 2\n ) failure_data\n GROUP BY DATE_FORMAT(deployment_finished_date, '%Y-%m')\n)\nSELECT \n COALESCE(ai.month, dm.month, cfr.month) AS 'Month',\n COALESCE(ai.active_users, 0) AS 'Q Dev Users',\n COALESCE(ai.ai_lines, 0) AS 'AI Code Lines',\n ai.acceptance_rate AS 'AI Acceptance Rate',\n COALESCE(ai.generated_tests, 0) AS 'AI Tests',\n COALESCE(ai.review_findings, 0) AS 'Review Findings',\n COALESCE(dm.deployments, 0) AS 'Deployments',\n ROUND(dm.avg_lead_time, 1) AS 'Lead Time (hours)',\n cfr.cfr AS 'Change Failure Rate'\nFROM ai_metrics ai\nLEFT JOIN dora_metrics dm ON ai.month = dm.month\nLEFT JOIN cfr_metrics cfr ON ai.month = cfr.month\nORDER BY ai.month DESC", + "rawSql": "WITH ai_metrics AS (\n SELECT \n DATE_FORMAT(date, '%Y-%m') AS month,\n COUNT(DISTINCT user_id) AS active_users,\n SUM(credits_used) AS credits,\n SUM(credits_used) / NULLIF(SUM(total_messages), 0) AS credits_per_message,\n SUM(total_messages) AS messages,\n SUM(chat_conversations) AS conversations\n FROM _tool_kiro_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_FORMAT(date, '%Y-%m')\n),\ndora_metrics AS (\n SELECT\n DATE_FORMAT(cdc.finished_date, '%Y-%m') AS month,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deployments,\n AVG(ppm.pr_cycle_time) / 60 AS avg_lead_time\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n LEFT JOIN cicd_deployment_commits cdc2 ON cdc.cicd_deployment_id = cdc2.cicd_deployment_id\n LEFT JOIN project_pr_metrics ppm ON ppm.deployment_commit_id = cdc2.id\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_FORMAT(cdc.finished_date, '%Y-%m')\n),\ncfr_metrics AS (\n SELECT\n DATE_FORMAT(deployment_finished_date, '%Y-%m') AS month,\n SUM(has_incident) / NULLIF(COUNT(deployment_id), 0) AS cfr\n FROM (\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN d.deployment_id ELSE NULL END) AS has_incident\n FROM (\n SELECT\n cdc.cicd_deployment_id AS deployment_id,\n MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE pm.project_name IN (${project})\n AND cdc.result = 'SUCCESS'\n AND cdc.environment = 'PRODUCTION'\n GROUP BY 1\n HAVING $__timeFilter(MAX(cdc.finished_date))\n ) d\n LEFT JOIN project_incident_deployment_relationships pim ON d.deployment_id = pim.deployment_id\n LEFT JOIN incidents i ON pim.id = i.id\n GROUP BY 1, 2\n ) failure_data\n GROUP BY DATE_FORMAT(deployment_finished_date, '%Y-%m')\n)\nSELECT \n COALESCE(ai.month, dm.month, cfr.month) AS 'Month',\n COALESCE(ai.active_users, 0) AS 'Kiro Users',\n COALESCE(ai.credits, 0) AS 'Credits Used',\n ai.credits_per_message AS 'Credits per Message',\n COALESCE(ai.messages, 0) AS 'Messages',\n COALESCE(ai.conversations, 0) AS 'Conversations',\n COALESCE(dm.deployments, 0) AS 'Deployments',\n ROUND(dm.avg_lead_time, 1) AS 'Lead Time (hours)',\n cfr.cfr AS 'Change Failure Rate'\nFROM ai_metrics ai\nLEFT JOIN dora_metrics dm ON ai.month = dm.month\nLEFT JOIN cfr_metrics cfr ON ai.month = cfr.month\nORDER BY ai.month DESC", "refId": "A" } ], - "title": "Monthly Q Dev vs DORA Metrics Comparison", + "title": "Monthly Kiro + DORA Metrics", "type": "table" } ], @@ -1161,7 +1116,7 @@ "refresh": "5m", "schemaVersion": 39, "tags": [ - "q_dev", + "kiro", "DORA", "AI", "correlation" @@ -1171,10 +1126,17 @@ { "current": { "selected": true, - "text": ["All"], - "value": ["$__all"] + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" }, - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "SELECT DISTINCT name FROM projects", "hide": 0, "includeAll": true, @@ -1197,7 +1159,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Q Dev + DORA Correlation", - "uid": "qdev_dora_correlation", + "title": "Kiro + DORA Correlation", + "uid": "kiro_dora_correlation", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_user_data.json b/grafana/dashboards/mysql/kiro_activity_detail.json similarity index 69% rename from grafana/dashboards/postgresql/qdev_user_data.json rename to grafana/dashboards/mysql/kiro_activity_detail.json index d446fca81db..ed5c615c4db 100644 --- a/grafana/dashboards/postgresql/qdev_user_data.json +++ b/grafana/dashboards/mysql/kiro_activity_detail.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Overview of key user metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Usage-report totals. Accepted LOC and acceptance rates are not exported by Kiro.", "fieldConfig": { "defaults": { "color": { @@ -66,13 +69,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", SUM(chat_ai_code_lines) AS \"Accepted Lines (Chat)\", SUM(inline_ai_code_lines) AS \"Accepted Lines (Inline Suggestion)\", CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Acceptance Rate (Inline Suggestion)\", SUM(code_review_findings_count) AS \"Findings (Code Review)\", SUM(code_fix_accepted_lines) AS \"Accepted Lines (Code Fix)\", CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) AS \"Acceptance Rate (Code Fix)\", SUM(transformation_lines_ingested) AS \"Ingested Lines (Java Transform)\", SUM(transformation_lines_generated) AS \"Generated Lines (Java Transform)\", SUM(inline_chat_accepted_line_additions) AS \"Accepted Lines (Inline Chat)\", CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) AS \"Acceptance Rate (Inline Chat)\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS 'Active Users', ROUND(SUM(credits_used), 1) AS 'Credits Used', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A", "select": [ [ @@ -115,8 +121,11 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily AI code line changes across all users", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Daily Kiro usage volume from the user report.", "fieldConfig": { "defaults": { "color": { @@ -195,13 +204,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(chat_ai_code_lines) AS \"Chat Accepted Lines\", SUM(code_fix_accepted_lines) AS \"Code Fix Accepted Lines\", SUM(code_fix_generated_lines) AS \"Code Fix Generated Lines\", SUM(transformation_lines_ingested) AS \"Java Transform Ingested Lines\", SUM(transformation_lines_generated) AS \"Java Transform Generated Lines\", SUM(inline_ai_code_lines) AS \"Inline Suggestion Accepted Lines\", SUM(inline_chat_accepted_line_additions) AS \"Inline Chat Accepted Line Additions\", SUM(inline_chat_accepted_line_deletions) AS \"Inline Chat Accepted Line Deletions\", SUM(inline_chat_dismissed_line_additions) AS \"Inline Chat Dismissed Line Additions\", SUM(inline_chat_dismissed_line_deletions) AS \"Inline Chat Dismissed Line Deletions\", SUM(inline_chat_rejected_line_additions) AS \"Inline Chat Rejected Line Additions\", SUM(inline_chat_rejected_line_deletions) AS \"Inline Chat Rejected Line Deletions\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(date) AS time, ROUND(SUM(credits_used), 1) AS 'Credits Used', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE(date) ORDER BY time", "refId": "A", "select": [ [ @@ -240,12 +252,15 @@ ] } ], - "title": "Daily AI Code Line Changes", + "title": "Daily Credits, Messages & Conversations", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily AI interaction trends across all users", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Chat events split into user turns and agent continuations.", "fieldConfig": { "defaults": { "color": { @@ -324,13 +339,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(chat_messages_sent) AS \"Chat Messages Sent\", SUM(code_fix_acceptance_event_count) AS \"Code Fix Accepted Event Count\", SUM(code_fix_generation_event_count) AS \"Code Fix Generated Event Count\", SUM(transformation_event_count) AS \"Java Transform Event Count\", SUM(inline_acceptance_count) AS \"Inline Suggestion Accepted Suggestions\", SUM(inline_suggestions_count) AS \"Inline Suggestion Count\", SUM(inline_chat_total_event_count) AS \"Inline Chat Total Suggestions\", SUM(inline_chat_acceptance_event_count) AS \"Inline Chat Accepted Suggestions\", SUM(inline_chat_dismissal_event_count) AS \"Inline Chat Dismissed Suggestions\", SUM(inline_chat_rejection_event_count) AS \"Inline Chat Rejected Suggestions\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(timestamp) AS time, COUNT(*) AS 'Chat Events', SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS 'User Turns', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A", "select": [ [ @@ -369,12 +387,15 @@ ] } ], - "title": "Daily AI Interactions", + "title": "Daily Chat Activity", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Code review metrics over time", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Authoritative per-model and routing-mode message counts.", "fieldConfig": { "defaults": { "color": { @@ -453,12 +474,15 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(code_fix_acceptance_event_count) AS \"Code Fix Accepted Event Count\", SUM(code_fix_generation_event_count) AS \"Code Fix Generated Event Count\", SUM(code_review_findings_count) AS \"Total Findings\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(date) AS time, model_name AS metric, SUM(message_count) AS value FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY DATE(date), model_name ORDER BY time", "refId": "A", "select": [ [ @@ -497,12 +521,15 @@ ] } ], - "title": "Code Review Metrics", + "title": "Daily Model / Route Messages", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily acceptance rate of AI suggestions", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Agent continuations divided by all chat events. This is not suggestion acceptance.", "fieldConfig": { "defaults": { "color": { @@ -581,13 +608,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) AS \"Code Fix\", CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Inline Suggestions\", CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) AS \"Inline Chat\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) AS 'Agent Continuation Rate' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A", "select": [ [ @@ -626,12 +656,15 @@ ] } ], - "title": "Daily AI Suggestion Acceptance Rate", + "title": "Daily Agent Continuation Rate", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "User AI interaction metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Per-user credits, messages, conversations, overage, and last activity.", "fieldConfig": { "defaults": { "color": { @@ -655,51 +688,7 @@ ] } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Acceptance Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inline Chat Accepted Events" - }, - "properties": [ - { - "id": "custom.width", - "value": 239 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Acceptance Rate (Inline Suggestion)" - }, - "properties": [ - { - "id": "custom.width", - "value": 172 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -724,13 +713,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", SUM(chat_ai_code_lines) AS \"Accepted Lines (Chat)\", SUM(transformation_lines_ingested) AS \"Lines Ingested (Java Transform)\", SUM(transformation_lines_generated) AS \"Lines Generated (Java Transform)\", SUM(transformation_event_count) AS \"Event Count (Java Transform)\", SUM(code_review_findings_count) AS \"Findings (Code Review)\", SUM(code_fix_accepted_lines) AS \"Accepted Lines (Code Fix)\", SUM(code_fix_generated_lines) AS \"Generated Lines (Code Fix)\", SUM(code_fix_acceptance_event_count) AS \"Accepted Count (Code Fix)\", SUM(code_fix_generation_event_count) AS \"Generated Count (Code Fix)\", ROUND(CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) * 100, 2) || '%' AS \"Acceptance Rate (Code Fix)\", SUM(inline_ai_code_lines) AS \"Accepted Lines (Inline Suggestion)\", SUM(inline_acceptance_count) AS \"Accepted Count (Inline Suggestion)\", SUM(inline_suggestions_count) AS \"Total Count (Inline Suggestion)\", ROUND(CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) * 100, 2) || '%' AS \"Acceptance Rate (Inline Suggestion)\", SUM(inline_chat_accepted_line_additions) AS \"Accepted Line Additions (Inline Chat)\", SUM(inline_chat_accepted_line_deletions) AS \"Accepted Line Deletions (Inline Chat)\", SUM(inline_chat_acceptance_event_count) AS \"Accepted Events (Inline Chat)\", SUM(inline_chat_total_event_count) AS \"Total Events (Inline Chat)\", ROUND(CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) * 100, 2) || '%' AS \"Acceptance Rate (Inline Chat)\", SUM(doc_generation_event_count) AS \"Doc Gen Events\", SUM(test_generation_event_count) AS \"Test Gen Events\", SUM(dev_accepted_lines) AS \"Dev Accepted Lines\", MIN(date) AS \"First Activity\", MAX(date) AS \"Last Activity\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"user_id\" ORDER BY SUM(inline_ai_code_lines) DESC NULLS LAST", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS 'User', MAX(subscription_tier) AS 'Tier', ROUND(SUM(credits_used), 1) AS 'Credits Used', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations', SUM(overage_credits_used) AS 'Overage Credits', MAX(date) AS 'Last Activity' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id ORDER BY SUM(credits_used) DESC", "refId": "A", "select": [ [ @@ -769,12 +761,15 @@ ] } ], - "title": "User Interactions", + "title": "Per-User Usage", "type": "table" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily doc generation events and accepted/rejected lines", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Messages by Kiro client type. Doc-generation metrics are not exported.", "fieldConfig": { "defaults": { "color": { @@ -853,13 +848,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(doc_generation_event_count) AS \"Doc Generation Events\", SUM(doc_generation_accepted_line_additions) AS \"Accepted Line Additions\", SUM(doc_generation_accepted_line_updates) AS \"Accepted Line Updates\", SUM(doc_generation_rejected_line_additions) AS \"Rejected Line Additions\", SUM(doc_generation_rejected_line_updates) AS \"Rejected Line Updates\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(date) AS time, client_type AS metric, SUM(total_messages) AS value FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE(date), client_type ORDER BY time", "refId": "A", "select": [ [ @@ -898,12 +896,15 @@ ] } ], - "title": "Doc Generation Metrics", + "title": "Daily Client Type Messages", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily test generation events and lines", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Completion requests, requests with returned suggestions, and returned lines. These are not acceptance events.", "fieldConfig": { "defaults": { "color": { @@ -982,13 +983,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(test_generation_event_count) AS \"Test Generation Events\", SUM(test_generation_accepted_tests) AS \"Accepted Tests\", SUM(test_generation_generated_tests) AS \"Generated Tests\", SUM(test_generation_accepted_lines) AS \"Accepted Lines\", SUM(test_generation_generated_lines) AS \"Generated Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(timestamp) AS time, COUNT(*) AS 'Completion Requests', SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS 'Requests with Results', SUM(returned_line_count) AS 'Returned Lines' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A", "select": [ [ @@ -1027,12 +1031,15 @@ ] } ], - "title": "Test Generation Metrics", + "title": "Daily Completion Activity", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily agentic dev events and lines", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Steering, spec-mode, and agent-continuation events. Kiro does not export accepted agentic LOC.", "fieldConfig": { "defaults": { "color": { @@ -1111,13 +1118,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(dev_generation_event_count) AS \"Dev Generation Events\", SUM(dev_acceptance_event_count) AS \"Dev Acceptance Events\", SUM(dev_generated_lines) AS \"Dev Generated Lines\", SUM(dev_accepted_lines) AS \"Dev Accepted Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS 'Steering', SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS 'Spec Mode', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A", "select": [ [ @@ -1156,7 +1166,7 @@ ] } ], - "title": "Dev (Agentic) Metrics", + "title": "Daily Agentic Signals", "type": "timeseries" } ], @@ -1164,7 +1174,7 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", + "kiro", "user_data" ], "templating": { @@ -1176,7 +1186,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Kiro Code Metrics Dashboard", - "uid": "qdev_user_data-pg", + "title": "Kiro Activity Detail Dashboard", + "uid": "kiro_user_data", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/mysql/qdev_executive.json b/grafana/dashboards/mysql/kiro_executive.json similarity index 74% rename from grafana/dashboards/mysql/qdev_executive.json rename to grafana/dashboards/mysql/kiro_executive.json index fffbc99ba60..32b2c45a377 100644 --- a/grafana/dashboards/mysql/qdev_executive.json +++ b/grafana/dashboards/mysql/kiro_executive.json @@ -27,7 +27,7 @@ "title": "Usage (New)", "tooltip": "Kiro Usage Dashboard - Credits & Messages (new format)", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" }, { "asDropdown": false, @@ -36,10 +36,10 @@ "keepTime": true, "tags": [], "targetBlank": true, - "title": "Feature Metrics (Legacy)", - "tooltip": "Kiro Legacy Feature Metrics (old format)", + "title": "Feature Metrics", + "tooltip": "Kiro Feature Metrics", "type": "link", - "url": "/d/qdev_feature_metrics" + "url": "/d/kiro_feature_metrics" }, { "asDropdown": false, @@ -51,7 +51,7 @@ "title": "Prompt Logging", "tooltip": "Kiro AI Activity Insights - Prompt Logging", "type": "link", - "url": "/d/qdev_logging" + "url": "/d/kiro_logging" } ], "panels": [ @@ -69,7 +69,10 @@ "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Distinct users with chat activity in the last 7 days (from prompt logging)", "fieldConfig": { "defaults": { @@ -116,11 +119,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) as 'WAU'\nFROM lake._tool_q_dev_chat_log\nWHERE timestamp >= DATE_SUB(NOW(), INTERVAL 7 DAY)", + "rawSql": "SELECT COUNT(DISTINCT user_id) as 'WAU'\nFROM lake._tool_kiro_chat_log\nWHERE timestamp >= DATE_SUB(NOW(), INTERVAL 7 DAY)", "refId": "A" } ], @@ -128,8 +134,11 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Average credits spent per accepted line of code (new report + legacy metrics)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Usage efficiency from Kiro reports. Accepted LOC is not exported.", "fieldConfig": { "defaults": { "color": { @@ -175,20 +184,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(SUM(d.total_accepted), 0), 2) as 'Credits per Accepted Line'\nFROM (\n SELECT user_id, date, SUM(credits_used) as credits_used\n FROM lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id, date\n) r\nJOIN (\n SELECT user_id, date,\n (inline_ai_code_lines + chat_ai_code_lines + code_fix_accepted_lines + dev_accepted_lines) as total_accepted\n FROM lake._tool_q_dev_user_data\n WHERE $__timeFilter(date)\n) d ON r.user_id = d.user_id AND r.date = d.date", + "rawSql": "SELECT ROUND(SUM(credits_used) / NULLIF(SUM(total_messages), 0), 2) AS 'Credits per Message' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Credits Efficiency (new + legacy)", + "title": "Credits per Message", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Percentage of inline suggestions accepted (from legacy feature metrics)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Share of chat events generated by agent self-continuation. Suggestion acceptance is not exported.", "fieldConfig": { "defaults": { "color": { @@ -234,19 +249,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 1) as 'Acceptance %'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0), 1) AS 'Agent Continuation %' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], - "title": "Inline Acceptance Rate (legacy)", + "title": "Agent Continuation Rate", "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Percentage of users who used steering rules (from prompt logging)", "fieldConfig": { "defaults": { @@ -294,11 +315,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0) as 'Steering %'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0) as 'Steering %'\nFROM lake._tool_kiro_chat_log\nWHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -315,11 +339,14 @@ }, "id": 101, "panels": [], - "title": "User Engagement (logging data: _tool_q_dev_chat_log)", + "title": "User Engagement", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Weekly active user count over time (from prompt logging)", "fieldConfig": { "defaults": { @@ -399,11 +426,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM (\n SELECT user_id, YEARWEEK(timestamp, 1) as yw\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n) t\nGROUP BY yw\nORDER BY time", + "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM (\n SELECT user_id, YEARWEEK(timestamp, 1) as yw\n FROM lake._tool_kiro_chat_log\n WHERE $__timeFilter(timestamp)\n) t\nGROUP BY yw\nORDER BY time", "refId": "A" } ], @@ -411,8 +441,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "New vs returning users by week (from prompt logging)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "New users use the native is_new_user flag; early rows without that field are reported as existing/unknown.", "fieldConfig": { "defaults": { "color": { @@ -491,15 +524,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n SUM(CASE WHEN yw = first_yw THEN 1 ELSE 0 END) as 'New Users',\n SUM(CASE WHEN yw != first_yw THEN 1 ELSE 0 END) as 'Returning Users'\nFROM (\n SELECT DISTINCT u.user_id, YEARWEEK(u.timestamp, 1) as yw, f.first_yw\n FROM lake._tool_q_dev_chat_log u\n JOIN (SELECT user_id, YEARWEEK(MIN(timestamp), 1) as first_yw FROM lake._tool_q_dev_chat_log GROUP BY user_id) f\n ON u.user_id = f.user_id\n WHERE $__timeFilter(u.timestamp)\n) weekly\nGROUP BY yw\nORDER BY time", + "rawSql": "SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time, COUNT(DISTINCT CASE WHEN is_new_user = 1 THEN user_id END) AS 'New Users', COUNT(DISTINCT CASE WHEN is_new_user = 0 OR is_new_user IS NULL THEN user_id END) AS 'Existing/Unknown Users' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) ORDER BY time", "refId": "A" } ], - "title": "New vs Returning Users (Weekly)", + "title": "New vs Existing/Unknown Users (Weekly)", "type": "timeseries" }, { @@ -512,11 +548,14 @@ }, "id": 102, "panels": [], - "title": "Credits & Subscription (new format: _tool_q_dev_user_report)", + "title": "Credits & Subscription", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Cumulative credits this month vs projected total (from new user_report)", "fieldConfig": { "defaults": { @@ -596,11 +635,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(SUM(credits_used)) OVER (ORDER BY date) as 'Cumulative Credits',\n (SELECT SUM(credits_used) / COUNT(DISTINCT date) * DAY(LAST_DAY(CURDATE()))\n FROM lake._tool_q_dev_user_report\n WHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())) as 'Projected Monthly'\nFROM lake._tool_q_dev_user_report\nWHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT\n date as time,\n SUM(SUM(credits_used)) OVER (ORDER BY date) as 'Cumulative Credits',\n (SELECT SUM(credits_used) / COUNT(DISTINCT date) * DAY(LAST_DAY(CURDATE()))\n FROM lake._tool_kiro_user_report\n WHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())) as 'Projected Monthly'\nFROM lake._tool_kiro_user_report\nWHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())\nGROUP BY date\nORDER BY date", "refId": "A" } ], @@ -608,7 +650,10 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Power tier users with no activity in the last 14 days (from new user_report)", "fieldConfig": { "defaults": { @@ -658,11 +703,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as 'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) < DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)", + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as 'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) < DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)", "refId": "A" } ], @@ -679,12 +727,15 @@ }, "id": 103, "panels": [], - "title": "Cross-Source: User Productivity (new report + legacy metrics)", + "title": "Per-User Usage & Efficiency", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Per-user productivity combining credits (new format) with feature metrics (legacy). Only shows users present in both data sources.", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Credits, messages, conversations, and credits per message. Accepted LOC is not exported.", "fieldConfig": { "defaults": { "color": { @@ -733,15 +784,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as 'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as 'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 2)\n ELSE NULL END as 'Credits/Line',\n CONCAT(ROUND(SUM(d.inline_acceptance_count) / NULLIF(SUM(d.inline_suggestions_count), 0) * 100, 1), '%') as 'Accept Rate',\n SUM(d.code_review_findings_count) as 'Review Findings',\n SUM(d.test_generation_event_count) as 'Test Gen Events',\n SUM(d.dev_accepted_lines) as 'Agentic Lines',\n MIN(d.date) as 'First Active',\n MAX(d.date) as 'Last Active'\nFROM lake._tool_q_dev_user_data d\nLEFT JOIN (\n SELECT user_id, date, SUM(credits_used) as credits_used, MAX(subscription_tier) as subscription_tier\n FROM lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id, date\n) r ON d.user_id = r.user_id AND d.date = r.date\nWHERE $__timeFilter(d.date)\nGROUP BY d.user_id\nORDER BY SUM(r.credits_used) DESC", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS 'User', MAX(subscription_tier) AS 'Tier', ROUND(SUM(credits_used), 1) AS 'Credits Used', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations', ROUND(SUM(credits_used) / NULLIF(SUM(total_messages), 0), 3) AS 'Credits per Message', MAX(date) AS 'Last Activity' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id ORDER BY SUM(credits_used) DESC", "refId": "A" } ], - "title": "User Productivity & Efficiency", + "title": "Per-User Usage & Efficiency", "type": "table" } ], @@ -749,9 +803,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "executive", - "kiro" + "kiro", + "executive" ], "templating": { "list": [] @@ -763,6 +816,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro Executive Dashboard", - "uid": "qdev_executive", + "uid": "kiro_executive", "version": 1 } diff --git a/grafana/dashboards/mysql/qdev_feature_metrics.json b/grafana/dashboards/mysql/kiro_feature_metrics.json similarity index 71% rename from grafana/dashboards/mysql/qdev_feature_metrics.json rename to grafana/dashboards/mysql/kiro_feature_metrics.json index 5156e652a34..58604d0234c 100644 --- a/grafana/dashboards/mysql/qdev_feature_metrics.json +++ b/grafana/dashboards/mysql/kiro_feature_metrics.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "High-level summary of legacy feature-level activity metrics (from by_user_analytic CSV reports)", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro usage-report metrics. Legacy accepted LOC and feature counters are not exported.", "fieldConfig": { "defaults": { "color": { @@ -66,15 +69,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(inline_suggestions_count) as 'Inline Suggestions',\n SUM(inline_acceptance_count) as 'Inline Accepted',\n SUM(chat_messages_sent) as 'Chat Messages',\n SUM(chat_ai_code_lines) as 'Chat AI Lines',\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Agentic Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS 'Active Users', SUM(total_messages) AS 'Messages', SUM(chat_conversations) AS 'Conversations', ROUND(SUM(credits_used), 1) AS 'Credits Used' FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Legacy Feature Metrics Overview", + "title": "Kiro Usage Overview", "type": "stat" }, { @@ -87,12 +93,15 @@ }, "id": 20, "panels": [], - "title": "Inline Suggestions", + "title": "Completions", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily inline suggestion and acceptance counts", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Returned suggestions are offers, not acceptance events.", "fieldConfig": { "defaults": { "color": { @@ -170,20 +179,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(inline_suggestions_count) as 'Suggestions',\n SUM(inline_acceptance_count) as 'Accepted',\n SUM(inline_ai_code_lines) as 'AI Code Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, COUNT(*) AS 'Completion Requests', SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS 'Requests with Results', SUM(returned_line_count) AS 'Returned Lines' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Inline Suggestions & Acceptance", + "title": "Completion Requests & Results", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Percentage of completion requests that returned at least one suggestion. Kiro does not export acceptance.", "fieldConfig": { "defaults": { "color": { @@ -261,15 +276,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) AS 'Suggestion Return Rate' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Acceptance Rate Trends", + "title": "Suggestion Return Rate", "type": "timeseries" }, { @@ -282,12 +300,15 @@ }, "id": 21, "panels": [], - "title": "Chat & Agentic (Dev)", + "title": "Chat & Agent Continuations", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily chat messages sent and AI-generated code lines from chat", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "User turns and agent self-continuations from chat logs.", "fieldConfig": { "defaults": { "color": { @@ -365,11 +386,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(chat_messages_sent) as 'Messages Sent',\n SUM(chat_messages_interacted) as 'Messages Interacted',\n SUM(chat_ai_code_lines) as 'AI Code Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, COUNT(*) AS 'Chat Events', SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS 'User Turns', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], @@ -377,8 +401,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Agentic (Dev) code generation and acceptance metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Agent continuations are empty-prompt records after tool calls; this measures autonomy, not accepted code.", "fieldConfig": { "defaults": { "color": { @@ -456,15 +483,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(dev_generation_event_count) as 'Generation Events',\n SUM(dev_generated_lines) as 'Generated Lines',\n SUM(dev_accepted_lines) as 'Accepted Lines',\n SUM(dev_acceptance_event_count) as 'Acceptance Events'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS 'User Turns', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) AS 'Agent Continuation Rate' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Agentic (Dev) Activity", + "title": "Agent Continuation Activity", "type": "timeseries" }, { @@ -477,12 +507,15 @@ }, "id": 22, "panels": [], - "title": "Code Review, Test Gen & Transformations", + "title": "Models, Prompts & Clients", "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Code review findings and test generation metrics over time", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Authoritative model and routing-mode message counts from daily reports.", "fieldConfig": { "defaults": { "color": { @@ -560,20 +593,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(code_review_succeeded_event_count) as 'Reviews Succeeded',\n SUM(code_review_failed_event_count) as 'Reviews Failed'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(date) AS time, model_name AS metric, SUM(message_count) AS value FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY DATE(date), model_name ORDER BY time", "refId": "A" } ], - "title": "Code Review Activity", + "title": "Model / Route Message Mix", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Test generation events and acceptance over time", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Daily average prompt and response lengths. Test-generation counts are not exported.", "fieldConfig": { "defaults": { "color": { @@ -651,20 +690,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(test_generation_generated_tests) as 'Tests Generated',\n SUM(test_generation_accepted_tests) as 'Tests Accepted',\n SUM(test_generation_generated_lines) as 'Lines Generated',\n SUM(test_generation_accepted_lines) as 'Lines Accepted'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(timestamp) AS time, ROUND(AVG(prompt_length)) AS 'Avg Prompt Length', ROUND(AVG(response_length)) AS 'Avg Response Length' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Test Generation Activity", + "title": "Prompt & Response Length", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Doc generation and code transformation events", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Messages by KIRO_IDE, KIRO_CLI, KIRO_WEB, and PLUGIN. Doc-generation metrics are not exported.", "fieldConfig": { "defaults": { "color": { @@ -742,20 +787,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(doc_generation_accepted_line_additions) as 'Doc Lines Accepted',\n SUM(transformation_event_count) as 'Transformation Events',\n SUM(transformation_lines_generated) as 'Transform Lines Generated'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT DATE(date) AS time, client_type AS metric, SUM(total_messages) AS value FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY DATE(date), client_type ORDER BY time", "refId": "A" } ], - "title": "Doc Generation & Transformations", + "title": "Client Type Message Mix", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Number of users who used each feature in the selected period", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Distinct users observed in chat, completion, steering, and spec-mode signals.", "fieldConfig": { "defaults": { "color": { @@ -838,11 +889,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT 'Chat' as Feature, COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) as Users FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Review', COUNT(DISTINCT CASE WHEN code_review_succeeded_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Doc Generation', COUNT(DISTINCT CASE WHEN doc_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Test Generation', COUNT(DISTINCT CASE WHEN test_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Dev (Agentic)', COUNT(DISTINCT CASE WHEN dev_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Transformation', COUNT(DISTINCT CASE WHEN transformation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT 'Chat' AS 'Feature', COUNT(DISTINCT user_id) AS 'Users' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Completion' AS 'Feature', COUNT(DISTINCT user_id) AS 'Users' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Steering' AS 'Feature', COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) AS 'Users' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Spec Mode' AS 'Feature', COUNT(DISTINCT CASE WHEN is_spec_mode = 1 THEN user_id END) AS 'Users' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -863,8 +917,11 @@ "type": "row" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Per-user breakdown of legacy feature-level metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Per-user chat, completion, autonomy, steering, and spec-mode activity.", "fieldConfig": { "defaults": { "color": { @@ -913,15 +970,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n SUM(inline_suggestions_count) as 'Suggestions',\n SUM(inline_acceptance_count) as 'Accepted',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 1), '%') as 'Accept %',\n SUM(chat_messages_sent) as 'Chat Msgs',\n SUM(chat_ai_code_lines) as 'Chat Lines',\n SUM(dev_accepted_lines) as 'Agentic Lines',\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_accepted_tests) as 'Tests Accepted',\n SUM(doc_generation_event_count) as 'Doc Gen',\n SUM(transformation_event_count) as 'Transforms',\n MIN(date) as 'First Active',\n MAX(date) as 'Last Active'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id\nORDER BY SUM(inline_suggestions_count) DESC", + "rawSql": "WITH users AS (SELECT user_id FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION SELECT user_id FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp)), chat AS (SELECT user_id, COUNT(*) AS chat_events, SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS user_turns, SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS agent_continuations, SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS steering, SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS spec_mode FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id), completion AS (SELECT user_id, COUNT(*) AS completion_requests, SUM(returned_line_count) AS returned_lines FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id), names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, u.user_id) AS 'User', COALESCE(c.chat_events, 0) AS 'Chat Events', COALESCE(c.user_turns, 0) AS 'User Turns', COALESCE(c.agent_continuations, 0) AS 'Agent Continuations', COALESCE(x.completion_requests, 0) AS 'Completion Requests', COALESCE(x.returned_lines, 0) AS 'Returned Lines', COALESCE(c.steering, 0) AS 'Steering', COALESCE(c.spec_mode, 0) AS 'Spec Mode' FROM users u LEFT JOIN chat c ON u.user_id = c.user_id LEFT JOIN completion x ON u.user_id = x.user_id LEFT JOIN names n ON u.user_id = n.user_id ORDER BY COALESCE(c.chat_events, 0) DESC", "refId": "A" } ], - "title": "Per-User Feature Metrics", + "title": "Per-User Kiro Activity", "type": "table" } ], @@ -929,8 +989,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "legacy", "kiro" ], "templating": { @@ -942,7 +1000,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Kiro Legacy Feature Metrics", - "uid": "qdev_feature_metrics", + "title": "Kiro Feature Metrics", + "uid": "kiro_feature_metrics", "version": 1 } diff --git a/grafana/dashboards/mysql/qdev_logging.json b/grafana/dashboards/mysql/kiro_logging.json similarity index 74% rename from grafana/dashboards/mysql/qdev_logging.json rename to grafana/dashboards/mysql/kiro_logging.json index aa7826ae243..538b907aab9 100644 --- a/grafana/dashboards/mysql/qdev_logging.json +++ b/grafana/dashboards/mysql/kiro_logging.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Overview of logging event metrics", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Chat users, user turns, agent continuations, and completion events.", "fieldConfig": { "defaults": { "color": { @@ -66,11 +69,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Users',\n (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id != '') as 'Conversations',\n (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Completion Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Completion Users',\n (SELECT SUM(code_reference_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Code References',\n (SELECT SUM(web_link_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Web Links Cited'", + "rawSql": "SELECT (SELECT COUNT(*) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS 'Chat Events', (SELECT COUNT(DISTINCT user_id) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS 'Chat Users', (SELECT SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS 'User Turns', (SELECT SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS 'Agent Continuations', (SELECT COUNT(*) FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp)) AS 'Completion Events'", "refId": "A" } ], @@ -78,7 +84,10 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Hourly distribution of AI usage activity (chat + completions)", "fieldConfig": { "defaults": { @@ -155,11 +164,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n LPAD(CAST(hour_of_day AS CHAR), 2, '0') as 'Hour',\n SUM(chat_count) as 'Chat Events',\n SUM(completion_count) as 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) as hour_of_day, COUNT(*) as chat_count, 0 as completion_count\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) as hour_of_day, 0 as chat_count, COUNT(*) as completion_count\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) combined\nGROUP BY hour_of_day\nORDER BY hour_of_day", + "rawSql": "SELECT\n LPAD(CAST(hour_of_day AS CHAR), 2, '0') as 'Hour',\n SUM(chat_count) as 'Chat Events',\n SUM(completion_count) as 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) as hour_of_day, COUNT(*) as chat_count, 0 as completion_count\n FROM lake._tool_kiro_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) as hour_of_day, 0 as chat_count, COUNT(*) as completion_count\n FROM lake._tool_kiro_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) combined\nGROUP BY hour_of_day\nORDER BY hour_of_day", "refId": "A" } ], @@ -167,7 +179,10 @@ "type": "barchart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Distribution of chat trigger types: MANUAL (chat window) vs INLINE_CHAT", "fieldConfig": { "defaults": { @@ -223,11 +238,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)'\n ELSE chat_trigger_type\n END as 'Trigger Type',\n COUNT(*) as 'Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY chat_trigger_type\nORDER BY COUNT(*) DESC", + "rawSql": "SELECT\n CASE\n WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)'\n ELSE chat_trigger_type\n END as 'Trigger Type',\n COUNT(*) as 'Events'\nFROM lake._tool_kiro_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY chat_trigger_type\nORDER BY COUNT(*) DESC", "refId": "A" } ], @@ -235,8 +253,11 @@ "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Distribution of model usage across chat events", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Authoritative model and routing-mode counts from daily reports; chat model_id is only partially populated.", "fieldConfig": { "defaults": { "color": { @@ -280,7 +301,7 @@ "calcs": [ "lastNotNull" ], - "fields": "/^Requests$/", + "fields": "/^Messages$/", "values": true }, "tooltip": { @@ -291,19 +312,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN model_id = '' OR model_id IS NULL THEN '(unknown)'\n ELSE model_id\n END as 'Model',\n COUNT(*) as 'Requests'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY model_id\nORDER BY COUNT(*) DESC", + "rawSql": "SELECT model_name AS 'Model / Route', SUM(message_count) AS 'Messages' FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY model_name ORDER BY SUM(message_count) DESC", "refId": "A" } ], - "title": "Model Usage Distribution", + "title": "Model / Route Message Distribution", "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Top file extensions used with inline completions", "fieldConfig": { "defaults": { @@ -359,11 +386,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN file_extension = '' THEN '(unknown)'\n ELSE file_extension\n END as 'File Type',\n COUNT(*) as 'Completions'\nFROM lake._tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "rawSql": "SELECT\n CASE\n WHEN file_extension = '' THEN '(unknown)'\n ELSE file_extension\n END as 'File Type',\n COUNT(*) as 'Completions'\nFROM lake._tool_kiro_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", "refId": "A" } ], @@ -371,8 +401,11 @@ "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Average number of chat events per conversation", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Conversation IDs are absent in sampled Kiro logs, so conversation depth cannot be computed.", "fieldConfig": { "defaults": { "color": { @@ -451,19 +484,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(timestamp) as time,\n COUNT(*) / NULLIF(COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END), 0) as 'Avg Turns per Conversation',\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as 'Unique Conversations',\n COUNT(*) as 'Total Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS 'User Turns', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS 'Agent Continuations', SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) AS 'Agent Continuation Rate' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Conversation Depth Analysis", + "title": "User Turns vs Agent Continuations", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Daily chat and completion events over time", "fieldConfig": { "defaults": { @@ -543,11 +582,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(chat) as 'Chat Events', SUM(completions) as 'Completion Events'\nFROM (\n SELECT DATE(timestamp) as time, COUNT(*) as chat, 0 as completions\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n UNION ALL\n SELECT DATE(timestamp) as time, 0 as chat, COUNT(*) as completions\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n) combined\nGROUP BY time\nORDER BY time", + "rawSql": "SELECT time, SUM(chat) as 'Chat Events', SUM(completions) as 'Completion Events'\nFROM (\n SELECT DATE(timestamp) as time, COUNT(*) as chat, 0 as completions\n FROM lake._tool_kiro_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n UNION ALL\n SELECT DATE(timestamp) as time, 0 as chat, COUNT(*) as completions\n FROM lake._tool_kiro_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n) combined\nGROUP BY time\nORDER BY time", "refId": "A" } ], @@ -555,8 +597,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Per-user logging activity summary", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Per-user chat and completion activity with display names resolved from usage reports.", "fieldConfig": { "defaults": { "color": { @@ -605,11 +650,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as 'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n u.conversations as 'Conversations',\n ROUND(u.chat_events / NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events, 0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as 'Avg Response Len',\n u.steering_count as 'Steering Uses',\n u.spec_count as 'Spec Mode Uses',\n u.code_ref_count as 'Code Refs',\n u.web_link_count as 'Web Links',\n u.models_used as 'Models Used',\n u.first_seen as 'First Seen',\n GREATEST(u.last_seen, COALESCE(c.last_seen, u.last_seen)) as 'Last Seen'\nFROM (\n SELECT\n user_id,\n MAX(display_name) as display_name,\n COUNT(*) as chat_events,\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as conversations,\n AVG(prompt_length) as avg_prompt_len,\n AVG(response_length) as avg_response_len,\n GROUP_CONCAT(DISTINCT CASE WHEN model_id != '' AND model_id IS NOT NULL THEN model_id END ORDER BY model_id SEPARATOR ', ') as models_used,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as steering_count,\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as spec_count,\n SUM(code_reference_count) as code_ref_count,\n SUM(web_link_count) as web_link_count,\n MIN(timestamp) as first_seen,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) u\nLEFT JOIN (\n SELECT\n user_id,\n COUNT(*) as completion_events,\n COUNT(DISTINCT file_name) as files_count,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) c ON u.user_id = c.user_id\nORDER BY u.user_id", + "rawSql": "WITH chat AS (SELECT user_id, COUNT(*) AS chat_events, SUM(CASE WHEN has_prompt = 1 THEN 1 ELSE 0 END) AS user_turns, SUM(CASE WHEN has_prompt = 0 THEN 1 ELSE 0 END) AS agent_continuations FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id), completion AS (SELECT user_id, COUNT(*) AS completion_events, SUM(returned_line_count) AS returned_lines FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id), names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, c.user_id) AS 'User', c.user_id AS 'User ID', c.chat_events AS 'Chat Events', c.user_turns AS 'User Turns', c.agent_continuations AS 'Agent Continuations', COALESCE(x.completion_events, 0) AS 'Completion Events', COALESCE(x.returned_lines, 0) AS 'Returned Lines' FROM chat c LEFT JOIN completion x ON c.user_id = x.user_id LEFT JOIN names n ON c.user_id = n.user_id ORDER BY c.chat_events DESC", "refId": "A" } ], @@ -617,7 +665,10 @@ "type": "table" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Distribution of Kiro feature adoption: Steering, Spec Mode, and Plain Chat", "fieldConfig": { "defaults": { @@ -673,11 +724,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as 'Using Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as 'Using Spec Mode',\n SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) as 'Plain Chat'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "rawSql": "SELECT\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as 'Using Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as 'Using Spec Mode',\n SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) as 'Plain Chat'\nFROM lake._tool_kiro_chat_log\nWHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -685,8 +739,11 @@ "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Top file extensions active during chat events", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Chat logs do not contain active file extensions; completion logs provide file types and returned lines.", "fieldConfig": { "defaults": { "color": { @@ -741,20 +798,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file active)'\n ELSE active_file_extension\n END as 'File Type',\n COUNT(*) as 'Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY active_file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS 'File Type', SUM(returned_line_count) AS 'Returned Lines' FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY file_extension ORDER BY SUM(returned_line_count) DESC", "refId": "A" } ], - "title": "Active File Types in Chat", + "title": "Returned Lines by File Type", "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "How often Kiro responses include code references and web links", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Kiro logs expose follow-up prompts but not code-reference or web-link counters.", "fieldConfig": { "defaults": { "color": { @@ -809,19 +872,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n SUM(CASE WHEN code_reference_count > 0 THEN 1 ELSE 0 END) as 'With Code References',\n SUM(CASE WHEN web_link_count > 0 THEN 1 ELSE 0 END) as 'With Web Links',\n SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) as 'With Followup Prompts',\n SUM(CASE WHEN code_reference_count = 0 AND web_link_count = 0 AND has_followup_prompts = 0 THEN 1 ELSE 0 END) as 'Plain Response'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "rawSql": "SELECT SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) AS 'With Follow-up Prompts', SUM(CASE WHEN has_followup_prompts = 0 THEN 1 ELSE 0 END) AS 'Without Follow-up Prompts' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], - "title": "Response Enrichment Breakdown", + "title": "Follow-up Prompt Breakdown", "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Average and maximum prompt/response lengths over time", "fieldConfig": { "defaults": { @@ -901,11 +970,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(timestamp) as time,\n AVG(prompt_length) as 'Avg Prompt Length',\n AVG(response_length) as 'Avg Response Length',\n MAX(prompt_length) as 'Max Prompt Length',\n MAX(response_length) as 'Max Response Length'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "rawSql": "SELECT\n DATE(timestamp) as time,\n AVG(prompt_length) as 'Avg Prompt Length',\n AVG(response_length) as 'Avg Response Length',\n MAX(prompt_length) as 'Max Prompt Length',\n MAX(response_length) as 'Max Response Length'\nFROM lake._tool_kiro_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", "refId": "A" } ], @@ -913,7 +985,10 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Average code context size provided to inline completions over time", "fieldConfig": { "defaults": { @@ -992,11 +1067,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(timestamp) as time,\n ROUND(AVG(left_context_length)) as 'Avg Left Context',\n ROUND(AVG(right_context_length)) as 'Avg Right Context',\n ROUND(AVG(left_context_length + right_context_length)) as 'Avg Total Context'\nFROM lake._tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "rawSql": "SELECT\n DATE(timestamp) as time,\n ROUND(AVG(left_context_length)) as 'Avg Left Context',\n ROUND(AVG(right_context_length)) as 'Avg Right Context',\n ROUND(AVG(left_context_length + right_context_length)) as 'Avg Total Context'\nFROM lake._tool_kiro_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", "refId": "A" } ], @@ -1004,8 +1082,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily trend of code references and web links in chat responses", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Daily responses with and without follow-up prompts. Code-reference and web-link counters are not exported.", "fieldConfig": { "defaults": { "color": { @@ -1083,15 +1164,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(timestamp) as time,\n SUM(code_reference_count) as 'Code References',\n SUM(web_link_count) as 'Web Links',\n SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) as 'Followup Prompts'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "rawSql": "SELECT DATE(timestamp) AS time, SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) AS 'With Follow-up Prompts', SUM(CASE WHEN has_followup_prompts = 0 THEN 1 ELSE 0 END) AS 'Without Follow-up Prompts' FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY DATE(timestamp) ORDER BY time", "refId": "A" } ], - "title": "Response Enrichment Trends", + "title": "Follow-up Prompt Trends", "type": "timeseries" } ], @@ -1099,9 +1183,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "logging", - "kiro" + "kiro", + "logging" ], "templating": { "list": [] @@ -1113,6 +1196,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro AI Activity Insights", - "uid": "qdev_logging", + "uid": "kiro_logging", "version": 1 } diff --git a/grafana/dashboards/mysql/qdev_user_report.json b/grafana/dashboards/mysql/kiro_user_report.json similarity index 82% rename from grafana/dashboards/mysql/qdev_user_report.json rename to grafana/dashboards/mysql/kiro_user_report.json index acd0f14b3b6..3941859756f 100644 --- a/grafana/dashboards/mysql/qdev_user_report.json +++ b/grafana/dashboards/mysql/kiro_user_report.json @@ -19,7 +19,10 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Overview of credits and usage metrics", "fieldConfig": { "defaults": { @@ -66,11 +69,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n SUM(credits_used) as 'Total Credits Used',\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(total_messages) as 'Total Messages',\n SUM(chat_conversations) as 'Total Conversations'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)", + "rawSql": "SELECT\n SUM(credits_used) as 'Total Credits Used',\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(total_messages) as 'Total Messages',\n SUM(chat_conversations) as 'Total Conversations'\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)", "refId": "A" } ], @@ -78,7 +84,10 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Daily credits consumed broken down by subscription tier", "fieldConfig": { "defaults": { @@ -158,11 +167,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n subscription_tier as metric,\n SUM(credits_used) as value\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY date, subscription_tier\nORDER BY date", + "rawSql": "SELECT\n date as time,\n subscription_tier as metric,\n SUM(credits_used) as value\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)\nGROUP BY date, subscription_tier\nORDER BY date", "refId": "A" } ], @@ -170,8 +182,11 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily messages and conversations broken down by client type", + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, + "description": "Daily messages by Kiro IDE, CLI, Web, and Plugin clients, plus conversations.", "fieldConfig": { "defaults": { "color": { @@ -250,11 +265,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) as 'Messages (IDE)',\n SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) as 'Messages (CLI)',\n SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) as 'Messages (Plugin)',\n SUM(chat_conversations) as 'Conversations'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT date AS time, SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) AS 'Messages (IDE)', SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) AS 'Messages (CLI)', SUM(CASE WHEN client_type = 'KIRO_WEB' THEN total_messages ELSE 0 END) AS 'Messages (Web)', SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) AS 'Messages (Plugin)', SUM(chat_conversations) AS 'Conversations' FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY date ORDER BY date", "refId": "A" } ], @@ -262,7 +280,10 @@ "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Distribution of users across subscription tiers", "fieldConfig": { "defaults": { @@ -318,11 +339,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n subscription_tier as 'Tier',\n COUNT(DISTINCT user_id) as 'Users'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier IS NOT NULL\n AND subscription_tier != ''\nGROUP BY subscription_tier\nORDER BY COUNT(DISTINCT user_id) DESC", + "rawSql": "SELECT\n subscription_tier as 'Tier',\n COUNT(DISTINCT user_id) as 'Users'\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier IS NOT NULL\n AND subscription_tier != ''\nGROUP BY subscription_tier\nORDER BY COUNT(DISTINCT user_id) DESC", "refId": "A" } ], @@ -330,7 +354,10 @@ "type": "piechart" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "description": "Per-user credits, messages, and subscription details", "fieldConfig": { "defaults": { @@ -429,11 +456,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "mysql", + "uid": "devlake-mysql-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, subscription_tier, client_type\nORDER BY user_id DESC", + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_kiro_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, subscription_tier, client_type\nORDER BY user_id DESC", "refId": "A" } ], @@ -445,9 +475,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "user_report", - "kiro" + "kiro", + "user_report" ], "templating": { "list": [] @@ -459,6 +488,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro Usage Dashboard", - "uid": "qdev_user_report", + "uid": "kiro_user_report", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/ai-model-roi.json b/grafana/dashboards/postgresql/ai-model-roi.json index ffbda08b227..1a1c1205f79 100644 --- a/grafana/dashboards/postgresql/ai-model-roi.json +++ b/grafana/dashboards/postgresql/ai-model-roi.json @@ -26,7 +26,7 @@ "targetBlank": true, "title": "Kiro Usage Dashboard", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" } ], "panels": [ @@ -87,7 +87,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], @@ -138,7 +138,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0), 0), 1) AS \"Credits / PR\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0), 0), 1) AS \"Credits / PR\" FROM _tool_kiro_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -189,7 +189,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0), 0), 1) AS \"Credits / Deploy\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0), 0), 1) AS \"Credits / Deploy\" FROM _tool_kiro_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -240,7 +240,7 @@ "datasource": "postgresql", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0), 0), 1) AS \"Credits / Issue\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0), 0), 1) AS \"Credits / Issue\" FROM _tool_kiro_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", "refId": "A" } ], @@ -316,7 +316,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -379,7 +379,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -442,7 +442,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -505,7 +505,7 @@ "datasource": "postgresql", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -517,7 +517,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", "kiro", "cost", "efficiency" @@ -531,7 +530,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "AI Cost-Efficiency (PostgreSQL)", - "uid": "ai_cost_efficiency-pg", + "title": "AI Model ROI", + "uid": "ai_model_roi-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/q-dev-dora.json b/grafana/dashboards/postgresql/kiro-dora.json similarity index 62% rename from grafana/dashboards/postgresql/q-dev-dora.json rename to grafana/dashboards/postgresql/kiro-dora.json index 2a5368236d1..fecbc0cb490 100644 --- a/grafana/dashboards/postgresql/q-dev-dora.json +++ b/grafana/dashboards/postgresql/kiro-dora.json @@ -36,10 +36,10 @@ "keepTime": true, "tags": [], "targetBlank": true, - "title": "Q Dev Dashboard", + "title": "Kiro Dashboard", "tooltip": "", "type": "link", - "url": "/d/qdev_user_data/q-dev-user-data-dashboard" + "url": "/d/kiro_user_data/kiro-user-data-dashboard" } ], "panels": [ @@ -61,7 +61,7 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "## AI-Powered DORA Dashboard\nThis dashboard correlates **Q Dev (AI coding assistant)** usage metrics with **DORA** performance indicators to help understand the impact of AI-assisted development on engineering efficiency.\n\n- **Left side**: Q Dev AI usage metrics (code generation, acceptance rate)\n- **Right side**: DORA metrics (Lead Time, Deployment Frequency, Change Failure Rate)\n- **Correlation charts**: Show trends over time to identify potential relationships", + "content": "## Kiro + DORA Dashboard\nThis dashboard compares **Kiro usage** with **DORA** performance indicators at monthly aggregates.\n\n- **Kiro metrics**: active users, credits, messages, conversations, and credits per message\n- **DORA metrics**: lead time, deployment frequency, and change failure rate\n- **Semantic boundary**: Kiro does not export accepted LOC, suggestion acceptance, generated tests, or code-review findings; no proxy is presented as an equivalent measure", "mode": "markdown" }, "pluginVersion": "13.0.2", @@ -82,8 +82,11 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Number of unique users who used Q Dev AI features", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Unique Kiro users in the selected period, from the daily usage report.", "fieldConfig": { "defaults": { "color": { @@ -128,20 +131,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Q Dev Users\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Kiro Users\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Q Dev Active Users", + "title": "Kiro Active Users", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Total AI-generated code lines accepted (Inline + Chat)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro exports credits, not accepted lines of code. Accepted LOC has no equivalent field.", "fieldConfig": { "defaults": { "color": { @@ -187,20 +196,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(inline_ai_code_lines + chat_ai_code_lines) AS \"AI Accepted Lines\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(SUM(credits_used), 1) AS \"Kiro Credits Used\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Total AI Code Lines", + "title": "Total Kiro Credits", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Acceptance rate of inline AI suggestions", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro does not export suggestion acceptance. Credits per message is a usage-efficiency ratio, not an acceptance rate.", "fieldConfig": { "defaults": { "color": { @@ -224,7 +239,7 @@ } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [] }, @@ -254,19 +269,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Acceptance Rate\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT CAST(SUM(credits_used) AS NUMERIC) / NULLIF(SUM(total_messages), 0) AS \"Credits per Message\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "AI Acceptance Rate", + "title": "Credits per Message", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Number of production deployments in selected period", "fieldConfig": { "defaults": { @@ -312,7 +333,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -324,7 +348,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Median lead time for changes in hours", "fieldConfig": { "defaults": { @@ -379,7 +406,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -391,7 +421,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Percentage of deployments that caused incidents", "fieldConfig": { "defaults": { @@ -446,7 +479,10 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, @@ -467,12 +503,15 @@ }, "id": 9, "panels": [], - "title": "AI Usage vs DORA Metrics Correlation", + "title": "Kiro Usage vs DORA Metrics", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Compare AI code generation trends with Lead Time for Changes. A negative correlation (AI lines up, Lead Time down) suggests AI is helping accelerate delivery.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro credits are compared with Lead Time. Kiro does not export accepted code lines.", "fieldConfig": { "defaults": { "color": { @@ -526,7 +565,7 @@ { "matcher": { "id": "byName", - "options": "AI Accepted Lines" + "options": "Kiro Credits" }, "properties": [ { @@ -545,29 +584,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Median Lead Time (hours)" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Lead Time (hours)" - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] } ] }, @@ -597,20 +613,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_monthly AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), lead_time_monthly AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, lt.month), '%Y-%m-%d') AS time, ai.ai_lines AS \"AI Accepted Lines\", ROUND(lt.avg_lead_time, 1) AS \"Median Lead Time (hours)\" FROM ai_monthly AS ai LEFT JOIN lead_time_monthly AS lt ON ai.month = lt.month WHERE NOT ai.month IS NULL OR NOT lt.month IS NULL ORDER BY time NULLS FIRST", + "rawSql": "WITH ai_monthly AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(credits_used) AS kiro_credits FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), lead_time_monthly AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, lt.month), '%Y-%m-%d') AS time, ai.kiro_credits AS \"Kiro Credits\", ROUND(lt.avg_lead_time, 1) AS \"Median Lead Time (hours)\" FROM ai_monthly AS ai LEFT JOIN lead_time_monthly AS lt ON ai.month = lt.month WHERE NOT ai.month IS NULL OR NOT lt.month IS NULL ORDER BY time NULLS FIRST", "refId": "A" } ], - "title": "AI Code Generation vs Lead Time Trend", + "title": "Kiro Credits vs Lead Time Trend", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Compare AI suggestion acceptance rate with deployment frequency. Higher acceptance rate may indicate better AI integration and potentially more deployments.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Usage-efficiency ratio versus deployment frequency. This is not suggestion acceptance.", "fieldConfig": { "defaults": { "color": { @@ -664,7 +686,7 @@ { "matcher": { "id": "byName", - "options": "AI Acceptance Rate" + "options": "Credits per Message" }, "properties": [ { @@ -673,7 +695,7 @@ }, { "id": "custom.axisLabel", - "value": "Acceptance Rate" + "value": "Credits per Message" }, { "id": "unit", @@ -687,29 +709,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Deployment Count" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Deployments" - }, - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] } ] }, @@ -739,20 +738,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_acceptance AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS acceptance_rate FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), deployment_count AS (SELECT TO_CHAR(CAST(MAX(cdc.finished_date) AS TIMESTAMP), 'YYYY-MM-01') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploy_count FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, dc.month), '%Y-%m-%d') AS time, ai.acceptance_rate AS \"AI Acceptance Rate\", dc.deploy_count AS \"Deployment Count\" FROM ai_acceptance AS ai LEFT JOIN deployment_count AS dc ON ai.month = dc.month WHERE NOT ai.month IS NULL OR NOT dc.month IS NULL ORDER BY time NULLS FIRST", + "rawSql": "WITH ai_acceptance AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(credits_used) AS NUMERIC) / NULLIF(NULLIF(SUM(total_messages), 0), 0) AS credits_per_message FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), deployment_count AS (SELECT TO_CHAR(CAST(MAX(cdc.finished_date) AS TIMESTAMP), 'YYYY-MM-01') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploy_count FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, dc.month), '%Y-%m-%d') AS time, ai.credits_per_message AS \"Credits per Message\", dc.deploy_count AS \"Deployment Count\" FROM ai_acceptance AS ai LEFT JOIN deployment_count AS dc ON ai.month = dc.month WHERE NOT ai.month IS NULL OR NOT dc.month IS NULL ORDER BY time NULLS FIRST", "refId": "A" } ], - "title": "AI Acceptance Rate vs Deployment Frequency", + "title": "Credits per Message vs Deployment Frequency", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Compare AI-generated tests with Change Failure Rate. More AI-generated tests might correlate with lower failure rates.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro does not export generated-test counts; total messages is the available activity-volume measure.", "fieldConfig": { "defaults": { "color": { @@ -806,7 +811,7 @@ { "matcher": { "id": "byName", - "options": "AI Generated Tests" + "options": "Kiro Messages" }, "properties": [ { @@ -825,33 +830,6 @@ } } ] - }, - { - "matcher": { - "id": "byName", - "options": "Change Failure Rate" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.axisLabel", - "value": "Failure Rate" - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] } ] }, @@ -881,20 +859,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_tests AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(test_generation_generated_tests) AS generated_tests FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), cfr_monthly AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, cfr.month), '%Y-%m-%d') AS time, ai.generated_tests AS \"AI Generated Tests\", cfr.cfr AS \"Change Failure Rate\" FROM ai_tests AS ai LEFT JOIN cfr_monthly AS cfr ON ai.month = cfr.month WHERE NOT ai.month IS NULL OR NOT cfr.month IS NULL ORDER BY time NULLS FIRST", + "rawSql": "WITH ai_tests AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(total_messages) AS kiro_messages FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), cfr_monthly AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, cfr.month), '%Y-%m-%d') AS time, ai.kiro_messages AS \"Kiro Messages\", cfr.cfr AS \"Change Failure Rate\" FROM ai_tests AS ai LEFT JOIN cfr_monthly AS cfr ON ai.month = cfr.month WHERE NOT ai.month IS NULL OR NOT cfr.month IS NULL ORDER BY time NULLS FIRST", "refId": "A" } ], - "title": "AI Test Generation vs Change Failure Rate", + "title": "Kiro Messages vs Change Failure Rate", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Compare active Q Dev users with Code Review findings. More AI-assisted code review might catch issues earlier.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro does not export code-review findings. Shows active users and credits by month.", "fieldConfig": { "defaults": { "color": { @@ -948,30 +932,7 @@ { "matcher": { "id": "byName", - "options": "Active Users" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "left" - }, - { - "id": "custom.axisLabel", - "value": "Users" - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Code Review Findings" + "options": "Kiro Credits" }, "properties": [ { @@ -1019,15 +980,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT TO_DATE(TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01'), '%Y-%m-%d') AS time, COUNT(DISTINCT user_id) AS \"Active Users\", SUM(code_review_findings_count) AS \"Code Review Findings\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') ORDER BY time NULLS FIRST", + "rawSql": "SELECT TO_DATE(TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01'), '%Y-%m-%d') AS time, COUNT(DISTINCT user_id) AS \"Active Users\", SUM(credits_used) AS \"Kiro Credits\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') ORDER BY time NULLS FIRST", "refId": "A" } ], - "title": "Q Dev Users vs Code Review Findings", + "title": "Kiro Users vs Credits", "type": "timeseries" }, { @@ -1044,8 +1008,11 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Monthly summary comparing Q Dev AI metrics with DORA metrics side by side", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Monthly Kiro usage metrics beside DORA metrics. LOC, acceptance, generated tests, and review findings are not exported by Kiro.", "fieldConfig": { "defaults": { "color": { @@ -1074,7 +1041,7 @@ { "matcher": { "id": "byName", - "options": "AI Acceptance Rate" + "options": "Credits per Message" }, "properties": [ { @@ -1099,47 +1066,6 @@ "value": 1 } ] - }, - { - "matcher": { - "id": "byName", - "options": "Change Failure Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-RdYlGr" - } - }, - { - "id": "max", - "value": 0.3 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Lead Time (hours)" - }, - "properties": [ - { - "id": "unit", - "value": "h" - } - ] } ] }, @@ -1171,15 +1097,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "WITH ai_metrics AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT user_id) AS active_users, SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines, CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS acceptance_rate, SUM(test_generation_generated_tests) AS generated_tests, SUM(code_review_findings_count) AS review_findings FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM')), dora_metrics AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deployments, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' LEFT JOIN cicd_deployment_commits AS cdc2 ON cdc.cicd_deployment_id = cdc2.cicd_deployment_id LEFT JOIN project_pr_metrics AS ppm ON ppm.deployment_commit_id = cdc2.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM')), cfr_metrics AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM')) SELECT COALESCE(ai.month, dm.month, cfr.month) AS \"Month\", COALESCE(ai.active_users, 0) AS \"Q Dev Users\", COALESCE(ai.ai_lines, 0) AS \"AI Code Lines\", ai.acceptance_rate AS \"AI Acceptance Rate\", COALESCE(ai.generated_tests, 0) AS \"AI Tests\", COALESCE(ai.review_findings, 0) AS \"Review Findings\", COALESCE(dm.deployments, 0) AS \"Deployments\", ROUND(dm.avg_lead_time, 1) AS \"Lead Time (hours)\", cfr.cfr AS \"Change Failure Rate\" FROM ai_metrics AS ai LEFT JOIN dora_metrics AS dm ON ai.month = dm.month LEFT JOIN cfr_metrics AS cfr ON ai.month = cfr.month ORDER BY ai.month DESC NULLS LAST", + "rawSql": "WITH ai_metrics AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT user_id) AS active_users, SUM(credits_used) AS credits, CAST(SUM(credits_used) AS NUMERIC) / NULLIF(NULLIF(SUM(total_messages), 0), 0) AS credits_per_message, SUM(total_messages) AS messages, SUM(chat_conversations) AS conversations FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM')), dora_metrics AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deployments, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' LEFT JOIN cicd_deployment_commits AS cdc2 ON cdc.cicd_deployment_id = cdc2.cicd_deployment_id LEFT JOIN project_pr_metrics AS ppm ON ppm.deployment_commit_id = cdc2.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM')), cfr_metrics AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM')) SELECT COALESCE(ai.month, dm.month, cfr.month) AS \"Month\", COALESCE(ai.active_users, 0) AS \"Kiro Users\", COALESCE(ai.credits, 0) AS \"Credits Used\", ai.credits_per_message AS \"Credits per Message\", COALESCE(ai.messages, 0) AS \"Messages\", COALESCE(ai.conversations, 0) AS \"Conversations\", COALESCE(dm.deployments, 0) AS \"Deployments\", ROUND(dm.avg_lead_time, 1) AS \"Lead Time (hours)\", cfr.cfr AS \"Change Failure Rate\" FROM ai_metrics AS ai LEFT JOIN dora_metrics AS dm ON ai.month = dm.month LEFT JOIN cfr_metrics AS cfr ON ai.month = cfr.month ORDER BY ai.month DESC NULLS LAST", "refId": "A" } ], - "title": "Monthly Q Dev vs DORA Metrics Comparison", + "title": "Monthly Kiro + DORA Metrics", "type": "table" } ], @@ -1187,7 +1116,7 @@ "refresh": "5m", "schemaVersion": 39, "tags": [ - "q_dev", + "kiro", "DORA", "AI", "correlation" @@ -1204,7 +1133,10 @@ "$__all" ] }, - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "definition": "SELECT DISTINCT name FROM projects", "hide": 0, "includeAll": true, @@ -1227,7 +1159,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Q Dev + DORA Correlation", - "uid": "qdev_dora_correlation-pg", + "title": "Kiro + DORA Correlation", + "uid": "kiro_dora_correlation-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/mysql/qdev_user_data.json b/grafana/dashboards/postgresql/kiro_activity_detail.json similarity index 70% rename from grafana/dashboards/mysql/qdev_user_data.json rename to grafana/dashboards/postgresql/kiro_activity_detail.json index affaf57244c..f67b94fd9a2 100644 --- a/grafana/dashboards/mysql/qdev_user_data.json +++ b/grafana/dashboards/postgresql/kiro_activity_detail.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Overview of key user metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Usage-report totals. Accepted LOC and acceptance rates are not exported by Kiro.", "fieldConfig": { "defaults": { "color": { @@ -66,13 +69,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Acceptance Rate (Inline Suggestion)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Acceptance Rate (Code Fix)',\n SUM(transformation_lines_ingested) as 'Ingested Lines (Java Transform)',\n SUM(transformation_lines_generated) as 'Generated Lines (Java Transform)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Lines (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Acceptance Rate (Inline Chat)'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", ROUND(SUM(credits_used), 1) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A", "select": [ [ @@ -115,8 +121,11 @@ "type": "stat" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily AI code line changes across all users", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Daily Kiro usage volume from the user report.", "fieldConfig": { "defaults": { "color": { @@ -195,13 +204,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(chat_ai_code_lines) as 'Chat Accepted Lines',\n SUM(code_fix_accepted_lines) as 'Code Fix Accepted Lines',\n SUM(code_fix_generated_lines) as 'Code Fix Generated Lines',\n SUM(transformation_lines_ingested) as 'Java Transform Ingested Lines',\n SUM(transformation_lines_generated) as 'Java Transform Generated Lines',\n SUM(inline_ai_code_lines) as 'Inline Suggestion Accepted Lines',\n SUM(inline_chat_accepted_line_additions) as 'Inline Chat Accepted Line Additions',\n SUM(inline_chat_accepted_line_deletions) as 'Inline Chat Accepted Line Deletions',\n SUM(inline_chat_dismissed_line_additions) as 'Inline Chat Dismissed Line Additions',\n SUM(inline_chat_dismissed_line_deletions) as 'Inline Chat Dismissed Line Deletions',\n SUM(inline_chat_rejected_line_additions) as 'Inline Chat Rejected Line Additions',\n SUM(inline_chat_rejected_line_deletions) as 'Inline Chat Rejected Line Deletions'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(date AS DATE) AS time, ROUND(SUM(credits_used), 1) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -240,12 +252,15 @@ ] } ], - "title": "Daily AI Code Line Changes", + "title": "Daily Credits, Messages & Conversations", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily AI interaction trends across all users", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Chat events split into user turns and agent continuations.", "fieldConfig": { "defaults": { "color": { @@ -324,13 +339,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(chat_messages_sent) as 'Chat Messages Sent',\n SUM(code_fix_acceptance_event_count) as 'Code Fix Accepted Event Count',\n SUM(code_fix_generation_event_count) as 'Code Fix Generated Event Count',\n SUM(transformation_event_count) as 'Java Transform Event Count',\n SUM(inline_acceptance_count) as 'Inline Suggestion Accepted Suggestions',\n SUM(inline_suggestions_count) as 'Inline Suggestion Count',\n SUM(inline_chat_total_event_count) as 'Inline Chat Total Suggestions',\n SUM(inline_chat_acceptance_event_count) as 'Inline Chat Accepted Suggestions',\n SUM(inline_chat_dismissal_event_count) as 'Inline Chat Dismissed Suggestions',\n SUM(inline_chat_rejection_event_count) as 'Inline Chat Rejected Suggestions'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS \"Chat Events\", SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS \"User Turns\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -369,12 +387,15 @@ ] } ], - "title": "Daily AI Interactions", + "title": "Daily Chat Activity", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Code review metrics over time", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Authoritative per-model and routing-mode message counts.", "fieldConfig": { "defaults": { "color": { @@ -453,12 +474,15 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(code_fix_acceptance_event_count) as 'Code Fix Accepted Event Count',\n SUM(code_fix_generation_event_count) as 'Code Fix Generated Event Count',\n SUM(code_review_findings_count) as 'Total Findings'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(date AS DATE) AS time, model_name AS metric, SUM(message_count) AS value FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE), model_name ORDER BY time", "refId": "A", "select": [ [ @@ -497,12 +521,15 @@ ] } ], - "title": "Code Review Metrics", + "title": "Daily Model / Route Messages", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily acceptance rate of AI suggestions", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Agent continuations divided by all chat events. This is not suggestion acceptance.", "fieldConfig": { "defaults": { "color": { @@ -581,13 +608,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CAST(SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0) AS \"Agent Continuation Rate\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -626,12 +656,15 @@ ] } ], - "title": "Daily AI Suggestion Acceptance Rate", + "title": "Daily Agent Continuation Rate", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "User AI interaction metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Per-user credits, messages, conversations, overage, and last activity.", "fieldConfig": { "defaults": { "color": { @@ -655,51 +688,7 @@ ] } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Acceptance Rate" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Inline Chat Accepted Events" - }, - "properties": [ - { - "id": "custom.width", - "value": 239 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Acceptance Rate (Inline Suggestion)" - }, - "properties": [ - { - "id": "custom.width", - "value": 172 - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -724,13 +713,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_generated_lines) as 'Generated Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) as 'Accepted Count (Code Fix)',\n SUM(code_fix_generation_event_count) as 'Generated Count (Code Fix)',\n CONCAT(ROUND(SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Code Fix)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) as 'Accepted Count (Inline Suggestion)',\n SUM(inline_suggestions_count) as 'Total Count (Inline Suggestion)',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Suggestion)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Line Additions (Inline Chat)',\n SUM(inline_chat_accepted_line_deletions) as 'Accepted Line Deletions (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) as 'Accepted Events (Inline Chat)',\n SUM(inline_chat_total_event_count) as 'Total Events (Inline Chat)',\n CONCAT(ROUND(SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Chat)',\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id\nORDER BY SUM(inline_ai_code_lines) DESC", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", MAX(subscription_tier) AS \"Tier\", ROUND(SUM(credits_used), 1) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", SUM(overage_credits_used) AS \"Overage Credits\", MAX(date) AS \"Last Activity\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id ORDER BY SUM(credits_used) DESC", "refId": "A", "select": [ [ @@ -769,12 +761,15 @@ ] } ], - "title": "User Interactions", + "title": "Per-User Usage", "type": "table" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily doc generation events and accepted/rejected lines", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Messages by Kiro client type. Doc-generation metrics are not exported.", "fieldConfig": { "defaults": { "color": { @@ -853,13 +848,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(doc_generation_event_count) as 'Doc Generation Events',\n SUM(doc_generation_accepted_line_additions) as 'Accepted Line Additions',\n SUM(doc_generation_accepted_line_updates) as 'Accepted Line Updates',\n SUM(doc_generation_rejected_line_additions) as 'Rejected Line Additions',\n SUM(doc_generation_rejected_line_updates) as 'Rejected Line Updates'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(date AS DATE) AS time, client_type AS metric, SUM(total_messages) AS value FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE), client_type ORDER BY time", "refId": "A", "select": [ [ @@ -898,12 +896,15 @@ ] } ], - "title": "Doc Generation Metrics", + "title": "Daily Client Type Messages", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily test generation events and lines", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Completion requests, requests with returned suggestions, and returned lines. These are not acceptance events.", "fieldConfig": { "defaults": { "color": { @@ -982,13 +983,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(test_generation_event_count) as 'Test Generation Events',\n SUM(test_generation_accepted_tests) as 'Accepted Tests',\n SUM(test_generation_generated_tests) as 'Generated Tests',\n SUM(test_generation_accepted_lines) as 'Accepted Lines',\n SUM(test_generation_generated_lines) as 'Generated Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS \"Completion Requests\", SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS \"Requests with Results\", SUM(returned_line_count) AS \"Returned Lines\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -1027,12 +1031,15 @@ ] } ], - "title": "Test Generation Metrics", + "title": "Daily Completion Activity", "type": "timeseries" }, { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, - "description": "Daily agentic dev events and lines", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Steering, spec-mode, and agent-continuation events. Kiro does not export accepted agentic LOC.", "fieldConfig": { "defaults": { "color": { @@ -1111,13 +1118,16 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(dev_generation_event_count) as 'Dev Generation Events',\n SUM(dev_acceptance_event_count) as 'Dev Acceptance Events',\n SUM(dev_generated_lines) as 'Dev Generated Lines',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Spec Mode\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A", "select": [ [ @@ -1156,7 +1166,7 @@ ] } ], - "title": "Dev (Agentic) Metrics", + "title": "Daily Agentic Signals", "type": "timeseries" } ], @@ -1164,7 +1174,7 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", + "kiro", "user_data" ], "templating": { @@ -1176,7 +1186,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Kiro Code Metrics Dashboard", - "uid": "qdev_user_data", + "title": "Kiro Activity Detail Dashboard", + "uid": "kiro_user_data-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_executive.json b/grafana/dashboards/postgresql/kiro_executive.json similarity index 71% rename from grafana/dashboards/postgresql/qdev_executive.json rename to grafana/dashboards/postgresql/kiro_executive.json index 8ea7bbb8c8f..9a13ba80109 100644 --- a/grafana/dashboards/postgresql/qdev_executive.json +++ b/grafana/dashboards/postgresql/kiro_executive.json @@ -27,7 +27,7 @@ "title": "Usage (New)", "tooltip": "Kiro Usage Dashboard - Credits & Messages (new format)", "type": "link", - "url": "/d/qdev_user_report" + "url": "/d/kiro_user_report" }, { "asDropdown": false, @@ -36,10 +36,10 @@ "keepTime": true, "tags": [], "targetBlank": true, - "title": "Feature Metrics (Legacy)", - "tooltip": "Kiro Legacy Feature Metrics (old format)", + "title": "Feature Metrics", + "tooltip": "Kiro Feature Metrics", "type": "link", - "url": "/d/qdev_feature_metrics" + "url": "/d/kiro_feature_metrics" }, { "asDropdown": false, @@ -51,7 +51,7 @@ "title": "Prompt Logging", "tooltip": "Kiro AI Activity Insights - Prompt Logging", "type": "link", - "url": "/d/qdev_logging" + "url": "/d/kiro_logging" } ], "panels": [ @@ -69,7 +69,10 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Distinct users with chat activity in the last 7 days (from prompt logging)", "fieldConfig": { "defaults": { @@ -116,11 +119,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"WAU\" FROM lake._tool_q_dev_chat_log WHERE timestamp >= NOW() - INTERVAL '7 DAY'", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"WAU\" FROM lake._tool_kiro_chat_log WHERE timestamp >= NOW() - INTERVAL '7 DAY'", "refId": "A" } ], @@ -128,8 +134,11 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Average credits spent per accepted line of code (new report + legacy metrics)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Usage efficiency from Kiro reports. Accepted LOC is not exported.", "fieldConfig": { "defaults": { "color": { @@ -175,20 +184,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(SUM(d.total_accepted), 0), 0), 2) AS \"Credits per Accepted Line\" FROM (SELECT user_id, date, SUM(credits_used) AS credits_used FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", date) AS r JOIN (SELECT user_id, \"date\", (inline_ai_code_lines + chat_ai_code_lines + code_fix_accepted_lines + dev_accepted_lines) AS total_accepted FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)) AS d ON r.user_id = d.user_id AND r.date = d.date", + "rawSql": "SELECT ROUND(CAST(SUM(credits_used) AS NUMERIC) / NULLIF(SUM(total_messages), 0), 2) AS \"Credits per Message\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Credits Efficiency (new + legacy)", + "title": "Credits per Message", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Percentage of inline suggestions accepted (from legacy feature metrics)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Share of chat events generated by agent self-continuation. Suggestion acceptance is not exported.", "fieldConfig": { "defaults": { "color": { @@ -234,19 +249,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) * 100, 1) AS \"Acceptance %\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT ROUND(CAST(SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0), 1) AS \"Agent Continuation %\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], - "title": "Inline Acceptance Rate (legacy)", + "title": "Agent Continuation Rate", "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Percentage of users who used steering rules (from prompt logging)", "fieldConfig": { "defaults": { @@ -294,11 +315,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) * 100 AS DECIMAL), 0) AS \"Steering %\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) * 100 AS DECIMAL), 0) AS \"Steering %\" FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -315,11 +339,14 @@ }, "id": 101, "panels": [], - "title": "User Engagement (logging data: _tool_q_dev_chat_log)", + "title": "User Engagement", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Weekly active user count over time (from prompt logging)", "fieldConfig": { "defaults": { @@ -399,11 +426,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT TO_DATE(yw || ' Monday', 'IYYYIW FMDay') AS time, COUNT(DISTINCT user_id) AS \"Active Users\" FROM (SELECT user_id, (EXTRACT(ISOYEAR FROM timestamp) * 100 + EXTRACT(WEEK FROM timestamp))::int AS yw FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS t GROUP BY \"yw\" ORDER BY time NULLS FIRST", + "rawSql": "SELECT TO_DATE(yw || ' Monday', 'IYYYIW FMDay') AS time, COUNT(DISTINCT user_id) AS \"Active Users\" FROM (SELECT user_id, (EXTRACT(ISOYEAR FROM timestamp) * 100 + EXTRACT(WEEK FROM timestamp))::int AS yw FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS t GROUP BY \"yw\" ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -411,8 +441,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "New vs returning users by week (from prompt logging)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "New users use the native is_new_user flag; early rows without that field are reported as existing/unknown.", "fieldConfig": { "defaults": { "color": { @@ -491,15 +524,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT TO_DATE(yw || ' Monday', 'IYYYIW FMDay') AS time, SUM(CASE WHEN yw = first_yw THEN 1 ELSE 0 END) AS \"New Users\", SUM(CASE WHEN yw <> first_yw THEN 1 ELSE 0 END) AS \"Returning Users\" FROM (SELECT DISTINCT u.user_id, (EXTRACT(ISOYEAR FROM u.timestamp) * 100 + EXTRACT(WEEK FROM u.timestamp))::int AS yw, f.first_yw FROM lake._tool_q_dev_chat_log AS u JOIN (SELECT user_id, (EXTRACT(ISOYEAR FROM MIN(timestamp)) * 100 + EXTRACT(WEEK FROM MIN(timestamp)))::int AS first_yw FROM lake._tool_q_dev_chat_log GROUP BY user_id) AS f ON u.user_id = f.user_id WHERE $__timeFilter(u.timestamp)) AS weekly GROUP BY yw ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(DISTINCT CASE WHEN is_new_user IS TRUE THEN user_id END) AS \"New Users\", COUNT(DISTINCT CASE WHEN is_new_user IS NOT TRUE THEN user_id END) AS \"Existing/Unknown Users\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time", "refId": "A" } ], - "title": "New vs Returning Users (Weekly)", + "title": "New vs Existing/Unknown Users (Weekly)", "type": "timeseries" }, { @@ -512,11 +548,14 @@ }, "id": 102, "panels": [], - "title": "Credits & Subscription (new format: _tool_q_dev_user_report)", + "title": "Credits & Subscription", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Cumulative credits this month vs projected total (from new user_report)", "fieldConfig": { "defaults": { @@ -596,11 +635,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(SUM(credits_used)) OVER (ORDER BY date NULLS FIRST) AS \"Cumulative Credits\", (SELECT CAST(SUM(credits_used) AS NUMERIC) / NULLIF(COUNT(DISTINCT date), 0) * EXTRACT(DAY FROM CAST(CAST(DATE_TRUNC('MONTH', CURRENT_DATE) + INTERVAL '1 MONTH' - INTERVAL '1 DAY' AS DATE) AS DATE)) FROM lake._tool_q_dev_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE))) AS \"Projected Monthly\" FROM lake._tool_q_dev_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE)) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT date AS time, SUM(SUM(credits_used)) OVER (ORDER BY date NULLS FIRST) AS \"Cumulative Credits\", (SELECT CAST(SUM(credits_used) AS NUMERIC) / NULLIF(COUNT(DISTINCT date), 0) * EXTRACT(DAY FROM CAST(CAST(DATE_TRUNC('MONTH', CURRENT_DATE) + INTERVAL '1 MONTH' - INTERVAL '1 DAY' AS DATE) AS DATE)) FROM lake._tool_kiro_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE))) AS \"Projected Monthly\" FROM lake._tool_kiro_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE)) GROUP BY \"date\" ORDER BY date NULLS FIRST", "refId": "A" } ], @@ -608,7 +650,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Power tier users with no activity in the last 14 days (from new user_report)", "fieldConfig": { "defaults": { @@ -658,11 +703,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", MAX(subscription_tier) AS \"Tier\", ROUND(SUM(credits_used), 1) AS \"Total Credits Used\", MAX(date) AS \"Last Activity\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) AND subscription_tier = 'POWER' GROUP BY \"user_id\" HAVING MAX(date) < NOW() - INTERVAL '14 DAY' ORDER BY MAX(date) NULLS FIRST", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", MAX(subscription_tier) AS \"Tier\", ROUND(SUM(credits_used), 1) AS \"Total Credits Used\", MAX(date) AS \"Last Activity\" FROM lake._tool_kiro_user_report WHERE $__timeFilter(date) AND subscription_tier = 'POWER' GROUP BY \"user_id\" HAVING MAX(date) < NOW() - INTERVAL '14 DAY' ORDER BY MAX(date) NULLS FIRST", "refId": "A" } ], @@ -679,12 +727,15 @@ }, "id": 103, "panels": [], - "title": "Cross-Source: User Productivity (new report + legacy metrics)", + "title": "Per-User Usage & Efficiency", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Per-user productivity combining credits (new format) with feature metrics (legacy). Only shows users present in both data sources.", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Credits, messages, conversations, and credits per message. Accepted LOC is not exported.", "fieldConfig": { "defaults": { "color": { @@ -733,15 +784,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(d.display_name), d.user_id) AS \"User\", COALESCE(MAX(r.subscription_tier), '') AS \"Tier\", ROUND(SUM(r.credits_used), 1) AS \"Credits Used\", SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) AS \"Total Accepted Lines\", CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0 THEN ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 0), 2) ELSE NULL END AS \"Credits/Line\", ROUND(CAST(SUM(d.inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(d.inline_suggestions_count), 0), 0) * 100, 1) || '%' AS \"Accept Rate\", SUM(d.code_review_findings_count) AS \"Review Findings\", SUM(d.test_generation_event_count) AS \"Test Gen Events\", SUM(d.dev_accepted_lines) AS \"Agentic Lines\", MIN(d.date) AS \"First Active\", MAX(d.date) AS \"Last Active\" FROM lake._tool_q_dev_user_data AS d LEFT JOIN (SELECT user_id, date, SUM(credits_used) AS credits_used, MAX(subscription_tier) AS subscription_tier FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", date) AS r ON d.user_id = r.user_id AND d.date = r.date WHERE $__timeFilter(d.date) GROUP BY d.user_id ORDER BY SUM(r.credits_used) DESC NULLS LAST", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", MAX(subscription_tier) AS \"Tier\", ROUND(SUM(credits_used), 1) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", ROUND(SUM(credits_used) / NULLIF(SUM(total_messages), 0), 3) AS \"Credits per Message\", MAX(date) AS \"Last Activity\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id ORDER BY SUM(credits_used) DESC", "refId": "A" } ], - "title": "User Productivity & Efficiency", + "title": "Per-User Usage & Efficiency", "type": "table" } ], @@ -749,9 +803,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "executive", - "kiro" + "kiro", + "executive" ], "templating": { "list": [] @@ -763,6 +816,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro Executive Dashboard", - "uid": "qdev_executive-pg", + "uid": "kiro_executive-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_feature_metrics.json b/grafana/dashboards/postgresql/kiro_feature_metrics.json similarity index 70% rename from grafana/dashboards/postgresql/qdev_feature_metrics.json rename to grafana/dashboards/postgresql/kiro_feature_metrics.json index 3ea698aa550..ac8b3a4872d 100644 --- a/grafana/dashboards/postgresql/qdev_feature_metrics.json +++ b/grafana/dashboards/postgresql/kiro_feature_metrics.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "High-level summary of legacy feature-level activity metrics (from by_user_analytic CSV reports)", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro usage-report metrics. Legacy accepted LOC and feature counters are not exported.", "fieldConfig": { "defaults": { "color": { @@ -66,15 +69,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", SUM(inline_suggestions_count) AS \"Inline Suggestions\", SUM(inline_acceptance_count) AS \"Inline Accepted\", SUM(chat_messages_sent) AS \"Chat Messages\", SUM(chat_ai_code_lines) AS \"Chat AI Lines\", SUM(code_review_findings_count) AS \"Review Findings\", SUM(test_generation_event_count) AS \"Test Gen Events\", SUM(dev_accepted_lines) AS \"Agentic Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", ROUND(SUM(credits_used), 1) AS \"Credits Used\" FROM _tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], - "title": "Legacy Feature Metrics Overview", + "title": "Kiro Usage Overview", "type": "stat" }, { @@ -87,12 +93,15 @@ }, "id": 20, "panels": [], - "title": "Inline Suggestions", + "title": "Completions", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily inline suggestion and acceptance counts", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Returned suggestions are offers, not acceptance events.", "fieldConfig": { "defaults": { "color": { @@ -170,20 +179,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(inline_suggestions_count) AS \"Suggestions\", SUM(inline_acceptance_count) AS \"Accepted\", SUM(inline_ai_code_lines) AS \"AI Code Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS \"Completion Requests\", SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS \"Requests with Results\", SUM(returned_line_count) AS \"Returned Lines\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Inline Suggestions & Acceptance", + "title": "Completion Requests & Results", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Percentage of completion requests that returned at least one suggestion. Kiro does not export acceptance.", "fieldConfig": { "defaults": { "color": { @@ -261,15 +276,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Inline Suggestions\", CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) AS \"Code Fix\", CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) AS \"Inline Chat\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CAST(SUM(CASE WHEN completions_count > 0 THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0) AS \"Suggestion Return Rate\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Acceptance Rate Trends", + "title": "Suggestion Return Rate", "type": "timeseries" }, { @@ -282,12 +300,15 @@ }, "id": 21, "panels": [], - "title": "Chat & Agentic (Dev)", + "title": "Chat & Agent Continuations", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily chat messages sent and AI-generated code lines from chat", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "User turns and agent self-continuations from chat logs.", "fieldConfig": { "defaults": { "color": { @@ -365,11 +386,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(chat_messages_sent) AS \"Messages Sent\", SUM(chat_messages_interacted) AS \"Messages Interacted\", SUM(chat_ai_code_lines) AS \"AI Code Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS \"Chat Events\", SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS \"User Turns\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], @@ -377,8 +401,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Agentic (Dev) code generation and acceptance metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Agent continuations are empty-prompt records after tool calls; this measures autonomy, not accepted code.", "fieldConfig": { "defaults": { "color": { @@ -456,15 +483,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(dev_generation_event_count) AS \"Generation Events\", SUM(dev_generated_lines) AS \"Generated Lines\", SUM(dev_accepted_lines) AS \"Accepted Lines\", SUM(dev_acceptance_event_count) AS \"Acceptance Events\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS \"User Turns\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\", CAST(SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0) AS \"Agent Continuation Rate\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Agentic (Dev) Activity", + "title": "Agent Continuation Activity", "type": "timeseries" }, { @@ -477,12 +507,15 @@ }, "id": 22, "panels": [], - "title": "Code Review, Test Gen & Transformations", + "title": "Models, Prompts & Clients", "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Code review findings and test generation metrics over time", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Authoritative model and routing-mode message counts from daily reports.", "fieldConfig": { "defaults": { "color": { @@ -560,20 +593,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(code_review_findings_count) AS \"Review Findings\", SUM(code_review_succeeded_event_count) AS \"Reviews Succeeded\", SUM(code_review_failed_event_count) AS \"Reviews Failed\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) AS time, model_name AS metric, SUM(message_count) AS value FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE), model_name ORDER BY time", "refId": "A" } ], - "title": "Code Review Activity", + "title": "Model / Route Message Mix", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Test generation events and acceptance over time", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Daily average prompt and response lengths. Test-generation counts are not exported.", "fieldConfig": { "defaults": { "color": { @@ -651,20 +690,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(test_generation_event_count) AS \"Test Gen Events\", SUM(test_generation_generated_tests) AS \"Tests Generated\", SUM(test_generation_accepted_tests) AS \"Tests Accepted\", SUM(test_generation_generated_lines) AS \"Lines Generated\", SUM(test_generation_accepted_lines) AS \"Lines Accepted\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, ROUND(AVG(prompt_length)) AS \"Avg Prompt Length\", ROUND(AVG(response_length)) AS \"Avg Response Length\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Test Generation Activity", + "title": "Prompt & Response Length", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Doc generation and code transformation events", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Messages by KIRO_IDE, KIRO_CLI, KIRO_WEB, and PLUGIN. Doc-generation metrics are not exported.", "fieldConfig": { "defaults": { "color": { @@ -742,20 +787,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(doc_generation_event_count) AS \"Doc Gen Events\", SUM(doc_generation_accepted_line_additions) AS \"Doc Lines Accepted\", SUM(transformation_event_count) AS \"Transformation Events\", SUM(transformation_lines_generated) AS \"Transform Lines Generated\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT CAST(date AS DATE) AS time, client_type AS metric, SUM(total_messages) AS value FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE), client_type ORDER BY time", "refId": "A" } ], - "title": "Doc Generation & Transformations", + "title": "Client Type Message Mix", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Number of users who used each feature in the selected period", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Distinct users observed in chat, completion, steering, and spec-mode signals.", "fieldConfig": { "defaults": { "color": { @@ -838,11 +889,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT 'Chat' AS \"Feature\", COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) AS \"Users\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Code Review', COUNT(DISTINCT CASE WHEN code_review_succeeded_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Doc Generation', COUNT(DISTINCT CASE WHEN doc_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Test Generation', COUNT(DISTINCT CASE WHEN test_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Dev (Agentic)', COUNT(DISTINCT CASE WHEN dev_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Transformation', COUNT(DISTINCT CASE WHEN transformation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "rawSql": "SELECT 'Chat' AS \"Feature\", COUNT(DISTINCT user_id) AS \"Users\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Completion' AS \"Feature\", COUNT(DISTINCT user_id) AS \"Users\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Steering' AS \"Feature\", COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) AS \"Users\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION ALL SELECT 'Spec Mode' AS \"Feature\", COUNT(DISTINCT CASE WHEN is_spec_mode = TRUE THEN user_id END) AS \"Users\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -863,8 +917,11 @@ "type": "row" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Per-user breakdown of legacy feature-level metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Per-user chat, completion, autonomy, steering, and spec-mode activity.", "fieldConfig": { "defaults": { "color": { @@ -913,15 +970,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", SUM(inline_suggestions_count) AS \"Suggestions\", SUM(inline_acceptance_count) AS \"Accepted\", ROUND(CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) * 100, 1) || '%' AS \"Accept %\", SUM(chat_messages_sent) AS \"Chat Msgs\", SUM(chat_ai_code_lines) AS \"Chat Lines\", SUM(dev_accepted_lines) AS \"Agentic Lines\", SUM(code_review_findings_count) AS \"Review Findings\", SUM(test_generation_accepted_tests) AS \"Tests Accepted\", SUM(doc_generation_event_count) AS \"Doc Gen\", SUM(transformation_event_count) AS \"Transforms\", MIN(date) AS \"First Active\", MAX(date) AS \"Last Active\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"user_id\" ORDER BY SUM(inline_suggestions_count) DESC NULLS LAST", + "rawSql": "WITH users AS (SELECT user_id FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) UNION SELECT user_id FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp)), chat AS (SELECT user_id, COUNT(*) AS chat_events, SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS user_turns, SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS agent_continuations, SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS steering, SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS spec_mode FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id), completion AS (SELECT user_id, COUNT(*) AS completion_requests, SUM(returned_line_count) AS returned_lines FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id), names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, u.user_id) AS \"User\", COALESCE(c.chat_events, 0) AS \"Chat Events\", COALESCE(c.user_turns, 0) AS \"User Turns\", COALESCE(c.agent_continuations, 0) AS \"Agent Continuations\", COALESCE(x.completion_requests, 0) AS \"Completion Requests\", COALESCE(x.returned_lines, 0) AS \"Returned Lines\", COALESCE(c.steering, 0) AS \"Steering\", COALESCE(c.spec_mode, 0) AS \"Spec Mode\" FROM users u LEFT JOIN chat c ON u.user_id = c.user_id LEFT JOIN completion x ON u.user_id = x.user_id LEFT JOIN names n ON u.user_id = n.user_id ORDER BY COALESCE(c.chat_events, 0) DESC", "refId": "A" } ], - "title": "Per-User Feature Metrics", + "title": "Per-User Kiro Activity", "type": "table" } ], @@ -929,8 +989,6 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "legacy", "kiro" ], "templating": { @@ -942,7 +1000,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Kiro Legacy Feature Metrics", - "uid": "qdev_feature_metrics-pg", + "title": "Kiro Feature Metrics", + "uid": "kiro_feature_metrics-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_logging.json b/grafana/dashboards/postgresql/kiro_logging.json similarity index 72% rename from grafana/dashboards/postgresql/qdev_logging.json rename to grafana/dashboards/postgresql/kiro_logging.json index 032b4fcfa16..c0b4396cf33 100644 --- a/grafana/dashboards/postgresql/qdev_logging.json +++ b/grafana/dashboards/postgresql/kiro_logging.json @@ -19,8 +19,11 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Overview of logging event metrics", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Chat users, user turns, agent continuations, and completion events.", "fieldConfig": { "defaults": { "color": { @@ -66,11 +69,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Events\", (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Users\", (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id <> '') AS \"Conversations\", (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) AS \"Completion Events\", (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) AS \"Completion Users\", (SELECT SUM(code_reference_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Code References\", (SELECT SUM(web_link_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Web Links Cited\"", + "rawSql": "SELECT (SELECT COUNT(*) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Events\", (SELECT COUNT(DISTINCT user_id) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Users\", (SELECT SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS \"User Turns\", (SELECT SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)) AS \"Agent Continuations\", (SELECT COUNT(*) FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp)) AS \"Completion Events\"", "refId": "A" } ], @@ -78,7 +84,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Hourly distribution of AI usage activity (chat + completions)", "fieldConfig": { "defaults": { @@ -155,11 +164,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT LPAD(CAST(hour_of_day AS TEXT), 2, '0') AS \"Hour\", SUM(chat_count) AS \"Chat Events\", SUM(completion_count) AS \"Completion Events\" FROM (SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, COUNT(*) AS chat_count, 0 AS completion_count FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) UNION ALL SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, 0 AS chat_count, COUNT(*) AS completion_count FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp)) AS combined GROUP BY hour_of_day ORDER BY hour_of_day NULLS FIRST", + "rawSql": "SELECT LPAD(CAST(hour_of_day AS TEXT), 2, '0') AS \"Hour\", SUM(chat_count) AS \"Chat Events\", SUM(completion_count) AS \"Completion Events\" FROM (SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, COUNT(*) AS chat_count, 0 AS completion_count FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) UNION ALL SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, 0 AS chat_count, COUNT(*) AS completion_count FROM lake._tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp)) AS combined GROUP BY hour_of_day ORDER BY hour_of_day NULLS FIRST", "refId": "A" } ], @@ -167,7 +179,10 @@ "type": "barchart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Distribution of chat trigger types: MANUAL (chat window) vs INLINE_CHAT", "fieldConfig": { "defaults": { @@ -223,11 +238,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)' ELSE chat_trigger_type END AS \"Trigger Type\", COUNT(*) AS \"Events\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"chat_trigger_type\" ORDER BY COUNT(*) DESC NULLS LAST", + "rawSql": "SELECT CASE WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)' ELSE chat_trigger_type END AS \"Trigger Type\", COUNT(*) AS \"Events\" FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"chat_trigger_type\" ORDER BY COUNT(*) DESC NULLS LAST", "refId": "A" } ], @@ -235,8 +253,11 @@ "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Distribution of model usage across chat events", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Authoritative model and routing-mode counts from daily reports; chat model_id is only partially populated.", "fieldConfig": { "defaults": { "color": { @@ -280,7 +301,7 @@ "calcs": [ "lastNotNull" ], - "fields": "/^Requests$/", + "fields": "/^Messages$/", "values": true }, "tooltip": { @@ -291,19 +312,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN model_id = '' OR model_id IS NULL THEN '(unknown)' ELSE model_id END AS \"Model\", COUNT(*) AS \"Requests\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"model_id\" ORDER BY COUNT(*) DESC NULLS LAST", + "rawSql": "SELECT model_name AS \"Model / Route\", SUM(message_count) AS \"Messages\" FROM _tool_kiro_user_model_messages WHERE $__timeFilter(date) GROUP BY model_name ORDER BY SUM(message_count) DESC", "refId": "A" } ], - "title": "Model Usage Distribution", + "title": "Model / Route Message Distribution", "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Top file extensions used with inline completions", "fieldConfig": { "defaults": { @@ -359,11 +386,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"File Type\", COUNT(*) AS \"Completions\" FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 15", + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"File Type\", COUNT(*) AS \"Completions\" FROM lake._tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 15", "refId": "A" } ], @@ -371,8 +401,11 @@ "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Average number of chat events per conversation", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Conversation IDs are absent in sampled Kiro logs, so conversation depth cannot be computed.", "fieldConfig": { "defaults": { "color": { @@ -451,19 +484,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CAST(COUNT(*) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT CASE WHEN conversation_id <> '' THEN conversation_id END), 0), 0) AS \"Avg Turns per Conversation\", COUNT(DISTINCT CASE WHEN conversation_id <> '' THEN conversation_id END) AS \"Unique Conversations\", COUNT(*) AS \"Total Chat Events\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS \"User Turns\", SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS \"Agent Continuations\", CAST(SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS NUMERIC) * 100.0 / NULLIF(COUNT(*), 0) AS \"Agent Continuation Rate\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Conversation Depth Analysis", + "title": "User Turns vs Agent Continuations", "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Daily chat and completion events over time", "fieldConfig": { "defaults": { @@ -543,11 +582,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(chat) AS \"Chat Events\", SUM(completions) AS \"Completion Events\" FROM (SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS chat, 0 AS completions FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) UNION ALL SELECT CAST(timestamp AS DATE) AS time, 0 AS chat, COUNT(*) AS completions FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE)) AS combined GROUP BY time ORDER BY time NULLS FIRST", + "rawSql": "SELECT time, SUM(chat) AS \"Chat Events\", SUM(completions) AS \"Completion Events\" FROM (SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS chat, 0 AS completions FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) UNION ALL SELECT CAST(timestamp AS DATE) AS time, 0 AS chat, COUNT(*) AS completions FROM lake._tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE)) AS combined GROUP BY time ORDER BY time NULLS FIRST", "refId": "A" } ], @@ -555,8 +597,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Per-user logging activity summary", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Per-user chat and completion activity with display names resolved from usage reports.", "fieldConfig": { "defaults": { "color": { @@ -605,11 +650,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(u.display_name, u.user_id) AS \"User\", u.user_id AS \"User ID\", u.chat_events AS \"Chat Events\", u.conversations AS \"Conversations\", ROUND(CAST(u.chat_events AS NUMERIC) / NULLIF(NULLIF(u.conversations, 0), 0), 1) AS \"Avg Turns\", COALESCE(c.completion_events, 0) AS \"Completion Events\", COALESCE(c.files_count, 0) AS \"Distinct Files\", ROUND(u.avg_prompt_len) AS \"Avg Prompt Len\", ROUND(u.avg_response_len) AS \"Avg Response Len\", u.steering_count AS \"Steering Uses\", u.spec_count AS \"Spec Mode Uses\", u.code_ref_count AS \"Code Refs\", u.web_link_count AS \"Web Links\", u.models_used AS \"Models Used\", u.first_seen AS \"First Seen\", GREATEST(u.last_seen, COALESCE(c.last_seen, u.last_seen)) AS \"Last Seen\" FROM (SELECT user_id, MAX(display_name) AS display_name, COUNT(*) AS chat_events, COUNT(DISTINCT CASE WHEN conversation_id <> '' THEN conversation_id END) AS conversations, AVG(prompt_length) AS avg_prompt_len, AVG(response_length) AS avg_response_len, STRING_AGG(DISTINCT CASE WHEN model_id <> '' AND NOT model_id IS NULL THEN model_id END, ', ' ORDER BY model_id NULLS FIRST) AS models_used, SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS steering_count, SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS spec_count, SUM(code_reference_count) AS code_ref_count, SUM(web_link_count) AS web_link_count, MIN(timestamp) AS first_seen, MAX(timestamp) AS last_seen FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id) AS u LEFT JOIN (SELECT user_id, COUNT(*) AS completion_events, COUNT(DISTINCT file_name) AS files_count, MAX(timestamp) AS last_seen FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id) AS c ON u.user_id = c.user_id ORDER BY u.user_id NULLS FIRST", + "rawSql": "WITH chat AS (SELECT user_id, COUNT(*) AS chat_events, SUM(CASE WHEN has_prompt = TRUE THEN 1 ELSE 0 END) AS user_turns, SUM(CASE WHEN has_prompt = FALSE THEN 1 ELSE 0 END) AS agent_continuations FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id), completion AS (SELECT user_id, COUNT(*) AS completion_events, SUM(returned_line_count) AS returned_lines FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id), names AS (SELECT user_id, MAX(display_name) AS display_name FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY user_id) SELECT COALESCE(n.display_name, c.user_id) AS \"User\", c.user_id AS \"User ID\", c.chat_events AS \"Chat Events\", c.user_turns AS \"User Turns\", c.agent_continuations AS \"Agent Continuations\", COALESCE(x.completion_events, 0) AS \"Completion Events\", COALESCE(x.returned_lines, 0) AS \"Returned Lines\" FROM chat c LEFT JOIN completion x ON c.user_id = x.user_id LEFT JOIN names n ON c.user_id = n.user_id ORDER BY c.chat_events DESC", "refId": "A" } ], @@ -617,7 +665,10 @@ "type": "table" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Distribution of Kiro feature adoption: Steering, Spec Mode, and Plain Chat", "fieldConfig": { "defaults": { @@ -673,11 +724,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Using Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Using Spec Mode\", SUM(CASE WHEN has_steering = FALSE AND is_spec_mode = FALSE THEN 1 ELSE 0 END) AS \"Plain Chat\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Using Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Using Spec Mode\", SUM(CASE WHEN has_steering = FALSE AND is_spec_mode = FALSE THEN 1 ELSE 0 END) AS \"Plain Chat\" FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], @@ -685,8 +739,11 @@ "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Top file extensions active during chat events", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Chat logs do not contain active file extensions; completion logs provide file types and returned lines.", "fieldConfig": { "defaults": { "color": { @@ -741,20 +798,26 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CASE WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file active)' ELSE active_file_extension END AS \"File Type\", COUNT(*) AS \"Chat Events\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"active_file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 15", + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"File Type\", SUM(returned_line_count) AS \"Returned Lines\" FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY file_extension ORDER BY SUM(returned_line_count) DESC", "refId": "A" } ], - "title": "Active File Types in Chat", + "title": "Returned Lines by File Type", "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "How often Kiro responses include code references and web links", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Kiro logs expose follow-up prompts but not code-reference or web-link counters.", "fieldConfig": { "defaults": { "color": { @@ -809,19 +872,25 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(CASE WHEN code_reference_count > 0 THEN 1 ELSE 0 END) AS \"With Code References\", SUM(CASE WHEN web_link_count > 0 THEN 1 ELSE 0 END) AS \"With Web Links\", SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"With Followup Prompts\", SUM(CASE WHEN code_reference_count = '0' AND web_link_count = '0' AND has_followup_prompts = FALSE THEN 1 ELSE 0 END) AS \"Plain Response\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "rawSql": "SELECT SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"With Follow-up Prompts\", SUM(CASE WHEN has_followup_prompts = FALSE THEN 1 ELSE 0 END) AS \"Without Follow-up Prompts\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp)", "refId": "A" } ], - "title": "Response Enrichment Breakdown", + "title": "Follow-up Prompt Breakdown", "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Average and maximum prompt/response lengths over time", "fieldConfig": { "defaults": { @@ -901,11 +970,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, AVG(prompt_length) AS \"Avg Prompt Length\", AVG(response_length) AS \"Avg Response Length\", MAX(prompt_length) AS \"Max Prompt Length\", MAX(response_length) AS \"Max Response Length\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, AVG(prompt_length) AS \"Avg Prompt Length\", AVG(response_length) AS \"Avg Response Length\", MAX(prompt_length) AS \"Max Prompt Length\", MAX(response_length) AS \"Max Response Length\" FROM lake._tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", "refId": "A" } ], @@ -913,7 +985,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Average code context size provided to inline completions over time", "fieldConfig": { "defaults": { @@ -992,11 +1067,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, ROUND(AVG(left_context_length)) AS \"Avg Left Context\", ROUND(AVG(right_context_length)) AS \"Avg Right Context\", ROUND(AVG(left_context_length + right_context_length)) AS \"Avg Total Context\" FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, ROUND(AVG(left_context_length)) AS \"Avg Left Context\", ROUND(AVG(right_context_length)) AS \"Avg Right Context\", ROUND(AVG(left_context_length + right_context_length)) AS \"Avg Total Context\" FROM lake._tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", "refId": "A" } ], @@ -1004,8 +1082,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily trend of code references and web links in chat responses", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Daily responses with and without follow-up prompts. Code-reference and web-link counters are not exported.", "fieldConfig": { "defaults": { "color": { @@ -1083,15 +1164,18 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(code_reference_count) AS \"Code References\", SUM(web_link_count) AS \"Web Links\", SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"Followup Prompts\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"With Follow-up Prompts\", SUM(CASE WHEN has_followup_prompts = FALSE THEN 1 ELSE 0 END) AS \"Without Follow-up Prompts\" FROM _tool_kiro_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY time", "refId": "A" } ], - "title": "Response Enrichment Trends", + "title": "Follow-up Prompt Trends", "type": "timeseries" } ], @@ -1099,9 +1183,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "logging", - "kiro" + "kiro", + "logging" ], "templating": { "list": [] @@ -1113,6 +1196,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro AI Activity Insights", - "uid": "qdev_logging-pg", + "uid": "kiro_logging-pg", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/postgresql/qdev_user_report.json b/grafana/dashboards/postgresql/kiro_user_report.json similarity index 82% rename from grafana/dashboards/postgresql/qdev_user_report.json rename to grafana/dashboards/postgresql/kiro_user_report.json index 592d8a75daf..4f720f1aaf4 100644 --- a/grafana/dashboards/postgresql/qdev_user_report.json +++ b/grafana/dashboards/postgresql/kiro_user_report.json @@ -19,7 +19,10 @@ "links": [], "panels": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Overview of credits and usage metrics", "fieldConfig": { "defaults": { @@ -66,11 +69,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(credits_used) AS \"Total Credits Used\", COUNT(DISTINCT user_id) AS \"Active Users\", SUM(total_messages) AS \"Total Messages\", SUM(chat_conversations) AS \"Total Conversations\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date)", + "rawSql": "SELECT SUM(credits_used) AS \"Total Credits Used\", COUNT(DISTINCT user_id) AS \"Active Users\", SUM(total_messages) AS \"Total Messages\", SUM(chat_conversations) AS \"Total Conversations\" FROM lake._tool_kiro_user_report WHERE $__timeFilter(date)", "refId": "A" } ], @@ -78,7 +84,10 @@ "type": "stat" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Daily credits consumed broken down by subscription tier", "fieldConfig": { "defaults": { @@ -158,11 +167,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, subscription_tier AS metric, SUM(credits_used) AS value FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"date\", \"subscription_tier\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT date AS time, subscription_tier AS metric, SUM(credits_used) AS value FROM lake._tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY \"date\", \"subscription_tier\" ORDER BY date NULLS FIRST", "refId": "A" } ], @@ -170,8 +182,11 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, - "description": "Daily messages and conversations broken down by client type", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Daily messages by Kiro IDE, CLI, Web, and Plugin clients, plus conversations.", "fieldConfig": { "defaults": { "color": { @@ -250,11 +265,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT date AS time, SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) AS \"Messages (IDE)\", SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) AS \"Messages (CLI)\", SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) AS \"Messages (Plugin)\", SUM(chat_conversations) AS \"Conversations\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "rawSql": "SELECT date AS time, SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) AS \"Messages (IDE)\", SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) AS \"Messages (CLI)\", SUM(CASE WHEN client_type = 'KIRO_WEB' THEN total_messages ELSE 0 END) AS \"Messages (Web)\", SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) AS \"Messages (Plugin)\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY date ORDER BY date", "refId": "A" } ], @@ -262,7 +280,10 @@ "type": "timeseries" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Distribution of users across subscription tiers", "fieldConfig": { "defaults": { @@ -318,11 +339,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT subscription_tier AS \"Tier\", COUNT(DISTINCT user_id) AS \"Users\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) AND NOT subscription_tier IS NULL AND subscription_tier <> '' GROUP BY \"subscription_tier\" ORDER BY COUNT(DISTINCT user_id) DESC NULLS LAST", + "rawSql": "SELECT subscription_tier AS \"Tier\", COUNT(DISTINCT user_id) AS \"Users\" FROM lake._tool_kiro_user_report WHERE $__timeFilter(date) AND NOT subscription_tier IS NULL AND subscription_tier <> '' GROUP BY \"subscription_tier\" ORDER BY COUNT(DISTINCT user_id) DESC NULLS LAST", "refId": "A" } ], @@ -330,7 +354,10 @@ "type": "piechart" }, { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "description": "Per-user credits, messages, and subscription details", "fieldConfig": { "defaults": { @@ -429,11 +456,14 @@ "pluginVersion": "13.0.2", "targets": [ { - "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", subscription_tier AS \"Tier\", client_type AS \"Client\", SUM(credits_used) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", SUM(overage_credits_used) AS \"Overage Credits\", CASE WHEN MAX(CAST(overage_enabled AS BIGINT)) = 1 THEN 'Yes' ELSE 'No' END AS \"Overage\", MIN(date) AS \"First Activity\", MAX(date) AS \"Last Activity\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", \"subscription_tier\", \"client_type\" ORDER BY user_id DESC NULLS LAST", + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", subscription_tier AS \"Tier\", client_type AS \"Client\", SUM(credits_used) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", SUM(overage_credits_used) AS \"Overage Credits\", CASE WHEN MAX(CAST(overage_enabled AS BIGINT)) = 1 THEN 'Yes' ELSE 'No' END AS \"Overage\", MIN(date) AS \"First Activity\", MAX(date) AS \"Last Activity\" FROM lake._tool_kiro_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", \"subscription_tier\", \"client_type\" ORDER BY user_id DESC NULLS LAST", "refId": "A" } ], @@ -445,9 +475,8 @@ "refresh": "5m", "schemaVersion": 41, "tags": [ - "q_dev", - "user_report", - "kiro" + "kiro", + "user_report" ], "templating": { "list": [] @@ -459,6 +488,6 @@ "timepicker": {}, "timezone": "utc", "title": "Kiro Usage Dashboard", - "uid": "qdev_user_report-pg", + "uid": "kiro_user_report-pg", "version": 1 -} \ No newline at end of file +}