From ec3ec3be0efe82fd9dcfeb728c2788ec4feea797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Thu, 24 Sep 2026 14:31:20 +0200 Subject: [PATCH 1/6] feat: add stars_count pipe for snapshot-based new-star counts (CM-1785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Uroš Marolt --- services/libs/tinybird/pipes/stars_count.pipe | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 services/libs/tinybird/pipes/stars_count.pipe diff --git a/services/libs/tinybird/pipes/stars_count.pipe b/services/libs/tinybird/pipes/stars_count.pipe new file mode 100644 index 0000000000..3008c8d4e2 --- /dev/null +++ b/services/libs/tinybird/pipes/stars_count.pipe @@ -0,0 +1,175 @@ +DESCRIPTION > + - `stars_count.pipe` provides "new stars" (period-specific, non-cumulative) analytics for a + project or curated collection, replacing `activities_count.pipe` for stars only (CM-1785). + - Built directly on top of `stars_cumulative_count.pipe`: in granularity mode, a period's "new + stars" is just that bucket's cumulative total minus the previous bucket's - correct regardless + of whether the underlying number came from a GitHub snapshot or a GitLab per-event count, so + there's no separate GitHub/GitLab branching in the bucketed mode at all. + - **Can go negative for a bucket** - GitHub's snapshot total is the platform's current net + count (stars minus unstars), so a bucket where unstars outpaced new stars nets negative. Real, + not a bug. GitLab's own contribution to the same bucket can't go negative (it's an event + tally), but the combined number can. + - **Backfill-inflation caveat (same as CM-1441's already-reworked pipes)**: the very first + bucket where a repo's snapshot history exists shows that repo's *entire* historical total as + "new", since there's no earlier snapshot to diff against. Repos whose CM-1463 backfill hasn't + reached back to the start of the requested range will show an inflated first-seen bucket. + - Exactly one of `project` / `collectionSlug` must be given, same contract as + `stars_cumulative_count.pipe` / `activities_filtered.pipe`. + - Parameters: `project`, `collectionSlug`, `repos`, `startDate`, `endDate`, `granularity` - + identical to `stars_cumulative_count.pipe`, passed straight through. + - Response: + - Without granularity: `activityCount` (single row) + - With granularity: `startDate`, `endDate`, `activityCount` per bucket + - Response field names deliberately match `activities_count.pipe` so `stars.ts` can swap the + pipe name with no other code changes. + +NODE stars_count_repo_scope +DESCRIPTION > + Same GitHub repo scoping as `stars_cumulative_count.pipe`'s `stars_repo_scope` node, + duplicated here since Tinybird nodes only see their own pipe file. + +SQL > + % + {% if defined(collectionSlug) %} + SELECT r.id AS repositoryId + FROM repositories r FINAL + WHERE + isNull (r.deletedAt) + AND r.enabled = true + AND r.excluded = false + AND r.segmentId IN (SELECT segmentId FROM segments_filtered_by_collection) + {% if defined(repos) %} + AND has( + arrayMap( + x -> lower(x), + {{ Array(repos, 'String', description="Repository URLs", required=False) }} + ), + lower(r.url) + ) + {% end %} + {% else %} + SELECT r.id AS repositoryId + FROM repositories r FINAL + INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id + WHERE + isNull (r.deletedAt) + AND r.enabled = true + AND r.excluded = false + AND isNull (i.deletedAt) + AND i.enabled = 1 + AND r.segmentId = (SELECT segmentId FROM segments_filtered) + {% if defined(repos) %} + AND has( + arrayMap( + x -> lower(x), + {{ Array(repos, 'String', description="Repository URLs", required=False) }} + ), + lower(r.url) + ) + {% end %} + {% end %} + +NODE stars_count_bucketed +DESCRIPTION > + Granularity mode: diffs consecutive buckets of `stars_cumulative_count`'s running total - "new + stars" in a bucket is just that bucket's cumulative total minus the previous bucket's. First + bucket in range diffs against 0 (see backfill-inflation caveat above). + +SQL > + % + {% if defined(granularity) %} + SELECT + startDate, + endDate, + toInt64(cumulativeActivityCount) - toInt64( + coalesce(lagInFrame(cumulativeActivityCount) OVER (ORDER BY startDate ASC), 0) + ) AS activityCount + FROM stars_cumulative_count + ORDER BY startDate ASC + {% else %} SELECT toDate(now()) AS startDate, toDate(now()) AS endDate, 0 AS activityCount + {% end %} + +NODE stars_count_summary +DESCRIPTION > + No-granularity mode: total "new stars" across [startDate, endDate]. GitHub = total as of + endDate minus total as of startDate (0 if a repo has no snapshot yet at startDate - same + backfill-inflation caveat as the bucketed mode, can also go negative on net unstars). GitLab = + plain per-event range count, `platform != 'github'`. + +SQL > + % + {% if not defined(granularity) %} + WITH + github_at_end AS ( + SELECT sum(starCount) AS total + FROM + ( + SELECT argMax(starCount, (capturedAt, updatedAt)) AS starCount + FROM repositoryStarSnapshots + WHERE + repositoryId IN (SELECT repositoryId FROM stars_count_repo_scope) + {% if defined(endDate) %} + AND capturedAt + <= {{ DateTime(endDate, description="Range end", required=False) }} + {% end %} + GROUP BY repositoryId + ) + ), + github_at_start AS ( + SELECT sum(starCount) AS total + FROM + ( + SELECT argMax(starCount, (capturedAt, updatedAt)) AS starCount + FROM repositoryStarSnapshots + WHERE + repositoryId IN (SELECT repositoryId FROM stars_count_repo_scope) + {% if defined(startDate) %} + AND capturedAt + <= {{ DateTime(startDate, description="Range start", required=False) }} + {% else %} AND 1 = 0 + {% end %} + GROUP BY repositoryId + ) + ), + gitlab_new AS ( + SELECT count() AS c + FROM + {% if defined(collectionSlug) %} activityRelations_collection_bucket_routing + {% else %} activityRelations_bucket_routing + {% end %} as a + WHERE + {% if defined(collectionSlug) %} + {% if defined(bucketId) %} + segmentId IN (SELECT segmentId FROM segments_filtered_by_collection) + AND a.collectionSlug = {{ String(collectionSlug) }} + {% else %} 1 = 0 + {% end %} + {% else %} segmentId = (SELECT segmentId FROM segments_filtered) + {% end %} + AND type = 'star' + AND platform != 'github' + AND channel NOT IN (SELECT channel FROM repos_to_channels_excluded) + {% if defined(repos) %} + AND channel IN (SELECT channel FROM repos_to_channels) + {% end %} + {% if defined(startDate) %} + AND timestamp + > {{ DateTime(startDate, description="Range start", required=False) }} + {% end %} + {% if defined(endDate) %} + AND timestamp < {{ DateTime(endDate, description="Range end", required=False) }} + {% end %} + ) + SELECT + toInt64(coalesce((SELECT total FROM github_at_end), 0)) + - toInt64(coalesce((SELECT total FROM github_at_start), 0)) + + (SELECT c FROM gitlab_new) AS activityCount + {% else %} SELECT 0 AS activityCount + {% end %} + +NODE stars_count +SQL > + % + {% if defined(granularity) %} SELECT startDate, endDate, activityCount FROM stars_count_bucketed + {% else %} SELECT activityCount FROM stars_count_summary + {% end %} From 322906367ec2bd870244e27b631ca06c20418b23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Fri, 25 Sep 2026 10:10:13 +0200 Subject: [PATCH 2/6] fix: grant insights-app-token, guard bucketId, restrict GitHub scope, fix GitLab endDate cutoff (CM-1785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-checked stars_count.pipe against the review findings fixed on the sibling stars_cumulative_count.pipe (PR #4764): - Add missing TOKEN "insights-app-token" READ grant - tb push doesn't auto-grant tokens to new pipes, same lesson as fac30216f. - stars_count_repo_scope's collectionSlug branch was missing the bucketId guard (falls back to 1 = 0 when bucketId isn't resolved yet) that stars_repo_scope already has, and that this same file's own gitlab_new CTE already has - an internal inconsistency, not just a gap versus the sibling pipe. - stars_count_repo_scope was missing the GitHub-only filter (r.url LIKE 'https://github.com/%') its own docstring already claimed existed. - gitlab_new's endDate cutoff was exclusive (<), silently dropping GitLab star events timestamped exactly at endDate - same boundary bug Copilot flagged and got fixed on the sibling pipe (0897a78bb). Signed-off-by: Uroš Marolt --- services/libs/tinybird/pipes/stars_count.pipe | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/services/libs/tinybird/pipes/stars_count.pipe b/services/libs/tinybird/pipes/stars_count.pipe index 3008c8d4e2..a0046fe9ce 100644 --- a/services/libs/tinybird/pipes/stars_count.pipe +++ b/services/libs/tinybird/pipes/stars_count.pipe @@ -23,6 +23,8 @@ DESCRIPTION > - Response field names deliberately match `activities_count.pipe` so `stars.ts` can swap the pipe name with no other code changes. +TOKEN "insights-app-token" READ + NODE stars_count_repo_scope DESCRIPTION > Same GitHub repo scoping as `stars_cumulative_count.pipe`'s `stars_repo_scope` node, @@ -34,18 +36,22 @@ SQL > SELECT r.id AS repositoryId FROM repositories r FINAL WHERE - isNull (r.deletedAt) - AND r.enabled = true - AND r.excluded = false - AND r.segmentId IN (SELECT segmentId FROM segments_filtered_by_collection) - {% if defined(repos) %} - AND has( - arrayMap( - x -> lower(x), - {{ Array(repos, 'String', description="Repository URLs", required=False) }} - ), - lower(r.url) - ) + {% if defined(bucketId) %} + isNull (r.deletedAt) + AND r.enabled = true + AND r.excluded = false + AND r.url LIKE 'https://github.com/%' + AND r.segmentId IN (SELECT segmentId FROM segments_filtered_by_collection) + {% if defined(repos) %} + AND has( + arrayMap( + x -> lower(x), + {{ Array(repos, 'String', description="Repository URLs", required=False) }} + ), + lower(r.url) + ) + {% end %} + {% else %} 1 = 0 {% end %} {% else %} SELECT r.id AS repositoryId @@ -55,6 +61,7 @@ SQL > isNull (r.deletedAt) AND r.enabled = true AND r.excluded = false + AND r.url LIKE 'https://github.com/%' AND isNull (i.deletedAt) AND i.enabled = 1 AND r.segmentId = (SELECT segmentId FROM segments_filtered) @@ -157,7 +164,8 @@ SQL > > {{ DateTime(startDate, description="Range start", required=False) }} {% end %} {% if defined(endDate) %} - AND timestamp < {{ DateTime(endDate, description="Range end", required=False) }} + AND timestamp + <= {{ DateTime(endDate, description="Range end", required=False) }} {% end %} ) SELECT From 00f975d101d3c326bfd504ec1c4bd0ca5b0cff0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Fri, 25 Sep 2026 11:08:49 +0200 Subject: [PATCH 3/6] fix: rename stars_count's terminal node to avoid pipe/node name collision (CM-1785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same bug as stars_cumulative_count.pipe (crowd.dev#4768): Tinybird rejects a node with the exact same name as its pipe, so this pipe would have failed to deploy on first push. Renamed the terminal node from stars_count to stars_count_final, matching the convention every other multi-node pipe in this repo follows. No other file references the node by name. Signed-off-by: Uroš Marolt --- services/libs/tinybird/pipes/stars_count.pipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/tinybird/pipes/stars_count.pipe b/services/libs/tinybird/pipes/stars_count.pipe index a0046fe9ce..2ecd607333 100644 --- a/services/libs/tinybird/pipes/stars_count.pipe +++ b/services/libs/tinybird/pipes/stars_count.pipe @@ -175,7 +175,7 @@ SQL > {% else %} SELECT 0 AS activityCount {% end %} -NODE stars_count +NODE stars_count_final SQL > % {% if defined(granularity) %} SELECT startDate, endDate, activityCount FROM stars_count_bucketed From a1fdda482163d0cba4445cdadd6c9307f69533e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Fri, 25 Sep 2026 11:42:55 +0200 Subject: [PATCH 4/6] docs: document bucketId requirement in stars_count's parameter contract (CM-1785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Copilot review: both collection branches (repo scoping and summary mode) return no rows unless bucketId is defined, but the pipe's own DESCRIPTION never mentioned it - a caller reading the docs could mistake an empty response for a valid zero. stars_cumulative_count.pipe already documents this same requirement for the same reason; stars_count.pipe just hadn't caught up. Signed-off-by: Uroš Marolt --- services/libs/tinybird/pipes/stars_count.pipe | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/libs/tinybird/pipes/stars_count.pipe b/services/libs/tinybird/pipes/stars_count.pipe index 2ecd607333..2045c669bb 100644 --- a/services/libs/tinybird/pipes/stars_count.pipe +++ b/services/libs/tinybird/pipes/stars_count.pipe @@ -14,9 +14,13 @@ DESCRIPTION > "new", since there's no earlier snapshot to diff against. Repos whose CM-1463 backfill hasn't reached back to the start of the requested range will show an inflated first-seen bucket. - Exactly one of `project` / `collectionSlug` must be given, same contract as - `stars_cumulative_count.pipe` / `activities_filtered.pipe`. - - Parameters: `project`, `collectionSlug`, `repos`, `startDate`, `endDate`, `granularity` - - identical to `stars_cumulative_count.pipe`, passed straight through. + `stars_cumulative_count.pipe` / `activities_filtered.pipe`. `bucketId` is required alongside + `collectionSlug` for the same reason `stars_cumulative_count.pipe` needs it - both collection + branches here return no rows unless it's defined. The insights backend's Tinybird client + already resolves and injects `bucketId` transparently for both `project` and `collectionSlug` + on every pipe call, same as it does for every other pipe today. + - Parameters: `project`, `collectionSlug`, `bucketId`, `repos`, `startDate`, `endDate`, + `granularity` - identical to `stars_cumulative_count.pipe`, passed straight through. - Response: - Without granularity: `activityCount` (single row) - With granularity: `startDate`, `endDate`, `activityCount` per bucket From edc8fd336d780eb3ec57c26f22e35bcc9645d5b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Fri, 25 Sep 2026 11:53:03 +0200 Subject: [PATCH 5/6] docs: correct stars_count summary-mode range notation to (startDate, endDate] (CM-1785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Copilot review: the SQL excludes a snapshot/event captured exactly at startDate (github_at_start folds it into the baseline, and the GitLab predicate is timestamp > startDate), so the actual range is (startDate, endDate], not [startDate, endDate] as previously documented. Signed-off-by: Uroš Marolt --- services/libs/tinybird/pipes/stars_count.pipe | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/libs/tinybird/pipes/stars_count.pipe b/services/libs/tinybird/pipes/stars_count.pipe index 2045c669bb..eef2c914a4 100644 --- a/services/libs/tinybird/pipes/stars_count.pipe +++ b/services/libs/tinybird/pipes/stars_count.pipe @@ -102,8 +102,9 @@ SQL > NODE stars_count_summary DESCRIPTION > - No-granularity mode: total "new stars" across [startDate, endDate]. GitHub = total as of - endDate minus total as of startDate (0 if a repo has no snapshot yet at startDate - same + No-granularity mode: total "new stars" across (startDate, endDate] - a star captured/counted + exactly at startDate belongs to the prior period's baseline, not this one. GitHub = total as + of endDate minus total as of startDate (0 if a repo has no snapshot yet at startDate - same backfill-inflation caveat as the bucketed mode, can also go negative on net unstars). GitLab = plain per-event range count, `platform != 'github'`. From 84145ff04f2601f9751221f81953b8e457330ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Fri, 25 Sep 2026 12:30:48 +0200 Subject: [PATCH 6/6] docs: drop JIRA ticket references from stars_count pipe description (CM-1785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback: pipe DESCRIPTION blocks shouldn't carry ticket numbers that go stale once the work ships. Reworded the same points without the CM-XXX references. Signed-off-by: Uroš Marolt --- services/libs/tinybird/pipes/stars_count.pipe | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/libs/tinybird/pipes/stars_count.pipe b/services/libs/tinybird/pipes/stars_count.pipe index eef2c914a4..24fe034030 100644 --- a/services/libs/tinybird/pipes/stars_count.pipe +++ b/services/libs/tinybird/pipes/stars_count.pipe @@ -1,6 +1,6 @@ DESCRIPTION > - `stars_count.pipe` provides "new stars" (period-specific, non-cumulative) analytics for a - project or curated collection, replacing `activities_count.pipe` for stars only (CM-1785). + project or curated collection, replacing `activities_count.pipe` for stars only. - Built directly on top of `stars_cumulative_count.pipe`: in granularity mode, a period's "new stars" is just that bucket's cumulative total minus the previous bucket's - correct regardless of whether the underlying number came from a GitHub snapshot or a GitLab per-event count, so @@ -9,10 +9,11 @@ DESCRIPTION > count (stars minus unstars), so a bucket where unstars outpaced new stars nets negative. Real, not a bug. GitLab's own contribution to the same bucket can't go negative (it's an event tally), but the combined number can. - - **Backfill-inflation caveat (same as CM-1441's already-reworked pipes)**: the very first - bucket where a repo's snapshot history exists shows that repo's *entire* historical total as - "new", since there's no earlier snapshot to diff against. Repos whose CM-1463 backfill hasn't - reached back to the start of the requested range will show an inflated first-seen bucket. + - **Backfill-inflation caveat (same as the already-reworked sibling star pipes)**: the very + first bucket where a repo's snapshot history exists shows that repo's *entire* historical + total as "new", since there's no earlier snapshot to diff against. Repos whose star-snapshot + backfill hasn't reached back to the start of the requested range will show an inflated + first-seen bucket. - Exactly one of `project` / `collectionSlug` must be given, same contract as `stars_cumulative_count.pipe` / `activities_filtered.pipe`. `bucketId` is required alongside `collectionSlug` for the same reason `stars_cumulative_count.pipe` needs it - both collection