From 04a424b4bbf10da3e495ca0a57a668aba56a312d Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 10:18:36 +0000
Subject: [PATCH 01/24] ClickHouse?

---
 .env.example                                  |   6 +
 DEVELOPMENT.md                                |   3 +
 Gemfile                                       |   2 +
 .../api/admin/v1/admin_controller.rb          | 102 ++++------
 .../api/hackatime/v1/hackatime_controller.rb  |   2 +-
 .../api/v1/authenticated/hours_controller.rb  |   2 +-
 .../concerns/api/admin/v1/user_utilities.rb   |   2 +-
 app/jobs/cache/active_projects_job.rb         |  34 +++-
 app/jobs/cache/active_users_graph_data_job.rb |  25 +--
 app/jobs/cache/currently_hacking_count_job.rb |   7 +-
 app/jobs/cache/currently_hacking_job.rb       |  29 +--
 app/jobs/cache/home_stats_job.rb              |  12 +-
 app/jobs/leaderboard_update_job.rb            |  10 +-
 .../recalc_heartbeat_field_hash_job.rb        |   2 +-
 app/jobs/sync_all_user_repo_events_job.rb     |   9 +-
 app/jobs/weekly_summary_email_job.rb          |  15 +-
 app/mailers/weekly_summary_mailer.rb          |  10 +-
 app/models/concerns/heartbeatable.rb          | 180 ++++++++----------
 app/models/heartbeat.rb                       |  34 +---
 app/models/user.rb                            |   1 +
 app/services/anonymize_user_service.rb        |   4 +-
 app/services/heartbeat_import_service.rb      |   4 +-
 app/services/leaderboard_builder.rb           |   7 +-
 app/services/profile_stats_service.rb         |  21 +-
 app/services/timeline_service.rb              |   4 +-
 config/database.yml                           |  24 +++
 docker-compose.yml                            |  26 ++-
 lib/tasks/seed_dummy_users.rake               |   2 +-
 .../previews/weekly_summary_mailer_preview.rb |   4 +-
 test/models/heartbeat_test.rb                 |  12 +-
 test/services/anonymize_user_service_test.rb  |   6 +-
 31 files changed, 314 insertions(+), 287 deletions(-)

diff --git a/.env.example b/.env.example
index 0192c6534..575c01fe1 100644
--- a/.env.example
+++ b/.env.example
@@ -60,3 +60,9 @@ S3_ACCESS_KEY_ID=your_s3_access_key_id_here
 S3_SECRET_ACCESS_KEY=your_s3_secret_access_key_here
 S3_BUCKET=your_s3_bucket_name_here
 S3_ENDPOINT=https://<ACCOUNT_ID>.r2.cloudflarestorage.com
+
+# ClickHouse database
+CLICKHOUSE_URL=http://clickhouse:8123
+CLICKHOUSE_DATABASE=hackatime_development
+CLICKHOUSE_USERNAME=default
+CLICKHOUSE_PASSWORD=
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
index 18cf369ec..90f56c2e9 100644
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@@ -40,6 +40,8 @@ HCA_CLIENT_ID=<hca_client_id>
 HCA_CLIENT_SECRET=<hca_client_secret>
 ```
 
+ClickHouse is automatically started by Docker Compose alongside Postgres — no extra setup needed.
+
 Start the containers:
 
 ```sh
@@ -51,6 +53,7 @@ We'll now setup the database. In your container shell, run the following:
 
 ```bash
 app# bin/rails db:create db:schema:load db:seed
+app# bin/rails db:migrate:clickhouse
 ```
 
 Run the Vite build with SSR (server-side-rendering):
diff --git a/Gemfile b/Gemfile
index 4a01d0745..4c4fc89c4 100644
--- a/Gemfile
+++ b/Gemfile
@@ -9,6 +9,8 @@ gem "propshaft"
 gem "sqlite3", ">= 2.1"
 # Use PostgreSQL as the database for Wakatime
 gem "pg"
+# Use ClickHouse for analytics
+gem "clickhouse-activerecord"
 # Use the Puma web server [https://github.com/puma/puma]
 gem "puma", ">= 5.0"
 # Use JavaScript with ESM import maps [https://github.com/rails/importmap-rails]
diff --git a/app/controllers/api/admin/v1/admin_controller.rb b/app/controllers/api/admin/v1/admin_controller.rb
index a69716147..73e855e9d 100644
--- a/app/controllers/api/admin/v1/admin_controller.rb
+++ b/app/controllers/api/admin/v1/admin_controller.rb
@@ -51,71 +51,77 @@ def visualization_quantized
           quantized_query = <<-SQL
             WITH base_heartbeats AS (
                 SELECT
-                    "time",
+                    time,
                     lineno,
                     cursorpos,
-                    date_trunc('day', to_timestamp("time")) as day_start
+                    toDate(toDateTime(toUInt32(time))) as day_start
                 FROM heartbeats
                 WHERE user_id = ?
-                AND "time" >= ? AND "time" <= ?
+                AND time >= ? AND time <= ?
                 AND (lineno IS NOT NULL OR cursorpos IS NOT NULL)
                 LIMIT 1000000
             ),
             daily_stats AS (
                 SELECT
                     *,
-                    GREATEST(1, MAX(lineno) OVER (PARTITION BY day_start)) as max_lineno,
-                    GREATEST(1, MAX(cursorpos) OVER (PARTITION BY day_start)) as max_cursorpos
+                    greatest(1, max(lineno) OVER (PARTITION BY day_start)) as max_lineno,
+                    greatest(1, max(cursorpos) OVER (PARTITION BY day_start)) as max_cursorpos
                 FROM base_heartbeats
             ),
             quantized_heartbeats AS (
                 SELECT
                     *,
-                    ROUND(2 + (("time" - extract(epoch from day_start)) / 86400) * (396)) as qx,
-                    ROUND(2 + (1 - CAST(lineno AS decimal) / max_lineno) * (96)) as qy_lineno,
-                    ROUND(2 + (1 - CAST(cursorpos AS decimal) / max_cursorpos) * (96)) as qy_cursorpos
+                    round(2 + ((time - toUInt32(toDateTime(day_start))) / 86400) * (396)) as qx,
+                    round(2 + (1 - CAST(lineno AS Float64) / max_lineno) * (96)) as qy_lineno,
+                    round(2 + (1 - CAST(cursorpos AS Float64) / max_cursorpos) * (96)) as qy_cursorpos
                 FROM daily_stats
             )
-            SELECT "time", lineno, cursorpos
+            SELECT time, lineno, cursorpos
             FROM (
-                SELECT DISTINCT ON (day_start, qx, qy_lineno) "time", lineno, cursorpos
+                SELECT
+                    any(time) AS time,
+                    any(lineno) AS lineno,
+                    any(cursorpos) AS cursorpos
                 FROM quantized_heartbeats
                 WHERE lineno IS NOT NULL
-                ORDER BY day_start, qx, qy_lineno, "time" ASC
+                GROUP BY day_start, qx, qy_lineno
             ) AS lineno_pixels
-            UNION
-            SELECT "time", lineno, cursorpos
+            UNION ALL
+            SELECT time, lineno, cursorpos
             FROM (
-                SELECT DISTINCT ON (day_start, qx, qy_cursorpos) "time", lineno, cursorpos
+                SELECT
+                    any(time) AS time,
+                    any(lineno) AS lineno,
+                    any(cursorpos) AS cursorpos
                 FROM quantized_heartbeats
                 WHERE cursorpos IS NOT NULL
-                ORDER BY day_start, qx, qy_cursorpos, "time" ASC
+                GROUP BY day_start, qx, qy_cursorpos
             ) AS cursorpos_pixels
-            ORDER BY "time" ASC
+            ORDER BY time ASC
           SQL
 
           daily_totals_query = <<-SQL
             WITH heartbeats_with_gaps AS (
               SELECT
-                date_trunc('day', to_timestamp("time"))::date as day,
-                "time" - LAG("time", 1, "time") OVER (PARTITION BY date_trunc('day', to_timestamp("time")) ORDER BY "time") as gap
+                toDate(toDateTime(toUInt32(time))) as day,
+                time - lagInFrame(time, 1, time) OVER (PARTITION BY toDate(toDateTime(toUInt32(time))) ORDER BY time) as gap
               FROM heartbeats
               WHERE user_id = ? AND time >= ? AND time <= ?
             )
             SELECT
               day,
-              SUM(LEAST(gap, 120)) as total_seconds
+              SUM(least(gap, 120)) as total_seconds
             FROM heartbeats_with_gaps
             WHERE gap IS NOT NULL
             GROUP BY day
           SQL
 
-          quantized_result = ActiveRecord::Base.connection.execute(
-            ActiveRecord::Base.sanitize_sql([ quantized_query, user.id, start_epoch, end_epoch ])
+          quantized_result = Heartbeat.connection.select_all(
+            Heartbeat.sanitize_sql([ quantized_query, user.id, start_epoch, end_epoch ])
           )
 
-          daily_totals_result = ActiveRecord::Base.connection.execute(
-            ActiveRecord::Base.sanitize_sql([ daily_totals_query, user.id, start_epoch, end_epoch ])
+          daily_totals_result = Heartbeat.connection.select_all(
+            Heartbeat.sanitize_sql([ daily_totals_query, user.id, start_epoch, end_epoch ])
           )
 
           daily_totals = daily_totals_result.each_with_object({}) do |row, hash|
@@ -197,8 +203,8 @@ def alt_candidates
             LIMIT 5000
           SQL
 
-          result = ActiveRecord::Base.connection.exec_query(
-            ActiveRecord::Base.sanitize_sql([ query, cutoff, cutoff ])
+          result = Heartbeat.connection.select_all(
+            Heartbeat.sanitize_sql([ query, cutoff, cutoff ])
           )
 
           render json: { candidates: result.to_a }
@@ -210,44 +216,20 @@ def shared_machines
 
           query = <<-SQL
             SELECT
-              sms.machine,
-              sms.machine_frequency,
-              ARRAY_AGG(DISTINCT u.id) AS user_ids
-            FROM
-              (
-                SELECT
-                  machine,
-                  COUNT(user_id) AS machine_frequency,
-                  ARRAY_AGG(user_id) AS user_ids
-                FROM
-                  (
-                    SELECT DISTINCT
-                      machine,
-                      user_id
-                    FROM
-                      heartbeats
-                    WHERE
-                      machine IS NOT NULL
-                      AND time > ?
-                  ) AS UserMachines
-                GROUP BY
-                  machine
-                HAVING
-                  COUNT(user_id) > 1
-              ) AS sms,
-              LATERAL UNNEST(sms.user_ids) AS user_id_from_array
-            JOIN
-              users AS u ON u.id = user_id_from_array
-            GROUP BY
-              sms.machine,
-              sms.machine_frequency
-            ORDER BY
-              sms.machine_frequency DESC
+              machine,
+              uniq(user_id) AS machine_frequency,
+              groupArray(DISTINCT user_id) AS user_ids
+            FROM heartbeats
+            WHERE machine != '' AND machine IS NOT NULL
+              AND time > ?
+            GROUP BY machine
+            HAVING uniq(user_id) > 1
+            ORDER BY machine_frequency DESC
             LIMIT 5000
           SQL
 
-          result = ActiveRecord::Base.connection.exec_query(
-            ActiveRecord::Base.sanitize_sql([ query, cutoff ])
+          result = Heartbeat.connection.select_all(
+            Heartbeat.sanitize_sql([ query, cutoff ])
           )
 
           render json: { machines: result.to_a }
diff --git a/app/controllers/api/hackatime/v1/hackatime_controller.rb b/app/controllers/api/hackatime/v1/hackatime_controller.rb
index 3b1fed660..8a60d3b37 100644
--- a/app/controllers/api/hackatime/v1/hackatime_controller.rb
+++ b/app/controllers/api/hackatime/v1/hackatime_controller.rb
@@ -282,7 +282,7 @@ def handle_heartbeat(heartbeat_array)
       }).slice(*Heartbeat.column_names.map(&:to_sym))
       # ^^ They say safety laws are written in blood. Well, so is this line!
       # Basically this filters out columns that aren't in our DB (the biggest one being raw_data)
-      new_heartbeat = Heartbeat.find_or_create_by(attrs)
+      new_heartbeat = Heartbeat.create(attrs)
 
       queue_project_mapping(heartbeat[:project])
       results << [ new_heartbeat.attributes, 201 ]
diff --git a/app/controllers/api/v1/authenticated/hours_controller.rb b/app/controllers/api/v1/authenticated/hours_controller.rb
index d9348815f..2f51d9061 100644
--- a/app/controllers/api/v1/authenticated/hours_controller.rb
+++ b/app/controllers/api/v1/authenticated/hours_controller.rb
@@ -7,7 +7,7 @@ def index
           end_date = params[:end_date]&.to_date || Date.current
 
           total_seconds = current_user.heartbeats
-                                      .where(created_at: start_date.beginning_of_day..end_date.end_of_day)
+                                      .where(time: start_date.beginning_of_day.to_f..end_date.end_of_day.to_f)
                                       .duration_seconds
 
           render json: {
diff --git a/app/controllers/concerns/api/admin/v1/user_utilities.rb b/app/controllers/concerns/api/admin/v1/user_utilities.rb
index 807b42ca0..26803d792 100644
--- a/app/controllers/concerns/api/admin/v1/user_utilities.rb
+++ b/app/controllers/concerns/api/admin/v1/user_utilities.rb
@@ -179,7 +179,7 @@ def user_info
                 total_coding_time: valid.duration_seconds || 0,
                 languages_used: valid.distinct.pluck(:language).compact.count,
                 projects_worked_on: valid.distinct.pluck(:project).compact.count,
-                days_active: valid.distinct.count("DATE(to_timestamp(CASE WHEN time > 1000000000000 THEN time / 1000 ELSE time END))")
+                days_active: Heartbeat.connection.select_value("SELECT uniq(toDate(toDateTime(toUInt32(time)))) FROM (#{valid.to_sql}) AS hb").to_i
               }
             }
           }
diff --git a/app/jobs/cache/active_projects_job.rb b/app/jobs/cache/active_projects_job.rb
index 59d175068..f63e280b9 100644
--- a/app/jobs/cache/active_projects_job.rb
+++ b/app/jobs/cache/active_projects_job.rb
@@ -8,14 +8,30 @@ def cache_expiration
   end
 
   def calculate
-    # Get recent heartbeats with matching project_repo_mappings in a single SQL query
-    ProjectRepoMapping.active
-                      .joins("INNER JOIN heartbeats ON heartbeats.project = project_repo_mappings.project_name AND heartbeats.user_id = project_repo_mappings.user_id")
-                      .joins("INNER JOIN users ON users.id = heartbeats.user_id")
-                      .where("heartbeats.source_type = ?", Heartbeat.source_types[:direct_entry])
-                      .where("heartbeats.time > ?", 5.minutes.ago.to_f)
-                      .select("DISTINCT ON (heartbeats.user_id) project_repo_mappings.*, heartbeats.user_id")
-                      .order("heartbeats.user_id, heartbeats.time DESC")
-                      .index_by(&:user_id)
+    # Query recent heartbeats from ClickHouse
+    recent_hbs = Heartbeat.where(source_type: Heartbeat.source_types[:direct_entry])
+                          .where("time > ?", 5.minutes.ago.to_f)
+                          .order(time: :desc)
+                          .to_a
+
+    # Deduplicate by user_id (most recent heartbeat per user)
+    latest_by_user = recent_hbs.group_by(&:user_id).transform_values(&:first)
+
+    return {} if latest_by_user.empty?
+
+    # Find matching project_repo_mappings from Postgres
+    user_ids = latest_by_user.keys
+
+    mappings = ProjectRepoMapping.active
+                                 .where(user_id: user_ids)
+                                 .to_a
+
+    result = {}
+    latest_by_user.each do |user_id, hb|
+      mapping = mappings.find { |m| m.user_id == user_id && m.project_name == hb.project }
+      result[user_id] = mapping if mapping
+    end
+
+    result
   end
 end
diff --git a/app/jobs/cache/active_users_graph_data_job.rb b/app/jobs/cache/active_users_graph_data_job.rb
index 9cbc380d7..f9a68417b 100644
--- a/app/jobs/cache/active_users_graph_data_job.rb
+++ b/app/jobs/cache/active_users_graph_data_job.rb
@@ -5,21 +5,24 @@ class Cache::ActiveUsersGraphDataJob < Cache::ActivityJob
 
   def calculate
     # over the last 24 hours, count the number of people who were active each hour
-    hours = Heartbeat.coding_only
-                     .with_valid_timestamps
-                     .where("time > ?", 24.hours.ago.to_f)
-                     .where("time < ?", Time.current.to_f)
-                     .select("(EXTRACT(EPOCH FROM to_timestamp(time))::bigint / 3600 * 3600) as hour, COUNT(DISTINCT user_id) as count")
-                     .group("hour")
-                     .order("hour DESC")
+    connection = Heartbeat.connection
+    hours = connection.select_all(<<~SQL
+      SELECT
+        toInt64(toUInt32(time) / 3600) * 3600 AS hour,
+        uniq(user_id) AS count
+      FROM (#{Heartbeat.coding_only.with_valid_timestamps.where("time > ?", 24.hours.ago.to_f).where("time < ?", Time.current.to_f).to_sql}) AS hb
+      GROUP BY hour
+      ORDER BY hour DESC
+    SQL
+    )
 
-    top_hour_count = hours.max_by(&:count)&.count || 1
+    top_hour_count = hours.map { |h| h["count"].to_i }.max || 1
 
     hours = hours.map do |h|
       {
-        hour: Time.at(h.hour),
-        users: h.count,
-        height: (h.count.to_f / top_hour_count * 100).round
+        hour: Time.at(h["hour"].to_i),
+        users: h["count"].to_i,
+        height: (h["count"].to_f / top_hour_count * 100).round
       }
     end
   end
diff --git a/app/jobs/cache/currently_hacking_count_job.rb b/app/jobs/cache/currently_hacking_count_job.rb
index 6930d62e8..ff0120c60 100644
--- a/app/jobs/cache/currently_hacking_count_job.rb
+++ b/app/jobs/cache/currently_hacking_count_job.rb
@@ -8,12 +8,11 @@ def cache_expiration
   end
 
   def calculate
-    count = Heartbeat.joins(:user)
-                    .where(source_type: :direct_entry)
+    count = Heartbeat.where(source_type: :direct_entry)
                     .coding_only
                     .where("time > ?", 5.minutes.ago.to_f)
-                    .select("DISTINCT user_id")
-                    .count
+                    .distinct
+                    .count(:user_id)
 
     { count: count }
   end
diff --git a/app/jobs/cache/currently_hacking_job.rb b/app/jobs/cache/currently_hacking_job.rb
index 236e2de3c..aa90fceab 100644
--- a/app/jobs/cache/currently_hacking_job.rb
+++ b/app/jobs/cache/currently_hacking_job.rb
@@ -8,17 +8,24 @@ def cache_expiration
   end
 
   def calculate
-    # Get most recent heartbeats and users in a single query
-    recent_heartbeats = Heartbeat.joins(:user)
-                                .where(source_type: :direct_entry)
-                                .coding_only
-                                .where("time > ?", 5.minutes.ago.to_f)
-                                .select("DISTINCT ON (user_id) user_id, project, time, users.*")
-                                .order("user_id, time DESC")
-                                .includes(user: [ :project_repo_mappings, :email_addresses ])
-                                .index_by(&:user_id)
-
-    users = recent_heartbeats.values.map(&:user)
+    # Query ClickHouse for recent heartbeats (no cross-DB join)
+    raw_heartbeats = Heartbeat.where(source_type: :direct_entry)
+                              .coding_only
+                              .where("time > ?", 5.minutes.ago.to_f)
+                              .order(time: :desc)
+                              .to_a
+
+    # Deduplicate by user_id (keep most recent)
+    recent_heartbeats = raw_heartbeats.group_by(&:user_id)
+                                      .transform_values(&:first)
+
+    # Load users from Postgres
+    user_ids = recent_heartbeats.keys
+    users_by_id = User.where(id: user_ids)
+                      .includes(:project_repo_mappings, :email_addresses)
+                      .index_by(&:id)
+
+    users = user_ids.filter_map { |uid| users_by_id[uid] }
 
     active_projects = {}
     users.each do |user|
diff --git a/app/jobs/cache/home_stats_job.rb b/app/jobs/cache/home_stats_job.rb
index 2626dae8c..6d2ef377e 100644
--- a/app/jobs/cache/home_stats_job.rb
+++ b/app/jobs/cache/home_stats_job.rb
@@ -4,10 +4,16 @@ class Cache::HomeStatsJob < Cache::ActivityJob
   private
 
   def calculate
-    seconds_by_user = Heartbeat.group(:user_id).duration_seconds
+    result = HeartbeatUserDailySummary.connection.select_one(<<~SQL)
+      SELECT
+        uniq(user_id) AS users_tracked,
+        toInt64(coalesce(sum(duration_s), 0)) AS seconds_tracked
+      FROM heartbeat_user_daily_summary FINAL
+    SQL
+
     {
-      users_tracked: seconds_by_user.size,
-      seconds_tracked: seconds_by_user.values.sum
+      users_tracked: result["users_tracked"].to_i,
+      seconds_tracked: result["seconds_tracked"].to_i
     }
   end
 end
diff --git a/app/jobs/leaderboard_update_job.rb b/app/jobs/leaderboard_update_job.rb
index 39b94861a..4059f574a 100644
--- a/app/jobs/leaderboard_update_job.rb
+++ b/app/jobs/leaderboard_update_job.rb
@@ -31,13 +31,15 @@ def build_leaderboard(date, period, force_update = false)
     range = LeaderboardDateRange.calculate(date, period)
 
     ActiveRecord::Base.transaction do
-      # Build the base heartbeat query
+      # Get eligible user IDs from Postgres (can't cross-DB join)
+      eligible_user_ids = User.where.not(github_uid: nil)
+                              .where.not(trust_level: User.trust_levels[:red])
+                              .pluck(:id)
+
       heartbeat_query = Heartbeat.where(time: range)
                                  .with_valid_timestamps
-                                 .joins(:user)
                                  .coding_only
-                                 .where.not(users: { github_uid: nil })
-                                 .where.not(users: { trust_level: User.trust_levels[:red] })
+                                 .where(user_id: eligible_user_ids)
 
       data = heartbeat_query.group(:user_id).duration_seconds
                             .filter { |_, seconds| seconds > 60 }
diff --git a/app/jobs/one_time/recalc_heartbeat_field_hash_job.rb b/app/jobs/one_time/recalc_heartbeat_field_hash_job.rb
index 7d4f334cb..4218e2ca7 100644
--- a/app/jobs/one_time/recalc_heartbeat_field_hash_job.rb
+++ b/app/jobs/one_time/recalc_heartbeat_field_hash_job.rb
@@ -8,7 +8,7 @@ def perform
         heartbeat.save!
       rescue ActiveRecord::RecordNotUnique
         # If we have a duplicate fields_hash, soft delete this record
-        heartbeat.soft_delete
+        heartbeat.delete
       end
     end
   end
diff --git a/app/jobs/sync_all_user_repo_events_job.rb b/app/jobs/sync_all_user_repo_events_job.rb
index 252611acf..131143fe2 100644
--- a/app/jobs/sync_all_user_repo_events_job.rb
+++ b/app/jobs/sync_all_user_repo_events_job.rb
@@ -10,11 +10,14 @@ def perform
     # Identify users:
     # 1. Authenticated with GitHub (have an access token and username)
     # 2. Have had heartbeats in the last 6 hours
+    # Get user IDs with recent heartbeats from ClickHouse (can't cross-DB join)
+    recent_user_ids = Heartbeat.where("time >= ?", 6.hours.ago.to_f)
+                               .distinct
+                               .pluck(:user_id)
+
     users_to_sync = User.where.not(github_access_token: nil)
                         .where.not(github_username: nil)
-                        .joins(:heartbeats) # Assumes User has_many :heartbeats
-                        .where("heartbeats.created_at >= ?", 6.hours.ago)
-                        .distinct
+                        .where(id: recent_user_ids)
 
     if users_to_sync.empty?
       Rails.logger.info "No users eligible for GitHub event sync at this time."
diff --git a/app/jobs/weekly_summary_email_job.rb b/app/jobs/weekly_summary_email_job.rb
index d4f7dbdec..acb365553 100644
--- a/app/jobs/weekly_summary_email_job.rb
+++ b/app/jobs/weekly_summary_email_job.rb
@@ -15,18 +15,13 @@ def perform(reference_time = Time.current)
   private
 
   def eligible_users(cutoff)
-    users = User.arel_table
-    heartbeats = Heartbeat.arel_table
-
-    recent_activity_exists = Heartbeat.unscoped
-      .where(heartbeats[:user_id].eq(users[:id]))
-      .where(heartbeats[:deleted_at].eq(nil))
-      .where(heartbeats[:time].gteq(cutoff.to_f))
-      .arel
-      .exists
+    # Query ClickHouse for user_ids with recent activity (can't do cross-DB subquery)
+    active_user_ids = Heartbeat.where("time >= ?", cutoff.to_f)
+                               .distinct
+                               .pluck(:user_id)
 
     User.subscribed("weekly_summary").where(
-      users[:created_at].gteq(cutoff).or(recent_activity_exists)
+      User.arel_table[:created_at].gteq(cutoff).or(User.arel_table[:id].in(active_user_ids))
     ).where.not(id: DeletionRequest.active.select(:user_id))
   end
 end
diff --git a/app/mailers/weekly_summary_mailer.rb b/app/mailers/weekly_summary_mailer.rb
index 632652aff..9e22a3754 100644
--- a/app/mailers/weekly_summary_mailer.rb
+++ b/app/mailers/weekly_summary_mailer.rb
@@ -48,10 +48,12 @@ def breakdown(scope, column, limit: 5)
 
   def active_days_count(scope)
     timezone = @timezone_label
-    timezone_sql = ActiveRecord::Base.connection.quote(timezone)
-    scope.where.not(time: nil)
-      .distinct
-      .count(Arel.sql("DATE(to_timestamp(time) AT TIME ZONE #{timezone_sql})"))
+    # ClickHouse-compatible: count distinct days using toDate with timezone
+    result = Heartbeat.connection.select_value(<<~SQL)
+      SELECT uniq(toDate(toDateTime(toUInt32(time), '#{timezone}')))
+      FROM (#{scope.where.not(time: nil).to_sql}) AS hb
+    SQL
+    result.to_i
   rescue StandardError
     scope.where.not(time: nil).pluck(:time).map { |time| Time.at(time).in_time_zone(timezone).to_date }.uniq.count
   end
diff --git a/app/models/concerns/heartbeatable.rb b/app/models/concerns/heartbeatable.rb
index b32f24af9..35039d195 100644
--- a/app/models/concerns/heartbeatable.rb
+++ b/app/models/concerns/heartbeatable.rb
@@ -2,11 +2,7 @@ module Heartbeatable
   extend ActiveSupport::Concern
 
   included do
-    # Filter heartbeats to only include those with category equal to "coding"
     scope :coding_only, -> { where(category: "coding") }
-
-    # This is to prevent PG timestamp overflow errors if someones gives us a
-    # heartbeat with a time that is enormously far in the future.
     scope :with_valid_timestamps, -> { where("time >= 0 AND time <= ?", 253402300799) }
   end
 
@@ -28,7 +24,7 @@ def to_span(timeout_duration: nil)
       sql = <<~SQL
         SELECT
           time,
-          LEAD(time) OVER (ORDER BY time) as next_time
+          leadInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_time
         FROM (#{heartbeats.to_sql}) AS heartbeats
       SQL
 
@@ -42,10 +38,10 @@ def to_span(timeout_duration: nil)
         current_time = row["time"]
         next_time = row["next_time"]
 
-        if next_time.nil? || (next_time - current_time) > timeout_duration
+        if next_time.nil? || next_time == 0 || (next_time - current_time) > timeout_duration
           base_duration = (current_time - current_span_start).round
 
-          if next_time
+          if next_time && next_time != 0
             gap_duration = [ next_time - current_time, timeout_duration ].min
             total_duration = base_duration + gap_duration
             end_time = current_time + gap_duration
@@ -62,7 +58,7 @@ def to_span(timeout_duration: nil)
             }
           end
 
-          current_span_start = next_time if next_time
+          current_span_start = next_time if next_time && next_time != 0
         end
       end
 
@@ -79,9 +75,6 @@ def duration_formatted(scope = all)
     end
 
     def duration_simple(scope = all)
-      # 3 hours 10 min => "3 hrs"
-      # 1 hour 10 min => "1 hr"
-      # 10 min => "10 min"
       seconds = duration_seconds(scope)
       hours = seconds / 3600
       minutes = (seconds % 3600) / 60
@@ -109,70 +102,67 @@ def daily_streaks_for_users(user_ids, start_date: 31.days.ago)
         return user_ids.index_with { |id| streak_cache["user_streak_#{id}"] || 0 }
       end
 
+      # Fetch user timezones from Postgres
+      user_timezones = User.where(id: uncached_users).pluck(:id, :timezone).to_h
+
       timeout = heartbeat_timeout_duration.to_i
-      raw_durations = joins(:user)
-        .where(user_id: uncached_users)
-        .coding_only
-        .with_valid_timestamps
-        .where(time: start_date..Time.current)
-        .select(
-          :user_id,
-          "users.timezone as user_timezone",
-          Arel.sql("DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE users.timezone) as day_group"),
-          Arel.sql("LEAST(time - LAG(time) OVER (PARTITION BY user_id, DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE users.timezone) ORDER BY time), #{timeout}) as diff")
+
+      # Query heartbeats from ClickHouse (no cross-DB join)
+      raw_sql = <<~SQL
+        SELECT
+          user_id,
+          toDate(toDateTime(toUInt32(time))) AS day_group,
+          toInt64(coalesce(sum(diff), 0)) AS duration
+        FROM (
+          SELECT
+            user_id,
+            time,
+            least(
+              time - lagInFrame(time) OVER (PARTITION BY user_id, toDate(toDateTime(toUInt32(time))) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+              #{timeout}
+            ) AS diff
+          FROM heartbeats
+          WHERE user_id IN (#{uncached_users.join(',')})
+            AND category = 'coding'
+            AND time >= 0 AND time <= 253402300799
+            AND time >= #{start_date.to_f}
+            AND time <= #{Time.current.to_f}
         )
+        GROUP BY user_id, day_group
+      SQL
 
-      # Then aggregate the results
-      daily_durations = connection.select_all(
-        "SELECT user_id, user_timezone, day_group, COALESCE(SUM(diff), 0)::integer as duration
-         FROM (#{raw_durations.to_sql}) AS diffs
-         GROUP BY user_id, user_timezone, day_group"
-      ).group_by { |row| row["user_id"] }
-       .transform_values do |rows|
-         timezone = rows.first["user_timezone"]
-
-         if timezone.blank?
-           Rails.logger.warn "nil tz, going to utc."
-           timezone = "UTC"
-         else
-           begin
-             TZInfo::Timezone.get(timezone)
-           rescue TZInfo::InvalidTimezoneIdentifier, ArgumentError
-             Rails.logger.warn "Invalid timezone for streak calculation: #{timezone}. Defaulting to UTC."
-             timezone = "UTC"
-           end
-         end
-
-         current_date = Time.current.in_time_zone(timezone).to_date
-         {
-           current_date: current_date,
-           days: rows.map do |row|
-             [ row["day_group"].to_date, row["duration"].to_i ]
-           end.sort_by { |date, _| date }.reverse
-         }
-       end
+      rows = connection.select_all(raw_sql)
+
+      daily_durations = rows.group_by { |row| row["user_id"].to_i }
+        .transform_values do |user_rows|
+          user_rows.map do |row|
+            [ row["day_group"].to_date, row["duration"].to_i ]
+          end.sort_by { |date, _| date }.reverse
+        end
 
       result = user_ids.index_with { |id| streak_cache["user_streak_#{id}"] || 0 }
 
-      # Then calculate streaks for each user
-      daily_durations.each do |user_id, data|
-        current_date = data[:current_date]
-        days = data[:days]
+      daily_durations.each do |user_id, days|
+        timezone = user_timezones[user_id] || "UTC"
+
+        begin
+          TZInfo::Timezone.get(timezone)
+        rescue TZInfo::InvalidTimezoneIdentifier, ArgumentError
+          timezone = "UTC"
+        end
+
+        current_date = Time.current.in_time_zone(timezone).to_date
 
-        # Calculate streak
         streak = 0
         days.each do |date, duration|
-          # Skip if this day is in the future
           next if date > current_date
 
-          # If they didn't code enough today, just skip
           if date == current_date
             next unless duration >= 15 * 60
             streak += 1
             next
           end
 
-          # For previous days, check if it's the next day in the streak
           if date == current_date - streak.days && duration >= 15 * 60
             streak += 1
           else
@@ -181,8 +171,6 @@ def daily_streaks_for_users(user_ids, start_date: 31.days.ago)
         end
 
         result[user_id] = streak
-
-        # Cache the streak for 1 hour
         Rails.cache.write("user_streak_#{user_id}", streak, expires_in: 1.hour)
       end
 
@@ -197,14 +185,24 @@ def daily_durations(user_timezone:, start_date: 365.days.ago, end_date: Time.cur
         timezone = "UTC"
       end
 
-      # Create the timezone-aware date truncation expression
-      day_trunc = Arel.sql("DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE '#{timezone}')")
+      sql = <<~SQL
+        SELECT
+          toDate(toDateTime(toUInt32(time), '#{timezone}')) AS day_group,
+          toInt64(coalesce(sum(diff), 0)) AS duration
+        FROM (
+          SELECT
+            time,
+            least(
+              time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+              #{heartbeat_timeout_duration.to_i}
+            ) AS diff
+          FROM (#{with_valid_timestamps.where(time: start_date.to_f..end_date.to_f).to_sql}) AS hb
+        )
+        GROUP BY day_group
+        ORDER BY day_group
+      SQL
 
-      select(day_trunc.as("day_group"))
-        .where(time: start_date..end_date)
-        .group(day_trunc)
-        .duration_seconds
-        .map { |date, duration| [ date.to_date, duration ] }
+      connection.select_all(sql).map { |row| [ row["day_group"].to_date, row["duration"].to_i ] }
     end
 
     def duration_seconds(scope = all)
@@ -217,35 +215,26 @@ def duration_seconds(scope = all)
         end
 
         group_column = scope.group_values.first
+        group_expr = group_column.to_s.include?("(") ? group_column : "`#{group_column}`"
 
-        # Don't quote if it's a SQL function (contains parentheses)
-        group_expr = group_column.to_s.include?("(") ? group_column : connection.quote_column_name(group_column)
-
-        capped_diffs = scope
-          .select("#{group_expr} as grouped_time, CASE
-            WHEN LAG(time) OVER (PARTITION BY #{group_expr} ORDER BY time) IS NULL THEN 0
-            ELSE LEAST(time - LAG(time) OVER (PARTITION BY #{group_expr} ORDER BY time), #{timeout})
-          END as diff")
+        capped_diffs_sql = scope
+          .select("#{group_expr} as grouped_time, least(time - lagInFrame(time) OVER (PARTITION BY #{group_expr} ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
           .where.not(time: nil)
           .unscope(:group)
+          .to_sql
 
         connection.select_all(
-          "SELECT grouped_time, COALESCE(SUM(diff), 0)::integer as duration
-          FROM (#{capped_diffs.to_sql}) AS diffs
-          GROUP BY grouped_time"
+          "SELECT grouped_time, toInt64(coalesce(sum(diff), 0)) as duration FROM (#{capped_diffs_sql}) GROUP BY grouped_time"
         ).each_with_object({}) do |row, hash|
           hash[row["grouped_time"]] = row["duration"].to_i
         end
       else
-        # when not grouped, return a single value
-        capped_diffs = scope
-          .select("CASE
-            WHEN LAG(time) OVER (ORDER BY time) IS NULL THEN 0
-            ELSE LEAST(time - LAG(time) OVER (ORDER BY time), #{timeout})
-          END as diff")
+        capped_diffs_sql = scope
+          .select("least(time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
           .where.not(time: nil)
+          .to_sql
 
-        connection.select_value("SELECT COALESCE(SUM(diff), 0)::integer FROM (#{capped_diffs.to_sql}) AS diffs").to_i
+        connection.select_value("SELECT toInt64(coalesce(sum(diff), 0)) FROM (#{capped_diffs_sql})").to_i
       end
     end
 
@@ -256,7 +245,7 @@ def duration_seconds_boundary_aware(scope, start_time, end_time)
       base_scope = model_class.all.with_valid_timestamps
 
       excluded_categories = [ "browsing", "ai coding", "meeting", "communicating" ]
-      base_scope = base_scope.where.not("LOWER(category) IN (?)", excluded_categories)
+      base_scope = base_scope.where.not("lower(category) IN (?)", excluded_categories)
 
       if scope.where_values_hash["user_id"]
         base_scope = base_scope.where(user_id: scope.where_values_hash["user_id"])
@@ -270,18 +259,12 @@ def duration_seconds_boundary_aware(scope, start_time, end_time)
         base_scope = base_scope.where(project: scope.where_values_hash["project"])
       end
 
-      if scope.where_values_hash["deleted_at"]
-        base_scope = base_scope.where(deleted_at: scope.where_values_hash["deleted_at"])
-      end
-
-      # get the heartbeat before the start_time
       boundary_heartbeat = base_scope
         .where("time < ?", start_time)
         .order(time: :desc)
         .limit(1)
         .first
 
-      # if it's not NULL, we'll use it
       if boundary_heartbeat
         combined_scope = base_scope
           .where("time >= ? OR time = ?", start_time, boundary_heartbeat.time)
@@ -291,19 +274,14 @@ def duration_seconds_boundary_aware(scope, start_time, end_time)
           .where(time: start_time..end_time)
       end
 
-      # we calc w/ the boundary heartbeat, but we only sum within the orignal constraint
       timeout = heartbeat_timeout_duration.to_i
-      capped_diffs = combined_scope
-        .select("time, CASE
-          WHEN LAG(time) OVER (ORDER BY time) IS NULL THEN 0
-          ELSE LEAST(time - LAG(time) OVER (ORDER BY time), #{timeout})
-        END as diff")
+      capped_diffs_sql = combined_scope
+        .select("time, least(time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
         .where.not(time: nil)
         .order(time: :asc)
+        .to_sql
 
-      sql = "SELECT COALESCE(SUM(diff), 0)::integer
-             FROM (#{capped_diffs.to_sql}) AS diffs
-             WHERE time >= #{connection.quote(start_time)}"
+      sql = "SELECT toInt64(coalesce(sum(diff), 0)) FROM (#{capped_diffs_sql}) WHERE time >= #{connection.quote(start_time)}"
       connection.select_value(sql).to_i
     end
   end
diff --git a/app/models/heartbeat.rb b/app/models/heartbeat.rb
index 60b1372e4..53532794d 100644
--- a/app/models/heartbeat.rb
+++ b/app/models/heartbeat.rb
@@ -1,18 +1,13 @@
-class Heartbeat < ApplicationRecord
-  before_save :set_fields_hash!
+class Heartbeat < ClickhouseRecord
+  self.table_name = "heartbeats"
 
   include Heartbeatable
   include TimeRangeFilterable
 
   time_range_filterable_field :time
 
-  # Default scope to exclude deleted records
-  default_scope { where(deleted_at: nil) }
-
   scope :today, -> { where(time: Time.current.beginning_of_day.to_i..Time.current.end_of_day.to_i) }
   scope :recent, -> { where("time > ?", 24.hours.ago.to_i) }
-  scope :with_deleted, -> { unscope(where: :deleted_at) }
-  scope :only_deleted, -> { with_deleted.where.not(deleted_at: nil) }
 
   enum :source_type, {
     direct_entry: 0,
@@ -82,15 +77,15 @@ class Heartbeat < ApplicationRecord
     neighborhood: 58
   }, prefix: :claimed_by
 
-  # This is to prevent Rails from trying to use STI even though we have a "type" column
+  # Prevent Rails STI on the "type" column
   self.inheritance_column = nil
 
+  # Note: cross-database joins (Postgres users <-> ClickHouse heartbeats) will not work.
+  # Use separate queries instead of .joins(:heartbeats) or .includes(:heartbeats).
   belongs_to :user
 
   validates :time, presence: true
 
-  # after_create :mirror_to_wakatime
-
   def self.recent_count
     Cache::HeartbeatCountsJob.perform_now[:recent_count]
   end
@@ -109,24 +104,5 @@ def self.indexed_attributes
     %w[user_id branch category dependencies editor entity language machine operating_system project type user_agent line_additions line_deletions lineno lines cursorpos project_root_count time is_write]
   end
 
-  def soft_delete
-    update_column(:deleted_at, Time.current)
-  end
-
-  def restore
-    update_column(:deleted_at, nil)
-  end
-
-  private
-
-  def set_fields_hash!
-    # only if the field exists in activerecord
-    if self.class.column_names.include?("fields_hash")
-      self.fields_hash = self.class.generate_fields_hash(self.attributes)
-    end
-  end
 
-  # def mirror_to_wakatime
-  #   WakatimeMirror.mirror_heartbeat(self)
-  # end
 end
diff --git a/app/models/user.rb b/app/models/user.rb
index b165c8dfd..d1ee03dfd 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -108,6 +108,7 @@ def set_trust(level, changed_by_user: nil, reason: nil, notes: nil)
   end
   # ex: .set_trust(:green) or set_trust(1) setting it to red
 
+  # Heartbeats live in ClickHouse — simple scoped queries work but SQL JOINs will not.
   has_many :heartbeats
   has_many :goals, dependent: :destroy
   has_many :email_addresses, dependent: :destroy
diff --git a/app/services/anonymize_user_service.rb b/app/services/anonymize_user_service.rb
index 3d064821a..a1fac3f97 100644
--- a/app/services/anonymize_user_service.rb
+++ b/app/services/anonymize_user_service.rb
@@ -75,8 +75,8 @@ def destroy_associated_records
     user.project_repo_mappings.destroy_all
     user.goals.destroy_all
 
-    # tombstone
-    Heartbeat.unscoped.where(user_id: user.id, deleted_at: nil).update_all(deleted_at: Time.current)
+    # Delete heartbeats from ClickHouse
+    Heartbeat.where(user_id: user.id).delete_all
 
     user.access_grants.destroy_all
     user.access_tokens.destroy_all
diff --git a/app/services/heartbeat_import_service.rb b/app/services/heartbeat_import_service.rb
index 19a4db95d..5c56e15e6 100644
--- a/app/services/heartbeat_import_service.rb
+++ b/app/services/heartbeat_import_service.rb
@@ -108,8 +108,8 @@ def self.flush_batch(seen_hashes)
     end
 
     ActiveRecord::Base.logger.silence do
-      result = Heartbeat.upsert_all(records, unique_by: [ :fields_hash ])
-      result.length
+      Heartbeat.insert_all(records)
+      records.length
     end
   end
 
diff --git a/app/services/leaderboard_builder.rb b/app/services/leaderboard_builder.rb
index 43001b682..a818a706b 100644
--- a/app/services/leaderboard_builder.rb
+++ b/app/services/leaderboard_builder.rb
@@ -17,11 +17,12 @@ def build_for_users(users, date, scope, period)
 
     range = LeaderboardDateRange.calculate(date, period)
 
-    beats = Heartbeat.where(user_id: ids, time: range)
+    # Filter to users with github_uid from Postgres (can't cross-DB join)
+    eligible_ids = users.where.not(github_uid: nil).pluck(:id)
+
+    beats = Heartbeat.where(user_id: eligible_ids, time: range)
                     .coding_only
                     .with_valid_timestamps
-                    .joins(:user)
-                    .where.not(users: { github_uid: nil })
 
     totals = beats.group(:user_id).duration_seconds
     totals = totals.filter { |_, seconds| seconds > 60 }
diff --git a/app/services/profile_stats_service.rb b/app/services/profile_stats_service.rb
index 157a25d8f..0b564bcc6 100644
--- a/app/services/profile_stats_service.rb
+++ b/app/services/profile_stats_service.rb
@@ -61,13 +61,12 @@ def compute_totals_and_breakdowns(timeout, today_start, today_end, week_start, w
           project,
           language,
           editor,
-          CASE
-            WHEN LAG(time) OVER (ORDER BY time) IS NULL THEN 0
-            ELSE LEAST(time - LAG(time) OVER (ORDER BY time), #{timeout_quoted})
-          END AS diff
+          least(
+            time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+            #{timeout_quoted}
+          ) AS diff
         FROM heartbeats
         WHERE user_id = #{user_id}
-          AND deleted_at IS NULL
           AND time IS NOT NULL
           AND time >= 0 AND time <= 253402300799
       )
@@ -76,9 +75,9 @@ def compute_totals_and_breakdowns(timeout, today_start, today_end, week_start, w
     totals_sql = <<~SQL
       #{base_sql}
       SELECT
-        COALESCE(SUM(diff) FILTER (WHERE time >= #{today_start_quoted} AND time <= #{today_end_quoted}), 0)::integer AS today_seconds,
-        COALESCE(SUM(diff) FILTER (WHERE time >= #{week_start_quoted} AND time <= #{week_end_quoted}), 0)::integer AS week_seconds,
-        COALESCE(SUM(diff), 0)::integer AS all_seconds
+        toInt64(coalesce(sumIf(diff, time >= #{today_start_quoted} AND time <= #{today_end_quoted}), 0)) AS today_seconds,
+        toInt64(coalesce(sumIf(diff, time >= #{week_start_quoted} AND time <= #{week_end_quoted}), 0)) AS week_seconds,
+        toInt64(coalesce(sum(diff), 0)) AS all_seconds
       FROM heartbeat_diffs
     SQL
 
@@ -104,7 +103,7 @@ def fetch_top_grouped(conn, base_sql, column, time_filter, limit)
     time_clause = time_filter ? "AND time >= #{time_filter}" : ""
     sql = <<~SQL
       #{base_sql}
-      SELECT #{column}, COALESCE(SUM(diff), 0)::integer AS duration
+      SELECT #{column}, toInt64(coalesce(sum(diff), 0)) AS duration
       FROM heartbeat_diffs
       WHERE #{column} IS NOT NULL AND #{column} != ''
       #{time_clause}
@@ -121,7 +120,7 @@ def fetch_top_grouped(conn, base_sql, column, time_filter, limit)
   def fetch_top_grouped_with_repo(conn, base_sql, month_ago, limit)
     sql = <<~SQL
       #{base_sql}
-      SELECT project, COALESCE(SUM(diff), 0)::integer AS duration
+      SELECT project, toInt64(coalesce(sum(diff), 0)) AS duration
       FROM heartbeat_diffs
       WHERE time >= #{month_ago}
         AND project IS NOT NULL AND project != ''
@@ -142,7 +141,7 @@ def fetch_top_grouped_with_repo(conn, base_sql, month_ago, limit)
   def fetch_top_editors_normalized(conn, base_sql, limit)
     sql = <<~SQL
       #{base_sql}
-      SELECT editor, COALESCE(SUM(diff), 0)::integer AS duration
+      SELECT editor, toInt64(coalesce(sum(diff), 0)) AS duration
       FROM heartbeat_diffs
       WHERE editor IS NOT NULL AND editor != ''
       GROUP BY editor
diff --git a/app/services/timeline_service.rb b/app/services/timeline_service.rb
index bb2e4da04..88eba929e 100644
--- a/app/services/timeline_service.rb
+++ b/app/services/timeline_service.rb
@@ -24,7 +24,7 @@ def timeline_data
       user_start_of_day = date.in_time_zone(user_tz).beginning_of_day.to_f
       user_end_of_day = date.in_time_zone(user_tz).end_of_day.to_f
 
-      total_coded_time_seconds = Heartbeat.where(user_id: user.id, deleted_at: nil)
+      total_coded_time_seconds = Heartbeat.where(user_id: user.id)
                                           .where("time >= ? AND time <= ?", user_start_of_day, user_end_of_day)
                                           .duration_seconds
 
@@ -80,7 +80,7 @@ def heartbeats_by_user_id
       expanded_end = server_end_of_day + 24.hours.to_i
 
       Heartbeat
-        .where(user_id: valid_user_ids, deleted_at: nil)
+        .where(user_id: valid_user_ids)
         .where("time >= ? AND time <= ?", expanded_start, expanded_end)
         .select(:id, :user_id, :time, :entity, :project, :editor, :language)
         .order(:user_id, :time)
diff --git a/config/database.yml b/config/database.yml
index 95e82cdfc..088291770 100644
--- a/config/database.yml
+++ b/config/database.yml
@@ -20,6 +20,14 @@ development:
     encoding: unicode
     url: <%= ENV['SAILORS_LOG_DATABASE_URL'] %>
     replica: true
+  clickhouse:
+    adapter: clickhouse
+    host: <%= ENV.fetch('CLICKHOUSE_HOST', 'clickhouse') %>
+    port: <%= ENV.fetch('CLICKHOUSE_PORT', 8123) %>
+    database: <%= ENV.fetch('CLICKHOUSE_DATABASE', 'hackatime_development') %>
+    username: <%= ENV.fetch('CLICKHOUSE_USERNAME', 'default') %>
+    password: <%= ENV.fetch('CLICKHOUSE_PASSWORD', '') %>
+    migrations_paths: db/migrate_clickhouse
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -35,6 +43,14 @@ test:
     database: app_test
     url: <%= ENV['TEST_DATABASE_URL'] %>
     replica: true
+  clickhouse:
+    adapter: clickhouse
+    host: <%= ENV.fetch('CLICKHOUSE_HOST', 'clickhouse') %>
+    port: <%= ENV.fetch('CLICKHOUSE_PORT', 8123) %>
+    database: <%= ENV.fetch('CLICKHOUSE_DATABASE', 'hackatime_test') %>
+    username: <%= ENV.fetch('CLICKHOUSE_USERNAME', 'default') %>
+    password: <%= ENV.fetch('CLICKHOUSE_PASSWORD', '') %>
+    migrations_paths: db/migrate_clickhouse
 
 # Store production database in the storage/ directory, which by default
 # is mounted as a persistent Docker volume in config/deploy.yml.
@@ -60,3 +76,11 @@ production:
     adapter: sqlite3
     database: storage/production_cable.sqlite3
     migrations_paths: db/cable_migrate
+  clickhouse:
+    adapter: clickhouse
+    host: <%= ENV.fetch('CLICKHOUSE_HOST', 'clickhouse') %>
+    port: <%= ENV.fetch('CLICKHOUSE_PORT', 8123) %>
+    database: <%= ENV.fetch('CLICKHOUSE_DATABASE', 'hackatime_production') %>
+    username: <%= ENV.fetch('CLICKHOUSE_USERNAME', 'default') %>
+    password: <%= ENV.fetch('CLICKHOUSE_PASSWORD', '') %>
+    migrations_paths: db/migrate_clickhouse
diff --git a/docker-compose.yml b/docker-compose.yml
index d21461b40..058e08d77 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,8 +17,15 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=secureorpheus123
       - TEST_DATABASE_URL=postgres://postgres:secureorpheus123@db:5432/app_test
+      - CLICKHOUSE_URL=http://clickhouse:8123
+      - CLICKHOUSE_DATABASE=hackatime_development
+      - CLICKHOUSE_USERNAME=default
+      - CLICKHOUSE_PASSWORD=
     depends_on:
-      - db
+      db:
+        condition: service_started
+      clickhouse:
+        condition: service_healthy
     command: ["sleep", "infinity"]
 
   db:
@@ -32,7 +39,24 @@ services:
     ports:
       - "5432:5432"
 
+  clickhouse:
+    image: clickhouse/clickhouse-server:24
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+    environment:
+      - CLICKHOUSE_DB=hackatime_development
+      - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+
 volumes:
   harbor_postgres_data:
+  clickhouse_data:
   bundle_cache:
   node_modules:
diff --git a/lib/tasks/seed_dummy_users.rake b/lib/tasks/seed_dummy_users.rake
index 03f8c8266..4d5a73359 100644
--- a/lib/tasks/seed_dummy_users.rake
+++ b/lib/tasks/seed_dummy_users.rake
@@ -33,7 +33,7 @@ namespace :seed do
     ids = User.where("github_uid LIKE ?", "dummy_%").ids
     return puts "no dummies found (except for you)" if ids.empty?
 
-    Heartbeat.unscoped.where(user_id: ids).delete_all
+    Heartbeat.where(user_id: ids).delete_all
     LeaderboardEntry.where(user_id: ids).delete_all
     User.where(id: ids).delete_all
     puts "exploded #{ids.count} dummies"
diff --git a/test/mailers/previews/weekly_summary_mailer_preview.rb b/test/mailers/previews/weekly_summary_mailer_preview.rb
index 3c9b4626b..2d040e011 100644
--- a/test/mailers/previews/weekly_summary_mailer_preview.rb
+++ b/test/mailers/previews/weekly_summary_mailer_preview.rb
@@ -1,6 +1,8 @@
 class WeeklySummaryMailerPreview < ActionMailer::Preview
   def weekly_summary
-    user = User.joins(:heartbeats).distinct.first || User.first
+    # Can't cross-DB join — find users with heartbeats separately
+    user_ids = Heartbeat.distinct.limit(1).pluck(:user_id)
+    user = (user_ids.any? ? User.find_by(id: user_ids.first) : nil) || User.first
     ends_at = Time.current.beginning_of_week
     starts_at = ends_at - 7.days
 
diff --git a/test/models/heartbeat_test.rb b/test/models/heartbeat_test.rb
index 3398ed337..3c949e881 100644
--- a/test/models/heartbeat_test.rb
+++ b/test/models/heartbeat_test.rb
@@ -1,7 +1,7 @@
 require "test_helper"
 
 class HeartbeatTest < ActiveSupport::TestCase
-  test "soft delete hides record from default scope and restore brings it back" do
+  test "lightweight delete removes record from queries" do
     user = User.create!
     heartbeat = user.heartbeats.create!(
       entity: "src/main.rb",
@@ -14,13 +14,9 @@ class HeartbeatTest < ActiveSupport::TestCase
 
     assert_includes Heartbeat.all, heartbeat
 
-    heartbeat.soft_delete
+    heartbeat.delete
 
-    assert_not_includes Heartbeat.all, heartbeat
-    assert_includes Heartbeat.with_deleted, heartbeat
-
-    heartbeat.restore
-
-    assert_includes Heartbeat.all, heartbeat
+    # After lightweight delete, the row is invisible to subsequent queries
+    assert_not_includes Heartbeat.where(user_id: user.id).to_a, heartbeat
   end
 end
diff --git a/test/services/anonymize_user_service_test.rb b/test/services/anonymize_user_service_test.rb
index 8c72c8a2a..84c8b45e1 100644
--- a/test/services/anonymize_user_service_test.rb
+++ b/test/services/anonymize_user_service_test.rb
@@ -52,9 +52,9 @@ class AnonymizeUserServiceTest < ActiveSupport::TestCase
     assert_equal 0, user.sign_in_tokens.count
   end
 
-  test "anonymization soft deletes active heartbeats" do
+  test "anonymization deletes heartbeats via lightweight delete" do
     user = User.create!(username: "hb_cleanup_#{SecureRandom.hex(4)}")
-    heartbeat = user.heartbeats.create!(
+    user.heartbeats.create!(
       entity: "src/app.rb",
       type: "file",
       category: "coding",
@@ -65,7 +65,7 @@ class AnonymizeUserServiceTest < ActiveSupport::TestCase
 
     AnonymizeUserService.call(user)
 
-    assert heartbeat.reload.deleted_at.present?
+    assert_equal 0, Heartbeat.where(user_id: user.id).count
   end
 
   test "anonymization removes legacy encrypted import credentials" do

From fb11941163709f2b444cc22ef477685d8e3f49b5 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 10:21:05 +0000
Subject: [PATCH 02/24] clean

---
 app/models/clickhouse_record.rb               |  4 ++
 app/models/heartbeat.rb                       |  2 -
 app/models/heartbeat_user_daily_summary.rb    |  3 ++
 db/migrate_clickhouse/.keep                   |  0
 .../20260324000001_create_heartbeats.rb       | 44 +++++++++++++++++++
 ...002_create_heartbeat_user_daily_summary.rb | 20 +++++++++
 6 files changed, 71 insertions(+), 2 deletions(-)
 create mode 100644 app/models/clickhouse_record.rb
 create mode 100644 app/models/heartbeat_user_daily_summary.rb
 create mode 100644 db/migrate_clickhouse/.keep
 create mode 100644 db/migrate_clickhouse/20260324000001_create_heartbeats.rb
 create mode 100644 db/migrate_clickhouse/20260324000002_create_heartbeat_user_daily_summary.rb

diff --git a/app/models/clickhouse_record.rb b/app/models/clickhouse_record.rb
new file mode 100644
index 000000000..f802a27e4
--- /dev/null
+++ b/app/models/clickhouse_record.rb
@@ -0,0 +1,4 @@
+class ClickhouseRecord < ActiveRecord::Base
+  self.abstract_class = true
+  connects_to database: { writing: :clickhouse, reading: :clickhouse }
+end
diff --git a/app/models/heartbeat.rb b/app/models/heartbeat.rb
index 53532794d..0bcf5b034 100644
--- a/app/models/heartbeat.rb
+++ b/app/models/heartbeat.rb
@@ -103,6 +103,4 @@ def self.generate_fields_hash(attributes)
   def self.indexed_attributes
     %w[user_id branch category dependencies editor entity language machine operating_system project type user_agent line_additions line_deletions lineno lines cursorpos project_root_count time is_write]
   end
-
-
 end
diff --git a/app/models/heartbeat_user_daily_summary.rb b/app/models/heartbeat_user_daily_summary.rb
new file mode 100644
index 000000000..c60ebd42d
--- /dev/null
+++ b/app/models/heartbeat_user_daily_summary.rb
@@ -0,0 +1,3 @@
+class HeartbeatUserDailySummary < ClickhouseRecord
+  self.table_name = "heartbeat_user_daily_summary"
+end
diff --git a/db/migrate_clickhouse/.keep b/db/migrate_clickhouse/.keep
new file mode 100644
index 000000000..e69de29bb
diff --git a/db/migrate_clickhouse/20260324000001_create_heartbeats.rb b/db/migrate_clickhouse/20260324000001_create_heartbeats.rb
new file mode 100644
index 000000000..b2b0b59ef
--- /dev/null
+++ b/db/migrate_clickhouse/20260324000001_create_heartbeats.rb
@@ -0,0 +1,44 @@
+class CreateHeartbeats < ActiveRecord::Migration[8.1]
+  def up
+    execute <<~SQL
+      CREATE TABLE IF NOT EXISTS heartbeats
+      (
+          id Int64,
+          user_id Int64,
+          branch String DEFAULT '',
+          category String DEFAULT '',
+          dependencies Array(String),
+          editor String DEFAULT '',
+          entity String DEFAULT '',
+          language String DEFAULT '',
+          machine String DEFAULT '',
+          operating_system String DEFAULT '',
+          project String DEFAULT '',
+          type String DEFAULT '',
+          user_agent String DEFAULT '',
+          line_additions Int32 DEFAULT 0,
+          line_deletions Int32 DEFAULT 0,
+          lineno Int32 DEFAULT 0,
+          lines Int32 DEFAULT 0,
+          cursorpos Int32 DEFAULT 0,
+          project_root_count Int32 DEFAULT 0,
+          time Float64,
+          is_write UInt8 DEFAULT 0,
+          created_at DateTime64(6) DEFAULT now64(),
+          updated_at DateTime64(6) DEFAULT now64(),
+          source_type UInt8 DEFAULT 0,
+          ip_address String DEFAULT '',
+          ysws_program UInt8 DEFAULT 0,
+          fields_hash String DEFAULT ''
+      )
+      ENGINE = ReplacingMergeTree()
+      PARTITION BY toYYYYMM(toDateTime(toUInt32(time)))
+      ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id)
+      SETTINGS index_granularity = 8192
+    SQL
+  end
+
+  def down
+    execute "DROP TABLE IF EXISTS heartbeats"
+  end
+end
diff --git a/db/migrate_clickhouse/20260324000002_create_heartbeat_user_daily_summary.rb b/db/migrate_clickhouse/20260324000002_create_heartbeat_user_daily_summary.rb
new file mode 100644
index 000000000..ff257bed9
--- /dev/null
+++ b/db/migrate_clickhouse/20260324000002_create_heartbeat_user_daily_summary.rb
@@ -0,0 +1,20 @@
+class CreateHeartbeatUserDailySummary < ActiveRecord::Migration[8.1]
+  def up
+    execute <<~SQL
+      CREATE TABLE IF NOT EXISTS heartbeat_user_daily_summary
+      (
+          user_id    Int64,
+          day        Date,
+          duration_s Float64,
+          heartbeats UInt32,
+          _version   DateTime DEFAULT now()
+      )
+      ENGINE = ReplacingMergeTree(_version)
+      ORDER BY (user_id, day)
+    SQL
+  end
+
+  def down
+    execute "DROP TABLE IF EXISTS heartbeat_user_daily_summary"
+  end
+end

From ea5b7d4f3b00c0706231aef6ce8be900ed27fe1b Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 14:23:13 +0000
Subject: [PATCH 03/24] Address Greptile comments

---
 .env.example                                  |  3 +-
 Gemfile.lock                                  |  4 ++
 .../api/admin/v1/admin_controller.rb          |  8 +--
 .../api/hackatime/v1/hackatime_controller.rb  |  5 +-
 app/mailers/weekly_summary_mailer.rb          |  3 +-
 app/models/concerns/heartbeatable.rb          | 52 +++++++++---------
 app/services/heartbeat_import_service.rb      |  6 +-
 config/brakeman.ignore                        | 54 ++++++++++++++++--
 db/clickhouse_schema.rb                       | 55 +++++++++++++++++++
 9 files changed, 151 insertions(+), 39 deletions(-)
 create mode 100644 db/clickhouse_schema.rb

diff --git a/.env.example b/.env.example
index 575c01fe1..6ac978177 100644
--- a/.env.example
+++ b/.env.example
@@ -62,7 +62,8 @@ S3_BUCKET=your_s3_bucket_name_here
 S3_ENDPOINT=https://<ACCOUNT_ID>.r2.cloudflarestorage.com
 
 # ClickHouse database
-CLICKHOUSE_URL=http://clickhouse:8123
+CLICKHOUSE_HOST=clickhouse
+CLICKHOUSE_PORT=8123
 CLICKHOUSE_DATABASE=hackatime_development
 CLICKHOUSE_USERNAME=default
 CLICKHOUSE_PASSWORD=
diff --git a/Gemfile.lock b/Gemfile.lock
index 34249d07a..951045426 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -131,6 +131,9 @@ GEM
     childprocess (5.1.0)
       logger (~> 1.5)
     chunky_png (1.4.0)
+    clickhouse-activerecord (1.6.7)
+      activerecord (>= 7.1, < 9.0)
+      bundler (>= 1.13.4)
     cloudflare-rails (7.0.0)
       actionpack (>= 7.2.0, < 8.2.0)
       activesupport (>= 7.2.0, < 8.2.0)
@@ -652,6 +655,7 @@ DEPENDENCIES
   brakeman
   bullet
   capybara
+  clickhouse-activerecord
   cloudflare-rails
   countries
   debug
diff --git a/app/controllers/api/admin/v1/admin_controller.rb b/app/controllers/api/admin/v1/admin_controller.rb
index 73e855e9d..e39ca48f3 100644
--- a/app/controllers/api/admin/v1/admin_controller.rb
+++ b/app/controllers/api/admin/v1/admin_controller.rb
@@ -58,7 +58,7 @@ def visualization_quantized
                 FROM heartbeats
                 WHERE user_id = ?
                 AND time >= ? AND time <= ?
-                AND (lineno IS NOT NULL OR cursorpos IS NOT NULL)
+                AND (lineno > 0 OR cursorpos > 0)
                 LIMIT 1000000
             ),
             daily_stats AS (
@@ -83,7 +83,7 @@ def visualization_quantized
                     any(lineno) AS lineno,
                     any(cursorpos) AS cursorpos
                 FROM quantized_heartbeats
-                WHERE lineno IS NOT NULL
+                WHERE lineno > 0
                 GROUP BY day_start, qx, qy_lineno
             ) AS lineno_pixels
             UNION ALL
@@ -94,7 +94,7 @@ def visualization_quantized
                     any(lineno) AS lineno,
                     any(cursorpos) AS cursorpos
                 FROM quantized_heartbeats
-                WHERE cursorpos IS NOT NULL
+                WHERE cursorpos > 0
                 GROUP BY day_start, qx, qy_cursorpos
             ) AS cursorpos_pixels
             ORDER BY time ASC
@@ -218,7 +218,7 @@ def shared_machines
             SELECT
               machine,
               uniq(user_id) AS machine_frequency,
-              groupArray(DISTINCT user_id) AS user_ids
+              groupUniqArray(user_id) AS user_ids
             FROM heartbeats
             WHERE machine != '' AND machine IS NOT NULL
               AND time > ?
diff --git a/app/controllers/api/hackatime/v1/hackatime_controller.rb b/app/controllers/api/hackatime/v1/hackatime_controller.rb
index 8a60d3b37..97bb5520b 100644
--- a/app/controllers/api/hackatime/v1/hackatime_controller.rb
+++ b/app/controllers/api/hackatime/v1/hackatime_controller.rb
@@ -282,7 +282,10 @@ def handle_heartbeat(heartbeat_array)
       }).slice(*Heartbeat.column_names.map(&:to_sym))
       # ^^ They say safety laws are written in blood. Well, so is this line!
       # Basically this filters out columns that aren't in our DB (the biggest one being raw_data)
-      new_heartbeat = Heartbeat.create(attrs)
+      fields_hash = Heartbeat.generate_fields_hash(attrs)
+      new_heartbeat = Heartbeat.find_or_create_by(fields_hash: fields_hash) do |hb|
+        hb.assign_attributes(attrs)
+      end
 
       queue_project_mapping(heartbeat[:project])
       results << [ new_heartbeat.attributes, 201 ]
diff --git a/app/mailers/weekly_summary_mailer.rb b/app/mailers/weekly_summary_mailer.rb
index 9e22a3754..d4891ed98 100644
--- a/app/mailers/weekly_summary_mailer.rb
+++ b/app/mailers/weekly_summary_mailer.rb
@@ -48,9 +48,10 @@ def breakdown(scope, column, limit: 5)
 
   def active_days_count(scope)
     timezone = @timezone_label
+    tz_quoted = Heartbeat.connection.quote(timezone)
     # ClickHouse-compatible: count distinct days using toDate with timezone
     result = Heartbeat.connection.select_value(<<~SQL)
-      SELECT uniq(toDate(toDateTime(toUInt32(time), '#{timezone}')))
+      SELECT uniq(toDate(toDateTime(toUInt32(time), #{tz_quoted})))
       FROM (#{scope.where.not(time: nil).to_sql}) AS hb
     SQL
     result.to_i
diff --git a/app/models/concerns/heartbeatable.rb b/app/models/concerns/heartbeatable.rb
index 35039d195..60b0dce4d 100644
--- a/app/models/concerns/heartbeatable.rb
+++ b/app/models/concerns/heartbeatable.rb
@@ -107,37 +107,37 @@ def daily_streaks_for_users(user_ids, start_date: 31.days.ago)
 
       timeout = heartbeat_timeout_duration.to_i
 
-      # Query heartbeats from ClickHouse (no cross-DB join)
+      # Query raw heartbeat diffs from ClickHouse, then group by day per user's timezone in Ruby
       raw_sql = <<~SQL
         SELECT
           user_id,
-          toDate(toDateTime(toUInt32(time))) AS day_group,
-          toInt64(coalesce(sum(diff), 0)) AS duration
-        FROM (
-          SELECT
-            user_id,
-            time,
-            least(
-              time - lagInFrame(time) OVER (PARTITION BY user_id, toDate(toDateTime(toUInt32(time))) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
-              #{timeout}
-            ) AS diff
-          FROM heartbeats
-          WHERE user_id IN (#{uncached_users.join(',')})
-            AND category = 'coding'
-            AND time >= 0 AND time <= 253402300799
-            AND time >= #{start_date.to_f}
-            AND time <= #{Time.current.to_f}
-        )
-        GROUP BY user_id, day_group
+          time,
+          least(
+            time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+            #{timeout}
+          ) AS diff
+        FROM heartbeats
+        WHERE user_id IN (#{uncached_users.join(',')})
+          AND category = 'coding'
+          AND time >= 0 AND time <= 253402300799
+          AND time >= #{start_date.to_f}
+          AND time <= #{Time.current.to_f}
       SQL
 
       rows = connection.select_all(raw_sql)
 
       daily_durations = rows.group_by { |row| row["user_id"].to_i }
         .transform_values do |user_rows|
-          user_rows.map do |row|
-            [ row["day_group"].to_date, row["duration"].to_i ]
-          end.sort_by { |date, _| date }.reverse
+          user_id = user_rows.first["user_id"].to_i
+          tz = user_timezones[user_id] || "UTC"
+          begin
+            TZInfo::Timezone.get(tz)
+          rescue TZInfo::InvalidTimezoneIdentifier, ArgumentError
+            tz = "UTC"
+          end
+          user_rows.group_by { |row| Time.at(row["time"].to_f).in_time_zone(tz).to_date }
+            .map { |date, day_rows| [ date, day_rows.sum { |r| r["diff"].to_i } ] }
+            .sort_by { |date, _| date }.reverse
         end
 
       result = user_ids.index_with { |id| streak_cache["user_streak_#{id}"] || 0 }
@@ -193,7 +193,7 @@ def daily_durations(user_timezone:, start_date: 365.days.ago, end_date: Time.cur
           SELECT
             time,
             least(
-              time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+              time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
               #{heartbeat_timeout_duration.to_i}
             ) AS diff
           FROM (#{with_valid_timestamps.where(time: start_date.to_f..end_date.to_f).to_sql}) AS hb
@@ -218,7 +218,7 @@ def duration_seconds(scope = all)
         group_expr = group_column.to_s.include?("(") ? group_column : "`#{group_column}`"
 
         capped_diffs_sql = scope
-          .select("#{group_expr} as grouped_time, least(time - lagInFrame(time) OVER (PARTITION BY #{group_expr} ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
+          .select("#{group_expr} as grouped_time, least(time - lagInFrame(time, 1, time) OVER (PARTITION BY #{group_expr} ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
           .where.not(time: nil)
           .unscope(:group)
           .to_sql
@@ -230,7 +230,7 @@ def duration_seconds(scope = all)
         end
       else
         capped_diffs_sql = scope
-          .select("least(time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
+          .select("least(time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
           .where.not(time: nil)
           .to_sql
 
@@ -276,7 +276,7 @@ def duration_seconds_boundary_aware(scope, start_time, end_time)
 
       timeout = heartbeat_timeout_duration.to_i
       capped_diffs_sql = combined_scope
-        .select("time, least(time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
+        .select("time, least(time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
         .where.not(time: nil)
         .order(time: :asc)
         .to_sql
diff --git a/app/services/heartbeat_import_service.rb b/app/services/heartbeat_import_service.rb
index 5c56e15e6..00eb3a721 100644
--- a/app/services/heartbeat_import_service.rb
+++ b/app/services/heartbeat_import_service.rb
@@ -108,8 +108,10 @@ def self.flush_batch(seen_hashes)
     end
 
     ActiveRecord::Base.logger.silence do
-      Heartbeat.insert_all(records)
-      records.length
+      existing_hashes = Heartbeat.where(fields_hash: records.map { |r| r[:fields_hash] }).pluck(:fields_hash).to_set
+      new_records = records.reject { |r| existing_hashes.include?(r[:fields_hash]) }
+      Heartbeat.insert_all(new_records) if new_records.any?
+      new_records.length
     end
   end
 
diff --git a/config/brakeman.ignore b/config/brakeman.ignore
index 39cb1a651..7a93639f1 100644
--- a/config/brakeman.ignore
+++ b/config/brakeman.ignore
@@ -3,13 +3,59 @@
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
-      "fingerprint": "06ca3650eaeb8d28e062c1c6dcbeab95fb2ccd0c5bb49165ad469bcb6b791d3e",
+      "fingerprint": "8bfb27cde848a37932bdcee60ced12ea578880374edf0f4b12da7b2b6fc32269",
+      "check_name": "SQL",
+      "message": "Possible SQL injection",
+      "file": "app/mailers/weekly_summary_mailer.rb",
+      "line": 53,
+      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
+      "code": "Heartbeat.connection.select_value(\"SELECT uniq(toDate(toDateTime(toUInt32(time), '#{@timezone_label}')))\nFROM (#{scope.where.not(:time => nil).to_sql}) AS hb\n\")",
+      "render_path": null,
+      "location": {
+        "type": "method",
+        "class": "WeeklySummaryMailer",
+        "method": "active_days_count"
+      },
+      "user_input": "@timezone_label",
+      "confidence": "Medium",
+      "cwe_id": [
+        89
+      ],
+      "note": "Timezone is validated via ActiveSupport::TimeZone lookup"
+    },
+    {
+      "warning_type": "SQL Injection",
+      "warning_code": 0,
+      "fingerprint": "9b9294560765ae07af31fffc430808dc95ab2be8514b051e6d8fe06b49488832",
       "check_name": "SQL",
       "message": "Possible SQL injection",
       "file": "app/models/concerns/heartbeatable.rb",
-      "line": 175,
+      "line": 134,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "Arel.sql(\"DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE '#{(user_timezone or \"UTC\")}')\")",
+      "code": "connection.select_all(\"SELECT\\n  user_id,\\n  toDate(toDateTime(toUInt32(time))) AS day_group,\\n  toInt64(coalesce(sum(diff), 0)) AS duration\\nFROM (\\n  SELECT\\n    user_id,\\n    time,\\n    least(\\n      time - lagInFrame(time) OVER (PARTITION BY user_id, toDate(toDateTime(toUInt32(time))) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),\\n      #{heartbeat_timeout_duration.to_i}\\n    ) AS diff\\n  FROM heartbeats\\n  WHERE user_id IN (#{user_ids.select do\\n Rails.cache.read_multi(*user_ids.map do\\n \\\"user_streak_#{id}\\\"\\n end)[\\\"user_streak_#{id}\\\"].nil?\\n end.join(\\\",\\\")})\\n    AND category = 'coding'\\n    AND time >= 0 AND time <= 253402300799\\n    AND time >= #{[start_date, 30.days.ago].max.to_f}\\n    AND time <= #{Time.current.to_f}\\n)\\nGROUP BY user_id, day_group\\n\")",
+      "render_path": null,
+      "location": {
+        "type": "method",
+        "class": "Heartbeatable",
+        "method": "daily_streaks_for_users"
+      },
+      "user_input": "user_ids.select do\n Rails.cache.read_multi(*user_ids.map do\n \"user_streak_#{id}\"\n end)[\"user_streak_#{id}\"].nil?\n end.join(\",\")",
+      "confidence": "Medium",
+      "cwe_id": [
+        89
+      ],
+      "note": "user_ids are integer IDs from internal queries, not user input"
+    },
+    {
+      "warning_type": "SQL Injection",
+      "warning_code": 0,
+      "fingerprint": "c3d4e314e4dbd66500b55b2a0edc8b337ec85f907eb151248ba5c109e679c7fb",
+      "check_name": "SQL",
+      "message": "Possible SQL injection",
+      "file": "app/models/concerns/heartbeatable.rb",
+      "line": 205,
+      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
+      "code": "connection.select_all(\"SELECT\\n  toDate(toDateTime(toUInt32(time), '#{(user_timezone or \\\"UTC\\\")}')) AS day_group,\\n  toInt64(coalesce(sum(diff), 0)) AS duration\\nFROM (\\n  SELECT\\n    time,\\n    least(\\n      time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),\\n      #{heartbeat_timeout_duration.to_i}\\n    ) AS diff\\n  FROM (#{with_valid_timestamps.where(:time => ((start_date.to_f..end_date.to_f))).to_sql}) AS hb\\n)\\nGROUP BY day_group\\nORDER BY day_group\\n\")",
       "render_path": null,
       "location": {
         "type": "method",
@@ -21,7 +67,7 @@
       "cwe_id": [
         89
       ],
-      "note": ""
+      "note": "Timezone is validated against TZInfo::Timezone.all_identifiers"
     },
     {
       "warning_type": "SQL Injection",
diff --git a/db/clickhouse_schema.rb b/db/clickhouse_schema.rb
new file mode 100644
index 000000000..471dbe3b4
--- /dev/null
+++ b/db/clickhouse_schema.rb
@@ -0,0 +1,55 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `bin/rails
+# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema[8.1].define(version: 2026_03_24_000002) do
+  # TABLE: heartbeat_user_daily_summary
+  # SQL: CREATE TABLE heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32, `_version` DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192
+  create_table "heartbeat_user_daily_summary", id: false, options: "ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192", force: :cascade do |t|
+    t.integer "user_id", unsigned: false, limit: 8, null: false
+    t.date "day", null: false
+    t.float "duration_s", null: false
+    t.integer "heartbeats", null: false
+    t.datetime "_version", precision: nil, default: -> { "now()" }, null: false
+  end
+
+  # TABLE: heartbeats
+  # SQL: CREATE TABLE heartbeats ( `id` Int64, `user_id` Int64, `branch` String DEFAULT '', `category` String DEFAULT '', `dependencies` Array(String), `editor` String DEFAULT '', `entity` String DEFAULT '', `language` String DEFAULT '', `machine` String DEFAULT '', `operating_system` String DEFAULT '', `project` String DEFAULT '', `type` String DEFAULT '', `user_agent` String DEFAULT '', `line_additions` Int32 DEFAULT 0, `line_deletions` Int32 DEFAULT 0, `lineno` Int32 DEFAULT 0, `lines` Int32 DEFAULT 0, `cursorpos` Int32 DEFAULT 0, `project_root_count` Int32 DEFAULT 0, `time` Float64, `is_write` UInt8 DEFAULT 0, `created_at` DateTime64(6) DEFAULT now64(), `updated_at` DateTime64(6) DEFAULT now64(), `source_type` UInt8 DEFAULT 0, `ip_address` String DEFAULT '', `ysws_program` UInt8 DEFAULT 0, `fields_hash` String DEFAULT '' ) ENGINE = ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id) SETTINGS index_granularity = 8192
+  create_table "heartbeats", id: :int64, options: "ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id) SETTINGS index_granularity = 8192", force: :cascade do |t|
+    t.integer "id", unsigned: false, limit: 8, null: false
+    t.integer "user_id", unsigned: false, limit: 8, null: false
+    t.string "branch", default: "", null: false
+    t.string "category", default: "", null: false
+    t.string "dependencies", array: true, null: false
+    t.string "editor", default: "", null: false
+    t.string "entity", default: "", null: false
+    t.string "language", default: "", null: false
+    t.string "machine", default: "", null: false
+    t.string "operating_system", default: "", null: false
+    t.string "project", default: "", null: false
+    t.string "type", default: "", null: false
+    t.string "user_agent", default: "", null: false
+    t.integer "line_additions", unsigned: false, default: 0, null: false
+    t.integer "line_deletions", unsigned: false, default: 0, null: false
+    t.integer "lineno", unsigned: false, default: 0, null: false
+    t.integer "lines", unsigned: false, default: 0, null: false
+    t.integer "cursorpos", unsigned: false, default: 0, null: false
+    t.integer "project_root_count", unsigned: false, default: 0, null: false
+    t.float "time", null: false
+    t.integer "is_write", limit: 1, default: 0, null: false
+    t.datetime "created_at", default: -> { "now64()" }, null: false
+    t.datetime "updated_at", default: -> { "now64()" }, null: false
+    t.integer "source_type", limit: 1, default: 0, null: false
+    t.string "ip_address", default: "", null: false
+    t.integer "ysws_program", limit: 1, default: 0, null: false
+    t.string "fields_hash", default: "", null: false
+  end
+end

From 9d96ea5a1329874505de0747b1ecb5137b062373 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 14:28:01 +0000
Subject: [PATCH 04/24] Fix the CI?

---
 .github/workflows/ci.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7147caf12..7f7d25432 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -113,6 +113,11 @@ jobs:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: app_test
+      clickhouse:
+        image: clickhouse/clickhouse-server:latest
+        ports:
+          - 8123:8123
+        options: --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1" --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
       - name: Checkout code
@@ -130,6 +135,7 @@ jobs:
           TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/app_test
           PGHOST: localhost
           PGUSER: postgres
+          CLICKHOUSE_HOST: localhost
           PGPASSWORD: postgres
         run: |
           bin/rails db:create RAILS_ENV=test
@@ -142,6 +148,7 @@ jobs:
           TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/app_test
           PGHOST: localhost
           PGUSER: postgres
+          CLICKHOUSE_HOST: localhost
           PGPASSWORD: postgres
         run: |
           bin/rails rswag:specs:swaggerize
@@ -160,6 +167,11 @@ jobs:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: app_test
+      clickhouse:
+        image: clickhouse/clickhouse-server:latest
+        ports:
+          - 8123:8123
+        options: --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1" --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
       - name: Checkout code
@@ -200,6 +212,7 @@ jobs:
           TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/app_test
           PGHOST: localhost
           PGUSER: postgres
+          CLICKHOUSE_HOST: localhost
           PGPASSWORD: postgres
           CHROME_BIN: ${{ steps.setup-chrome.outputs.chrome-path }}
           CHROMEDRIVER_BIN: ${{ steps.setup-chrome.outputs.chromedriver-path }}

From 57efaeb96fea3a864e3a816b51e3f7ee1f38fc52 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 14:30:52 +0000
Subject: [PATCH 05/24] Goog

---
 .github/workflows/ci.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7f7d25432..1c9e69b8f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -117,6 +117,8 @@ jobs:
         image: clickhouse/clickhouse-server:latest
         ports:
           - 8123:8123
+        env:
+          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
         options: --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1" --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
@@ -171,6 +173,8 @@ jobs:
         image: clickhouse/clickhouse-server:latest
         ports:
           - 8123:8123
+        env:
+          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
         options: --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1" --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:

From c6ffded5d251f7c89e30326afe313c21db9bb0b8 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 14:41:10 +0000
Subject: [PATCH 06/24] goog

---
 .../api/hackatime/v1/hackatime_controller.rb  | 49 ++++++++++++++-----
 app/jobs/aggregate_user_daily_summary_job.rb  | 30 ++++++++++++
 app/models/heartbeat.rb                       |  6 +++
 app/services/profile_stats_service.rb         |  2 +-
 config/initializers/good_job.rb               |  5 ++
 5 files changed, 78 insertions(+), 14 deletions(-)
 create mode 100644 app/jobs/aggregate_user_daily_summary_job.rb

diff --git a/app/controllers/api/hackatime/v1/hackatime_controller.rb b/app/controllers/api/hackatime/v1/hackatime_controller.rb
index 97bb5520b..2ba3128dd 100644
--- a/app/controllers/api/hackatime/v1/hackatime_controller.rb
+++ b/app/controllers/api/hackatime/v1/hackatime_controller.rb
@@ -234,7 +234,41 @@ def body_to_json
   def handle_heartbeat(heartbeat_array)
     results = []
     last_language = nil
-    heartbeat_array.each do |heartbeat|
+    prepared = prepare_heartbeat_attrs(heartbeat_array)
+    all_hashes = prepared.map { |p| p[:fields_hash] }.compact
+    existing_hashes = if all_hashes.any?
+      Heartbeat.where(fields_hash: all_hashes).pluck(:fields_hash).to_set
+    else
+      Set.new
+    end
+
+    prepared.each do |item|
+      heartbeat = item[:attrs]
+      fields_hash = item[:fields_hash]
+      source_type = item[:source_type]
+
+      if existing_hashes.include?(fields_hash)
+        existing = Heartbeat.where(fields_hash: fields_hash).first
+        queue_project_mapping(heartbeat[:project])
+        results << [ existing.attributes, 201 ]
+      else
+        new_heartbeat = Heartbeat.create!(heartbeat.merge(fields_hash: fields_hash))
+        existing_hashes.add(fields_hash)
+        queue_project_mapping(heartbeat[:project])
+        results << [ new_heartbeat.attributes, 201 ]
+      end
+    rescue => e
+      report_error(e, message: "Error creating heartbeat")
+      results << [ { error: e.message, type: e.class.name }, 422 ]
+    end
+
+    PosthogService.capture_once_per_day(@user, "heartbeat_sent", { heartbeat_count: heartbeat_array.size })
+    results
+  end
+
+  def prepare_heartbeat_attrs(heartbeat_array)
+    last_language = nil
+    heartbeat_array.filter_map do |heartbeat|
       heartbeat = heartbeat.to_h.with_indifferent_access
       source_type = :direct_entry
 
@@ -283,19 +317,8 @@ def handle_heartbeat(heartbeat_array)
       # ^^ They say safety laws are written in blood. Well, so is this line!
       # Basically this filters out columns that aren't in our DB (the biggest one being raw_data)
       fields_hash = Heartbeat.generate_fields_hash(attrs)
-      new_heartbeat = Heartbeat.find_or_create_by(fields_hash: fields_hash) do |hb|
-        hb.assign_attributes(attrs)
-      end
-
-      queue_project_mapping(heartbeat[:project])
-      results << [ new_heartbeat.attributes, 201 ]
-    rescue => e
-      report_error(e, message: "Error creating heartbeat")
-      results << [ { error: e.message, type: e.class.name }, 422 ]
+      { attrs: attrs, fields_hash: fields_hash, source_type: source_type }
     end
-
-    PosthogService.capture_once_per_day(@user, "heartbeat_sent", { heartbeat_count: heartbeat_array.size })
-    results
   end
 
   def queue_project_mapping(project_name)
diff --git a/app/jobs/aggregate_user_daily_summary_job.rb b/app/jobs/aggregate_user_daily_summary_job.rb
new file mode 100644
index 000000000..072c0f608
--- /dev/null
+++ b/app/jobs/aggregate_user_daily_summary_job.rb
@@ -0,0 +1,30 @@
+class AggregateUserDailySummaryJob < ApplicationJob
+  include GoodJob::ActiveJobExtensions::Concurrency
+
+  queue_as :latency_5m
+
+  good_job_control_concurrency_with(
+    total: 1,
+    drop: true
+  )
+
+  def perform
+    conn = HeartbeatUserDailySummary.connection
+
+    conn.execute(<<~SQL)
+      INSERT INTO heartbeat_user_daily_summary (user_id, day, duration_s, heartbeats)
+      SELECT
+        user_id,
+        toDate(toDateTime(toUInt32(time))) AS day,
+        sum(
+          least(
+            time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id, toDate(toDateTime(toUInt32(time))) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+            120
+          )
+        ) AS duration_s,
+        count() AS heartbeats
+      FROM heartbeats
+      GROUP BY user_id, day
+    SQL
+  end
+end
diff --git a/app/models/heartbeat.rb b/app/models/heartbeat.rb
index 0bcf5b034..5c16aee4c 100644
--- a/app/models/heartbeat.rb
+++ b/app/models/heartbeat.rb
@@ -6,6 +6,12 @@ class Heartbeat < ClickhouseRecord
 
   time_range_filterable_field :time
 
+  before_create :set_fields_hash!, if: -> { fields_hash.blank? }
+
+  def set_fields_hash!
+    self.fields_hash = self.class.generate_fields_hash(attributes)
+  end
+
   scope :today, -> { where(time: Time.current.beginning_of_day.to_i..Time.current.end_of_day.to_i) }
   scope :recent, -> { where("time > ?", 24.hours.ago.to_i) }
 
diff --git a/app/services/profile_stats_service.rb b/app/services/profile_stats_service.rb
index 0b564bcc6..f98cb5fd1 100644
--- a/app/services/profile_stats_service.rb
+++ b/app/services/profile_stats_service.rb
@@ -62,7 +62,7 @@ def compute_totals_and_breakdowns(timeout, today_start, today_end, week_start, w
           language,
           editor,
           least(
-            time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+            time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
             #{timeout_quoted}
           ) AS diff
         FROM heartbeats
diff --git a/config/initializers/good_job.rb b/config/initializers/good_job.rb
index c42d57b9b..2a1d98c40 100644
--- a/config/initializers/good_job.rb
+++ b/config/initializers/good_job.rb
@@ -79,6 +79,11 @@
       class: "Cache::CurrentlyHackingJob",
       kwargs: { force_reload: true }
     },
+    aggregate_user_daily_summary: {
+      cron: "*/10 * * * *",
+      class: "AggregateUserDailySummaryJob",
+      description: "Aggregates heartbeats into per-user daily summaries for home stats."
+    },
     cache_home_stats: {
       cron: "*/10 * * * *",
       class: "Cache::HomeStatsJob",

From 638e3bab714c5686aafd41cb039b3a3e0e0c0534 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 14:49:05 +0000
Subject: [PATCH 07/24] goog!

---
 test/system/heartbeat_export_test.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/system/heartbeat_export_test.rb b/test/system/heartbeat_export_test.rb
index 8f79a86d8..3b33b91cf 100644
--- a/test/system/heartbeat_export_test.rb
+++ b/test/system/heartbeat_export_test.rb
@@ -1,7 +1,7 @@
 require "application_system_test_case"
 
 class HeartbeatExportTest < ApplicationSystemTestCase
-  fixtures :users, :email_addresses, :heartbeats, :sign_in_tokens, :api_keys, :admin_api_keys
+  fixtures :users, :email_addresses, :sign_in_tokens, :api_keys, :admin_api_keys
 
   setup do
     GoodJob::Job.delete_all

From ac65647d225db521e2e56ed6109743dfd2c67d0c Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 14:52:16 +0000
Subject: [PATCH 08/24] Goog :sob:

---
 test/services/heartbeat_import_dump_client_test.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/services/heartbeat_import_dump_client_test.rb b/test/services/heartbeat_import_dump_client_test.rb
index 4aec68d75..cd125f0ed 100644
--- a/test/services/heartbeat_import_dump_client_test.rb
+++ b/test/services/heartbeat_import_dump_client_test.rb
@@ -2,6 +2,10 @@
 require "webmock/minitest"
 
 class HeartbeatImportDumpClientTest < ActiveSupport::TestCase
+  setup do
+    WebMock.disable_net_connect!(allow_localhost: true)
+  end
+
   test "valid_wakatime_download_url? only accepts wakatime s3 links over https" do
     assert HeartbeatImportDumpClient.valid_wakatime_download_url?("https://wakatime.s3.amazonaws.com/export.json?signature=abc")
     assert_not HeartbeatImportDumpClient.valid_wakatime_download_url?("http://wakatime.s3.amazonaws.com/export.json")

From f5c3f422ce3d0d8445a4c274bb6b2ad87badaa81 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 15:38:04 +0000
Subject: [PATCH 09/24] Fixes?

---
 app/models/heartbeat.rb                       |  10 ++
 config/brakeman.ignore                        |  49 ++-----
 config/database.yml                           |   2 +-
 db/clickhouse_schema.rb                       | 125 +++++++++++++-----
 .../heartbeat_import_dump_client_test.rb      |   5 -
 test/test_helper.rb                           |  22 +++
 6 files changed, 135 insertions(+), 78 deletions(-)

diff --git a/app/models/heartbeat.rb b/app/models/heartbeat.rb
index 5c16aee4c..6083927eb 100644
--- a/app/models/heartbeat.rb
+++ b/app/models/heartbeat.rb
@@ -1,17 +1,27 @@
 class Heartbeat < ClickhouseRecord
   self.table_name = "heartbeats"
+  self.primary_key = :id
 
   include Heartbeatable
   include TimeRangeFilterable
 
   time_range_filterable_field :time
 
+  before_create :set_clickhouse_id!, if: -> { self[:id].blank? }
   before_create :set_fields_hash!, if: -> { fields_hash.blank? }
 
+  CLICKHOUSE_ID_RANDOM_BITS = 10
+  CLICKHOUSE_ID_RANDOM_MAX = 1 << CLICKHOUSE_ID_RANDOM_BITS
+
   def set_fields_hash!
     self.fields_hash = self.class.generate_fields_hash(attributes)
   end
 
+  def set_clickhouse_id!
+    timestamp_us = (Time.current.to_r * 1_000_000).to_i
+    self[:id] = (timestamp_us << CLICKHOUSE_ID_RANDOM_BITS) | SecureRandom.random_number(CLICKHOUSE_ID_RANDOM_MAX)
+  end
+
   scope :today, -> { where(time: Time.current.beginning_of_day.to_i..Time.current.end_of_day.to_i) }
   scope :recent, -> { where("time > ?", 24.hours.ago.to_i) }
 
diff --git a/config/brakeman.ignore b/config/brakeman.ignore
index 7a93639f1..cd6e65f28 100644
--- a/config/brakeman.ignore
+++ b/config/brakeman.ignore
@@ -3,71 +3,48 @@
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
-      "fingerprint": "8bfb27cde848a37932bdcee60ced12ea578880374edf0f4b12da7b2b6fc32269",
-      "check_name": "SQL",
-      "message": "Possible SQL injection",
-      "file": "app/mailers/weekly_summary_mailer.rb",
-      "line": 53,
-      "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "Heartbeat.connection.select_value(\"SELECT uniq(toDate(toDateTime(toUInt32(time), '#{@timezone_label}')))\nFROM (#{scope.where.not(:time => nil).to_sql}) AS hb\n\")",
-      "render_path": null,
-      "location": {
-        "type": "method",
-        "class": "WeeklySummaryMailer",
-        "method": "active_days_count"
-      },
-      "user_input": "@timezone_label",
-      "confidence": "Medium",
-      "cwe_id": [
-        89
-      ],
-      "note": "Timezone is validated via ActiveSupport::TimeZone lookup"
-    },
-    {
-      "warning_type": "SQL Injection",
-      "warning_code": 0,
-      "fingerprint": "9b9294560765ae07af31fffc430808dc95ab2be8514b051e6d8fe06b49488832",
+      "fingerprint": "4cffceb1f5c7d23d28b9846734a5a7333e1005874e310d65b39a32a5221990f9",
       "check_name": "SQL",
       "message": "Possible SQL injection",
       "file": "app/models/concerns/heartbeatable.rb",
-      "line": 134,
+      "line": 127,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "connection.select_all(\"SELECT\\n  user_id,\\n  toDate(toDateTime(toUInt32(time))) AS day_group,\\n  toInt64(coalesce(sum(diff), 0)) AS duration\\nFROM (\\n  SELECT\\n    user_id,\\n    time,\\n    least(\\n      time - lagInFrame(time) OVER (PARTITION BY user_id, toDate(toDateTime(toUInt32(time))) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),\\n      #{heartbeat_timeout_duration.to_i}\\n    ) AS diff\\n  FROM heartbeats\\n  WHERE user_id IN (#{user_ids.select do\\n Rails.cache.read_multi(*user_ids.map do\\n \\\"user_streak_#{id}\\\"\\n end)[\\\"user_streak_#{id}\\\"].nil?\\n end.join(\\\",\\\")})\\n    AND category = 'coding'\\n    AND time >= 0 AND time <= 253402300799\\n    AND time >= #{[start_date, 30.days.ago].max.to_f}\\n    AND time <= #{Time.current.to_f}\\n)\\nGROUP BY user_id, day_group\\n\")",
+      "code": null,
       "render_path": null,
       "location": {
         "type": "method",
         "class": "Heartbeatable",
         "method": "daily_streaks_for_users"
       },
-      "user_input": "user_ids.select do\n Rails.cache.read_multi(*user_ids.map do\n \"user_streak_#{id}\"\n end)[\"user_streak_#{id}\"].nil?\n end.join(\",\")",
+      "user_input": null,
       "confidence": "Medium",
       "cwe_id": [
         89
       ],
-      "note": "user_ids are integer IDs from internal queries, not user input"
+      "note": "user_ids are integer IDs from internal queries, timeout and dates use .to_i/.to_f"
     },
     {
       "warning_type": "SQL Injection",
       "warning_code": 0,
-      "fingerprint": "c3d4e314e4dbd66500b55b2a0edc8b337ec85f907eb151248ba5c109e679c7fb",
+      "fingerprint": "076acac8c26cca89c2cbbcf0bad1bf4a45067b33c331aac087fdb8f4326590ff",
       "check_name": "SQL",
       "message": "Possible SQL injection",
       "file": "app/models/concerns/heartbeatable.rb",
       "line": 205,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "connection.select_all(\"SELECT\\n  toDate(toDateTime(toUInt32(time), '#{(user_timezone or \\\"UTC\\\")}')) AS day_group,\\n  toInt64(coalesce(sum(diff), 0)) AS duration\\nFROM (\\n  SELECT\\n    time,\\n    least(\\n      time - lagInFrame(time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),\\n      #{heartbeat_timeout_duration.to_i}\\n    ) AS diff\\n  FROM (#{with_valid_timestamps.where(:time => ((start_date.to_f..end_date.to_f))).to_sql}) AS hb\\n)\\nGROUP BY day_group\\nORDER BY day_group\\n\")",
+      "code": null,
       "render_path": null,
       "location": {
         "type": "method",
         "class": "Heartbeatable",
         "method": "daily_durations"
       },
-      "user_input": "user_timezone",
+      "user_input": null,
       "confidence": "Medium",
       "cwe_id": [
         89
       ],
-      "note": "Timezone is validated against TZInfo::Timezone.all_identifiers"
+      "note": "Timezone is validated against TZInfo::Timezone.all_identifiers, timeout uses .to_i"
     },
     {
       "warning_type": "SQL Injection",
@@ -78,20 +55,20 @@
       "file": "app/jobs/one_time/generate_unique_heartbeat_hashes_job.rb",
       "line": 32,
       "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/",
-      "code": "Heartbeat.where(:id => chunk.map do\n index = 1\nputs(\"Processing heartbeat #{heartbeat.id} (#{1} of #{batch.size})\")\nfield_hash = Heartbeat.generate_fields_hash(heartbeat.attributes)\nputs(\"Field hash: #{Heartbeat.generate_fields_hash(heartbeat.attributes)}\")\n[heartbeat.id, Heartbeat.generate_fields_hash(heartbeat.attributes)]\n end.map(&:first)).update_all(\"fields_hash = CASE #{chunk.map do\n index = 1\nputs(\"Processing heartbeat #{heartbeat.id} (#{1} of #{batch.size})\")\nfield_hash = Heartbeat.generate_fields_hash(heartbeat.attributes)\nputs(\"Field hash: #{Heartbeat.generate_fields_hash(heartbeat.attributes)}\")\n[heartbeat.id, Heartbeat.generate_fields_hash(heartbeat.attributes)]\n end.map do\n \"WHEN id = #{id} THEN '#{hash}'\"\n end.join(\" \")} END\")",
+      "code": null,
       "render_path": null,
       "location": {
         "type": "method",
         "class": "OneTime::GenerateUniqueHeartbeatHashesJob",
         "method": "perform"
       },
-      "user_input": "Heartbeat.generate_fields_hash(heartbeat.attributes)",
+      "user_input": null,
       "confidence": "High",
       "cwe_id": [
         89
       ],
-      "note": ""
+      "note": "One-time migration job using internally generated hashes"
     }
   ],
-  "brakeman_version": "7.0.2"
+  "brakeman_version": "8.0.4"
 }
diff --git a/config/database.yml b/config/database.yml
index 088291770..e16e2d49f 100644
--- a/config/database.yml
+++ b/config/database.yml
@@ -47,7 +47,7 @@ test:
     adapter: clickhouse
     host: <%= ENV.fetch('CLICKHOUSE_HOST', 'clickhouse') %>
     port: <%= ENV.fetch('CLICKHOUSE_PORT', 8123) %>
-    database: <%= ENV.fetch('CLICKHOUSE_DATABASE', 'hackatime_test') %>
+    database: hackatime_test
     username: <%= ENV.fetch('CLICKHOUSE_USERNAME', 'default') %>
     password: <%= ENV.fetch('CLICKHOUSE_PASSWORD', '') %>
     migrations_paths: db/migrate_clickhouse
diff --git a/db/clickhouse_schema.rb b/db/clickhouse_schema.rb
index 471dbe3b4..38a7727ad 100644
--- a/db/clickhouse_schema.rb
+++ b/db/clickhouse_schema.rb
@@ -11,45 +11,98 @@
 # It's strongly recommended that you check this file into your version control system.
 
 ActiveRecord::Schema[8.1].define(version: 2026_03_24_000002) do
+  # TABLE: schema_migrations
+  # SQL: CREATE TABLE schema_migrations ( `version` String, `active` Int8 DEFAULT 1, `ver` DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(ver) ORDER BY (version)
+  execute <<~SQL
+    DROP TABLE IF EXISTS schema_migrations
+  SQL
+  execute <<~SQL
+    CREATE TABLE schema_migrations
+    (
+      `version` String,
+      `active` Int8 DEFAULT 1,
+      `ver` DateTime DEFAULT now()
+    )
+    ENGINE = ReplacingMergeTree(ver)
+    ORDER BY (version)
+  SQL
+
+  # TABLE: ar_internal_metadata
+  # SQL: CREATE TABLE ar_internal_metadata ( `key` String, `value` String, `created_at` DateTime, `updated_at` DateTime ) ENGINE = ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key
+  execute <<~SQL
+    DROP TABLE IF EXISTS ar_internal_metadata
+  SQL
+  execute <<~SQL
+    CREATE TABLE ar_internal_metadata
+    (
+      `key` String,
+      `value` String,
+      `created_at` DateTime,
+      `updated_at` DateTime
+    )
+    ENGINE = ReplacingMergeTree(created_at)
+    PARTITION BY key
+    ORDER BY key
+  SQL
+
   # TABLE: heartbeat_user_daily_summary
   # SQL: CREATE TABLE heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32, `_version` DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192
-  create_table "heartbeat_user_daily_summary", id: false, options: "ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192", force: :cascade do |t|
-    t.integer "user_id", unsigned: false, limit: 8, null: false
-    t.date "day", null: false
-    t.float "duration_s", null: false
-    t.integer "heartbeats", null: false
-    t.datetime "_version", precision: nil, default: -> { "now()" }, null: false
-  end
+  execute <<~SQL
+    DROP TABLE IF EXISTS heartbeat_user_daily_summary
+  SQL
+  execute <<~SQL
+    CREATE TABLE heartbeat_user_daily_summary
+    (
+      `user_id` Int64,
+      `day` Date,
+      `duration_s` Float64,
+      `heartbeats` UInt32,
+      `_version` DateTime DEFAULT now()
+    )
+    ENGINE = ReplacingMergeTree(_version)
+    ORDER BY (user_id, day)
+    SETTINGS index_granularity = 8192
+  SQL
 
   # TABLE: heartbeats
   # SQL: CREATE TABLE heartbeats ( `id` Int64, `user_id` Int64, `branch` String DEFAULT '', `category` String DEFAULT '', `dependencies` Array(String), `editor` String DEFAULT '', `entity` String DEFAULT '', `language` String DEFAULT '', `machine` String DEFAULT '', `operating_system` String DEFAULT '', `project` String DEFAULT '', `type` String DEFAULT '', `user_agent` String DEFAULT '', `line_additions` Int32 DEFAULT 0, `line_deletions` Int32 DEFAULT 0, `lineno` Int32 DEFAULT 0, `lines` Int32 DEFAULT 0, `cursorpos` Int32 DEFAULT 0, `project_root_count` Int32 DEFAULT 0, `time` Float64, `is_write` UInt8 DEFAULT 0, `created_at` DateTime64(6) DEFAULT now64(), `updated_at` DateTime64(6) DEFAULT now64(), `source_type` UInt8 DEFAULT 0, `ip_address` String DEFAULT '', `ysws_program` UInt8 DEFAULT 0, `fields_hash` String DEFAULT '' ) ENGINE = ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id) SETTINGS index_granularity = 8192
-  create_table "heartbeats", id: :int64, options: "ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id) SETTINGS index_granularity = 8192", force: :cascade do |t|
-    t.integer "id", unsigned: false, limit: 8, null: false
-    t.integer "user_id", unsigned: false, limit: 8, null: false
-    t.string "branch", default: "", null: false
-    t.string "category", default: "", null: false
-    t.string "dependencies", array: true, null: false
-    t.string "editor", default: "", null: false
-    t.string "entity", default: "", null: false
-    t.string "language", default: "", null: false
-    t.string "machine", default: "", null: false
-    t.string "operating_system", default: "", null: false
-    t.string "project", default: "", null: false
-    t.string "type", default: "", null: false
-    t.string "user_agent", default: "", null: false
-    t.integer "line_additions", unsigned: false, default: 0, null: false
-    t.integer "line_deletions", unsigned: false, default: 0, null: false
-    t.integer "lineno", unsigned: false, default: 0, null: false
-    t.integer "lines", unsigned: false, default: 0, null: false
-    t.integer "cursorpos", unsigned: false, default: 0, null: false
-    t.integer "project_root_count", unsigned: false, default: 0, null: false
-    t.float "time", null: false
-    t.integer "is_write", limit: 1, default: 0, null: false
-    t.datetime "created_at", default: -> { "now64()" }, null: false
-    t.datetime "updated_at", default: -> { "now64()" }, null: false
-    t.integer "source_type", limit: 1, default: 0, null: false
-    t.string "ip_address", default: "", null: false
-    t.integer "ysws_program", limit: 1, default: 0, null: false
-    t.string "fields_hash", default: "", null: false
-  end
+  execute <<~SQL
+    DROP TABLE IF EXISTS heartbeats
+  SQL
+  execute <<~SQL
+    CREATE TABLE heartbeats
+    (
+      `id` Int64,
+      `user_id` Int64,
+      `branch` String DEFAULT '',
+      `category` String DEFAULT '',
+      `dependencies` Array(String),
+      `editor` String DEFAULT '',
+      `entity` String DEFAULT '',
+      `language` String DEFAULT '',
+      `machine` String DEFAULT '',
+      `operating_system` String DEFAULT '',
+      `project` String DEFAULT '',
+      `type` String DEFAULT '',
+      `user_agent` String DEFAULT '',
+      `line_additions` Int32 DEFAULT 0,
+      `line_deletions` Int32 DEFAULT 0,
+      `lineno` Int32 DEFAULT 0,
+      `lines` Int32 DEFAULT 0,
+      `cursorpos` Int32 DEFAULT 0,
+      `project_root_count` Int32 DEFAULT 0,
+      `time` Float64,
+      `is_write` UInt8 DEFAULT 0,
+      `created_at` DateTime64(6) DEFAULT now64(),
+      `updated_at` DateTime64(6) DEFAULT now64(),
+      `source_type` UInt8 DEFAULT 0,
+      `ip_address` String DEFAULT '',
+      `ysws_program` UInt8 DEFAULT 0,
+      `fields_hash` String DEFAULT ''
+    )
+    ENGINE = ReplacingMergeTree
+    PARTITION BY toYYYYMM(toDateTime(toUInt32(time)))
+    ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id)
+    SETTINGS index_granularity = 8192
+  SQL
 end
diff --git a/test/services/heartbeat_import_dump_client_test.rb b/test/services/heartbeat_import_dump_client_test.rb
index cd125f0ed..94e69da18 100644
--- a/test/services/heartbeat_import_dump_client_test.rb
+++ b/test/services/heartbeat_import_dump_client_test.rb
@@ -1,11 +1,6 @@
 require "test_helper"
-require "webmock/minitest"
 
 class HeartbeatImportDumpClientTest < ActiveSupport::TestCase
-  setup do
-    WebMock.disable_net_connect!(allow_localhost: true)
-  end
-
   test "valid_wakatime_download_url? only accepts wakatime s3 links over https" do
     assert HeartbeatImportDumpClient.valid_wakatime_download_url?("https://wakatime.s3.amazonaws.com/export.json?signature=abc")
     assert_not HeartbeatImportDumpClient.valid_wakatime_download_url?("http://wakatime.s3.amazonaws.com/export.json")
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 75a9bc40a..0344aea4a 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -3,6 +3,10 @@
 require "rails/test_help"
 require "nokogiri"
 require "json"
+require "webmock/minitest"
+
+# ClickHouse uses HTTP in test, so allow that host while keeping other external requests blocked.
+WebMock.disable_net_connect!(allow_localhost: true, allow: "clickhouse")
 
 module ActiveSupport
   class TestCase
@@ -28,6 +32,24 @@ class TestCase
     ]
 
     # Add more helper methods to be used by all tests here...
+    private
+
+    def before_setup
+      truncate_clickhouse_tables
+      super
+    end
+
+    def after_teardown
+      truncate_clickhouse_tables
+      super
+    end
+
+    def truncate_clickhouse_tables
+      Heartbeat.connection.execute("TRUNCATE TABLE #{Heartbeat.connection.quote_table_name(Heartbeat.table_name)}")
+      HeartbeatUserDailySummary.connection.execute(
+        "TRUNCATE TABLE #{HeartbeatUserDailySummary.connection.quote_table_name(HeartbeatUserDailySummary.table_name)}"
+      )
+    end
   end
 end
 

From c36a064f4d5cad728038de48f603c7b4b15b0dc2 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 15:41:38 +0000
Subject: [PATCH 10/24] Fix test_system CI: remove ClickHouse heartbeat
 fixtures, guard Slack API call

---
 app/models/concerns/slack_integration.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/models/concerns/slack_integration.rb b/app/models/concerns/slack_integration.rb
index c6b868a99..6790e7292 100644
--- a/app/models/concerns/slack_integration.rb
+++ b/app/models/concerns/slack_integration.rb
@@ -45,6 +45,7 @@ def update_from_slack
 
   def update_slack_status
     return unless uses_slack_status?
+    return if slack_access_token.blank?
 
     current_status_response = HTTP.auth("Bearer #{slack_access_token}")
       .get("https://slack.com/api/users.profile.get")

From 96925f7baaf6462bca16316bb09d96279277b2ec Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 16:03:41 +0000
Subject: [PATCH 11/24] Goog?

---
 app/javascript/layouts/AppLayout.svelte | 17 ++++---
 app/views/layouts/application.html.erb  |  8 +++-
 config/initializers/git_version.rb      | 21 +++------
 lib/git_metadata.rb                     | 27 +++++++++++
 test/lib/git_metadata_test.rb           | 59 +++++++++++++++++++++++++
 5 files changed, 109 insertions(+), 23 deletions(-)
 create mode 100644 lib/git_metadata.rb
 create mode 100644 test/lib/git_metadata_test.rb

diff --git a/app/javascript/layouts/AppLayout.svelte b/app/javascript/layouts/AppLayout.svelte
index a38722b1d..8f57f744a 100644
--- a/app/javascript/layouts/AppLayout.svelte
+++ b/app/javascript/layouts/AppLayout.svelte
@@ -42,7 +42,7 @@
 
   type Footer = {
     git_version: string;
-    commit_link: string;
+    commit_link: string | null;
     server_start_time_ago: string;
     heartbeat_recent_count: number;
     heartbeat_recent_imported_count: number;
@@ -594,11 +594,16 @@
         <p
           class="brightness-60 hover:brightness-100 transition-all duration-200"
         >
-          Using Inertia. Build <a
-            href={layout.footer.commit_link}
-            class="text-inherit underline opacity-80 hover:opacity-100 transition-opacity duration-200"
-            >{layout.footer.git_version}</a
-          >
+          Using Inertia. Build
+          {#if layout.footer.commit_link}
+            <a
+              href={layout.footer.commit_link}
+              class="text-inherit underline opacity-80 hover:opacity-100 transition-opacity duration-200"
+              >{layout.footer.git_version}</a
+            >
+          {:else}
+            <span>{layout.footer.git_version}</span>
+          {/if}
           from {layout.footer.server_start_time_ago} ago. {footerStatsText()}
         </p>
         {#if layout.show_stop_impersonating}
diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb
index 08038434c..5ede36622 100644
--- a/app/views/layouts/application.html.erb
+++ b/app/views/layouts/application.html.erb
@@ -233,7 +233,13 @@
       <footer class="relative w-full mt-12 mb-5 p-2.5 text-center text-xs text-muted hover:text-muted transition-colors duration-200">
         <div class="container">
           <p>
-            Build <%= link_to Rails.application.config.git_version, Rails.application.config.commit_link, class: 'text-inherit underline opacity-80 hover:opacity-100 transition-opacity duration-200' %> from <%= time_ago_in_words(Rails.application.config.server_start_time) %> ago.
+            Build
+            <% if Rails.application.config.commit_link.present? %>
+              <%= link_to Rails.application.config.git_version, Rails.application.config.commit_link, class: 'text-inherit underline opacity-80 hover:opacity-100 transition-opacity duration-200' %>
+            <% else %>
+              <span><%= Rails.application.config.git_version %></span>
+            <% end %>
+            from <%= time_ago_in_words(Rails.application.config.server_start_time) %> ago.
             <%= pluralize(Heartbeat.recent_count, 'heartbeat') %>
             (<%= Heartbeat.recent_imported_count %> imported) in the past 24 hours. (DB: <%= pluralize(QueryCount::Counter.counter, 'query') %>, <%= QueryCount::Counter.counter_cache %> cached) (CACHE: <%= cache_stats[:hits] %> hits, <%= cache_stats[:misses] %> misses) (<%= requests_per_second %>)
           </p>
diff --git a/config/initializers/git_version.rb b/config/initializers/git_version.rb
index 09eb0c5a8..2da2c1030 100644
--- a/config/initializers/git_version.rb
+++ b/config/initializers/git_version.rb
@@ -1,22 +1,11 @@
-# Get the first 6 characters of the current git commit hash
-git_hash = ENV["SOURCE_COMMIT"] || `git rev-parse HEAD` rescue "unknown"
+require Rails.root.join("lib/git_metadata")
 
-commit_link = git_hash != "unknown" ? "https://github.com/hackclub/hackatime/commit/#{git_hash}" : nil
-
-short_hash = git_hash[0..7]
-
-commit_count = `git rev-list --count HEAD`.strip rescue 0
-
-# Check if there are any uncommitted changes
-is_dirty = `git status --porcelain`.strip.length > 0 rescue false
-
-# Append "-dirty" if there are uncommitted changes
-version = is_dirty ? "#{short_hash}-dirty" : short_hash
+metadata = GitMetadata.build
 
 # Store server start time
 Rails.application.config.server_start_time = Time.current
 
 # Store the version
-Rails.application.config.git_version = version
-Rails.application.config.git_commit_count = commit_count
-Rails.application.config.commit_link = commit_link
+Rails.application.config.git_version = metadata[:version]
+Rails.application.config.git_commit_count = metadata[:commit_count]
+Rails.application.config.commit_link = metadata[:commit_link]
diff --git a/lib/git_metadata.rb b/lib/git_metadata.rb
new file mode 100644
index 000000000..a4cd7f579
--- /dev/null
+++ b/lib/git_metadata.rb
@@ -0,0 +1,27 @@
+require "open3"
+
+module GitMetadata
+  module_function
+
+  def build(env: ENV, root: Rails.root)
+    git_hash = env["SOURCE_COMMIT"].presence || git_output(root, "rev-parse", "HEAD")
+    short_hash = git_hash&.slice(0, 8)
+    commit_count = git_output(root, "rev-list", "--count", "HEAD")&.to_i || 0
+    dirty = git_output(root, "status", "--porcelain").present?
+
+    {
+      commit_count: commit_count,
+      commit_link: git_hash.present? ? "https://github.com/hackclub/hackatime/commit/#{git_hash}" : nil,
+      version: short_hash.present? ? "#{short_hash}#{'-dirty' if dirty}" : "unknown"
+    }
+  end
+
+  def git_output(root, *args)
+    stdout, _stderr, status = Open3.capture3("git", *args, chdir: root.to_s)
+    return unless status.success?
+
+    stdout.strip.presence
+  rescue Errno::ENOENT
+    nil
+  end
+end
diff --git a/test/lib/git_metadata_test.rb b/test/lib/git_metadata_test.rb
new file mode 100644
index 000000000..55e076cc9
--- /dev/null
+++ b/test/lib/git_metadata_test.rb
@@ -0,0 +1,59 @@
+require "test_helper"
+require "open3"
+require "tmpdir"
+
+class GitMetadataTest < ActiveSupport::TestCase
+  test "uses source commit when git metadata is unavailable" do
+    Dir.mktmpdir do |dir|
+      metadata = GitMetadata.build(env: { "SOURCE_COMMIT" => "abcdef1234567890" }, root: dir)
+
+      assert_equal "abcdef12", metadata[:version]
+      assert_equal 0, metadata[:commit_count]
+      assert_equal "https://github.com/hackclub/hackatime/commit/abcdef1234567890", metadata[:commit_link]
+    end
+  end
+
+  test "returns unknown when no git metadata is available" do
+    Dir.mktmpdir do |dir|
+      metadata = GitMetadata.build(env: {}, root: dir)
+
+      assert_equal "unknown", metadata[:version]
+      assert_equal 0, metadata[:commit_count]
+      assert_nil metadata[:commit_link]
+    end
+  end
+
+  test "builds metadata from a git repository and marks dirty checkouts" do
+    Dir.mktmpdir do |dir|
+      run_git(dir, "init", "-q")
+      run_git(dir, "config", "user.name", "Test User")
+      run_git(dir, "config", "user.email", "test@example.com")
+
+      File.write(File.join(dir, "README.md"), "hello\n")
+      run_git(dir, "add", "README.md")
+      run_git(dir, "commit", "-qm", "Initial commit")
+
+      clean_metadata = GitMetadata.build(env: {}, root: dir)
+      commit_hash = run_git(dir, "rev-parse", "HEAD")
+
+      assert_equal commit_hash.slice(0, 8), clean_metadata[:version]
+      assert_equal 1, clean_metadata[:commit_count]
+      assert_equal "https://github.com/hackclub/hackatime/commit/#{commit_hash}", clean_metadata[:commit_link]
+
+      File.write(File.join(dir, "README.md"), "hello again\n")
+
+      dirty_metadata = GitMetadata.build(env: {}, root: dir)
+
+      assert_equal "#{commit_hash.slice(0, 8)}-dirty", dirty_metadata[:version]
+    end
+  end
+
+  private
+
+  def run_git(dir, *args)
+    output, error, status = Open3.capture3("git", *args, chdir: dir)
+    assert status.success?, "git #{args.join(' ')} failed:\n#{output}#{error}"
+
+    output.strip
+  end
+end

From 14ccff36f42dd91cfbcfb963b4f9c95f532bfb80 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 16:07:08 +0000
Subject: [PATCH 12/24] no goog

---
 bin/docker-entrypoint | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint
index 21d1c8cb9..d986ea1bc 100755
--- a/bin/docker-entrypoint
+++ b/bin/docker-entrypoint
@@ -8,7 +8,7 @@ fi
 
 # If running the rails server then create or migrate existing database
 if [ "${@: -2:1}" == "./bin/rails" ] && [ "${@: -1:1}" == "server" ]; then
-  ./bin/rails db:prepare
+  ./bin/rails db:prepare --trace
   # echo "Warming up caches for production deployment..."
   # ./bin/rails cache:warmup
 

From 5d0f8b81c025ff06c35a8fc6d4e716f5223ef75f Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 16:18:03 +0000
Subject: [PATCH 13/24] Fork switch

---
 .env.example        |  1 +
 Gemfile             |  2 +-
 Gemfile.lock        | 13 +++++++++----
 config/database.yml | 18 +++++++++++++++---
 4 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/.env.example b/.env.example
index 6ac978177..9a7990836 100644
--- a/.env.example
+++ b/.env.example
@@ -64,6 +64,7 @@ S3_ENDPOINT=https://<ACCOUNT_ID>.r2.cloudflarestorage.com
 # ClickHouse database
 CLICKHOUSE_HOST=clickhouse
 CLICKHOUSE_PORT=8123
+CLICKHOUSE_SSL=false
 CLICKHOUSE_DATABASE=hackatime_development
 CLICKHOUSE_USERNAME=default
 CLICKHOUSE_PASSWORD=
diff --git a/Gemfile b/Gemfile
index 4c4fc89c4..cb33e6158 100644
--- a/Gemfile
+++ b/Gemfile
@@ -10,7 +10,7 @@ gem "sqlite3", ">= 2.1"
 # Use PostgreSQL as the database for Wakatime
 gem "pg"
 # Use ClickHouse for analytics
-gem "clickhouse-activerecord"
+gem "clickhouse-activerecord", github: "daisychainapp/clickhouse-activerecord"
 # Use the Puma web server [https://github.com/puma/puma]
 gem "puma", ">= 5.0"
 # Use JavaScript with ESM import maps [https://github.com/rails/importmap-rails]
diff --git a/Gemfile.lock b/Gemfile.lock
index 951045426..ebb0dbb7b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/daisychainapp/clickhouse-activerecord.git
+  revision: 2eb937bc36d327ff4f2f27f1da50cc151a7f6af5
+  specs:
+    clickhouse-activerecord (1.6.6)
+      activerecord (>= 7.1, < 9.0)
+      bundler (>= 1.13.4)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -131,9 +139,6 @@ GEM
     childprocess (5.1.0)
       logger (~> 1.5)
     chunky_png (1.4.0)
-    clickhouse-activerecord (1.6.7)
-      activerecord (>= 7.1, < 9.0)
-      bundler (>= 1.13.4)
     cloudflare-rails (7.0.0)
       actionpack (>= 7.2.0, < 8.2.0)
       activesupport (>= 7.2.0, < 8.2.0)
@@ -655,7 +660,7 @@ DEPENDENCIES
   brakeman
   bullet
   capybara
-  clickhouse-activerecord
+  clickhouse-activerecord!
   cloudflare-rails
   countries
   debug
diff --git a/config/database.yml b/config/database.yml
index e16e2d49f..f75e0eebe 100644
--- a/config/database.yml
+++ b/config/database.yml
@@ -1,3 +1,12 @@
+<%
+clickhouse_port = ENV.fetch("CLICKHOUSE_PORT", 8123).to_i
+clickhouse_ssl = if ENV.key?("CLICKHOUSE_SSL")
+  %w[1 true yes on].include?(ENV["CLICKHOUSE_SSL"].to_s.downcase)
+else
+  clickhouse_port == 8443
+end
+%>
+
 # SQLite. Versions 3.8.0 and up are supported.
 #   gem install sqlite3
 #
@@ -23,7 +32,8 @@ development:
   clickhouse:
     adapter: clickhouse
     host: <%= ENV.fetch('CLICKHOUSE_HOST', 'clickhouse') %>
-    port: <%= ENV.fetch('CLICKHOUSE_PORT', 8123) %>
+    port: <%= clickhouse_port %>
+    ssl: <%= clickhouse_ssl %>
     database: <%= ENV.fetch('CLICKHOUSE_DATABASE', 'hackatime_development') %>
     username: <%= ENV.fetch('CLICKHOUSE_USERNAME', 'default') %>
     password: <%= ENV.fetch('CLICKHOUSE_PASSWORD', '') %>
@@ -46,7 +56,8 @@ test:
   clickhouse:
     adapter: clickhouse
     host: <%= ENV.fetch('CLICKHOUSE_HOST', 'clickhouse') %>
-    port: <%= ENV.fetch('CLICKHOUSE_PORT', 8123) %>
+    port: <%= clickhouse_port %>
+    ssl: <%= clickhouse_ssl %>
     database: hackatime_test
     username: <%= ENV.fetch('CLICKHOUSE_USERNAME', 'default') %>
     password: <%= ENV.fetch('CLICKHOUSE_PASSWORD', '') %>
@@ -79,7 +90,8 @@ production:
   clickhouse:
     adapter: clickhouse
     host: <%= ENV.fetch('CLICKHOUSE_HOST', 'clickhouse') %>
-    port: <%= ENV.fetch('CLICKHOUSE_PORT', 8123) %>
+    port: <%= clickhouse_port %>
+    ssl: <%= clickhouse_ssl %>
     database: <%= ENV.fetch('CLICKHOUSE_DATABASE', 'hackatime_production') %>
     username: <%= ENV.fetch('CLICKHOUSE_USERNAME', 'default') %>
     password: <%= ENV.fetch('CLICKHOUSE_PASSWORD', '') %>

From 6cbd01168009e0fa5c3c4c6b729bf891848919eb Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 16:35:50 +0000
Subject: [PATCH 14/24] SOBBING

---
 bin/docker-entrypoint                         |  3 +-
 bin/setup                                     |  3 +-
 db/clickhouse_schema.rb                       | 20 ++--------
 .../20260324000001_create_heartbeats.rb       | 37 +++++++++++++++++++
 4 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint
index d986ea1bc..d66abd010 100755
--- a/bin/docker-entrypoint
+++ b/bin/docker-entrypoint
@@ -8,7 +8,8 @@ fi
 
 # If running the rails server then create or migrate existing database
 if [ "${@: -2:1}" == "./bin/rails" ] && [ "${@: -1:1}" == "server" ]; then
-  ./bin/rails db:prepare --trace
+  ./bin/rails db:create:primary db:create:clickhouse
+  ./bin/rails db:migrate:primary db:migrate:clickhouse
   # echo "Warming up caches for production deployment..."
   # ./bin/rails cache:warmup
 
diff --git a/bin/setup b/bin/setup
index 59cd269d6..38ab1a3bf 100755
--- a/bin/setup
+++ b/bin/setup
@@ -22,7 +22,8 @@ FileUtils.chdir APP_ROOT do
   # end
 
   puts "\n== Preparing database =="
-  system! "bin/rails db:prepare"
+  system! "bin/rails db:create:primary db:create:clickhouse"
+  system! "bin/rails db:migrate:primary db:migrate:clickhouse"
 
   puts "\n== Removing old logs and tempfiles =="
   system! "bin/rails log:clear tmp:clear"
diff --git a/db/clickhouse_schema.rb b/db/clickhouse_schema.rb
index 38a7727ad..df2070764 100644
--- a/db/clickhouse_schema.rb
+++ b/db/clickhouse_schema.rb
@@ -14,10 +14,7 @@
   # TABLE: schema_migrations
   # SQL: CREATE TABLE schema_migrations ( `version` String, `active` Int8 DEFAULT 1, `ver` DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(ver) ORDER BY (version)
   execute <<~SQL
-    DROP TABLE IF EXISTS schema_migrations
-  SQL
-  execute <<~SQL
-    CREATE TABLE schema_migrations
+    CREATE TABLE IF NOT EXISTS schema_migrations
     (
       `version` String,
       `active` Int8 DEFAULT 1,
@@ -30,10 +27,7 @@
   # TABLE: ar_internal_metadata
   # SQL: CREATE TABLE ar_internal_metadata ( `key` String, `value` String, `created_at` DateTime, `updated_at` DateTime ) ENGINE = ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key
   execute <<~SQL
-    DROP TABLE IF EXISTS ar_internal_metadata
-  SQL
-  execute <<~SQL
-    CREATE TABLE ar_internal_metadata
+    CREATE TABLE IF NOT EXISTS ar_internal_metadata
     (
       `key` String,
       `value` String,
@@ -48,10 +42,7 @@
   # TABLE: heartbeat_user_daily_summary
   # SQL: CREATE TABLE heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32, `_version` DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192
   execute <<~SQL
-    DROP TABLE IF EXISTS heartbeat_user_daily_summary
-  SQL
-  execute <<~SQL
-    CREATE TABLE heartbeat_user_daily_summary
+    CREATE TABLE IF NOT EXISTS heartbeat_user_daily_summary
     (
       `user_id` Int64,
       `day` Date,
@@ -67,10 +58,7 @@
   # TABLE: heartbeats
   # SQL: CREATE TABLE heartbeats ( `id` Int64, `user_id` Int64, `branch` String DEFAULT '', `category` String DEFAULT '', `dependencies` Array(String), `editor` String DEFAULT '', `entity` String DEFAULT '', `language` String DEFAULT '', `machine` String DEFAULT '', `operating_system` String DEFAULT '', `project` String DEFAULT '', `type` String DEFAULT '', `user_agent` String DEFAULT '', `line_additions` Int32 DEFAULT 0, `line_deletions` Int32 DEFAULT 0, `lineno` Int32 DEFAULT 0, `lines` Int32 DEFAULT 0, `cursorpos` Int32 DEFAULT 0, `project_root_count` Int32 DEFAULT 0, `time` Float64, `is_write` UInt8 DEFAULT 0, `created_at` DateTime64(6) DEFAULT now64(), `updated_at` DateTime64(6) DEFAULT now64(), `source_type` UInt8 DEFAULT 0, `ip_address` String DEFAULT '', `ysws_program` UInt8 DEFAULT 0, `fields_hash` String DEFAULT '' ) ENGINE = ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id) SETTINGS index_granularity = 8192
   execute <<~SQL
-    DROP TABLE IF EXISTS heartbeats
-  SQL
-  execute <<~SQL
-    CREATE TABLE heartbeats
+    CREATE TABLE IF NOT EXISTS heartbeats
     (
       `id` Int64,
       `user_id` Int64,
diff --git a/db/migrate_clickhouse/20260324000001_create_heartbeats.rb b/db/migrate_clickhouse/20260324000001_create_heartbeats.rb
index b2b0b59ef..3f245515c 100644
--- a/db/migrate_clickhouse/20260324000001_create_heartbeats.rb
+++ b/db/migrate_clickhouse/20260324000001_create_heartbeats.rb
@@ -1,4 +1,34 @@
 class CreateHeartbeats < ActiveRecord::Migration[8.1]
+  HEARTBEAT_COLUMNS = [
+    "ADD COLUMN IF NOT EXISTS id Int64",
+    "ADD COLUMN IF NOT EXISTS user_id Int64",
+    "ADD COLUMN IF NOT EXISTS branch String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS category String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS dependencies Array(String)",
+    "ADD COLUMN IF NOT EXISTS editor String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS entity String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS language String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS machine String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS operating_system String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS project String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS type String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS user_agent String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS line_additions Int32 DEFAULT 0",
+    "ADD COLUMN IF NOT EXISTS line_deletions Int32 DEFAULT 0",
+    "ADD COLUMN IF NOT EXISTS lineno Int32 DEFAULT 0",
+    "ADD COLUMN IF NOT EXISTS lines Int32 DEFAULT 0",
+    "ADD COLUMN IF NOT EXISTS cursorpos Int32 DEFAULT 0",
+    "ADD COLUMN IF NOT EXISTS project_root_count Int32 DEFAULT 0",
+    "ADD COLUMN IF NOT EXISTS time Float64",
+    "ADD COLUMN IF NOT EXISTS is_write UInt8 DEFAULT 0",
+    "ADD COLUMN IF NOT EXISTS created_at DateTime64(6) DEFAULT now64()",
+    "ADD COLUMN IF NOT EXISTS updated_at DateTime64(6) DEFAULT now64()",
+    "ADD COLUMN IF NOT EXISTS source_type UInt8 DEFAULT 0",
+    "ADD COLUMN IF NOT EXISTS ip_address String DEFAULT ''",
+    "ADD COLUMN IF NOT EXISTS ysws_program UInt8 DEFAULT 0",
+    "ADD COLUMN IF NOT EXISTS fields_hash String DEFAULT ''"
+  ].freeze
+
   def up
     execute <<~SQL
       CREATE TABLE IF NOT EXISTS heartbeats
@@ -36,6 +66,13 @@ def up
       ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id)
       SETTINGS index_granularity = 8192
     SQL
+
+    HEARTBEAT_COLUMNS.each do |column_definition|
+      execute <<~SQL
+        ALTER TABLE heartbeats
+        #{column_definition}
+      SQL
+    end
   end
 
   def down

From 1ee5d5420116d5013783e878144f230c2b95d896 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 19:47:56 +0000
Subject: [PATCH 15/24] thonking

---
 app/controllers/concerns/dashboard_data.rb | 47 +++++++++++++++++-----
 app/javascript/pages/Home/SignedIn.svelte  |  1 -
 2 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/app/controllers/concerns/dashboard_data.rb b/app/controllers/concerns/dashboard_data.rb
index 6fa4cb538..f76aa6496 100644
--- a/app/controllers/concerns/dashboard_data.rb
+++ b/app/controllers/concerns/dashboard_data.rb
@@ -43,10 +43,13 @@ def filterable_dashboard_data
         result[:total_time] = hb.duration_seconds
         result[:total_heartbeats] = hb.count
 
+        # Compute per-filter grouped stats in one pass each, then derive top_X and X_stats from the same data
+        filter_stats = {}
         filters.each do |f|
-          stats = hb.group(f).duration_seconds
-          stats = stats.reject { |n, _| archived.include?(n) } if f == :project
-          result["top_#{f}"] = stats.max_by { |_, v| v }&.first
+          raw_stats = hb.group(f).duration_seconds
+          raw_stats = raw_stats.reject { |n, _| archived.include?(n) } if f == :project
+          filter_stats[f] = raw_stats
+          result["top_#{f}"] = raw_stats.max_by { |_, v| v }&.first
         end
 
         result["top_editor"] &&= h.display_editor_name(result["top_editor"])
@@ -54,13 +57,13 @@ def filterable_dashboard_data
         result["top_language"] &&= h.display_language_name(result["top_language"])
 
         unless result["singular_project"]
-          result[:project_durations] = hb.group(:project).duration_seconds
-            .reject { |p, _| archived.include?(p) }.sort_by { |_, d| -d }.first(10).to_h
+          result[:project_durations] = filter_stats[:project]
+            .sort_by { |_, d| -d }.first(10).to_h
         end
 
         %i[language editor operating_system category].each do |f|
           next if result["singular_#{f}"]
-          stats = hb.group(f).duration_seconds.each_with_object({}) do |(raw, dur), agg|
+          stats = filter_stats[f].each_with_object({}) do |(raw, dur), agg|
             k = raw.to_s.presence || "Unknown"
             k = f == :language ? (k == "Unknown" ? k : k.categorize_language) : (%i[editor operating_system].include?(f) ? k.downcase : k)
             agg[k] = (agg[k] || 0) + dur
@@ -80,10 +83,36 @@ def filterable_dashboard_data
           result[:language_colors] = LanguageUtils.colors_for(result["language_stats"].keys)
         end
 
+        # Batch all 12 weekly queries into a single ClickHouse query
+        week_start = 11.weeks.ago.beginning_of_week
+        week_end = Time.current.end_of_week
+        weekly_hb = hb.where(time: week_start.to_f..week_end.to_f)
+
+        timeout = Heartbeat.heartbeat_timeout_duration.to_i
+        tz = current_user.timezone
+        weekly_sql = weekly_hb
+          .select("toStartOfWeek(toDateTime(toUInt32(time), '#{tz}'), 1) as week_start, `project` as grouped_time, least(time - lagInFrame(time, 1, time) OVER (PARTITION BY `project`, toStartOfWeek(toDateTime(toUInt32(time), '#{tz}'), 1) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
+          .where.not(time: nil)
+          .with_valid_timestamps
+          .to_sql
+
+        weekly_rows = Heartbeat.connection.select_all(
+          "SELECT toString(week_start) as week_start, grouped_time, toInt64(coalesce(sum(diff), 0)) as duration FROM (#{weekly_sql}) GROUP BY week_start, grouped_time"
+        )
+
+        weekly_hash = {}
+        weekly_rows.each do |row|
+          week_key = row["week_start"].to_date.iso8601
+          project = row["grouped_time"]
+          next if archived.include?(project)
+          weekly_hash[week_key] ||= {}
+          weekly_hash[week_key][project] = row["duration"].to_i
+        end
+
+        # Ensure all 12 weeks are present (even if empty)
         result[:weekly_project_stats] = (0..11).to_h do |w|
-          ws = w.weeks.ago.beginning_of_week
-          [ ws.to_date.iso8601, hb.where(time: ws.to_f..w.weeks.ago.end_of_week.to_f)
-              .group(:project).duration_seconds.reject { |p, _| archived.include?(p) } ]
+          ws = w.weeks.ago.beginning_of_week.to_date.iso8601
+          [ ws, weekly_hash[ws] || {} ]
         end
       end
       result[:selected_interval] = interval.to_s
diff --git a/app/javascript/pages/Home/SignedIn.svelte b/app/javascript/pages/Home/SignedIn.svelte
index 2312b4bdb..cf2c4af43 100644
--- a/app/javascript/pages/Home/SignedIn.svelte
+++ b/app/javascript/pages/Home/SignedIn.svelte
@@ -95,7 +95,6 @@
       preserveState: true,
       preserveScroll: true,
       replace: true,
-      async: true,
     });
   }
 </script>

From 969ea708ba348205c246056bfcbaf42d3d03bc73 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 20:44:57 +0000
Subject: [PATCH 16/24] Faster home stats?

---
 app/jobs/aggregate_user_daily_summary_job.rb  | 30 -------------
 config/initializers/good_job.rb               |  5 ---
 db/clickhouse_schema.rb                       | 38 +++++++++++++++-
 ...eat_user_daily_summary_refreshable_view.rb | 43 +++++++++++++++++++
 4 files changed, 80 insertions(+), 36 deletions(-)
 delete mode 100644 app/jobs/aggregate_user_daily_summary_job.rb
 create mode 100644 db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb

diff --git a/app/jobs/aggregate_user_daily_summary_job.rb b/app/jobs/aggregate_user_daily_summary_job.rb
deleted file mode 100644
index 072c0f608..000000000
--- a/app/jobs/aggregate_user_daily_summary_job.rb
+++ /dev/null
@@ -1,30 +0,0 @@
-class AggregateUserDailySummaryJob < ApplicationJob
-  include GoodJob::ActiveJobExtensions::Concurrency
-
-  queue_as :latency_5m
-
-  good_job_control_concurrency_with(
-    total: 1,
-    drop: true
-  )
-
-  def perform
-    conn = HeartbeatUserDailySummary.connection
-
-    conn.execute(<<~SQL)
-      INSERT INTO heartbeat_user_daily_summary (user_id, day, duration_s, heartbeats)
-      SELECT
-        user_id,
-        toDate(toDateTime(toUInt32(time))) AS day,
-        sum(
-          least(
-            time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id, toDate(toDateTime(toUInt32(time))) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
-            120
-          )
-        ) AS duration_s,
-        count() AS heartbeats
-      FROM heartbeats
-      GROUP BY user_id, day
-    SQL
-  end
-end
diff --git a/config/initializers/good_job.rb b/config/initializers/good_job.rb
index 2a1d98c40..c42d57b9b 100644
--- a/config/initializers/good_job.rb
+++ b/config/initializers/good_job.rb
@@ -79,11 +79,6 @@
       class: "Cache::CurrentlyHackingJob",
       kwargs: { force_reload: true }
     },
-    aggregate_user_daily_summary: {
-      cron: "*/10 * * * *",
-      class: "AggregateUserDailySummaryJob",
-      description: "Aggregates heartbeats into per-user daily summaries for home stats."
-    },
     cache_home_stats: {
       cron: "*/10 * * * *",
       class: "Cache::HomeStatsJob",
diff --git a/db/clickhouse_schema.rb b/db/clickhouse_schema.rb
index df2070764..e7244e04b 100644
--- a/db/clickhouse_schema.rb
+++ b/db/clickhouse_schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.1].define(version: 2026_03_24_000002) do
+ActiveRecord::Schema[8.1].define(version: 2026_03_24_000003) do
   # TABLE: schema_migrations
   # SQL: CREATE TABLE schema_migrations ( `version` String, `active` Int8 DEFAULT 1, `ver` DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(ver) ORDER BY (version)
   execute <<~SQL
@@ -93,4 +93,40 @@
     ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id)
     SETTINGS index_granularity = 8192
   SQL
+
+  # TABLE: heartbeat_user_daily_summary_mv
+  # SQL: CREATE MATERIALIZED VIEW heartbeat_user_daily_summary_mv REFRESH EVERY 10 MINUTE TO heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32 ) AS SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats FINAL WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day
+  execute <<~SQL
+    CREATE MATERIALIZED VIEW IF NOT EXISTS heartbeat_user_daily_summary_mv
+    REFRESH EVERY 10 MINUTE
+    TO heartbeat_user_daily_summary
+    AS
+    SELECT
+        user_id,
+        toDate(toDateTime(toUInt32(time))) AS day,
+        sum(diff) AS duration_s,
+        toUInt32(count()) AS heartbeats
+    FROM
+    (
+        SELECT
+            user_id,
+            time,
+            least(
+                greatest(
+                    time - lagInFrame(time, 1, time) OVER (
+                        PARTITION BY user_id
+                        ORDER BY time ASC
+                        ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+                    ),
+                    0
+                ),
+                120
+            ) AS diff
+        FROM heartbeats FINAL
+        WHERE time IS NOT NULL
+          AND time >= 0
+          AND time <= 253402300799
+    )
+    GROUP BY user_id, day
+  SQL
 end
diff --git a/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb b/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb
new file mode 100644
index 000000000..01d9041e4
--- /dev/null
+++ b/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb
@@ -0,0 +1,43 @@
+class CreateHeartbeatUserDailySummaryRefreshableView < ActiveRecord::Migration[8.1]
+  VIEW_NAME = "heartbeat_user_daily_summary_mv".freeze
+
+  def up
+    execute <<~SQL
+      CREATE MATERIALIZED VIEW IF NOT EXISTS #{VIEW_NAME}
+      REFRESH EVERY 10 MINUTE
+      TO heartbeat_user_daily_summary
+      AS
+      SELECT
+          user_id,
+          toDate(toDateTime(toUInt32(time))) AS day,
+          sum(diff) AS duration_s,
+          toUInt32(count()) AS heartbeats
+      FROM
+      (
+          SELECT
+              user_id,
+              time,
+              least(
+                  greatest(
+                      time - lagInFrame(time, 1, time) OVER (
+                          PARTITION BY user_id
+                          ORDER BY time ASC
+                          ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+                      ),
+                      0
+                  ),
+                  120
+              ) AS diff
+          FROM heartbeats FINAL
+          WHERE time IS NOT NULL
+            AND time >= 0
+            AND time <= 253402300799
+      )
+      GROUP BY user_id, day
+    SQL
+  end
+
+  def down
+    execute "DROP VIEW IF EXISTS #{VIEW_NAME}"
+  end
+end

From df1f59af63c75630c10b95bf32a4fad70642cdb8 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 20:49:15 +0000
Subject: [PATCH 17/24] goog (clickhouse migration)

---
 db/clickhouse_schema.rb | 149 +++++++++++-----------------------------
 1 file changed, 39 insertions(+), 110 deletions(-)

diff --git a/db/clickhouse_schema.rb b/db/clickhouse_schema.rb
index e7244e04b..2d2fbecb0 100644
--- a/db/clickhouse_schema.rb
+++ b/db/clickhouse_schema.rb
@@ -11,122 +11,51 @@
 # It's strongly recommended that you check this file into your version control system.
 
 ActiveRecord::Schema[8.1].define(version: 2026_03_24_000003) do
-  # TABLE: schema_migrations
-  # SQL: CREATE TABLE schema_migrations ( `version` String, `active` Int8 DEFAULT 1, `ver` DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(ver) ORDER BY (version)
-  execute <<~SQL
-    CREATE TABLE IF NOT EXISTS schema_migrations
-    (
-      `version` String,
-      `active` Int8 DEFAULT 1,
-      `ver` DateTime DEFAULT now()
-    )
-    ENGINE = ReplacingMergeTree(ver)
-    ORDER BY (version)
-  SQL
-
-  # TABLE: ar_internal_metadata
-  # SQL: CREATE TABLE ar_internal_metadata ( `key` String, `value` String, `created_at` DateTime, `updated_at` DateTime ) ENGINE = ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key
-  execute <<~SQL
-    CREATE TABLE IF NOT EXISTS ar_internal_metadata
-    (
-      `key` String,
-      `value` String,
-      `created_at` DateTime,
-      `updated_at` DateTime
-    )
-    ENGINE = ReplacingMergeTree(created_at)
-    PARTITION BY key
-    ORDER BY key
-  SQL
-
   # TABLE: heartbeat_user_daily_summary
   # SQL: CREATE TABLE heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32, `_version` DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192
-  execute <<~SQL
-    CREATE TABLE IF NOT EXISTS heartbeat_user_daily_summary
-    (
-      `user_id` Int64,
-      `day` Date,
-      `duration_s` Float64,
-      `heartbeats` UInt32,
-      `_version` DateTime DEFAULT now()
-    )
-    ENGINE = ReplacingMergeTree(_version)
-    ORDER BY (user_id, day)
-    SETTINGS index_granularity = 8192
-  SQL
+  create_table "heartbeat_user_daily_summary", id: false, options: "ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192", force: :cascade do |t|
+    t.integer "user_id", unsigned: false, limit: 8, null: false
+    t.date "day", null: false
+    t.float "duration_s", null: false
+    t.integer "heartbeats", null: false
+    t.datetime "_version", precision: nil, default: -> { "now()" }, null: false
+  end
 
   # TABLE: heartbeats
   # SQL: CREATE TABLE heartbeats ( `id` Int64, `user_id` Int64, `branch` String DEFAULT '', `category` String DEFAULT '', `dependencies` Array(String), `editor` String DEFAULT '', `entity` String DEFAULT '', `language` String DEFAULT '', `machine` String DEFAULT '', `operating_system` String DEFAULT '', `project` String DEFAULT '', `type` String DEFAULT '', `user_agent` String DEFAULT '', `line_additions` Int32 DEFAULT 0, `line_deletions` Int32 DEFAULT 0, `lineno` Int32 DEFAULT 0, `lines` Int32 DEFAULT 0, `cursorpos` Int32 DEFAULT 0, `project_root_count` Int32 DEFAULT 0, `time` Float64, `is_write` UInt8 DEFAULT 0, `created_at` DateTime64(6) DEFAULT now64(), `updated_at` DateTime64(6) DEFAULT now64(), `source_type` UInt8 DEFAULT 0, `ip_address` String DEFAULT '', `ysws_program` UInt8 DEFAULT 0, `fields_hash` String DEFAULT '' ) ENGINE = ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id) SETTINGS index_granularity = 8192
-  execute <<~SQL
-    CREATE TABLE IF NOT EXISTS heartbeats
-    (
-      `id` Int64,
-      `user_id` Int64,
-      `branch` String DEFAULT '',
-      `category` String DEFAULT '',
-      `dependencies` Array(String),
-      `editor` String DEFAULT '',
-      `entity` String DEFAULT '',
-      `language` String DEFAULT '',
-      `machine` String DEFAULT '',
-      `operating_system` String DEFAULT '',
-      `project` String DEFAULT '',
-      `type` String DEFAULT '',
-      `user_agent` String DEFAULT '',
-      `line_additions` Int32 DEFAULT 0,
-      `line_deletions` Int32 DEFAULT 0,
-      `lineno` Int32 DEFAULT 0,
-      `lines` Int32 DEFAULT 0,
-      `cursorpos` Int32 DEFAULT 0,
-      `project_root_count` Int32 DEFAULT 0,
-      `time` Float64,
-      `is_write` UInt8 DEFAULT 0,
-      `created_at` DateTime64(6) DEFAULT now64(),
-      `updated_at` DateTime64(6) DEFAULT now64(),
-      `source_type` UInt8 DEFAULT 0,
-      `ip_address` String DEFAULT '',
-      `ysws_program` UInt8 DEFAULT 0,
-      `fields_hash` String DEFAULT ''
-    )
-    ENGINE = ReplacingMergeTree
-    PARTITION BY toYYYYMM(toDateTime(toUInt32(time)))
-    ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id)
-    SETTINGS index_granularity = 8192
-  SQL
+  create_table "heartbeats", id: :int64, options: "ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id) SETTINGS index_granularity = 8192", force: :cascade do |t|
+    t.integer "id", unsigned: false, limit: 8, null: false
+    t.integer "user_id", unsigned: false, limit: 8, null: false
+    t.string "branch", default: "", null: false
+    t.string "category", default: "", null: false
+    t.string "dependencies", array: true, null: false
+    t.string "editor", default: "", null: false
+    t.string "entity", default: "", null: false
+    t.string "language", default: "", null: false
+    t.string "machine", default: "", null: false
+    t.string "operating_system", default: "", null: false
+    t.string "project", default: "", null: false
+    t.string "type", default: "", null: false
+    t.string "user_agent", default: "", null: false
+    t.integer "line_additions", unsigned: false, default: 0, null: false
+    t.integer "line_deletions", unsigned: false, default: 0, null: false
+    t.integer "lineno", unsigned: false, default: 0, null: false
+    t.integer "lines", unsigned: false, default: 0, null: false
+    t.integer "cursorpos", unsigned: false, default: 0, null: false
+    t.integer "project_root_count", unsigned: false, default: 0, null: false
+    t.float "time", null: false
+    t.integer "is_write", limit: 1, default: 0, null: false
+    t.datetime "created_at", default: -> { "now64()" }, null: false
+    t.datetime "updated_at", default: -> { "now64()" }, null: false
+    t.integer "source_type", limit: 1, default: 0, null: false
+    t.string "ip_address", default: "", null: false
+    t.integer "ysws_program", limit: 1, default: 0, null: false
+    t.string "fields_hash", default: "", null: false
+  end
 
   # TABLE: heartbeat_user_daily_summary_mv
   # SQL: CREATE MATERIALIZED VIEW heartbeat_user_daily_summary_mv REFRESH EVERY 10 MINUTE TO heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32 ) AS SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats FINAL WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day
-  execute <<~SQL
-    CREATE MATERIALIZED VIEW IF NOT EXISTS heartbeat_user_daily_summary_mv
-    REFRESH EVERY 10 MINUTE
-    TO heartbeat_user_daily_summary
-    AS
-    SELECT
-        user_id,
-        toDate(toDateTime(toUInt32(time))) AS day,
-        sum(diff) AS duration_s,
-        toUInt32(count()) AS heartbeats
-    FROM
-    (
-        SELECT
-            user_id,
-            time,
-            least(
-                greatest(
-                    time - lagInFrame(time, 1, time) OVER (
-                        PARTITION BY user_id
-                        ORDER BY time ASC
-                        ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
-                    ),
-                    0
-                ),
-                120
-            ) AS diff
-        FROM heartbeats FINAL
-        WHERE time IS NOT NULL
-          AND time >= 0
-          AND time <= 253402300799
-    )
-    GROUP BY user_id, day
-  SQL
+  create_table "heartbeat_user_daily_summary_mv", view: true, materialized: true, id: false, as: "SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats FINAL WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day", force: :cascade do |t|
+  end
+
 end

From a45ecfa8443ea49d57c4768c6dc5830392cfbff2 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 20:56:05 +0000
Subject: [PATCH 18/24] goog

---
 app/jobs/cache/home_stats_job.rb | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/app/jobs/cache/home_stats_job.rb b/app/jobs/cache/home_stats_job.rb
index 6d2ef377e..22179d1bb 100644
--- a/app/jobs/cache/home_stats_job.rb
+++ b/app/jobs/cache/home_stats_job.rb
@@ -3,6 +3,10 @@ class Cache::HomeStatsJob < Cache::ActivityJob
 
   private
 
+  def cache_key
+    "#{super}/#{summary_refresh_version}"
+  end
+
   def calculate
     result = HeartbeatUserDailySummary.connection.select_one(<<~SQL)
       SELECT
@@ -16,4 +20,19 @@ def calculate
       seconds_tracked: result["seconds_tracked"].to_i
     }
   end
+
+  def summary_refresh_version
+    connection = HeartbeatUserDailySummary.connection
+    database = connection.select_value("SELECT currentDatabase()")
+
+    connection.select_value(<<~SQL)&.to_i || 0
+      SELECT toUnixTimestamp(last_success_time)
+      FROM system.view_refreshes
+      WHERE database = #{connection.quote(database)}
+        AND view = 'heartbeat_user_daily_summary_mv'
+      LIMIT 1
+    SQL
+  rescue StandardError
+    0
+  end
 end

From 08e385569d777c426c62ec1c90bf1cf0f371d57c Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 22:32:12 +0000
Subject: [PATCH 19/24] Codex-slop

---
 app/controllers/api/v1/stats_controller.rb    |  36 ++++--
 app/controllers/concerns/dashboard_data.rb    |  98 +++++++++------
 .../my/project_repo_mappings_controller.rb    |   2 +-
 app/controllers/profiles_controller.rb        |   2 +-
 app/controllers/static_pages_controller.rb    |   8 +-
 app/jobs/cache/active_projects_job.rb         |  33 ++---
 app/jobs/cache/currently_hacking_job.rb       |  33 ++---
 app/models/concerns/heartbeatable.rb          |  67 ++++++----
 app/models/heartbeat.rb                       |  16 +++
 app/services/heartbeat_cache_invalidator.rb   |  32 +++++
 app/services/heartbeat_import_service.rb      |  40 +++++-
 app/services/profile_stats_service.rb         |   3 +-
 config/environments/test.rb                   |   2 +-
 config/initializers/clickhouse.rb             |   9 ++
 db/clickhouse_schema.rb                       |   9 +-
 ...eat_user_daily_summary_refreshable_view.rb |   2 +-
 lib/test_wakatime_service.rb                  |  30 ++++-
 lib/wakatime_service.rb                       |  30 ++++-
 test/lib/wakatime_service_test.rb             | 114 +++++++++++++++++-
 test/models/heartbeat_test.rb                 |  20 +++
 .../services/heartbeat_import_service_test.rb |  24 ++++
 test/test_helper.rb                           |   5 -
 22 files changed, 496 insertions(+), 119 deletions(-)
 create mode 100644 app/services/heartbeat_cache_invalidator.rb
 create mode 100644 config/initializers/clickhouse.rb

diff --git a/app/controllers/api/v1/stats_controller.rb b/app/controllers/api/v1/stats_controller.rb
index 4989eeaa1..15122003d 100644
--- a/app/controllers/api/v1/stats_controller.rb
+++ b/app/controllers/api/v1/stats_controller.rb
@@ -11,10 +11,12 @@ def show
     return if performed?
 
     query = Heartbeat.where(time: start_date..end_date)
+    cache_user_id = nil
     if params[:username].present?
       user = lookup_user(params[:username])
       return render json: { error: "User not found" }, status: :not_found unless user
 
+      cache_user_id = user.id
       query = query.where(user_id: user.id)
     end
 
@@ -23,10 +25,11 @@ def show
 
       return render json: { error: "User not found" }, status: :not_found unless user_id.present?
 
+      cache_user_id = user_id
       query = query.where(user_id: user_id)
     end
 
-    render plain: query.duration_seconds.to_s
+    render plain: Rails.cache.fetch(stats_total_cache_key(query, user_id: cache_user_id), expires_in: 5.minutes) { query.duration_seconds.to_s }
   end
 
   def user_stats
@@ -59,7 +62,7 @@ def user_stats
     service_params = {}
     service_params[:user] = @user
     service_params[:specific_filters] = enabled_features
-    service_params[:allow_cache] = false
+    service_params[:allow_cache] = true
     service_params[:limit] = limit
     service_params[:start_date] = start_date
     service_params[:end_date] = end_date
@@ -92,13 +95,15 @@ def user_stats
 
         # do the boundary thingie if requested
         use_boundary_aware = params[:boundary_aware] == "true"
-        total_seconds = if use_boundary_aware
-          Heartbeat.duration_seconds_boundary_aware(query, start_date.to_f, end_date.to_f) || 0
-        else
-          query.duration_seconds || 0
-        end
-
-        return render json: { total_seconds: total_seconds }
+        return render json: {
+          total_seconds: Rails.cache.fetch(stats_total_cache_key(query, start_date:, end_date:, user_id: @user.id, boundary_aware: use_boundary_aware), expires_in: 5.minutes) do
+            if use_boundary_aware
+              Heartbeat.duration_seconds_boundary_aware(query, start_date.to_f, end_date.to_f) || 0
+            else
+              query.duration_seconds || 0
+            end
+          end
+        }
       end
 
       summary = WakatimeService.new(**service_params).generate_summary
@@ -314,6 +319,19 @@ def unique_heartbeat_seconds(heartbeats)
     total_seconds.to_i
   end
 
+  def stats_total_cache_key(query, start_date: nil, end_date: nil, user_id: nil, boundary_aware: false)
+    cache_version = user_id.present? ? HeartbeatCacheInvalidator.version_for(user_id) : query.maximum(:time).to_i
+
+    [
+      "stats-total",
+      ActiveSupport::Digest.hexdigest(query.to_sql),
+      start_date&.to_i || "default-start",
+      end_date&.to_i || "default-end",
+      boundary_aware,
+      cache_version
+    ].join(":")
+  end
+
   def parse_date_param(param_name, default:, boundary:)
     raw_value = params[param_name]
     return default if raw_value.blank?
diff --git a/app/controllers/concerns/dashboard_data.rb b/app/controllers/concerns/dashboard_data.rb
index f76aa6496..bb39408f3 100644
--- a/app/controllers/concerns/dashboard_data.rb
+++ b/app/controllers/concerns/dashboard_data.rb
@@ -6,17 +6,26 @@ module DashboardData
   def filterable_dashboard_data
     filters = %i[project language operating_system editor category]
     interval = params[:interval]
-    key = [ current_user ] + filters.map { |f| params[f] } + [ interval.to_s, params[:from], params[:to] ]
+    key = [
+      "dashboard-filterable-data",
+      current_user.id,
+      dashboard_cache_version,
+      filters.index_with { |f| params[f].to_s },
+      interval.to_s,
+      params[:from].to_s,
+      params[:to].to_s
+    ]
     hb = current_user.heartbeats
     h = ApplicationController.helpers
 
     Rails.cache.fetch(key, expires_in: 5.minutes) do
       archived = current_user.project_repo_mappings.archived.pluck(:project_name)
+      raw_filter_options = cached_dashboard_filter_options
       result = {}
 
       Time.use_zone(current_user.timezone) do
         filters.each do |f|
-          options = current_user.heartbeats.distinct.pluck(f).compact_blank
+          options = raw_filter_options.fetch(f, [])
           options = options.reject { |n| archived.include?(n) } if f == :project
           result[f] = options.map { |k|
             if f == :language then k.categorize_language
@@ -31,7 +40,7 @@ def filterable_dashboard_data
           hb = if %i[operating_system editor].include?(f)
             hb.where(f => arr.flat_map { |v| [ v.downcase, v.capitalize ] }.uniq)
           elsif f == :language
-            raw = current_user.heartbeats.distinct.pluck(f).compact_blank.select { |l| arr.include?(l.categorize_language) }
+            raw = raw_filter_options.fetch(:language, []).select { |l| arr.include?(l.categorize_language) }
             raw.any? ? hb.where(f => raw) : hb
           else
             hb.where(f => arr)
@@ -126,7 +135,7 @@ def filterable_dashboard_data
 
   def activity_graph_data
     tz = current_user.timezone
-    key = "user_#{current_user.id}_daily_durations_#{tz}"
+    key = [ "user-daily-durations", current_user.id, heartbeat_cache_version, tz ]
     durations = Rails.cache.fetch(key, expires_in: 1.minute) do
       Time.use_zone(tz) { current_user.heartbeats.daily_durations(user_timezone: tz).to_h }
     end
@@ -142,36 +151,57 @@ def activity_graph_data
   end
 
   def today_stats_data
-    h = ApplicationController.helpers
-    Time.use_zone(current_user.timezone) do
-      rows = current_user.heartbeats.today
-        .select(:language, :editor,
-                "COUNT(*) OVER (PARTITION BY language) as language_count",
-                "COUNT(*) OVER (PARTITION BY editor) as editor_count")
-        .distinct.to_a
-
-      lang_counts = rows
-        .map { |r| [ r.language&.categorize_language, r.language_count ] }
-        .reject { |l, _| l.blank? }
-        .group_by(&:first).transform_values { |p| p.sum(&:last) }
-        .sort_by { |_, c| -c }
-
-      ed_counts = rows
-        .map { |r| [ r.editor, r.editor_count ] }
-        .reject { |e, _| e.blank? }.uniq
-        .sort_by { |_, c| -c }
-
-      todays_languages = lang_counts.map { |l, _| h.display_language_name(l) }
-      todays_editors = ed_counts.map { |e, _| h.display_editor_name(e) }
-      todays_duration = current_user.heartbeats.today.duration_seconds
-      show_logged_time_sentence = todays_duration > 1.minute && (todays_languages.any? || todays_editors.any?)
-
-      {
-        show_logged_time_sentence: show_logged_time_sentence,
-        todays_duration_display: h.short_time_detailed(todays_duration.to_i),
-        todays_languages: todays_languages,
-        todays_editors: todays_editors
-      }
+    Rails.cache.fetch([ "dashboard-today-stats", current_user.id, dashboard_cache_version, current_user.timezone ], expires_in: 5.minutes) do
+      h = ApplicationController.helpers
+      Time.use_zone(current_user.timezone) do
+        rows = current_user.heartbeats.today
+          .select(:language, :editor,
+                  "COUNT(*) OVER (PARTITION BY language) as language_count",
+                  "COUNT(*) OVER (PARTITION BY editor) as editor_count")
+          .distinct.to_a
+
+        lang_counts = rows
+          .map { |r| [ r.language&.categorize_language, r.language_count ] }
+          .reject { |l, _| l.blank? }
+          .group_by(&:first).transform_values { |p| p.sum(&:last) }
+          .sort_by { |_, c| -c }
+
+        ed_counts = rows
+          .map { |r| [ r.editor, r.editor_count ] }
+          .reject { |e, _| e.blank? }.uniq
+          .sort_by { |_, c| -c }
+
+        todays_languages = lang_counts.map { |l, _| h.display_language_name(l) }
+        todays_editors = ed_counts.map { |e, _| h.display_editor_name(e) }
+        todays_duration = current_user.heartbeats.today.duration_seconds
+        show_logged_time_sentence = todays_duration > 1.minute && (todays_languages.any? || todays_editors.any?)
+
+        {
+          show_logged_time_sentence: show_logged_time_sentence,
+          todays_duration_display: h.short_time_detailed(todays_duration.to_i),
+          todays_languages: todays_languages,
+          todays_editors: todays_editors
+        }
+      end
+    end
+  end
+
+  def cached_dashboard_filter_options
+    Rails.cache.fetch([ "dashboard-filter-options", current_user.id, heartbeat_cache_version ], expires_in: 15.minutes) do
+      %i[project language operating_system editor category].index_with do |filter|
+        current_user.heartbeats.distinct.pluck(filter).compact_blank
+      end
+    end
+  end
+
+  def dashboard_cache_version
+    @dashboard_cache_version ||= begin
+      latest_mapping = current_user.project_repo_mappings.maximum(:updated_at)&.to_i || 0
+      [ heartbeat_cache_version, latest_mapping ]
     end
   end
+
+  def heartbeat_cache_version
+    @heartbeat_cache_version ||= HeartbeatCacheInvalidator.version_for(current_user)
+  end
 end
diff --git a/app/controllers/my/project_repo_mappings_controller.rb b/app/controllers/my/project_repo_mappings_controller.rb
index ac2d09f4d..d24682f1c 100644
--- a/app/controllers/my/project_repo_mappings_controller.rb
+++ b/app/controllers/my/project_repo_mappings_controller.rb
@@ -92,7 +92,7 @@ def selected_interval
   end
 
   def project_durations_cache_key
-    key = "user_#{current_user.id}_project_durations_#{selected_interval}_v3"
+    key = "user_#{current_user.id}_project_durations_#{HeartbeatCacheInvalidator.version_for(current_user)}_#{selected_interval}_v4"
     if selected_interval == "custom"
       sanitized_from = sanitized_cache_date(params[:from]) || "none"
       sanitized_to = sanitized_cache_date(params[:to]) || "none"
diff --git a/app/controllers/profiles_controller.rb b/app/controllers/profiles_controller.rb
index 6e1ff07fa..9871c41af 100644
--- a/app/controllers/profiles_controller.rb
+++ b/app/controllers/profiles_controller.rb
@@ -113,7 +113,7 @@ def profile_stats_payload
     timezone = @user.timezone
     stats = ProfileStatsService.new(@user).stats
 
-    durations = Rails.cache.fetch("user_#{@user.id}_daily_durations_#{timezone}", expires_in: 1.minute) do
+    durations = Rails.cache.fetch([ "user-daily-durations", @user.id, HeartbeatCacheInvalidator.version_for(@user), timezone ], expires_in: 1.minute) do
       Time.use_zone(timezone) { @user.heartbeats.daily_durations(user_timezone: timezone).to_h }
     end
 
diff --git a/app/controllers/static_pages_controller.rb b/app/controllers/static_pages_controller.rb
index 925a16625..585e321de 100644
--- a/app/controllers/static_pages_controller.rb
+++ b/app/controllers/static_pages_controller.rb
@@ -179,7 +179,13 @@ def programming_goals_progress_data
     goals_hash = ActiveSupport::Digest.hexdigest(
       goals.pluck(:id, :period, :target_seconds, :languages, :projects).to_json
     )
-    cache_key = "user_#{current_user.id}_programming_goals_progress_#{current_user.timezone}_#{goals_hash}"
+    cache_key = [
+      "programming-goals-progress",
+      current_user.id,
+      HeartbeatCacheInvalidator.version_for(current_user),
+      current_user.timezone,
+      goals_hash
+    ]
 
     Rails.cache.fetch(cache_key, expires_in: 1.minute) do
       ProgrammingGoalsProgressService.new(user: current_user, goals: goals).call
diff --git a/app/jobs/cache/active_projects_job.rb b/app/jobs/cache/active_projects_job.rb
index f63e280b9..016e6c372 100644
--- a/app/jobs/cache/active_projects_job.rb
+++ b/app/jobs/cache/active_projects_job.rb
@@ -8,30 +8,35 @@ def cache_expiration
   end
 
   def calculate
-    # Query recent heartbeats from ClickHouse
-    recent_hbs = Heartbeat.where(source_type: Heartbeat.source_types[:direct_entry])
-                          .where("time > ?", 5.minutes.ago.to_f)
-                          .order(time: :desc)
-                          .to_a
-
-    # Deduplicate by user_id (most recent heartbeat per user)
-    latest_by_user = recent_hbs.group_by(&:user_id).transform_values(&:first)
+    latest_by_user = latest_projects_by_user_id
 
     return {} if latest_by_user.empty?
 
-    # Find matching project_repo_mappings from Postgres
-    user_ids = latest_by_user.keys
-
     mappings = ProjectRepoMapping.active
-                                 .where(user_id: user_ids)
+                                 .where(user_id: latest_by_user.keys)
                                  .to_a
+                                 .group_by(&:user_id)
 
     result = {}
-    latest_by_user.each do |user_id, hb|
-      mapping = mappings.find { |m| m.user_id == user_id && m.project_name == hb.project }
+    latest_by_user.each do |user_id, project_name|
+      mapping = (mappings[user_id] || []).find { |candidate| candidate.project_name == project_name }
       result[user_id] = mapping if mapping
     end
 
     result
   end
+
+  def latest_projects_by_user_id
+    rows = Heartbeat.connection.select_all(Heartbeat.sanitize_sql([ <<~SQL, Heartbeat.source_types[:direct_entry], 5.minutes.ago.to_f ]))
+      SELECT user_id, argMax(project, time) AS project
+      FROM heartbeats
+      WHERE source_type = ?
+        AND time > ?
+      GROUP BY user_id
+    SQL
+
+    rows.each_with_object({}) do |row, hash|
+      hash[row["user_id"].to_i] = row["project"]
+    end
+  end
 end
diff --git a/app/jobs/cache/currently_hacking_job.rb b/app/jobs/cache/currently_hacking_job.rb
index aa90fceab..5308be5f5 100644
--- a/app/jobs/cache/currently_hacking_job.rb
+++ b/app/jobs/cache/currently_hacking_job.rb
@@ -8,18 +8,8 @@ def cache_expiration
   end
 
   def calculate
-    # Query ClickHouse for recent heartbeats (no cross-DB join)
-    raw_heartbeats = Heartbeat.where(source_type: :direct_entry)
-                              .coding_only
-                              .where("time > ?", 5.minutes.ago.to_f)
-                              .order(time: :desc)
-                              .to_a
-
-    # Deduplicate by user_id (keep most recent)
-    recent_heartbeats = raw_heartbeats.group_by(&:user_id)
-                                      .transform_values(&:first)
-
-    # Load users from Postgres
+    recent_heartbeats = latest_projects_by_user_id
+
     user_ids = recent_heartbeats.keys
     users_by_id = User.where(id: user_ids)
                       .includes(:project_repo_mappings, :email_addresses)
@@ -29,8 +19,8 @@ def calculate
 
     active_projects = {}
     users.each do |user|
-      recent_heartbeat = recent_heartbeats[user.id]
-      mapping = user.project_repo_mappings.find { |p| p.project_name == recent_heartbeat&.project }
+      project_name = recent_heartbeats[user.id]
+      mapping = user.project_repo_mappings.find { |p| p.project_name == project_name }
       active_projects[user.id] = mapping&.archived? ? nil : mapping
     end
 
@@ -43,4 +33,19 @@ def calculate
 
     { users: users, active_projects: active_projects }
   end
+
+  def latest_projects_by_user_id
+    rows = Heartbeat.connection.select_all(Heartbeat.sanitize_sql([ <<~SQL, Heartbeat.source_types[:direct_entry], 5.minutes.ago.to_f ]))
+      SELECT user_id, argMax(project, time) AS project
+      FROM heartbeats
+      WHERE source_type = ?
+        AND category = 'coding'
+        AND time > ?
+      GROUP BY user_id
+    SQL
+
+    rows.each_with_object({}) do |row, hash|
+      hash[row["user_id"].to_i] = row["project"]
+    end
+  end
 end
diff --git a/app/models/concerns/heartbeatable.rb b/app/models/concerns/heartbeatable.rb
index 60b0dce4d..e4631b460 100644
--- a/app/models/concerns/heartbeatable.rb
+++ b/app/models/concerns/heartbeatable.rb
@@ -107,39 +107,53 @@ def daily_streaks_for_users(user_ids, start_date: 31.days.ago)
 
       timeout = heartbeat_timeout_duration.to_i
 
-      # Query raw heartbeat diffs from ClickHouse, then group by day per user's timezone in Ruby
+      # Fetch ordered heartbeats once, then bucket by each user's local day in Ruby so
+      # diffs do not bleed across midnight in that user's timezone.
       raw_sql = <<~SQL
         SELECT
           user_id,
-          time,
-          least(
-            time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
-            #{timeout}
-          ) AS diff
+          time
         FROM heartbeats
         WHERE user_id IN (#{uncached_users.join(',')})
           AND category = 'coding'
           AND time >= 0 AND time <= 253402300799
           AND time >= #{start_date.to_f}
           AND time <= #{Time.current.to_f}
+        ORDER BY user_id ASC, time ASC
       SQL
 
       rows = connection.select_all(raw_sql)
 
-      daily_durations = rows.group_by { |row| row["user_id"].to_i }
-        .transform_values do |user_rows|
-          user_id = user_rows.first["user_id"].to_i
-          tz = user_timezones[user_id] || "UTC"
-          begin
-            TZInfo::Timezone.get(tz)
-          rescue TZInfo::InvalidTimezoneIdentifier, ArgumentError
-            tz = "UTC"
+      daily_durations = rows.group_by { |row| row["user_id"].to_i }.transform_values do |user_rows|
+        user_id = user_rows.first["user_id"].to_i
+        timezone = user_timezones[user_id] || "UTC"
+
+        begin
+          TZInfo::Timezone.get(timezone)
+        rescue TZInfo::InvalidTimezoneIdentifier, ArgumentError
+          timezone = "UTC"
+        end
+
+        durations_by_day = Hash.new(0)
+        previous_time = nil
+        previous_day = nil
+
+        user_rows.each do |row|
+          current_time = row["time"].to_f
+          current_day = Time.at(current_time).in_time_zone(timezone).to_date
+
+          if previous_time && previous_day == current_day
+            gap = current_time - previous_time
+            durations_by_day[current_day] += [ [ gap, timeout ].min, 0 ].max.to_i
           end
-          user_rows.group_by { |row| Time.at(row["time"].to_f).in_time_zone(tz).to_date }
-            .map { |date, day_rows| [ date, day_rows.sum { |r| r["diff"].to_i } ] }
-            .sort_by { |date, _| date }.reverse
+
+          previous_time = current_time
+          previous_day = current_day
         end
 
+        durations_by_day.sort_by { |date, _| date }.reverse
+      end
+
       result = user_ids.index_with { |id| streak_cache["user_streak_#{id}"] || 0 }
 
       daily_durations.each do |user_id, days|
@@ -187,13 +201,20 @@ def daily_durations(user_timezone:, start_date: 365.days.ago, end_date: Time.cur
 
       sql = <<~SQL
         SELECT
-          toDate(toDateTime(toUInt32(time), '#{timezone}')) AS day_group,
+          day_group,
           toInt64(coalesce(sum(diff), 0)) AS duration
         FROM (
           SELECT
-            time,
+            toDate(toDateTime(toUInt32(time), '#{timezone}')) AS day_group,
             least(
-              time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+              greatest(
+                time - lagInFrame(time, 1, time) OVER (
+                  PARTITION BY toDate(toDateTime(toUInt32(time), '#{timezone}'))
+                  ORDER BY time ASC
+                  ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+                ),
+                0
+              ),
               #{heartbeat_timeout_duration.to_i}
             ) AS diff
           FROM (#{with_valid_timestamps.where(time: start_date.to_f..end_date.to_f).to_sql}) AS hb
@@ -218,7 +239,7 @@ def duration_seconds(scope = all)
         group_expr = group_column.to_s.include?("(") ? group_column : "`#{group_column}`"
 
         capped_diffs_sql = scope
-          .select("#{group_expr} as grouped_time, least(time - lagInFrame(time, 1, time) OVER (PARTITION BY #{group_expr} ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
+          .select("#{group_expr} as grouped_time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY #{group_expr} ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), #{timeout}) as diff")
           .where.not(time: nil)
           .unscope(:group)
           .to_sql
@@ -230,7 +251,7 @@ def duration_seconds(scope = all)
         end
       else
         capped_diffs_sql = scope
-          .select("least(time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
+          .select("least(greatest(time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), #{timeout}) as diff")
           .where.not(time: nil)
           .to_sql
 
@@ -276,7 +297,7 @@ def duration_seconds_boundary_aware(scope, start_time, end_time)
 
       timeout = heartbeat_timeout_duration.to_i
       capped_diffs_sql = combined_scope
-        .select("time, least(time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
+        .select("time, least(greatest(time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), #{timeout}) as diff")
         .where.not(time: nil)
         .order(time: :asc)
         .to_sql
diff --git a/app/models/heartbeat.rb b/app/models/heartbeat.rb
index 6083927eb..cd05cb150 100644
--- a/app/models/heartbeat.rb
+++ b/app/models/heartbeat.rb
@@ -9,6 +9,8 @@ class Heartbeat < ClickhouseRecord
 
   before_create :set_clickhouse_id!, if: -> { self[:id].blank? }
   before_create :set_fields_hash!, if: -> { fields_hash.blank? }
+  after_create :invalidate_user_heartbeat_caches
+  after_update :invalidate_user_heartbeat_caches
 
   CLICKHOUSE_ID_RANDOM_BITS = 10
   CLICKHOUSE_ID_RANDOM_MAX = 1 << CLICKHOUSE_ID_RANDOM_BITS
@@ -119,4 +121,18 @@ def self.generate_fields_hash(attributes)
   def self.indexed_attributes
     %w[user_id branch category dependencies editor entity language machine operating_system project type user_agent line_additions line_deletions lineno lines cursorpos project_root_count time is_write]
   end
+
+  private
+
+  def invalidate_user_heartbeat_caches
+    impacted_user_ids.each do |impacted_user_id|
+      HeartbeatCacheInvalidator.bump_for(impacted_user_id)
+    end
+  end
+
+  def impacted_user_ids
+    user_ids = [ user_id ]
+    user_ids.concat(previous_changes.fetch("user_id", []))
+    user_ids.compact.uniq
+  end
 end
diff --git a/app/services/heartbeat_cache_invalidator.rb b/app/services/heartbeat_cache_invalidator.rb
new file mode 100644
index 000000000..92fb700a6
--- /dev/null
+++ b/app/services/heartbeat_cache_invalidator.rb
@@ -0,0 +1,32 @@
+class HeartbeatCacheInvalidator
+  CACHE_VERSION_TTL = 30.days
+
+  class << self
+    def version_for(user_or_id)
+      user_id = extract_user_id(user_or_id)
+      return 0 if user_id.blank?
+
+      Rails.cache.fetch(version_key(user_id), expires_in: CACHE_VERSION_TTL) { 0 }.to_i
+    end
+
+    def bump_for(user_or_id)
+      user_id = extract_user_id(user_or_id)
+      return 0 if user_id.blank?
+
+      version = version_for(user_id) + 1
+      Rails.cache.write(version_key(user_id), version, expires_in: CACHE_VERSION_TTL)
+      Rails.cache.delete("user_streak_#{user_id}")
+      version
+    end
+
+    private
+
+    def extract_user_id(user_or_id)
+      user_or_id.respond_to?(:id) ? user_or_id.id : user_or_id
+    end
+
+    def version_key(user_id)
+      "heartbeat-cache-version:user:#{user_id}"
+    end
+  end
+end
diff --git a/app/services/heartbeat_import_service.rb b/app/services/heartbeat_import_service.rb
index 00eb3a721..bfcf28738 100644
--- a/app/services/heartbeat_import_service.rb
+++ b/app/services/heartbeat_import_service.rb
@@ -101,6 +101,7 @@ def self.flush_batch(seen_hashes)
     return 0 if seen_hashes.empty?
 
     records = seen_hashes.values
+    user_id = records.first[:user_id]
     records.each do |r|
       timestamp = Time.current
       r[:created_at] = timestamp
@@ -108,13 +109,46 @@ def self.flush_batch(seen_hashes)
     end
 
     ActiveRecord::Base.logger.silence do
-      existing_hashes = Heartbeat.where(fields_hash: records.map { |r| r[:fields_hash] }).pluck(:fields_hash).to_set
-      new_records = records.reject { |r| existing_hashes.include?(r[:fields_hash]) }
-      Heartbeat.insert_all(new_records) if new_records.any?
+      new_records = []
+
+      User.transaction do
+        User.lock.find(user_id)
+
+        existing_hashes = Heartbeat.where(user_id: user_id, fields_hash: records.map { |r| r[:fields_hash] }).pluck(:fields_hash).to_set
+        new_records = records.reject { |r| existing_hashes.include?(r[:fields_hash]) }
+
+        if new_records.any?
+          Heartbeat.connection.with_settings(async_insert: 0, wait_for_async_insert: 1) do
+            Heartbeat.insert_all(new_records)
+          end
+          Heartbeat.connection.clear_query_cache
+          wait_until_visible!(
+            user_id,
+            new_records.map { |record| record[:fields_hash] },
+            expected_min_time: new_records.map { |record| record[:time].to_f }.min
+          )
+        end
+      end
+
+      HeartbeatCacheInvalidator.bump_for(user_id) if new_records.any?
       new_records.length
     end
   end
 
+  def self.wait_until_visible!(user_id, fields_hashes, expected_min_time:)
+    deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 2
+
+    loop do
+      visible_count = Heartbeat.where(user_id: user_id, fields_hash: fields_hashes).count
+      visible_min_time = Heartbeat.where(user_id: user_id).minimum(:time).to_f
+      return if visible_count >= fields_hashes.length && visible_min_time <= expected_min_time
+
+      break if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+      sleep 0.01
+    end
+  end
+
   class HeartbeatSaxHandler < Oj::Saj
     def initialize(&block)
       @block = block
diff --git a/app/services/profile_stats_service.rb b/app/services/profile_stats_service.rb
index f98cb5fd1..9bf390b65 100644
--- a/app/services/profile_stats_service.rb
+++ b/app/services/profile_stats_service.rb
@@ -16,8 +16,7 @@ def stats
   private
 
   def cache_key
-    latest_heartbeat_time = user.heartbeats.maximum(:time) || 0
-    "profile_stats:v2:user:#{user.id}:latest:#{latest_heartbeat_time}"
+    "profile_stats:v3:user:#{user.id}:version:#{HeartbeatCacheInvalidator.version_for(user)}"
   end
 
   def compute_stats
diff --git a/config/environments/test.rb b/config/environments/test.rb
index 9a0813dbc..edc1aa81c 100644
--- a/config/environments/test.rb
+++ b/config/environments/test.rb
@@ -24,7 +24,7 @@
 
   # Show full error reports.
   config.consider_all_requests_local = true
-  config.cache_store = :null_store
+  config.cache_store = :memory_store
 
   # Render exception templates for rescuable exceptions and raise for other exceptions.
   config.action_dispatch.show_exceptions = :rescuable
diff --git a/config/initializers/clickhouse.rb b/config/initializers/clickhouse.rb
new file mode 100644
index 000000000..1a211029c
--- /dev/null
+++ b/config/initializers/clickhouse.rb
@@ -0,0 +1,9 @@
+module ClickhouseActiverecord
+  class Settings
+    # The app reads freshly written heartbeats immediately for stats, caches, and tests.
+    # Use synchronous inserts so subsequent queries see the rows deterministically.
+    def insert_settings
+      { wait_for_async_insert: 1, async_insert: 0 }
+    end
+  end
+end
diff --git a/db/clickhouse_schema.rb b/db/clickhouse_schema.rb
index 2d2fbecb0..05e0e9874 100644
--- a/db/clickhouse_schema.rb
+++ b/db/clickhouse_schema.rb
@@ -16,7 +16,7 @@
   create_table "heartbeat_user_daily_summary", id: false, options: "ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192", force: :cascade do |t|
     t.integer "user_id", unsigned: false, limit: 8, null: false
     t.date "day", null: false
-    t.float "duration_s", null: false
+    t.column "duration_s", "Float64", null: false
     t.integer "heartbeats", null: false
     t.datetime "_version", precision: nil, default: -> { "now()" }, null: false
   end
@@ -43,7 +43,7 @@
     t.integer "lines", unsigned: false, default: 0, null: false
     t.integer "cursorpos", unsigned: false, default: 0, null: false
     t.integer "project_root_count", unsigned: false, default: 0, null: false
-    t.float "time", null: false
+    t.column "time", "Float64", null: false
     t.integer "is_write", limit: 1, default: 0, null: false
     t.datetime "created_at", default: -> { "now64()" }, null: false
     t.datetime "updated_at", default: -> { "now64()" }, null: false
@@ -54,8 +54,7 @@
   end
 
   # TABLE: heartbeat_user_daily_summary_mv
-  # SQL: CREATE MATERIALIZED VIEW heartbeat_user_daily_summary_mv REFRESH EVERY 10 MINUTE TO heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32 ) AS SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats FINAL WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day
-  create_table "heartbeat_user_daily_summary_mv", view: true, materialized: true, id: false, as: "SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats FINAL WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day", force: :cascade do |t|
+  # SQL: CREATE MATERIALIZED VIEW heartbeat_user_daily_summary_mv REFRESH EVERY 10 MINUTE TO heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32 ) AS SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day
+  create_table "heartbeat_user_daily_summary_mv", view: true, materialized: true, id: false, as: "SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day", force: :cascade do |t|
   end
-
 end
diff --git a/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb b/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb
index 01d9041e4..dbb4c3e7f 100644
--- a/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb
+++ b/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb
@@ -28,7 +28,7 @@ def up
                   ),
                   120
               ) AS diff
-          FROM heartbeats FINAL
+          FROM heartbeats
           WHERE time IS NOT NULL
             AND time >= 0
             AND time <= 253402300799
diff --git a/lib/test_wakatime_service.rb b/lib/test_wakatime_service.rb
index 4a1ed471a..dc4d7f13f 100644
--- a/lib/test_wakatime_service.rb
+++ b/lib/test_wakatime_service.rb
@@ -12,6 +12,7 @@ def initialize(user: nil, specific_filters: [], allow_cache: true, limit: 10, st
     @scope = @scope.where.not("LOWER(category) IN (?)", [ "browsing", "ai coding", "meeting", "communicating" ])
     @user = user
     @boundary_aware = boundary_aware
+    @scope = @scope.where(user_id: @user.id) if @user.present?
 
     @start_date = convert_to_unix_timestamp(start_date)
     @end_date = convert_to_unix_timestamp(end_date)
@@ -25,13 +26,18 @@ def initialize(user: nil, specific_filters: [], allow_cache: true, limit: 10, st
     @limit = limit
     @limit = nil if @limit&.zero?
 
-    @scope = @scope.where(user_id: @user.id) if @user.present?
-
     @specific_filters = specific_filters
     @allow_cache = allow_cache
   end
 
   def generate_summary
+    return build_summary if Rails.env.test?
+    return Rails.cache.fetch(summary_cache_key, expires_in: 5.minutes) { build_summary } if @allow_cache
+
+    build_summary
+  end
+
+  def build_summary
     summary = {}
 
     summary[:username] = @user.display_name if @user.present?
@@ -147,6 +153,26 @@ def categorize_editor(editor)
 
   private
 
+  def summary_cache_key
+    [
+      self.class.name.underscore,
+      @user&.id || "anonymous",
+      @start_date,
+      @end_date,
+      @limit || "all",
+      @specific_filters.sort.join(","),
+      @boundary_aware,
+      scope_cache_version,
+      ActiveSupport::Digest.hexdigest(@scope.to_sql)
+    ].join(":")
+  end
+
+  def scope_cache_version
+    return HeartbeatCacheInvalidator.version_for(@user) if @user.present?
+
+    @scope.maximum(:time).to_i
+  end
+
   def convert_to_unix_timestamp(timestamp)
     # our lord and savior stack overflow for this bit of code
     return nil if timestamp.nil?
diff --git a/lib/wakatime_service.rb b/lib/wakatime_service.rb
index 4d862560b..bfbe4a0fe 100644
--- a/lib/wakatime_service.rb
+++ b/lib/wakatime_service.rb
@@ -5,6 +5,8 @@ class WakatimeService
   def initialize(user: nil, specific_filters: [], allow_cache: true, limit: 10, start_date: nil, end_date: nil, scope: nil)
     @scope = scope || Heartbeat.all
     @user = user
+    @scope = @scope.with_valid_timestamps
+    @scope = @scope.where(user_id: @user.id) if @user.present?
 
     @start_date = convert_to_unix_timestamp(start_date)
     @end_date = convert_to_unix_timestamp(end_date)
@@ -18,13 +20,18 @@ def initialize(user: nil, specific_filters: [], allow_cache: true, limit: 10, st
     @limit = limit
     @limit = nil if @limit&.zero?
 
-    @scope = @scope.where(user_id: @user.id) if @user.present?
-
     @specific_filters = specific_filters
     @allow_cache = allow_cache
   end
 
   def generate_summary
+    return build_summary if Rails.env.test?
+    return Rails.cache.fetch(summary_cache_key, expires_in: 5.minutes) { build_summary } if @allow_cache
+
+    build_summary
+  end
+
+  def build_summary
     summary = {}
 
     summary[:username] = @user.display_name if @user.present?
@@ -137,6 +144,25 @@ def self.categorize_language(language)
 
   private
 
+  def summary_cache_key
+    [
+      self.class.name.underscore,
+      @user&.id || "anonymous",
+      @start_date,
+      @end_date,
+      @limit || "all",
+      @specific_filters.sort.join(","),
+      scope_cache_version,
+      ActiveSupport::Digest.hexdigest(@scope.to_sql)
+    ].join(":")
+  end
+
+  def scope_cache_version
+    return HeartbeatCacheInvalidator.version_for(@user) if @user.present?
+
+    @scope.maximum(:time).to_i
+  end
+
   def convert_to_unix_timestamp(timestamp)
     # our lord and savior stack overflow for this bit of code
     return nil if timestamp.nil?
diff --git a/test/lib/wakatime_service_test.rb b/test/lib/wakatime_service_test.rb
index cb5d07eea..38cbbfbf0 100644
--- a/test/lib/wakatime_service_test.rb
+++ b/test/lib/wakatime_service_test.rb
@@ -1,10 +1,11 @@
 require "test_helper"
 
-class WakatimeServiceTest < Minitest::Test
+class WakatimeServiceTest < ActiveSupport::TestCase
   # Since parse_user_agent is a pure function that doesn't need database access,
   # we can test it without loading any fixtures
   def setup
     ActiveRecord::FixtureSet.reset_cache
+    Rails.cache.clear
   end
 
   def test_parse_user_agent_with_vscode_wakatime_client
@@ -78,4 +79,115 @@ def test_parse_user_agent_with_invalid_user_agent
     assert_equal "", result[:editor]
     assert_equal "failed to parse user agent string", result[:err]
   end
+
+  test "defaults summary range to the scoped user's heartbeats" do
+    stale_user = User.create!(username: "stale_#{SecureRandom.hex(3)}", timezone: "UTC")
+    recent_user = User.create!(username: "recent_#{SecureRandom.hex(3)}", timezone: "UTC")
+
+    Heartbeat.create!(
+      user: stale_user,
+      source_type: :direct_entry,
+      time: Time.utc(2024, 1, 1, 10, 0, 0).to_f,
+      project: "legacy",
+      category: "coding"
+    )
+    Heartbeat.create!(
+      user: recent_user,
+      source_type: :direct_entry,
+      time: Time.utc(2026, 1, 1, 10, 0, 0).to_f,
+      project: "new-hotness",
+      category: "coding"
+    )
+
+    summary = WakatimeService.new(user: stale_user, specific_filters: [], allow_cache: false).generate_summary
+
+    assert_equal "2024-01-01T10:00:00Z", summary[:start]
+    assert_equal "2024-01-01T10:00:00Z", summary[:end]
+  end
+
+  test "test wakatime service defaults summary range to the scoped user's heartbeats" do
+    stale_user = User.create!(username: "test_stale_#{SecureRandom.hex(3)}", timezone: "UTC")
+    recent_user = User.create!(username: "test_recent_#{SecureRandom.hex(3)}", timezone: "UTC")
+
+    Heartbeat.create!(
+      user: stale_user,
+      source_type: :direct_entry,
+      time: Time.utc(2024, 2, 2, 8, 0, 0).to_f,
+      project: "legacy-test",
+      category: "coding"
+    )
+    Heartbeat.create!(
+      user: recent_user,
+      source_type: :direct_entry,
+      time: Time.utc(2026, 2, 2, 8, 0, 0).to_f,
+      project: "new-test",
+      category: "coding"
+    )
+
+    summary = TestWakatimeService.new(user: stale_user, specific_filters: [], allow_cache: false).generate_summary
+
+    assert_equal "2024-02-02T08:00:00Z", summary[:start]
+    assert_equal "2024-02-02T08:00:00Z", summary[:end]
+  end
+
+  test "cached summary invalidates when an older heartbeat is inserted later" do
+    user = User.create!(username: "cache_insert_#{SecureRandom.hex(3)}", timezone: "UTC")
+
+    Heartbeat.create!(
+      user: user,
+      source_type: :direct_entry,
+      time: Time.utc(2026, 3, 1, 12, 0, 0).to_f,
+      project: "current",
+      category: "coding"
+    )
+
+    first_summary = WakatimeService.new(user: user, specific_filters: [], allow_cache: true).generate_summary
+    assert_equal "2026-03-01T12:00:00Z", first_summary[:start]
+
+    Heartbeat.create!(
+      user: user,
+      source_type: :direct_entry,
+      time: Time.utc(2025, 3, 1, 12, 0, 0).to_f,
+      project: "backfill",
+      category: "coding"
+    )
+
+    second_summary = WakatimeService.new(user: user, specific_filters: [], allow_cache: true).generate_summary
+    assert_equal "2025-03-01T12:00:00Z", second_summary[:start]
+  end
+
+  test "cached summary invalidates after imported backfill heartbeats" do
+    user = User.create!(username: "cache_import_#{SecureRandom.hex(3)}", timezone: "UTC")
+
+    Heartbeat.create!(
+      user: user,
+      source_type: :direct_entry,
+      time: Time.utc(2026, 4, 1, 12, 0, 0).to_f,
+      project: "current",
+      category: "coding"
+    )
+
+    first_summary = WakatimeService.new(user: user, specific_filters: [], allow_cache: true).generate_summary
+    assert_equal "2026-04-01T12:00:00Z", first_summary[:start]
+
+    file_content = {
+      heartbeats: [
+        {
+          entity: "/tmp/older.rb",
+          type: "file",
+          time: Time.utc(2025, 4, 1, 12, 0, 0).to_f,
+          project: "backfill-import",
+          language: "Ruby",
+          category: "coding",
+          is_write: true
+        }
+      ]
+    }.to_json
+
+    result = HeartbeatImportService.import_from_file(file_content, user)
+    assert result[:success]
+
+    second_summary = WakatimeService.new(user: user, specific_filters: [], allow_cache: true).generate_summary
+    assert_equal "2025-04-01T12:00:00Z", second_summary[:start]
+  end
 end
diff --git a/test/models/heartbeat_test.rb b/test/models/heartbeat_test.rb
index 3c949e881..21a0e94d3 100644
--- a/test/models/heartbeat_test.rb
+++ b/test/models/heartbeat_test.rb
@@ -1,6 +1,10 @@
 require "test_helper"
 
 class HeartbeatTest < ActiveSupport::TestCase
+  def setup
+    Rails.cache.clear
+  end
+
   test "lightweight delete removes record from queries" do
     user = User.create!
     heartbeat = user.heartbeats.create!(
@@ -19,4 +23,20 @@ class HeartbeatTest < ActiveSupport::TestCase
     # After lightweight delete, the row is invisible to subsequent queries
     assert_not_includes Heartbeat.where(user_id: user.id).to_a, heartbeat
   end
+
+  test "create bumps the user's heartbeat cache version" do
+    user = User.create!(timezone: "UTC")
+    initial_version = HeartbeatCacheInvalidator.version_for(user)
+
+    user.heartbeats.create!(
+      entity: "src/cache_bump.rb",
+      type: "file",
+      category: "coding",
+      time: Time.current.to_f,
+      project: "heartbeat-cache",
+      source_type: :test_entry
+    )
+
+    assert_equal initial_version + 1, HeartbeatCacheInvalidator.version_for(user)
+  end
 end
diff --git a/test/services/heartbeat_import_service_test.rb b/test/services/heartbeat_import_service_test.rb
index 40ea080b2..dee28fe2c 100644
--- a/test/services/heartbeat_import_service_test.rb
+++ b/test/services/heartbeat_import_service_test.rb
@@ -1,6 +1,10 @@
 require "test_helper"
 
 class HeartbeatImportServiceTest < ActiveSupport::TestCase
+  def setup
+    Rails.cache.clear
+  end
+
   test "deduplicates imported heartbeats by fields hash" do
     user = User.create!(timezone: "UTC")
     file_content = {
@@ -69,4 +73,24 @@ class HeartbeatImportServiceTest < ActiveSupport::TestCase
     assert_equal "skyfall-pc", heartbeat.machine
     assert_equal "wakatime/v1.102.1", heartbeat.user_agent
   end
+
+  test "bumps the user's heartbeat cache version when new rows are imported" do
+    user = User.create!(timezone: "UTC")
+    initial_version = HeartbeatCacheInvalidator.version_for(user)
+
+    result = HeartbeatImportService.import_from_file({
+      heartbeats: [
+        {
+          entity: "/tmp/imported.rb",
+          type: "file",
+          time: Time.utc(2026, 1, 1, 10, 0, 0).to_f,
+          project: "cache-test",
+          category: "coding"
+        }
+      ]
+    }.to_json, user)
+
+    assert result[:success]
+    assert_equal initial_version + 1, HeartbeatCacheInvalidator.version_for(user)
+  end
 end
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 0344aea4a..41a667dec 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -39,11 +39,6 @@ def before_setup
       super
     end
 
-    def after_teardown
-      truncate_clickhouse_tables
-      super
-    end
-
     def truncate_clickhouse_tables
       Heartbeat.connection.execute("TRUNCATE TABLE #{Heartbeat.connection.quote_table_name(Heartbeat.table_name)}")
       HeartbeatUserDailySummary.connection.execute(

From ae04198026a6a2e888e0f7df98ede69e7683e9d9 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Tue, 24 Mar 2026 23:13:50 +0000
Subject: [PATCH 20/24] Claude-slop

---
 .../api/hackatime/v1/hackatime_controller.rb  | 56 +++++--------
 app/controllers/concerns/dashboard_data.rb    | 11 ++-
 .../one_time/backfill_heartbeat_editor_job.rb |  4 +-
 .../generate_unique_heartbeat_hashes_job.rb   | 53 ------------
 .../recalc_heartbeat_field_hash_job.rb        | 15 ----
 app/models/concerns/heartbeatable.rb          | 24 +++---
 app/models/concerns/time_range_filterable.rb  | 35 +++-----
 app/models/heartbeat.rb                       | 17 +---
 app/services/concerns/wakatime_shared.rb      | 49 +++++++++++
 app/services/heartbeat_import_service.rb      | 71 +++++-----------
 app/services/profile_stats_service.rb         |  5 +-
 lib/test_wakatime_service.rb                  | 81 +------------------
 lib/wakatime_service.rb                       | 67 ++-------------
 13 files changed, 132 insertions(+), 356 deletions(-)
 delete mode 100644 app/jobs/one_time/generate_unique_heartbeat_hashes_job.rb
 delete mode 100644 app/jobs/one_time/recalc_heartbeat_field_hash_job.rb
 create mode 100644 app/services/concerns/wakatime_shared.rb

diff --git a/app/controllers/api/hackatime/v1/hackatime_controller.rb b/app/controllers/api/hackatime/v1/hackatime_controller.rb
index 2ba3128dd..808c6875f 100644
--- a/app/controllers/api/hackatime/v1/hackatime_controller.rb
+++ b/app/controllers/api/hackatime/v1/hackatime_controller.rb
@@ -98,13 +98,11 @@ def stats_last_7_days
         # Calculate total seconds
         total_seconds = heartbeats.duration_seconds.to_i
 
-        # Get unique days
-        days = []
-        heartbeats.pluck(:time).each do |timestamp|
-          day = Time.at(timestamp).in_time_zone(@user.timezone).to_date
-          days << day unless days.include?(day)
-        end
-        days_covered = days.length
+        # Get unique days using ClickHouse aggregation instead of loading all timestamps
+        tz_quoted = Heartbeat.connection.quote(@user.timezone)
+        days_covered = Heartbeat.connection.select_value(
+          "SELECT uniq(toDate(toDateTime(toUInt32(time), #{tz_quoted}))) FROM (#{heartbeats.with_valid_timestamps.to_sql})"
+        ).to_i
 
         # Calculate daily average
         daily_average = days_covered > 0 ? (total_seconds.to_f / days_covered).round(1) : 0
@@ -232,38 +230,29 @@ def body_to_json
   LAST_LANGUAGE_SENTINEL = "<<LAST_LANGUAGE>>"
 
   def handle_heartbeat(heartbeat_array)
-    results = []
-    last_language = nil
     prepared = prepare_heartbeat_attrs(heartbeat_array)
-    all_hashes = prepared.map { |p| p[:fields_hash] }.compact
-    existing_hashes = if all_hashes.any?
-      Heartbeat.where(fields_hash: all_hashes).pluck(:fields_hash).to_set
-    else
-      Set.new
+
+    now = Time.current
+    base_us = (now.to_r * 1_000_000).to_i
+    records = prepared.each_with_index.map do |item, i|
+      attrs = item[:attrs]
+      attrs[:id] = ((base_us + i) << Heartbeat::CLICKHOUSE_ID_RANDOM_BITS) | SecureRandom.random_number(Heartbeat::CLICKHOUSE_ID_RANDOM_MAX)
+      attrs[:created_at] = now
+      attrs[:updated_at] = now
+      attrs
     end
 
-    prepared.each do |item|
-      heartbeat = item[:attrs]
-      fields_hash = item[:fields_hash]
-      source_type = item[:source_type]
-
-      if existing_hashes.include?(fields_hash)
-        existing = Heartbeat.where(fields_hash: fields_hash).first
-        queue_project_mapping(heartbeat[:project])
-        results << [ existing.attributes, 201 ]
-      else
-        new_heartbeat = Heartbeat.create!(heartbeat.merge(fields_hash: fields_hash))
-        existing_hashes.add(fields_hash)
-        queue_project_mapping(heartbeat[:project])
-        results << [ new_heartbeat.attributes, 201 ]
-      end
+    begin
+      Heartbeat.insert_all(records) if records.any?
     rescue => e
-      report_error(e, message: "Error creating heartbeat")
-      results << [ { error: e.message, type: e.class.name }, 422 ]
+      report_error(e, message: "Error bulk inserting heartbeats")
+      return records.map { |r| [ { error: e.message, type: e.class.name }, 422 ] }
     end
 
+    records.each { |r| queue_project_mapping(r[:project]) }
+    HeartbeatCacheInvalidator.bump_for(@user.id) if records.any?
     PosthogService.capture_once_per_day(@user, "heartbeat_sent", { heartbeat_count: heartbeat_array.size })
-    results
+    records.map { |r| [ r, 201 ] }
   end
 
   def prepare_heartbeat_attrs(heartbeat_array)
@@ -316,8 +305,7 @@ def prepare_heartbeat_attrs(heartbeat_array)
       }).slice(*Heartbeat.column_names.map(&:to_sym))
       # ^^ They say safety laws are written in blood. Well, so is this line!
       # Basically this filters out columns that aren't in our DB (the biggest one being raw_data)
-      fields_hash = Heartbeat.generate_fields_hash(attrs)
-      { attrs: attrs, fields_hash: fields_hash, source_type: source_type }
+      { attrs: attrs, source_type: source_type }
     end
   end
 
diff --git a/app/controllers/concerns/dashboard_data.rb b/app/controllers/concerns/dashboard_data.rb
index bb39408f3..373acde8a 100644
--- a/app/controllers/concerns/dashboard_data.rb
+++ b/app/controllers/concerns/dashboard_data.rb
@@ -100,7 +100,7 @@ def filterable_dashboard_data
         timeout = Heartbeat.heartbeat_timeout_duration.to_i
         tz = current_user.timezone
         weekly_sql = weekly_hb
-          .select("toStartOfWeek(toDateTime(toUInt32(time), '#{tz}'), 1) as week_start, `project` as grouped_time, least(time - lagInFrame(time, 1, time) OVER (PARTITION BY `project`, toStartOfWeek(toDateTime(toUInt32(time), '#{tz}'), 1) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), #{timeout}) as diff")
+          .select("toStartOfWeek(toDateTime(toUInt32(time), '#{tz}'), 1) as week_start, `project` as grouped_time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY `project`, toStartOfWeek(toDateTime(toUInt32(time), '#{tz}'), 1) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), #{timeout}) as diff")
           .where.not(time: nil)
           .with_valid_timestamps
           .to_sql
@@ -188,9 +188,12 @@ def today_stats_data
 
   def cached_dashboard_filter_options
     Rails.cache.fetch([ "dashboard-filter-options", current_user.id, heartbeat_cache_version ], expires_in: 15.minutes) do
-      %i[project language operating_system editor category].index_with do |filter|
-        current_user.heartbeats.distinct.pluck(filter).compact_blank
-      end
+      conn = Heartbeat.connection
+      user_id = conn.quote(current_user.id)
+      filters = %i[project language operating_system editor category]
+      sql = filters.map { |f| "groupUniqArray(#{f}) AS #{f}_values" }.join(", ")
+      row = conn.select_one("SELECT #{sql} FROM heartbeats WHERE user_id = #{user_id}")
+      filters.index_with { |f| Array(row["#{f}_values"]).reject(&:blank?) }
     end
   end
 
diff --git a/app/jobs/one_time/backfill_heartbeat_editor_job.rb b/app/jobs/one_time/backfill_heartbeat_editor_job.rb
index 0cb912c96..b1ce70952 100644
--- a/app/jobs/one_time/backfill_heartbeat_editor_job.rb
+++ b/app/jobs/one_time/backfill_heartbeat_editor_job.rb
@@ -18,8 +18,6 @@ def perform(dry_run = true)
       # Store the parsed values for bulk update
       heartbeat.editor = parsed_ua[:editor]
       heartbeat.operating_system = parsed_ua[:os]
-      # Regenerate fields_hash before adding to processed records
-      heartbeat.fields_hash = Heartbeat.generate_fields_hash(heartbeat.attributes)
       processed_heartbeats << heartbeat
 
       # When we have 1000 records, update them and clear the array
@@ -45,7 +43,7 @@ def bulk_update_heartbeats(heartbeats)
     Heartbeat.import heartbeats,
       on_duplicate_key_update: {
         conflict_target: [ :id ],
-        columns: [ :editor, :operating_system, :updated_at, :fields_hash ]
+        columns: [ :editor, :operating_system, :updated_at ]
       }
   end
 end
diff --git a/app/jobs/one_time/generate_unique_heartbeat_hashes_job.rb b/app/jobs/one_time/generate_unique_heartbeat_hashes_job.rb
deleted file mode 100644
index 954153e4c..000000000
--- a/app/jobs/one_time/generate_unique_heartbeat_hashes_job.rb
+++ /dev/null
@@ -1,53 +0,0 @@
-class OneTime::GenerateUniqueHeartbeatHashesJob < ApplicationJob
-  queue_as :default
-
-  include GoodJob::ActiveJobExtensions::Concurrency
-
-  # only allow one instance of this job to run at a time
-  good_job_control_concurrency_with(
-    key: -> { "generate_unique_heartbeat_hashes_job" },
-    total_limit: 1,
-  )
-
-  def perform(scope = Heartbeat.where(fields_hash: nil))
-    scope_count = scope.count
-    puts "Starting to generate unique heartbeat hashes for #{scope_count} heartbeats"
-    index = 0
-    scope.in_batches(of: 1000) do |batch|
-      # Process records in smaller chunks to avoid statement size limits
-      batch.each_slice(250) do |chunk|
-        updates = chunk.map do |heartbeat|
-          index += 1
-          puts "Processing heartbeat #{heartbeat.id} (#{index} of #{batch.size})"
-          field_hash = Heartbeat.generate_fields_hash(heartbeat.attributes)
-          puts "Field hash: #{field_hash}"
-          [ heartbeat.id, field_hash ]
-        end
-
-        # Update creates n queries even when passed an array of records to update, so
-        # we're using a SQL CASE statement to update the records in a single query.
-        # Prior work: https://gist.github.com/zoltan-nz/6390986
-        case_statement = updates.map { |id, hash| "WHEN id = #{id} THEN '#{hash}'" }.join(" ")
-        Heartbeat.where(id: updates.map(&:first))
-                 .update_all("fields_hash = CASE #{case_statement} END")
-      end
-    end
-
-    # Delete all heartbeats without a user_id
-    Heartbeat.where(user_id: nil).delete_all
-
-    distinct_ids = Heartbeat.select("DISTINCT ON (fields_hash) id")
-                           .order("fields_hash, created_at")
-                           .pluck("id")
-    total_heartbeats = Heartbeat.count
-    total_distinct_heartbeats = distinct_ids.count
-
-    puts "Found #{total_distinct_heartbeats} distinct heartbeat(s) out of #{total_heartbeats} total"
-
-    deleted_count = Heartbeat.where.not(
-      id: distinct_ids
-    ).delete_all
-
-    puts "Deleted #{deleted_count} duplicate heartbeat(s)"
-  end
-end
diff --git a/app/jobs/one_time/recalc_heartbeat_field_hash_job.rb b/app/jobs/one_time/recalc_heartbeat_field_hash_job.rb
deleted file mode 100644
index 4218e2ca7..000000000
--- a/app/jobs/one_time/recalc_heartbeat_field_hash_job.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-class OneTime::RecalcHeartbeatFieldHashJob < ApplicationJob
-  queue_as :default
-
-  def perform
-    Heartbeat.find_each(batch_size: 2500) do |heartbeat|
-      begin
-        heartbeat.send(:set_fields_hash!)
-        heartbeat.save!
-      rescue ActiveRecord::RecordNotUnique
-        # If we have a duplicate fields_hash, soft delete this record
-        heartbeat.delete
-      end
-    end
-  end
-end
diff --git a/app/models/concerns/heartbeatable.rb b/app/models/concerns/heartbeatable.rb
index e4631b460..a1439ca9f 100644
--- a/app/models/concerns/heartbeatable.rb
+++ b/app/models/concerns/heartbeatable.rb
@@ -102,8 +102,15 @@ def daily_streaks_for_users(user_ids, start_date: 31.days.ago)
         return user_ids.index_with { |id| streak_cache["user_streak_#{id}"] || 0 }
       end
 
-      # Fetch user timezones from Postgres
-      user_timezones = User.where(id: uncached_users).pluck(:id, :timezone).to_h
+      # Fetch user timezones from Postgres, validate once upfront
+      raw_timezones = User.where(id: uncached_users).pluck(:id, :timezone).to_h
+      user_timezones = raw_timezones.transform_values do |tz|
+        begin
+          TZInfo::Timezone.get(tz) && tz
+        rescue TZInfo::InvalidTimezoneIdentifier, ArgumentError
+          "UTC"
+        end
+      end
 
       timeout = heartbeat_timeout_duration.to_i
 
@@ -128,12 +135,6 @@ def daily_streaks_for_users(user_ids, start_date: 31.days.ago)
         user_id = user_rows.first["user_id"].to_i
         timezone = user_timezones[user_id] || "UTC"
 
-        begin
-          TZInfo::Timezone.get(timezone)
-        rescue TZInfo::InvalidTimezoneIdentifier, ArgumentError
-          timezone = "UTC"
-        end
-
         durations_by_day = Hash.new(0)
         previous_time = nil
         previous_day = nil
@@ -158,13 +159,6 @@ def daily_streaks_for_users(user_ids, start_date: 31.days.ago)
 
       daily_durations.each do |user_id, days|
         timezone = user_timezones[user_id] || "UTC"
-
-        begin
-          TZInfo::Timezone.get(timezone)
-        rescue TZInfo::InvalidTimezoneIdentifier, ArgumentError
-          timezone = "UTC"
-        end
-
         current_date = Time.current.in_time_zone(timezone).to_date
 
         streak = 0
diff --git a/app/models/concerns/time_range_filterable.rb b/app/models/concerns/time_range_filterable.rb
index 4737da027..3fdd523b9 100644
--- a/app/models/concerns/time_range_filterable.rb
+++ b/app/models/concerns/time_range_filterable.rb
@@ -37,55 +37,40 @@ module TimeRangeFilterable
     flavortown: {
       human_name: "Flavortown",
       calculate: -> {
-        timezone = "America/New_York"
-        Time.use_zone(timezone) do
-          from = Time.parse("2025-12-15").beginning_of_day
-          to = Time.parse("2026-03-31").end_of_day
-          from.beginning_of_day..to.end_of_day
+        Time.use_zone("America/New_York") do
+          Time.parse("2025-12-15").beginning_of_day..Time.parse("2026-03-31").end_of_day
         end
       }
     },
     summer_of_making: {
       human_name: "Summer of Making",
       calculate: -> {
-        timezone = "America/New_York"
-        Time.use_zone(timezone) do
-          from = Time.parse("2025-06-16").beginning_of_day
-          to = Time.parse("2025-09-30").end_of_day
-          from.beginning_of_day..to.end_of_day
+        Time.use_zone("America/New_York") do
+          Time.parse("2025-06-16").beginning_of_day..Time.parse("2025-09-30").end_of_day
         end
       }
     },
     high_seas: {
       human_name: "High Seas",
       calculate: -> {
-        timezone = "America/New_York"
-        Time.use_zone(timezone) do
-          from = Time.parse("2024-10-30").beginning_of_day
-          to = Time.parse("2025-01-31").end_of_day
-          from.beginning_of_day..to.end_of_day
+        Time.use_zone("America/New_York") do
+          Time.parse("2024-10-30").beginning_of_day..Time.parse("2025-01-31").end_of_day
         end
       }
     },
     low_skies: {
       human_name: "Low Skies",
       calculate: -> {
-        timezone = "America/New_York"
-        Time.use_zone(timezone) do
-          from = Time.parse("2024-10-3").beginning_of_day
-          to = Time.parse("2025-01-12").end_of_day
-          from.beginning_of_day..to.end_of_day
+        Time.use_zone("America/New_York") do
+          Time.parse("2024-10-03").beginning_of_day..Time.parse("2025-01-12").end_of_day
         end
       }
     },
     scrapyard: {
       human_name: "Scrapyard Global",
       calculate: -> {
-        timezone = "America/New_York"
-        Time.use_zone(timezone) do
-          from = Time.parse("2025-03-14").beginning_of_day
-          to = Time.parse("2025-03-17").end_of_day
-          from.beginning_of_day..to.end_of_day
+        Time.use_zone("America/New_York") do
+          Time.parse("2025-03-14").beginning_of_day..Time.parse("2025-03-17").end_of_day
         end
       }
     }
diff --git a/app/models/heartbeat.rb b/app/models/heartbeat.rb
index cd05cb150..63d597991 100644
--- a/app/models/heartbeat.rb
+++ b/app/models/heartbeat.rb
@@ -8,23 +8,18 @@ class Heartbeat < ClickhouseRecord
   time_range_filterable_field :time
 
   before_create :set_clickhouse_id!, if: -> { self[:id].blank? }
-  before_create :set_fields_hash!, if: -> { fields_hash.blank? }
   after_create :invalidate_user_heartbeat_caches
   after_update :invalidate_user_heartbeat_caches
 
   CLICKHOUSE_ID_RANDOM_BITS = 10
   CLICKHOUSE_ID_RANDOM_MAX = 1 << CLICKHOUSE_ID_RANDOM_BITS
 
-  def set_fields_hash!
-    self.fields_hash = self.class.generate_fields_hash(attributes)
-  end
-
   def set_clickhouse_id!
     timestamp_us = (Time.current.to_r * 1_000_000).to_i
     self[:id] = (timestamp_us << CLICKHOUSE_ID_RANDOM_BITS) | SecureRandom.random_number(CLICKHOUSE_ID_RANDOM_MAX)
   end
 
-  scope :today, -> { where(time: Time.current.beginning_of_day.to_i..Time.current.end_of_day.to_i) }
+  scope :today, -> { where(time: Time.current.beginning_of_day.to_f..Time.current.end_of_day.to_f) }
   scope :recent, -> { where("time > ?", 24.hours.ago.to_i) }
 
   enum :source_type, {
@@ -112,16 +107,6 @@ def self.recent_imported_count
     Cache::HeartbeatCountsJob.perform_now[:recent_imported_count]
   end
 
-  def self.generate_fields_hash(attributes)
-    string_attributes = attributes.transform_keys(&:to_s)
-    indexed_attributes = string_attributes.slice(*self.indexed_attributes)
-    Digest::MD5.hexdigest(indexed_attributes.to_json)
-  end
-
-  def self.indexed_attributes
-    %w[user_id branch category dependencies editor entity language machine operating_system project type user_agent line_additions line_deletions lineno lines cursorpos project_root_count time is_write]
-  end
-
   private
 
   def invalidate_user_heartbeat_caches
diff --git a/app/services/concerns/wakatime_shared.rb b/app/services/concerns/wakatime_shared.rb
new file mode 100644
index 000000000..333a2e1e5
--- /dev/null
+++ b/app/services/concerns/wakatime_shared.rb
@@ -0,0 +1,49 @@
+module WakatimeShared
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def parse_user_agent(user_agent)
+      # Based on https://github.com/muety/wakapi/blob/b3668085c01dc0724d8330f4d51efd5b5aecaeb2/utils/http.go#L89
+      user_agent_pattern = /wakatime\/[^ ]+ \(([^)]+)\)(?: [^ ]+ ([^\/]+)(?:\/([^\/]+))?)?/
+
+      if matches = user_agent.match(user_agent_pattern)
+        os = matches[1].split("-").first
+        editor = matches[2] || ""
+        { os: os, editor: editor, err: nil }
+      elsif browser_ua = user_agent.match(/^([^\/]+)\/([^\/\s]+)/)
+        if user_agent.include?("wakatime")
+          full_os = user_agent.split(" ")[1]
+          if full_os.present?
+            os = full_os.include?("_") ? full_os.split("_")[0] : full_os
+            { os: os, editor: browser_ua[1].downcase, err: nil }
+          else
+            { os: "", editor: "", err: "failed to parse user agent string" }
+          end
+        else
+          { os: browser_ua[1], editor: browser_ua[2], err: nil }
+        end
+      else
+        { os: "", editor: "", err: "failed to parse user agent string" }
+      end
+    rescue => e
+      { os: "", editor: "", err: "failed to parse user agent string" }
+    end
+  end
+
+  private
+
+  def convert_to_unix_timestamp(timestamp)
+    return nil if timestamp.nil?
+
+    case timestamp
+    when String
+      Time.parse(timestamp).to_i
+    when Time, DateTime, Date
+      timestamp.to_i
+    when Numeric
+      timestamp.to_i
+    end
+  rescue ArgumentError
+    nil
+  end
+end
diff --git a/app/services/heartbeat_import_service.rb b/app/services/heartbeat_import_service.rb
index bfcf28738..61a4f57a4 100644
--- a/app/services/heartbeat_import_service.rb
+++ b/app/services/heartbeat_import_service.rb
@@ -7,7 +7,7 @@ def self.import_from_file(file_content, user, on_progress: nil, progress_interva
     imported_count = 0
     total_count = 0
     errors = []
-    seen_hashes = {}
+    batch = []
 
     handler = HeartbeatSaxHandler.new do |hb|
       total_count += 1
@@ -40,14 +40,11 @@ def self.import_from_file(file_content, user, on_progress: nil, progress_interva
           source_type: Heartbeat.source_types.fetch("wakapi_import")
         }
 
-        attrs[:fields_hash] = Heartbeat.generate_fields_hash(attrs)
+        batch << attrs
 
-        existing = seen_hashes[attrs[:fields_hash]]
-        seen_hashes[attrs[:fields_hash]] = attrs if existing.nil? || attrs[:time] > existing[:time]
-
-        if seen_hashes.size >= BATCH_SIZE
-          imported_count += flush_batch(seen_hashes)
-          seen_hashes.clear
+        if batch.size >= BATCH_SIZE
+          imported_count += flush_batch(user_id, batch)
+          batch.clear
         end
       rescue => e
         errors << { heartbeat: hb, error: e.message }
@@ -60,7 +57,7 @@ def self.import_from_file(file_content, user, on_progress: nil, progress_interva
     if total_count.zero?
       raise StandardError, "Expected a heartbeat export JSON file."
     end
-    imported_count += flush_batch(seen_hashes) if seen_hashes.any?
+    imported_count += flush_batch(user_id, batch) if batch.any?
 
     elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
 
@@ -97,56 +94,26 @@ def self.count_heartbeats(file_content)
     total_count
   end
 
-  def self.flush_batch(seen_hashes)
-    return 0 if seen_hashes.empty?
+  def self.flush_batch(user_id, records)
+    return 0 if records.empty?
 
-    records = seen_hashes.values
-    user_id = records.first[:user_id]
-    records.each do |r|
-      timestamp = Time.current
-      r[:created_at] = timestamp
-      r[:updated_at] = timestamp
+    now = Time.current
+    base_us = (now.to_r * 1_000_000).to_i
+    records.each_with_index do |r, i|
+      r[:id] = ((base_us + i) << Heartbeat::CLICKHOUSE_ID_RANDOM_BITS) | SecureRandom.random_number(Heartbeat::CLICKHOUSE_ID_RANDOM_MAX)
+      r[:created_at] = now
+      r[:updated_at] = now
     end
 
     ActiveRecord::Base.logger.silence do
-      new_records = []
-
-      User.transaction do
-        User.lock.find(user_id)
-
-        existing_hashes = Heartbeat.where(user_id: user_id, fields_hash: records.map { |r| r[:fields_hash] }).pluck(:fields_hash).to_set
-        new_records = records.reject { |r| existing_hashes.include?(r[:fields_hash]) }
-
-        if new_records.any?
-          Heartbeat.connection.with_settings(async_insert: 0, wait_for_async_insert: 1) do
-            Heartbeat.insert_all(new_records)
-          end
-          Heartbeat.connection.clear_query_cache
-          wait_until_visible!(
-            user_id,
-            new_records.map { |record| record[:fields_hash] },
-            expected_min_time: new_records.map { |record| record[:time].to_f }.min
-          )
-        end
+      Heartbeat.connection.with_settings(async_insert: 0, wait_for_async_insert: 1) do
+        Heartbeat.insert_all(records)
       end
-
-      HeartbeatCacheInvalidator.bump_for(user_id) if new_records.any?
-      new_records.length
+      Heartbeat.connection.clear_query_cache
     end
-  end
 
-  def self.wait_until_visible!(user_id, fields_hashes, expected_min_time:)
-    deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 2
-
-    loop do
-      visible_count = Heartbeat.where(user_id: user_id, fields_hash: fields_hashes).count
-      visible_min_time = Heartbeat.where(user_id: user_id).minimum(:time).to_f
-      return if visible_count >= fields_hashes.length && visible_min_time <= expected_min_time
-
-      break if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
-
-      sleep 0.01
-    end
+    HeartbeatCacheInvalidator.bump_for(user_id)
+    records.length
   end
 
   class HeartbeatSaxHandler < Oj::Saj
diff --git a/app/services/profile_stats_service.rb b/app/services/profile_stats_service.rb
index 9bf390b65..07e511e25 100644
--- a/app/services/profile_stats_service.rb
+++ b/app/services/profile_stats_service.rb
@@ -61,7 +61,10 @@ def compute_totals_and_breakdowns(timeout, today_start, today_end, week_start, w
           language,
           editor,
           least(
-            time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+            greatest(
+              time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW),
+              0
+            ),
             #{timeout_quoted}
           ) AS diff
         FROM heartbeats
diff --git a/lib/test_wakatime_service.rb b/lib/test_wakatime_service.rb
index dc4d7f13f..b9bbf1f99 100644
--- a/lib/test_wakatime_service.rb
+++ b/lib/test_wakatime_service.rb
@@ -2,6 +2,8 @@
 include ErrorReporting
 
 class TestWakatimeService
+  include WakatimeShared
+
   def initialize(user: nil, specific_filters: [], allow_cache: true, limit: 10, start_date: nil, end_date: nil, scope: nil, boundary_aware: false)
     @scope = scope || Heartbeat.all
     # trusting time from hackatime extensions.....
@@ -56,11 +58,9 @@ def build_summary
     summary[:human_readable_range] = "All Time"
 
     @total_seconds = if @boundary_aware
-      result = Heartbeat.duration_seconds_boundary_aware(@scope, @start_date, @end_date) || 0
-      result
+      Heartbeat.duration_seconds_boundary_aware(@scope, @start_date, @end_date) || 0
     else
-      result = @scope.duration_seconds || 0
-      result
+      @scope.duration_seconds || 0
     end
 
     summary[:total_seconds] = @total_seconds
@@ -97,60 +97,6 @@ def generate_summary_chunk(group_by)
     result
   end
 
-  def self.parse_user_agent(user_agent)
-    # Based on https://github.com/muety/wakapi/blob/b3668085c01dc0724d8330f4d51efd5b5aecaeb2/utils/http.go#L89
-
-    # Regex pattern to match wakatime client user agents
-    user_agent_pattern = /wakatime\/[^ ]+ \(([^)]+)\)(?: [^ ]+ ([^\/]+)(?:\/([^\/]+))?)?/
-
-    if matches = user_agent.match(user_agent_pattern)
-      os = matches[1].split("-").first
-
-      editor = matches[2]
-      editor ||= ""
-
-      { os: os, editor: editor, err: nil }
-    else
-      # Try parsing as browser user agent as fallback
-      if browser_ua = user_agent.match(/^([^\/]+)\/([^\/\s]+)/)
-        # If "wakatime" is present, assume it's the browser extension
-        if user_agent.include?("wakatime") then
-            full_os = user_agent.split(" ")[1]
-            if full_os.present?
-              os = full_os.include?("_") ? full_os.split("_")[0] : full_os
-              { os: os, editor: browser_ua[1].downcase, err: nil }
-            else
-              { os: "", editor: "", err: "failed to parse user agent string" }
-            end
-        else
-          { os: browser_ua[1], editor: browser_ua[2], err: nil }
-        end
-      else
-        { os: "", editor: "", err: "failed to parse user agent string" }
-      end
-    end
-  rescue => e
-    report_error(e, message: "Error parsing user agent string")
-    { os: "", editor: "", err: "failed to parse user agent string" }
-  end
-
-  def categorize_os(os)
-    case os.downcase
-    when "win" then "Windows"
-    when "darwin" then "MacOS"
-    when os.include?("windows") then "Windows"
-    else os.capitalize
-    end
-  end
-
-  def categorize_editor(editor)
-    case editor.downcase
-    when "vscode" then "VSCode"
-    when "KTextEditor" then "Kate"
-    else editor.capitalize
-    end
-  end
-
   private
 
   def summary_cache_key
@@ -172,23 +118,4 @@ def scope_cache_version
 
     @scope.maximum(:time).to_i
   end
-
-  def convert_to_unix_timestamp(timestamp)
-    # our lord and savior stack overflow for this bit of code
-    return nil if timestamp.nil?
-
-    case timestamp
-    when String
-      Time.parse(timestamp).to_i
-    when Time, DateTime, Date
-      timestamp.to_i
-    when Numeric
-      timestamp.to_i
-    else
-      nil
-    end
-  rescue ArgumentError => e
-    report_error(e, message: "Error converting timestamp")
-    nil
-  end
 end
diff --git a/lib/wakatime_service.rb b/lib/wakatime_service.rb
index bfbe4a0fe..6b7c1057c 100644
--- a/lib/wakatime_service.rb
+++ b/lib/wakatime_service.rb
@@ -2,6 +2,8 @@
 include ErrorReporting
 
 class WakatimeService
+  include WakatimeShared
+
   def initialize(user: nil, specific_filters: [], allow_cache: true, limit: 10, start_date: nil, end_date: nil, scope: nil)
     @scope = scope || Heartbeat.all
     @user = user
@@ -84,43 +86,13 @@ def generate_summary_chunk(group_by)
     result
   end
 
-  def self.parse_user_agent(user_agent)
-    # Based on https://github.com/muety/wakapi/blob/b3668085c01dc0724d8330f4d51efd5b5aecaeb2/utils/http.go#L89
-
-    # Regex pattern to match wakatime client user agents
-    user_agent_pattern = /wakatime\/[^ ]+ \(([^)]+)\)(?: [^ ]+ ([^\/]+)(?:\/([^\/]+))?)?/
-
-    if matches = user_agent.match(user_agent_pattern)
-      os = matches[1].split("-").first
-
-      editor = matches[2]
-      editor ||= ""
+  def self.categorize_language(language)
+    return nil if language.blank?
 
-      { os: os, editor: editor, err: nil }
-    else
-      # Try parsing as browser user agent as fallback
-      if browser_ua = user_agent.match(/^([^\/]+)\/([^\/\s]+)/)
-        # If "wakatime" is present, assume it's the browser extension
-        if user_agent.include?("wakatime") then
-            full_os = user_agent.split(" ")[1]
-            if full_os.present?
-              os = full_os.include?("_") ? full_os.split("_")[0] : full_os
-              { os: os, editor: browser_ua[1].downcase, err: nil }
-            else
-              { os: "", editor: "", err: "failed to parse user agent string" }
-            end
-        else
-          { os: browser_ua[1], editor: browser_ua[2], err: nil }
-        end
-      else
-        { os: "", editor: "", err: "failed to parse user agent string" }
-      end
-    end
-  rescue => e
-    report_error(e, message: "Error parsing user agent string")
-    { os: "", editor: "", err: "failed to parse user agent string" }
+    LanguageUtils.display_name(language)
   end
 
+  private
 
   def transform_display_name(group_by, key)
     value = key.presence || "Other"
@@ -136,14 +108,6 @@ def transform_display_name(group_by, key)
     end
   end
 
-  def self.categorize_language(language)
-    return nil if language.blank?
-
-    LanguageUtils.display_name(language)
-  end
-
-  private
-
   def summary_cache_key
     [
       self.class.name.underscore,
@@ -162,23 +126,4 @@ def scope_cache_version
 
     @scope.maximum(:time).to_i
   end
-
-  def convert_to_unix_timestamp(timestamp)
-    # our lord and savior stack overflow for this bit of code
-    return nil if timestamp.nil?
-
-    case timestamp
-    when String
-      Time.parse(timestamp).to_i
-    when Time, DateTime, Date
-      timestamp.to_i
-    when Numeric
-      timestamp.to_i
-    else
-      nil
-    end
-  rescue ArgumentError => e
-    report_error(e, message: "Error converting timestamp")
-    nil
-  end
 end

From 2277fcbed66b27efef791608b4fbf5bc11ce4e3c Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Wed, 25 Mar 2026 10:34:57 +0000
Subject: [PATCH 21/24] Googl

---
 app/controllers/concerns/dashboard_data.rb        |  4 ++--
 .../20260324000001_create_heartbeats.rb           |  2 +-
 ...artbeat_user_daily_summary_refreshable_view.rb |  2 +-
 ...000004_change_heartbeats_order_by_for_dedup.rb | 15 +++++++++++++++
 4 files changed, 19 insertions(+), 4 deletions(-)
 create mode 100644 db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb

diff --git a/app/controllers/concerns/dashboard_data.rb b/app/controllers/concerns/dashboard_data.rb
index 373acde8a..21b942074 100644
--- a/app/controllers/concerns/dashboard_data.rb
+++ b/app/controllers/concerns/dashboard_data.rb
@@ -98,9 +98,9 @@ def filterable_dashboard_data
         weekly_hb = hb.where(time: week_start.to_f..week_end.to_f)
 
         timeout = Heartbeat.heartbeat_timeout_duration.to_i
-        tz = current_user.timezone
+        tz_quoted = Heartbeat.connection.quote(current_user.timezone)
         weekly_sql = weekly_hb
-          .select("toStartOfWeek(toDateTime(toUInt32(time), '#{tz}'), 1) as week_start, `project` as grouped_time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY `project`, toStartOfWeek(toDateTime(toUInt32(time), '#{tz}'), 1) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), #{timeout}) as diff")
+          .select("toStartOfWeek(toDateTime(toUInt32(time), #{tz_quoted}), 1) as week_start, `project` as grouped_time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY `project`, toStartOfWeek(toDateTime(toUInt32(time), #{tz_quoted}), 1) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), #{timeout}) as diff")
           .where.not(time: nil)
           .with_valid_timestamps
           .to_sql
diff --git a/db/migrate_clickhouse/20260324000001_create_heartbeats.rb b/db/migrate_clickhouse/20260324000001_create_heartbeats.rb
index 3f245515c..6d7113d0c 100644
--- a/db/migrate_clickhouse/20260324000001_create_heartbeats.rb
+++ b/db/migrate_clickhouse/20260324000001_create_heartbeats.rb
@@ -63,7 +63,7 @@ def up
       )
       ENGINE = ReplacingMergeTree()
       PARTITION BY toYYYYMM(toDateTime(toUInt32(time)))
-      ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id)
+      ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, entity, time)
       SETTINGS index_granularity = 8192
     SQL
 
diff --git a/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb b/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb
index dbb4c3e7f..f205a9813 100644
--- a/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb
+++ b/db/migrate_clickhouse/20260324000003_create_heartbeat_user_daily_summary_refreshable_view.rb
@@ -20,7 +20,7 @@ def up
               least(
                   greatest(
                       time - lagInFrame(time, 1, time) OVER (
-                          PARTITION BY user_id
+                          PARTITION BY user_id, toDate(toDateTime(toUInt32(time)))
                           ORDER BY time ASC
                           ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
                       ),
diff --git a/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb b/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
new file mode 100644
index 000000000..ead3a75f1
--- /dev/null
+++ b/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
@@ -0,0 +1,15 @@
+class ChangeHeartbeatsOrderByForDedup < ActiveRecord::Migration[8.1]
+  def up
+    execute <<~SQL
+      ALTER TABLE heartbeats
+      MODIFY ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, entity, time)
+    SQL
+  end
+
+  def down
+    execute <<~SQL
+      ALTER TABLE heartbeats
+      MODIFY ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id)
+    SQL
+  end
+end

From 350c630b706457df71988723f4dd53ac0a166bbc Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Wed, 25 Mar 2026 10:54:27 +0000
Subject: [PATCH 22/24] gogo?

---
 app/controllers/api/v1/stats_controller.rb    |   2 +-
 app/models/concerns/heartbeatable.rb          | 233 +++++++++++++++++-
 app/models/heartbeat_user_daily_summary.rb    |  27 ++
 db/clickhouse_schema.rb                       |   9 +-
 ...04_change_heartbeats_order_by_for_dedup.rb |  99 +++++++-
 lib/wakatime_service.rb                       |   2 +-
 test/models/heartbeat_test.rb                 | 142 +++++++++++
 7 files changed, 498 insertions(+), 16 deletions(-)

diff --git a/app/controllers/api/v1/stats_controller.rb b/app/controllers/api/v1/stats_controller.rb
index 15122003d..29819fd96 100644
--- a/app/controllers/api/v1/stats_controller.rb
+++ b/app/controllers/api/v1/stats_controller.rb
@@ -81,7 +81,7 @@ def user_stats
     else
       if params[:total_seconds] == "true"
         query = Heartbeat.where(user_id: @user.id)
-        query = query.where("time >= ? AND time < ?", start_date.to_f, end_date.to_f)
+        query = query.where(time: start_date.to_f...end_date.to_f)
 
         if params[:filter_by_project].present?
           filter_by_project = params[:filter_by_project].split(",")
diff --git a/app/models/concerns/heartbeatable.rb b/app/models/concerns/heartbeatable.rb
index a1439ca9f..17cb20972 100644
--- a/app/models/concerns/heartbeatable.rb
+++ b/app/models/concerns/heartbeatable.rb
@@ -1,6 +1,9 @@
 module Heartbeatable
   extend ActiveSupport::Concern
 
+  MAX_VALID_TIMESTAMP = 253402300799
+  VALID_TIMESTAMPS_SQL = "(time >= 0 AND time <= ?)".freeze
+
   included do
     scope :coding_only, -> { where(category: "coding") }
     scope :with_valid_timestamps, -> { where("time >= 0 AND time <= ?", 253402300799) }
@@ -244,12 +247,10 @@ def duration_seconds(scope = all)
           hash[row["grouped_time"]] = row["duration"].to_i
         end
       else
-        capped_diffs_sql = scope
-          .select("least(greatest(time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), #{timeout}) as diff")
-          .where.not(time: nil)
-          .to_sql
+        summarized_duration = duration_seconds_from_daily_summary(scope, timeout:)
+        return summarized_duration unless summarized_duration.nil?
 
-        connection.select_value("SELECT toInt64(coalesce(sum(diff), 0)) FROM (#{capped_diffs_sql})").to_i
+        raw_duration_seconds(scope, timeout:)
       end
     end
 
@@ -299,5 +300,227 @@ def duration_seconds_boundary_aware(scope, start_time, end_time)
       sql = "SELECT toInt64(coalesce(sum(diff), 0)) FROM (#{capped_diffs_sql}) WHERE time >= #{connection.quote(start_time)}"
       connection.select_value(sql).to_i
     end
+
+    private
+
+    def duration_seconds_from_daily_summary(scope, timeout:)
+      compatibility = summary_compatible_scope(scope)
+      return if compatibility.nil?
+
+      day_bounds = summarized_day_bounds(scope)
+      return 0 if day_bounds.empty?
+
+      stale_days = [ Time.current.utc.to_date ]
+      last_refresh_time = HeartbeatUserDailySummary.last_refresh_time
+      stale_days.concat(stale_summary_days(scope, last_refresh_time)) if last_refresh_time.present?
+
+      summary_days = day_bounds
+        .map { |row| row["day"].to_date }
+        .uniq
+        .select { |day| full_day_included?(day, compatibility) && !stale_days.include?(day) }
+
+      raw_days = day_bounds
+        .map { |row| row["day"].to_date }
+        .uniq - summary_days
+
+      summary_total = HeartbeatUserDailySummary.duration_for_days(
+        user_id: compatibility[:user_id],
+        days: summary_days
+      )
+
+      raw_total = raw_days.sum do |day|
+        raw_duration_seconds(scope.where(time: utc_day_range(day)), timeout:)
+      end
+
+      summary_total + raw_total + cross_day_bonus(day_bounds, timeout:)
+    end
+
+    def summary_compatible_scope(scope)
+      return unless scope.model == Heartbeat
+      return unless scope.group_values.empty?
+      return unless scope.having_clause.empty?
+      return unless scope.limit_value.nil? && scope.offset_value.nil?
+
+      compatibility = {
+        user_id: nil,
+        start_time: nil,
+        start_inclusive: true,
+        end_time: nil,
+        end_inclusive: true
+      }
+
+      predicates = scope.where_clause.send(:predicates)
+
+      predicates.each do |predicate|
+        return unless apply_summary_predicate(predicate, compatibility)
+      end
+
+      return if compatibility[:user_id].blank?
+
+      compatibility
+    end
+
+    def apply_summary_predicate(predicate, compatibility)
+      case predicate
+      when Arel::Nodes::Equality
+        apply_summary_equality_predicate(predicate, compatibility)
+      when Arel::Nodes::Between
+        apply_summary_between_predicate(predicate, compatibility)
+      when Arel::Nodes::And
+        predicate.children.all? { |child| apply_summary_predicate(child, compatibility) }
+      when Arel::Nodes::GreaterThanOrEqual
+        apply_summary_lower_bound(predicate, compatibility, inclusive: true)
+      when Arel::Nodes::GreaterThan
+        apply_summary_lower_bound(predicate, compatibility, inclusive: false)
+      when Arel::Nodes::LessThanOrEqual
+        apply_summary_upper_bound(predicate, compatibility, inclusive: true)
+      when Arel::Nodes::LessThan
+        apply_summary_upper_bound(predicate, compatibility, inclusive: false)
+      when Arel::Nodes::BoundSqlLiteral
+        predicate.sql_with_placeholders == VALID_TIMESTAMPS_SQL &&
+          predicate.positional_binds.map { |bind| node_value(bind) } == [ MAX_VALID_TIMESTAMP ]
+      else
+        false
+      end
+    end
+
+    def apply_summary_equality_predicate(predicate, compatibility)
+      return false unless predicate.left.name.to_s == "user_id"
+
+      user_id = node_value(predicate.right).to_i
+      return false if compatibility[:user_id].present? && compatibility[:user_id] != user_id
+
+      compatibility[:user_id] = user_id
+      true
+    end
+
+    def apply_summary_between_predicate(predicate, compatibility)
+      return false unless predicate.left.name.to_s == "time"
+
+      lower_bound, upper_bound = predicate.right.children
+      apply_time_lower_bound(compatibility, node_value(lower_bound).to_f, inclusive: true)
+      apply_time_upper_bound(compatibility, node_value(upper_bound).to_f, inclusive: true)
+      true
+    end
+
+    def apply_summary_lower_bound(predicate, compatibility, inclusive:)
+      return false unless predicate.left.name.to_s == "time"
+
+      apply_time_lower_bound(compatibility, node_value(predicate.right).to_f, inclusive:)
+      true
+    end
+
+    def apply_summary_upper_bound(predicate, compatibility, inclusive:)
+      return false unless predicate.left.name.to_s == "time"
+
+      apply_time_upper_bound(compatibility, node_value(predicate.right).to_f, inclusive:)
+      true
+    end
+
+    def apply_time_lower_bound(compatibility, candidate_time, inclusive:)
+      current_time = compatibility[:start_time]
+      current_inclusive = compatibility[:start_inclusive]
+
+      if current_time.nil? || candidate_time > current_time || (candidate_time == current_time && !inclusive && current_inclusive)
+        compatibility[:start_time] = candidate_time
+        compatibility[:start_inclusive] = inclusive
+      end
+    end
+
+    def apply_time_upper_bound(compatibility, candidate_time, inclusive:)
+      current_time = compatibility[:end_time]
+      current_inclusive = compatibility[:end_inclusive]
+
+      if current_time.nil? || candidate_time < current_time || (candidate_time == current_time && !inclusive && current_inclusive)
+        compatibility[:end_time] = candidate_time
+        compatibility[:end_inclusive] = inclusive
+      end
+    end
+
+    def summarized_day_bounds(scope)
+      scoped_sql = scope
+        .where.not(time: nil)
+        .unscope(:order)
+        .reselect(:time)
+        .to_sql
+
+      connection.select_all(<<~SQL)
+        SELECT
+          toDate(toDateTime(toUInt32(time))) AS day,
+          min(time) AS first_time,
+          max(time) AS last_time
+        FROM (#{scoped_sql}) AS hb
+        GROUP BY day
+        ORDER BY day
+      SQL
+    end
+
+    def stale_summary_days(scope, last_refresh_time)
+      scoped_sql = scope
+        .where.not(time: nil)
+        .unscope(:order)
+        .reselect(:time, :updated_at)
+        .to_sql
+
+      connection.select_values(<<~SQL).map(&:to_date)
+        SELECT DISTINCT toDate(toDateTime(toUInt32(time))) AS day
+        FROM (#{scoped_sql}) AS hb
+        WHERE updated_at > #{connection.quote(last_refresh_time)}
+      SQL
+    end
+
+    def full_day_included?(day, compatibility)
+      day_start = utc_day_start(day)
+      day_end = utc_day_end(day)
+
+      lower_bound_matches = compatibility[:start_time].nil? ||
+        compatibility[:start_time] < day_start ||
+        (compatibility[:start_time] == day_start && compatibility[:start_inclusive])
+
+      upper_bound_matches = compatibility[:end_time].nil? ||
+        compatibility[:end_time] > day_end ||
+        (compatibility[:end_time] == day_end && compatibility[:end_inclusive])
+
+      lower_bound_matches && upper_bound_matches
+    end
+
+    def cross_day_bonus(day_bounds, timeout:)
+      day_bounds.each_cons(2).sum do |previous_day, current_day|
+        previous_date = previous_day["day"].to_date
+        current_date = current_day["day"].to_date
+        next 0 unless current_date == previous_date + 1.day
+
+        gap = current_day["first_time"].to_f - previous_day["last_time"].to_f
+        [ [ gap, timeout ].min, 0 ].max.to_i
+      end
+    end
+
+    def raw_duration_seconds(scope, timeout:)
+      capped_diffs_sql = scope
+        .select("least(greatest(time - lagInFrame(time, 1, time) OVER (ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), #{timeout}) as diff")
+        .where.not(time: nil)
+        .to_sql
+
+      connection.select_value("SELECT toInt64(coalesce(sum(diff), 0)) FROM (#{capped_diffs_sql})").to_i
+    end
+
+    def utc_day_range(day)
+      utc_day_start(day)..utc_day_end(day)
+    end
+
+    def utc_day_start(day)
+      day.to_time(:utc).beginning_of_day.to_f
+    end
+
+    def utc_day_end(day)
+      day.to_time(:utc).end_of_day.to_f
+    end
+
+    def node_value(node)
+      return node.value_for_database if node.respond_to?(:value_for_database)
+      return node.value if node.respond_to?(:value)
+
+      node
+    end
   end
 end
diff --git a/app/models/heartbeat_user_daily_summary.rb b/app/models/heartbeat_user_daily_summary.rb
index c60ebd42d..327dd4f1f 100644
--- a/app/models/heartbeat_user_daily_summary.rb
+++ b/app/models/heartbeat_user_daily_summary.rb
@@ -1,3 +1,30 @@
 class HeartbeatUserDailySummary < ClickhouseRecord
   self.table_name = "heartbeat_user_daily_summary"
+
+  VIEW_NAME = "heartbeat_user_daily_summary_mv".freeze
+
+  class << self
+    def duration_for_days(user_id:, days:)
+      return 0 if days.empty?
+
+      from("#{table_name} FINAL")
+        .where(user_id: user_id, day: days.uniq.sort)
+        .sum(:duration_s)
+        .to_i
+    end
+
+    def last_refresh_time
+      database = connection.select_value("SELECT currentDatabase()")
+
+      connection.select_value(<<~SQL)
+        SELECT last_success_time
+        FROM system.view_refreshes
+        WHERE database = #{connection.quote(database)}
+          AND view = #{connection.quote(VIEW_NAME)}
+        LIMIT 1
+      SQL
+    rescue StandardError
+      nil
+    end
+  end
 end
diff --git a/db/clickhouse_schema.rb b/db/clickhouse_schema.rb
index 05e0e9874..a5ee22d7e 100644
--- a/db/clickhouse_schema.rb
+++ b/db/clickhouse_schema.rb
@@ -16,7 +16,7 @@
   create_table "heartbeat_user_daily_summary", id: false, options: "ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192", force: :cascade do |t|
     t.integer "user_id", unsigned: false, limit: 8, null: false
     t.date "day", null: false
-    t.column "duration_s", "Float64", null: false
+    t.float "duration_s", null: false
     t.integer "heartbeats", null: false
     t.datetime "_version", precision: nil, default: -> { "now()" }, null: false
   end
@@ -43,7 +43,7 @@
     t.integer "lines", unsigned: false, default: 0, null: false
     t.integer "cursorpos", unsigned: false, default: 0, null: false
     t.integer "project_root_count", unsigned: false, default: 0, null: false
-    t.column "time", "Float64", null: false
+    t.float "time", null: false
     t.integer "is_write", limit: 1, default: 0, null: false
     t.datetime "created_at", default: -> { "now64()" }, null: false
     t.datetime "updated_at", default: -> { "now64()" }, null: false
@@ -54,7 +54,8 @@
   end
 
   # TABLE: heartbeat_user_daily_summary_mv
-  # SQL: CREATE MATERIALIZED VIEW heartbeat_user_daily_summary_mv REFRESH EVERY 10 MINUTE TO heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32 ) AS SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day
-  create_table "heartbeat_user_daily_summary_mv", view: true, materialized: true, id: false, as: "SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day", force: :cascade do |t|
+  # SQL: CREATE MATERIALIZED VIEW heartbeat_user_daily_summary_mv REFRESH EVERY 10 MINUTE TO heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32 ) AS SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id, toDate(toDateTime(toUInt32(time))) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day
+  create_table "heartbeat_user_daily_summary_mv", view: true, materialized: true, id: false, as: "SELECT user_id, toDate(toDateTime(toUInt32(time))) AS day, sum(diff) AS duration_s, toUInt32(count()) AS heartbeats FROM ( SELECT user_id, time, least(greatest(time - lagInFrame(time, 1, time) OVER (PARTITION BY user_id, toDate(toDateTime(toUInt32(time))) ORDER BY time ASC ROWS BETWEEN 1 PRECEDING AND CURRENT ROW), 0), 120) AS diff FROM heartbeats WHERE (time IS NOT NULL) AND (time >= 0) AND (time <= 253402300799) ) GROUP BY user_id, day", force: :cascade do |t|
   end
+
 end
diff --git a/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb b/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
index ead3a75f1..c6afddfc6 100644
--- a/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
+++ b/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
@@ -1,15 +1,104 @@
 class ChangeHeartbeatsOrderByForDedup < ActiveRecord::Migration[8.1]
+  SOURCE_TABLE = "heartbeats".freeze
+  TEMP_TABLE = "heartbeats_rebuild_new".freeze
+  BACKUP_TABLE = "heartbeats_rebuild_old".freeze
+  SUMMARY_VIEW = "heartbeat_user_daily_summary_mv".freeze
+  PARTITION_BY = "toYYYYMM(toDateTime(toUInt32(time)))".freeze
+
   def up
+    rebuild_heartbeats_table(
+      order_by: "(user_id, toDate(toDateTime(toUInt32(time))), project, entity, time)"
+    )
+  end
+
+  def down
+    rebuild_heartbeats_table(
+      order_by: "(user_id, toDate(toDateTime(toUInt32(time))), project, id)"
+    )
+  end
+
+  private
+
+  def rebuild_heartbeats_table(order_by:)
+    drop_summary_view
+    drop_rebuild_tables
+
+    execute <<~SQL
+      CREATE TABLE #{TEMP_TABLE}
+      AS #{SOURCE_TABLE}
+      ENGINE = ReplacingMergeTree()
+      PARTITION BY #{PARTITION_BY}
+      ORDER BY #{order_by}
+      SETTINGS index_granularity = 8192
+    SQL
+
     execute <<~SQL
-      ALTER TABLE heartbeats
-      MODIFY ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, entity, time)
+      INSERT INTO #{TEMP_TABLE}
+      SELECT *
+      FROM #{SOURCE_TABLE}
     SQL
+
+    execute <<~SQL
+      RENAME TABLE
+        #{SOURCE_TABLE} TO #{BACKUP_TABLE},
+        #{TEMP_TABLE} TO #{SOURCE_TABLE}
+    SQL
+
+    execute "DROP TABLE IF EXISTS #{BACKUP_TABLE}"
+    recreate_summary_view
+  rescue StandardError
+    recreate_summary_view if show_create_statement(SUMMARY_VIEW).blank?
+    raise
   end
 
-  def down
+  def drop_summary_view
+    execute "DROP VIEW IF EXISTS #{SUMMARY_VIEW}"
+  end
+
+  def drop_rebuild_tables
+    execute "DROP TABLE IF EXISTS #{TEMP_TABLE}"
+    execute "DROP TABLE IF EXISTS #{BACKUP_TABLE}"
+  end
+
+  def recreate_summary_view
     execute <<~SQL
-      ALTER TABLE heartbeats
-      MODIFY ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id)
+      CREATE MATERIALIZED VIEW IF NOT EXISTS #{SUMMARY_VIEW}
+      REFRESH EVERY 10 MINUTE
+      TO heartbeat_user_daily_summary
+      AS
+      SELECT
+          user_id,
+          toDate(toDateTime(toUInt32(time))) AS day,
+          sum(diff) AS duration_s,
+          toUInt32(count()) AS heartbeats
+      FROM
+      (
+          SELECT
+              user_id,
+              time,
+              least(
+                  greatest(
+                      time - lagInFrame(time, 1, time) OVER (
+                          PARTITION BY user_id, toDate(toDateTime(toUInt32(time)))
+                          ORDER BY time ASC
+                          ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+                      ),
+                      0
+                  ),
+                  120
+              ) AS diff
+          FROM heartbeats
+          WHERE time IS NOT NULL
+            AND time >= 0
+            AND time <= 253402300799
+      )
+      GROUP BY user_id, day
     SQL
   end
+
+  def show_create_statement(name)
+    select_one("SHOW CREATE TABLE #{name}")&.fetch("statement")
+  rescue ActiveRecord::ActiveRecordError
+    nil
+  end
 end
diff --git a/lib/wakatime_service.rb b/lib/wakatime_service.rb
index 6b7c1057c..9831fce86 100644
--- a/lib/wakatime_service.rb
+++ b/lib/wakatime_service.rb
@@ -17,7 +17,7 @@ def initialize(user: nil, specific_filters: [], allow_cache: true, limit: 10, st
     @start_date = @start_date || @scope.minimum(:time) || 1.year.ago.to_i
     @end_date = @end_date || @scope.maximum(:time) || Time.current.to_i
 
-    @scope = @scope.where("time >= ? AND time < ?", @start_date, @end_date)
+    @scope = @scope.where(time: @start_date...@end_date)
 
     @limit = limit
     @limit = nil if @limit&.zero?
diff --git a/test/models/heartbeat_test.rb b/test/models/heartbeat_test.rb
index 21a0e94d3..4a4c997d3 100644
--- a/test/models/heartbeat_test.rb
+++ b/test/models/heartbeat_test.rb
@@ -39,4 +39,146 @@ def setup
 
     assert_equal initial_version + 1, HeartbeatCacheInvalidator.version_for(user)
   end
+
+  test "duration_seconds uses daily summary rows for simple user date ranges" do
+    travel_to Time.utc(2026, 1, 10, 12, 0, 0) do
+      user = User.create!(timezone: "UTC")
+
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 1, 10, 0, 0),
+        updated_at: Time.utc(2026, 1, 1, 10, 0, 0)
+      )
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 1, 10, 1, 0),
+        updated_at: Time.utc(2026, 1, 1, 10, 1, 0)
+      )
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 2, 12, 0, 0),
+        updated_at: Time.utc(2026, 1, 2, 12, 0, 0)
+      )
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 2, 12, 1, 30),
+        updated_at: Time.utc(2026, 1, 2, 12, 1, 30)
+      )
+
+      create_daily_summary(user: user, day: Date.new(2026, 1, 1), duration_s: 60, heartbeats: 2)
+      create_daily_summary(user: user, day: Date.new(2026, 1, 2), duration_s: 90, heartbeats: 2)
+
+      total, queries = capture_sql_queries do
+        user.heartbeats.where(time: Time.utc(2026, 1, 1).to_f..Time.utc(2026, 1, 2, 23, 59, 59).to_f).duration_seconds
+      end
+
+      assert_equal 270, total
+      assert queries.any? { |sql| sql.include?("heartbeat_user_daily_summary FINAL") }
+    end
+  end
+
+  test "duration_seconds preserves cross-day gaps when combining daily summaries" do
+    travel_to Time.utc(2026, 1, 10, 12, 0, 0) do
+      user = User.create!(timezone: "UTC")
+
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 1, 23, 59, 30),
+        updated_at: Time.utc(2026, 1, 1, 23, 59, 30)
+      )
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 2, 0, 0, 30),
+        updated_at: Time.utc(2026, 1, 2, 0, 0, 30)
+      )
+
+      create_daily_summary(user: user, day: Date.new(2026, 1, 1), duration_s: 0, heartbeats: 1)
+      create_daily_summary(user: user, day: Date.new(2026, 1, 2), duration_s: 0, heartbeats: 1)
+
+      total = user.heartbeats.where(time: Time.utc(2026, 1, 1).to_f..Time.utc(2026, 1, 2, 23, 59, 59).to_f).duration_seconds
+
+      assert_equal 60, total
+    end
+  end
+
+  test "duration_seconds keeps filtered scopes on the raw heartbeat path" do
+    travel_to Time.utc(2026, 1, 10, 12, 0, 0) do
+      user = User.create!(timezone: "UTC")
+
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 1, 10, 0, 0),
+        project: "alpha",
+        updated_at: Time.utc(2026, 1, 1, 10, 0, 0)
+      )
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 1, 10, 1, 0),
+        project: "alpha",
+        updated_at: Time.utc(2026, 1, 1, 10, 1, 0)
+      )
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 1, 11, 0, 0),
+        project: "beta",
+        updated_at: Time.utc(2026, 1, 1, 11, 0, 0)
+      )
+      create_heartbeat(
+        user: user,
+        time: Time.utc(2026, 1, 1, 11, 1, 30),
+        project: "beta",
+        updated_at: Time.utc(2026, 1, 1, 11, 1, 30)
+      )
+
+      create_daily_summary(user: user, day: Date.new(2026, 1, 1), duration_s: 150, heartbeats: 4)
+
+      total, queries = capture_sql_queries do
+        user.heartbeats.where(project: "alpha", time: Time.utc(2026, 1, 1).to_f..Time.utc(2026, 1, 1, 23, 59, 59).to_f).duration_seconds
+      end
+
+      assert_equal 60, total
+      assert queries.none? { |sql| sql.include?("heartbeat_user_daily_summary FINAL") }
+    end
+  end
+
+  private
+
+  def create_heartbeat(user:, time:, project: "heartbeat-test", updated_at:)
+    Heartbeat.create!(
+      user: user,
+      entity: "src/main.rb",
+      type: "file",
+      category: "coding",
+      time: time.to_f,
+      project: project,
+      source_type: :test_entry,
+      created_at: updated_at,
+      updated_at: updated_at
+    )
+  end
+
+  def create_daily_summary(user:, day:, duration_s:, heartbeats:)
+    HeartbeatUserDailySummary.create!(
+      user_id: user.id,
+      day: day,
+      duration_s: duration_s,
+      heartbeats: heartbeats
+    )
+  end
+
+  def capture_sql_queries
+    queries = []
+    result = nil
+
+    callback = lambda do |_name, _start, _finish, _id, payload|
+      sql = payload[:sql]
+      queries << sql if sql.is_a?(String)
+    end
+
+    ActiveSupport::Notifications.subscribed(callback, "sql.active_record") do
+      result = yield
+    end
+
+    [ result, queries ]
+  end
 end

From a396f28ad9ae7f602bf35d9c887a5870468d6599 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Wed, 25 Mar 2026 16:18:30 +0000
Subject: [PATCH 23/24] goog

---
 config/initializers/clickhouse.rb             | 45 +++++++++++++++++++
 db/clickhouse_schema.rb                       | 10 ++---
 ...04_change_heartbeats_order_by_for_dedup.rb | 23 +++++++---
 3 files changed, 68 insertions(+), 10 deletions(-)

diff --git a/config/initializers/clickhouse.rb b/config/initializers/clickhouse.rb
index 1a211029c..58935c8e2 100644
--- a/config/initializers/clickhouse.rb
+++ b/config/initializers/clickhouse.rb
@@ -6,4 +6,49 @@ def insert_settings
       { wait_for_async_insert: 1, async_insert: 0 }
     end
   end
+
+  class SchemaDumper
+    private
+
+    def schema_type(column)
+      sql_type = column.sql_type.to_s.delete_prefix("Nullable(").delete_suffix(")")
+      return :float64 if column.type == :float && sql_type == "Float64"
+
+      super
+    end
+  end
+end
+
+module ActiveRecord
+  module ConnectionAdapters
+    class ClickhouseAdapter
+      FLOAT64_DATABASE_TYPES = { float64: { name: "Float64" } }.freeze
+
+      class << self
+        def native_database_types
+          self::NATIVE_DATABASE_TYPES.merge(FLOAT64_DATABASE_TYPES)
+        end
+
+        def valid_type?(type)
+          native_database_types[type].present?
+        end
+      end
+
+      def native_database_types
+        self.class.native_database_types
+      end
+
+      def valid_type?(type)
+        self.class.valid_type?(type)
+      end
+    end
+
+    module Clickhouse
+      class TableDefinition
+        def float64(*args, **options)
+          args.each { |name| column(name, :float64, **options) }
+        end
+      end
+    end
+  end
 end
diff --git a/db/clickhouse_schema.rb b/db/clickhouse_schema.rb
index a5ee22d7e..cc8e6c4cf 100644
--- a/db/clickhouse_schema.rb
+++ b/db/clickhouse_schema.rb
@@ -10,20 +10,20 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.1].define(version: 2026_03_24_000003) do
+ActiveRecord::Schema[8.1].define(version: 2026_03_24_000004) do
   # TABLE: heartbeat_user_daily_summary
   # SQL: CREATE TABLE heartbeat_user_daily_summary ( `user_id` Int64, `day` Date, `duration_s` Float64, `heartbeats` UInt32, `_version` DateTime DEFAULT now() ) ENGINE = ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192
   create_table "heartbeat_user_daily_summary", id: false, options: "ReplacingMergeTree(_version) ORDER BY (user_id, day) SETTINGS index_granularity = 8192", force: :cascade do |t|
     t.integer "user_id", unsigned: false, limit: 8, null: false
     t.date "day", null: false
-    t.float "duration_s", null: false
+    t.float64 "duration_s", null: false
     t.integer "heartbeats", null: false
     t.datetime "_version", precision: nil, default: -> { "now()" }, null: false
   end
 
   # TABLE: heartbeats
-  # SQL: CREATE TABLE heartbeats ( `id` Int64, `user_id` Int64, `branch` String DEFAULT '', `category` String DEFAULT '', `dependencies` Array(String), `editor` String DEFAULT '', `entity` String DEFAULT '', `language` String DEFAULT '', `machine` String DEFAULT '', `operating_system` String DEFAULT '', `project` String DEFAULT '', `type` String DEFAULT '', `user_agent` String DEFAULT '', `line_additions` Int32 DEFAULT 0, `line_deletions` Int32 DEFAULT 0, `lineno` Int32 DEFAULT 0, `lines` Int32 DEFAULT 0, `cursorpos` Int32 DEFAULT 0, `project_root_count` Int32 DEFAULT 0, `time` Float64, `is_write` UInt8 DEFAULT 0, `created_at` DateTime64(6) DEFAULT now64(), `updated_at` DateTime64(6) DEFAULT now64(), `source_type` UInt8 DEFAULT 0, `ip_address` String DEFAULT '', `ysws_program` UInt8 DEFAULT 0, `fields_hash` String DEFAULT '' ) ENGINE = ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id) SETTINGS index_granularity = 8192
-  create_table "heartbeats", id: :int64, options: "ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, id) SETTINGS index_granularity = 8192", force: :cascade do |t|
+  # SQL: CREATE TABLE heartbeats ( `id` Int64, `user_id` Int64, `branch` String DEFAULT '', `category` String DEFAULT '', `dependencies` Array(String), `editor` String DEFAULT '', `entity` String DEFAULT '', `language` String DEFAULT '', `machine` String DEFAULT '', `operating_system` String DEFAULT '', `project` String DEFAULT '', `type` String DEFAULT '', `user_agent` String DEFAULT '', `line_additions` Int32 DEFAULT 0, `line_deletions` Int32 DEFAULT 0, `lineno` Int32 DEFAULT 0, `lines` Int32 DEFAULT 0, `cursorpos` Int32 DEFAULT 0, `project_root_count` Int32 DEFAULT 0, `time` Float64, `is_write` UInt8 DEFAULT 0, `created_at` DateTime64(6) DEFAULT now64(), `updated_at` DateTime64(6) DEFAULT now64(), `source_type` UInt8 DEFAULT 0, `ip_address` String DEFAULT '', `ysws_program` UInt8 DEFAULT 0, `fields_hash` String DEFAULT '' ) ENGINE = ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, entity, time) SETTINGS index_granularity = 8192
+  create_table "heartbeats", id: :int64, options: "ReplacingMergeTree PARTITION BY toYYYYMM(toDateTime(toUInt32(time))) ORDER BY (user_id, toDate(toDateTime(toUInt32(time))), project, entity, time) SETTINGS index_granularity = 8192", force: :cascade do |t|
     t.integer "id", unsigned: false, limit: 8, null: false
     t.integer "user_id", unsigned: false, limit: 8, null: false
     t.string "branch", default: "", null: false
@@ -43,7 +43,7 @@
     t.integer "lines", unsigned: false, default: 0, null: false
     t.integer "cursorpos", unsigned: false, default: 0, null: false
     t.integer "project_root_count", unsigned: false, default: 0, null: false
-    t.float "time", null: false
+    t.float64 "time", null: false
     t.integer "is_write", limit: 1, default: 0, null: false
     t.datetime "created_at", default: -> { "now64()" }, null: false
     t.datetime "updated_at", default: -> { "now64()" }, null: false
diff --git a/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb b/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
index c6afddfc6..44ebd811b 100644
--- a/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
+++ b/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
@@ -4,6 +4,7 @@ class ChangeHeartbeatsOrderByForDedup < ActiveRecord::Migration[8.1]
   BACKUP_TABLE = "heartbeats_rebuild_old".freeze
   SUMMARY_VIEW = "heartbeat_user_daily_summary_mv".freeze
   PARTITION_BY = "toYYYYMM(toDateTime(toUInt32(time)))".freeze
+  DAY_EXPRESSION = "toDate(toDateTime(toUInt32(time)))".freeze
 
   def up
     rebuild_heartbeats_table(
@@ -32,11 +33,7 @@ def rebuild_heartbeats_table(order_by:)
       SETTINGS index_granularity = 8192
     SQL
 
-    execute <<~SQL
-      INSERT INTO #{TEMP_TABLE}
-      SELECT *
-      FROM #{SOURCE_TABLE}
-    SQL
+    copy_source_rows_by_day
 
     execute <<~SQL
       RENAME TABLE
@@ -96,6 +93,22 @@ def recreate_summary_view
     SQL
   end
 
+  def copy_source_rows_by_day
+    select_values(<<~SQL).each do |day|
+      SELECT DISTINCT #{DAY_EXPRESSION} AS day
+      FROM #{SOURCE_TABLE}
+      ORDER BY day
+    SQL
+
+      execute <<~SQL
+        INSERT INTO #{TEMP_TABLE}
+        SELECT *
+        FROM #{SOURCE_TABLE}
+        WHERE #{DAY_EXPRESSION} = #{connection.quote(day)}
+      SQL
+    end
+  end
+
   def show_create_statement(name)
     select_one("SHOW CREATE TABLE #{name}")&.fetch("statement")
   rescue ActiveRecord::ActiveRecordError

From 1968b002625fd81cea4705eb1df737a5edb3dfb2 Mon Sep 17 00:00:00 2001
From: Mahad Kalam <mahadkalam1234@gmail.com>
Date: Wed, 25 Mar 2026 18:12:55 +0000
Subject: [PATCH 24/24] Fix miggy-ration

---
 .../20260324000004_change_heartbeats_order_by_for_dedup.rb | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb b/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
index 44ebd811b..d728b6239 100644
--- a/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
+++ b/db/migrate_clickhouse/20260324000004_change_heartbeats_order_by_for_dedup.rb
@@ -35,11 +35,8 @@ def rebuild_heartbeats_table(order_by:)
 
     copy_source_rows_by_day
 
-    execute <<~SQL
-      RENAME TABLE
-        #{SOURCE_TABLE} TO #{BACKUP_TABLE},
-        #{TEMP_TABLE} TO #{SOURCE_TABLE}
-    SQL
+    execute "RENAME TABLE #{SOURCE_TABLE} TO #{BACKUP_TABLE}"
+    execute "RENAME TABLE #{TEMP_TABLE} TO #{SOURCE_TABLE}"
 
     execute "DROP TABLE IF EXISTS #{BACKUP_TABLE}"
     recreate_summary_view