From 9181d2acca2c93960601304cec18b5e66543b494 Mon Sep 17 00:00:00 2001 From: Andrew Atkinson Date: Fri, 6 Jun 2025 15:15:21 -0500 Subject: [PATCH] Add rollups table Use ankane/rollups Basic examples with Driver counts by day Pre-calculate these so lookups are nearly instant --- Gemfile | 1 + Gemfile.lock | 6 ++ README.md | 1 - config/application.rb | 5 +- db/migrate/20250606201409_create_rollups.rb | 12 ++++ db/structure.sql | 61 ++++++++++++++++++++ docs/rollups.md | 24 ++++++++ erd.pdf | Bin 55857 -> 55857 bytes lib/tasks/fake_data_generator.rake | 14 ++++- 9 files changed, 119 insertions(+), 5 deletions(-) create mode 100644 db/migrate/20250606201409_create_rollups.rb create mode 100644 docs/rollups.md diff --git a/Gemfile b/Gemfile index 7390283..987254a 100644 --- a/Gemfile +++ b/Gemfile @@ -28,6 +28,7 @@ gem 'scenic' # manage DB views, materialized views gem 'strong_migrations' # Use safe Migration patterns gem 'rubocop' +gem 'rollups' group :development, :test do gem 'active_record_doctor' diff --git a/Gemfile.lock b/Gemfile.lock index 40200c0..bbf3d47 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -152,6 +152,8 @@ GEM google-protobuf (4.31.0-x86_64-linux-gnu) bigdecimal rake (>= 13) + groupdate (6.7.0) + activesupport (>= 7.1) i18n (1.14.7) concurrent-ruby (~> 1.0) importmap-rails (1.2.3) @@ -289,6 +291,9 @@ GEM io-console (~> 0.5) require_all (3.0.0) rexml (3.4.1) + rollups (0.5.0) + activesupport (>= 7.1) + groupdate (>= 6.1) rubocop (1.75.7) json (~> 2.3) language_server-protocol (~> 3.17.0.2) @@ -384,6 +389,7 @@ DEPENDENCIES rails-erd rails-pg-extras rails_best_practices + rollups rubocop scenic sprockets-rails (~> 3.4) diff --git a/README.md b/README.md index dc1fc8c..d88ab34 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,6 @@ PostgreSQL 16 or greater is required. Installation may be via Homebrew, although ### PostgresApp - Once installed, from the Menu Bar app, choose "Open Postgres" then click the "+" icon to create a new PostgreSQL 16 server - ## Ruby Run `cat .ruby-version` from the Rideshare directory to find the needed version of Ruby. diff --git a/config/application.rb b/config/application.rb index b2ff765..fa5b460 100644 --- a/config/application.rb +++ b/config/application.rb @@ -43,6 +43,7 @@ class Application < Rails::Application # set a timezone. Times are generally stored as # timestamps without a time zone. This application # would need to treat times based on the user's timezone. + config.time_zone = 'Central Time (US & Canada)' # Enable Query Logging @@ -63,6 +64,8 @@ class Application < Rails::Application # Consider timestamps in the local time zone # This is because the app used "timestamp without time zone" columns and times are # stored in the local timezone (CST). - config.active_record.default_timezone = :local + + # Needed for ankane/rollups + config.active_record.default_timezone = :utc end end diff --git a/db/migrate/20250606201409_create_rollups.rb b/db/migrate/20250606201409_create_rollups.rb new file mode 100644 index 0000000..283e6b7 --- /dev/null +++ b/db/migrate/20250606201409_create_rollups.rb @@ -0,0 +1,12 @@ +class CreateRollups < ActiveRecord::Migration[7.2] + def change + create_table :rollups do |t| + t.string :name, null: false + t.string :interval, null: false + t.datetime :time, null: false + t.jsonb :dimensions, null: false, default: {} + t.float :value + end + add_index :rollups, [:name, :interval, :time, :dimensions], unique: true + end +end diff --git a/db/structure.sql b/db/structure.sql index 8da33d7..4aded82 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -27,6 +27,7 @@ DROP INDEX IF EXISTS rideshare.index_trips_on_driver_id; DROP INDEX IF EXISTS rideshare.index_trip_requests_on_start_location_id; DROP INDEX IF EXISTS rideshare.index_trip_requests_on_rider_id; DROP INDEX IF EXISTS rideshare.index_trip_requests_on_end_location_id; +DROP INDEX IF EXISTS rideshare.index_rollups_on_name_and_interval_and_time_and_dimensions; DROP INDEX IF EXISTS rideshare.index_locations_on_address; DROP INDEX IF EXISTS rideshare.index_fast_search_results_on_driver_id; ALTER TABLE IF EXISTS ONLY rideshare.vehicles DROP CONSTRAINT IF EXISTS vehicles_pkey; @@ -36,6 +37,7 @@ ALTER TABLE IF EXISTS ONLY rideshare.trips DROP CONSTRAINT IF EXISTS trips_pkey; ALTER TABLE IF EXISTS ONLY rideshare.trip_requests DROP CONSTRAINT IF EXISTS trip_requests_pkey; ALTER TABLE IF EXISTS ONLY rideshare.trip_positions DROP CONSTRAINT IF EXISTS trip_positions_pkey; ALTER TABLE IF EXISTS ONLY rideshare.schema_migrations DROP CONSTRAINT IF EXISTS schema_migrations_pkey; +ALTER TABLE IF EXISTS ONLY rideshare.rollups DROP CONSTRAINT IF EXISTS rollups_pkey; ALTER TABLE IF EXISTS ONLY rideshare.vehicle_reservations DROP CONSTRAINT IF EXISTS non_overlapping_vehicle_registration; ALTER TABLE IF EXISTS ONLY rideshare.locations DROP CONSTRAINT IF EXISTS locations_pkey; ALTER TABLE IF EXISTS rideshare.trips DROP CONSTRAINT IF EXISTS chk_rails_4743ddc2d2; @@ -46,6 +48,7 @@ ALTER TABLE IF EXISTS rideshare.users ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.trips ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.trip_requests ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.trip_positions ALTER COLUMN id DROP DEFAULT; +ALTER TABLE IF EXISTS rideshare.rollups ALTER COLUMN id DROP DEFAULT; ALTER TABLE IF EXISTS rideshare.locations ALTER COLUMN id DROP DEFAULT; DROP SEQUENCE IF EXISTS rideshare.vehicles_id_seq; DROP TABLE IF EXISTS rideshare.vehicles; @@ -59,6 +62,8 @@ DROP SEQUENCE IF EXISTS rideshare.trip_positions_id_seq; DROP TABLE IF EXISTS rideshare.trip_positions; DROP VIEW IF EXISTS rideshare.search_results; DROP TABLE IF EXISTS rideshare.schema_migrations; +DROP SEQUENCE IF EXISTS rideshare.rollups_id_seq; +DROP TABLE IF EXISTS rideshare.rollups; DROP SEQUENCE IF EXISTS rideshare.locations_id_seq; DROP TABLE IF EXISTS rideshare.locations; DROP MATERIALIZED VIEW IF EXISTS rideshare.fast_search_results; @@ -291,6 +296,39 @@ CREATE SEQUENCE rideshare.locations_id_seq ALTER SEQUENCE rideshare.locations_id_seq OWNED BY rideshare.locations.id; +-- +-- Name: rollups; Type: TABLE; Schema: rideshare; Owner: - +-- + +CREATE TABLE rideshare.rollups ( + id bigint NOT NULL, + name character varying NOT NULL, + "interval" character varying NOT NULL, + "time" timestamp(6) without time zone NOT NULL, + dimensions jsonb DEFAULT '{}'::jsonb NOT NULL, + value double precision +); + + +-- +-- Name: rollups_id_seq; Type: SEQUENCE; Schema: rideshare; Owner: - +-- + +CREATE SEQUENCE rideshare.rollups_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: rollups_id_seq; Type: SEQUENCE OWNED BY; Schema: rideshare; Owner: - +-- + +ALTER SEQUENCE rideshare.rollups_id_seq OWNED BY rideshare.rollups.id; + + -- -- Name: schema_migrations; Type: TABLE; Schema: rideshare; Owner: - -- @@ -491,6 +529,13 @@ ALTER SEQUENCE rideshare.vehicles_id_seq OWNED BY rideshare.vehicles.id; ALTER TABLE ONLY rideshare.locations ALTER COLUMN id SET DEFAULT nextval('rideshare.locations_id_seq'::regclass); +-- +-- Name: rollups id; Type: DEFAULT; Schema: rideshare; Owner: - +-- + +ALTER TABLE ONLY rideshare.rollups ALTER COLUMN id SET DEFAULT nextval('rideshare.rollups_id_seq'::regclass); + + -- -- Name: trip_positions id; Type: DEFAULT; Schema: rideshare; Owner: - -- @@ -565,6 +610,14 @@ ALTER TABLE ONLY rideshare.vehicle_reservations ADD CONSTRAINT non_overlapping_vehicle_registration EXCLUDE USING gist (int4range(vehicle_id, vehicle_id, '[]'::text) WITH =, tstzrange(starts_at, ends_at) WITH &&) WHERE ((NOT canceled)); +-- +-- Name: rollups rollups_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - +-- + +ALTER TABLE ONLY rideshare.rollups + ADD CONSTRAINT rollups_pkey PRIMARY KEY (id); + + -- -- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: rideshare; Owner: - -- @@ -635,6 +688,13 @@ CREATE UNIQUE INDEX index_fast_search_results_on_driver_id ON rideshare.fast_sea CREATE UNIQUE INDEX index_locations_on_address ON rideshare.locations USING btree (address); +-- +-- Name: index_rollups_on_name_and_interval_and_time_and_dimensions; Type: INDEX; Schema: rideshare; Owner: - +-- + +CREATE UNIQUE INDEX index_rollups_on_name_and_interval_and_time_and_dimensions ON rideshare.rollups USING btree (name, "interval", "time", dimensions); + + -- -- Name: index_trip_requests_on_end_location_id; Type: INDEX; Schema: rideshare; Owner: - -- @@ -776,6 +836,7 @@ ALTER TABLE ONLY rideshare.trip_requests SET search_path TO rideshare; INSERT INTO "schema_migrations" (version) VALUES +('20250606201409'), ('20231220043547'), ('20231218215836'), ('20231213045957'), diff --git a/docs/rollups.md b/docs/rollups.md new file mode 100644 index 0000000..bf3f34a --- /dev/null +++ b/docs/rollups.md @@ -0,0 +1,24 @@ +# Rollups + + +## Example +- Rollups for new drivers created on the platform + +Populate drivers over the last week: +```sh +bin/rails data_generators:drivers +``` + +Create rollups: +```sh +rails runner 'Driver.rollup("New drivers")' + +Driver.rollup("New drivers", range: 1.week.ago.all_week) +``` + +Check driver counts for a day: +``` +Rollup.series("New drivers") + +Rollup.where(time: Date.yesterday).series("New drivers").values.last.to_i +``` diff --git a/erd.pdf b/erd.pdf index 0847dbf78aa0e9b21aa85b3ededf390df8990b7a..7afd51f0f38b2e13eebf68896b3ca753437fe3ef 100644 GIT binary patch delta 292 zcmV+<0o(quv;(oU1F(eGe=%#rFcgG$|B72mF?)tqoLyxA3L!2j%k-qnEtS_yyh$YM!=1wNu|(s^fkWVaOc$Q8_bM qsT`Jf&r?{Gev0nG6&3!~+X^qfOnYk=E^sRsM88qjZv3;C*qQ;JFpsnV delta 292 zcmV+<0o(quv;(oU1F(eGf5B^mFc`(}{uS>sriW=_joKnT)Iu3!P}<(c4j~3B3?oS# z>wmvQYsbjx`|-ZK_xLDCzzQ%Dq@aujti-Cwf~rDtxeXG@k+;$W0Ft+^G5A0Vf)hR` zRI9PKpuu)KZ5*{~MjZf3fi7to{CxYw2_NWD|IoO~-fY$HgV?t@9FjUOYng z*2xXBhyI||IH0Qh;OE(6q6<1Zs(Eh)?