From 30f13ccbe7b3cb3d63dbff3879bfc36467d8a782 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Jul 2026 06:03:48 +0000 Subject: [PATCH 1/2] Rebuild the metrics ingest, storage and read paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Metrics stay what they are: a small set of operator-curated, human-readable series that give context during an incident. The mechanics behind them are brought up to date. Ingest no longer discards data. CreateMetricPoint used to fetch the latest point, and if it fell inside the metric's threshold, increment its counter and throw the submitted value away — for an average-type metric that was data loss by design, and the read-then-branch had no lock, so concurrent writers left duplicate rows behind. Observations now accumulate into a fixed-width bucket through a single upsert statement that maintains a running sum_value alongside counter. calc_type finally means something: it is how observations combine within a bucket. A point resolves to the total of its observations, or their mean, and calculated_value returns that instead of the unrendered value * counter. For sum metrics the number is unchanged; for average metrics it is now correct. Behaviour changes, both deliberate: - Metric points recorded through the ingest path dispatch no model events, so MetricPointCreated and the webhook behind it no longer fire per observation. The write is an upsert and loads no model. Counter increments already fired nothing, so consumers only ever saw an arbitrary subsample of what was submitted. - Metric points are pruned after cachet.metrics.retention_days, 90 by default, by the existing model:prune schedule. Set it to null to keep every point forever and let the table grow without bound. Also in this change: - A RecordsMetricObservations contract, taking MetricObservation DTOs singly or in batch, with Cachet's bucketing recorder bound as the default and a POST metrics/{metric}/points/batch endpoint in front of it. Monitoring packages that need to record elsewhere, or bucket differently, rebind the contract. - A unique (metric_id, created_at) index, which replaces the metric_id index every ingest and render was range-scanning, and enforces one point per bucket. Rows duplicated by the old racy write are collapsed first, summing their totals into the oldest of them. - The bucket width is frozen at write time. Editing a metric's threshold changes how later observations are bucketed and leaves the buckets already written alone. - The status page drew all four chart windows from 30 days of raw points inlined into the HTML. The hour and day windows now come from 24 hours of raw buckets and the week and month windows from hourly totals rolled up in the database, behind the same 30 second cache. No rollup table: with the composite index and retention in place, GROUP BY at read time is enough until hosted telemetry says otherwise. - Charts ordered by places, the decimal-places column, where Filament and MCP order by order. Fixed. The metric points endpoint had no default sort at all, which the new index exposed; it now sorts by id. - ?include=points loaded the unbounded relation. It is capped at cachet.metrics.max_included_points, 100 by default; the paginated metric points endpoint still walks the full history. - Metrics take an optional component_id, which groups a chart under the component it describes. Display only: component health has its own pruned telemetry in component_checks, and metrics never drive component status. Checks are operational, metrics are communicative. - Cachet's cache keys are documented as tenant-agnostic — they identify the audience a value was built for, never the installation — so hosts sharing a cache store separate installations with the store's prefix rather than each feature growing its own hook. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BP5RBEiV6NoeQsGPQLC1s1 --- config/cachet.php | 23 ++ database/factories/MetricPointFactory.php | 10 + ...1_add_sum_value_to_metric_points_table.php | 39 +++ ...dd_bucket_index_to_metric_points_table.php | 103 +++++++ ...0003_add_component_id_to_metrics_table.php | 35 +++ resources/lang/en/metric.php | 3 + resources/views/components/metric.blade.php | 6 + resources/views/components/metrics.blade.php | 20 +- routes/api.php | 3 + src/Actions/Metric/CreateMetricPoint.php | 27 +- .../Metric/RecordMetricObservations.php | 260 ++++++++++++++++++ src/CachetCoreServiceProvider.php | 7 +- src/Concerns/RecordsMetricObservations.php | 31 +++ src/Data/Metrics/MetricObservation.php | 35 +++ .../Metric/CreateMetricPointsRequestData.php | 38 +++ .../Metric/CreateMetricRequestData.php | 6 + .../Metric/UpdateMetricRequestData.php | 6 + .../Resources/Metrics/MetricResource.php | 11 + src/Http/Controllers/Api/MetricController.php | 9 +- .../Controllers/Api/MetricPointController.php | 39 ++- src/Http/Resources/Metric.php | 4 +- src/Http/Resources/MetricPoint.php | 1 + src/Mcp/Concerns/PresentsResources.php | 2 + src/Models/Component.php | 14 + src/Models/Metric.php | 38 ++- src/Models/MetricPoint.php | 149 +++++++++- src/View/Components/Metrics.php | 156 +++++++++-- tests/Feature/Api/MetricPointTest.php | 85 ++++++ tests/Feature/Api/MetricTest.php | 66 +++++ .../Actions/Metric/CreateMetricPointTest.php | 111 +++++--- .../Metric/RecordMetricObservationsTest.php | 175 ++++++++++++ .../MetricPointBucketMigrationTest.php | 79 ++++++ tests/Unit/Models/MetricPointTest.php | 89 ++++++ tests/Unit/Models/MetricTest.php | 33 +++ tests/Unit/Views/MetricsTest.php | 92 +++++++ 35 files changed, 1718 insertions(+), 87 deletions(-) create mode 100644 database/migrations/2026_07_25_000001_add_sum_value_to_metric_points_table.php create mode 100644 database/migrations/2026_07_25_000002_add_bucket_index_to_metric_points_table.php create mode 100644 database/migrations/2026_07_25_000003_add_component_id_to_metrics_table.php create mode 100644 src/Actions/Metric/RecordMetricObservations.php create mode 100644 src/Concerns/RecordsMetricObservations.php create mode 100644 src/Data/Metrics/MetricObservation.php create mode 100644 src/Data/Requests/Metric/CreateMetricPointsRequestData.php create mode 100644 tests/Unit/Actions/Metric/RecordMetricObservationsTest.php create mode 100644 tests/Unit/Database/MetricPointBucketMigrationTest.php create mode 100644 tests/Unit/Models/MetricPointTest.php diff --git a/config/cachet.php b/config/cachet.php index 3d429c0c..de598bbb 100644 --- a/config/cachet.php +++ b/config/cachet.php @@ -214,6 +214,29 @@ 'prune_checks_after_days' => env('CACHET_PRUNE_CHECKS_AFTER_DAYS', 30), ], + /* + |-------------------------------------------------------------------------- + | Cachet Metrics + |-------------------------------------------------------------------------- + | + | Metrics are a curated, human-readable display series, not a time series + | database. "retention_days" is how long metric points are kept before + | the model:prune scheduled task removes them; set it to null to keep + | every point forever, and expect the table to grow without bound. + | + | "max_included_points" caps how many points the API will attach to a + | metric through "?include=points". Use the metric points endpoint, + | which is paginated, to walk the full history of a metric. + | + */ + 'metrics' => [ + 'retention_days' => env('CACHET_METRICS_RETENTION_DAYS', 90), + + 'max_included_points' => env('CACHET_METRICS_MAX_INCLUDED_POINTS', 100), + + 'max_batch_points' => env('CACHET_METRICS_MAX_BATCH_POINTS', 1000), + ], + /* |-------------------------------------------------------------------------- | Cachet Webhooks diff --git a/database/factories/MetricPointFactory.php b/database/factories/MetricPointFactory.php index 8cec8dfc..d3ce2f29 100644 --- a/database/factories/MetricPointFactory.php +++ b/database/factories/MetricPointFactory.php @@ -5,6 +5,7 @@ use Cachet\Models\Metric; use Cachet\Models\MetricPoint; use Illuminate\Database\Eloquent\Factories\Factory; +use Illuminate\Support\Carbon; /** * @extends Factory @@ -13,6 +14,14 @@ class MetricPointFactory extends Factory { protected $model = MetricPoint::class; + /** + * How many points this factory has made. + * + * A metric holds at most one point per timestamp, so generated points + * are spread a minute apart rather than all landing on now(). + */ + private static int $made = 0; + /** * Define the model's default state. * @@ -24,6 +33,7 @@ public function definition(): array 'metric_id' => Metric::factory(), 'value' => 1, 'counter' => 1, + 'created_at' => fn (): Carbon => Carbon::now()->startOfMinute()->subMinutes(self::$made++ % 1440), ]; } } diff --git a/database/migrations/2026_07_25_000001_add_sum_value_to_metric_points_table.php b/database/migrations/2026_07_25_000001_add_sum_value_to_metric_points_table.php new file mode 100644 index 00000000..8eb3a022 --- /dev/null +++ b/database/migrations/2026_07_25_000001_add_sum_value_to_metric_points_table.php @@ -0,0 +1,39 @@ +decimal('sum_value', 20, 3)->default(0)->after('value'); + }); + + $connection = Schema::getConnection(); + $grammar = $connection->getQueryGrammar(); + + $connection->statement(sprintf( + 'update %s set %s = %s * %s', + $grammar->wrapTable('metric_points'), + $grammar->wrap('sum_value'), + $grammar->wrap('value'), + $grammar->wrap('counter'), + )); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('metric_points', function (Blueprint $table) { + $table->dropColumn('sum_value'); + }); + } +}; diff --git a/database/migrations/2026_07_25_000002_add_bucket_index_to_metric_points_table.php b/database/migrations/2026_07_25_000002_add_bucket_index_to_metric_points_table.php new file mode 100644 index 00000000..5be29095 --- /dev/null +++ b/database/migrations/2026_07_25_000002_add_bucket_index_to_metric_points_table.php @@ -0,0 +1,103 @@ +collapseDuplicateBuckets(); + + if (Schema::hasIndex('metric_points', ['metric_id'])) { + Schema::table('metric_points', function (Blueprint $table) { + $table->dropIndex(['metric_id']); + }); + } + + Schema::table('metric_points', function (Blueprint $table) { + $table->unique(['metric_id', 'created_at']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('metric_points', function (Blueprint $table) { + $table->dropUnique(['metric_id', 'created_at']); + $table->index('metric_id'); + }); + } + + /** + * Merge rows sharing a metric and timestamp into the oldest of them. + * + * Each pass removes the buckets it collapses, so re-querying the first + * chunk of duplicates always makes progress. + */ + private function collapseDuplicateBuckets(): void + { + while (true) { + $duplicates = DB::table('metric_points') + ->select('metric_id', 'created_at') + ->whereNotNull('created_at') + ->groupBy('metric_id', 'created_at') + ->havingRaw('count(*) > 1') + ->limit(self::CHUNK) + ->get(); + + if ($duplicates->isEmpty()) { + return; + } + + foreach ($duplicates as $duplicate) { + $this->collapseBucket($duplicate->metric_id, $duplicate->created_at); + } + } + } + + /** + * Collapse a single duplicated bucket. + */ + private function collapseBucket(int|string $metricId, string $createdAt): void + { + /** @var Collection $points */ + $points = DB::table('metric_points') + ->where('metric_id', $metricId) + ->where('created_at', $createdAt) + ->orderBy('id') + ->get(); + + if ($points->count() < 2) { + return; + } + + DB::table('metric_points')->where('id', $points->first()->id)->update([ + 'value' => $points->last()->value, + 'sum_value' => $points->sum(fn (object $point): float => (float) $point->sum_value), + 'counter' => $points->sum(fn (object $point): int => (int) $point->counter), + ]); + + DB::table('metric_points') + ->whereIn('id', $points->skip(1)->pluck('id')->all()) + ->delete(); + } +}; diff --git a/database/migrations/2026_07_25_000003_add_component_id_to_metrics_table.php b/database/migrations/2026_07_25_000003_add_component_id_to_metrics_table.php new file mode 100644 index 00000000..d00d0f17 --- /dev/null +++ b/database/migrations/2026_07_25_000003_add_component_id_to_metrics_table.php @@ -0,0 +1,35 @@ +unsignedInteger('component_id')->nullable()->after('order'); + + $table->index('component_id'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('metrics', function (Blueprint $table) { + $table->dropIndex(['component_id']); + $table->dropColumn('component_id'); + }); + } +}; diff --git a/resources/lang/en/metric.php b/resources/lang/en/metric.php index 70c4b968..01a9c7bb 100644 --- a/resources/lang/en/metric.php +++ b/resources/lang/en/metric.php @@ -13,6 +13,7 @@ 'default_view' => 'Default view', 'threshold' => 'Threshold', 'order' => 'Order', + 'component' => 'Component', 'visible' => 'Visible', 'points_count' => 'Points count', 'created_at' => 'Created at', @@ -32,6 +33,8 @@ 'calc_type_label' => 'Metric type', 'places_label' => 'Places', 'threshold_label' => 'Threshold', + 'component_label' => 'Component', + 'component_helper_text' => 'Display this metric alongside a component. Metrics never affect a component\'s status.', 'visible_label' => 'Visible', 'display_chart_label' => 'Display chart', diff --git a/resources/views/components/metric.blade.php b/resources/views/components/metric.blade.php index 2b897a92..2ee77f04 100644 --- a/resources/views/components/metric.blade.php +++ b/resources/views/components/metric.blade.php @@ -13,6 +13,12 @@ class="group relative overflow-hidden rounded-lg bg-white shadow-sm ring-1 ring-
+ @if($metric->component) +
+ {{ $metric->component->name }} +
+ @endif +

{{ $metric->name }} diff --git a/resources/views/components/metrics.blade.php b/resources/views/components/metrics.blade.php index a1186963..7250a074 100644 --- a/resources/views/components/metrics.blade.php +++ b/resources/views/components/metrics.blade.php @@ -49,20 +49,26 @@ function getThemeColors() { let themeColors = getThemeColors() - function init() { - // Parse metric points - const metricPoints = this.metric.metric_points.map((point) => { + function parsePoints(points) { + return (points ?? []).map((point) => { return { x: new Date(point.x), y: point.y, } }) + } + + function init() { + // The hour and day windows are drawn from raw buckets; the week and + // month windows from hourly totals rolled up in the database. + const rawPoints = parsePoints(this.metric.chart_points?.raw) + const hourlyPoints = parsePoints(this.metric.chart_points?.hourly) // Filter points based on the selected period - this.points[0] = metricPoints.filter((point) => point.x >= previousHour) - this.points[1] = metricPoints.filter((point) => point.x >= previous24Hours) - this.points[2] = metricPoints.filter((point) => point.x >= previous7Days) - this.points[3] = metricPoints.filter((point) => point.x >= previous30Days) + this.points[0] = rawPoints.filter((point) => point.x >= previousHour) + this.points[1] = rawPoints.filter((point) => point.x >= previous24Hours) + this.points[2] = hourlyPoints.filter((point) => point.x >= previous7Days) + this.points[3] = hourlyPoints.filter((point) => point.x >= previous30Days) // Initialize chart const chart = new Chart(this.$refs.canvas, { diff --git a/routes/api.php b/routes/api.php index c997a324..e2100eb1 100644 --- a/routes/api.php +++ b/routes/api.php @@ -59,6 +59,9 @@ ]) ->scoped(['updateable_id']); + Route::post('metrics/{metric}/points/batch', [MetricPointController::class, 'storeBatch']) + ->name('metrics.points.batch'); + Route::apiResource('metrics.points', MetricPointController::class, [ 'except' => ['index', 'show', 'update'], ]) diff --git a/src/Actions/Metric/CreateMetricPoint.php b/src/Actions/Metric/CreateMetricPoint.php index 0c90f51d..7993c5ad 100644 --- a/src/Actions/Metric/CreateMetricPoint.php +++ b/src/Actions/Metric/CreateMetricPoint.php @@ -2,30 +2,31 @@ namespace Cachet\Actions\Metric; +use Cachet\Concerns\RecordsMetricObservations; +use Cachet\Data\Metrics\MetricObservation; use Cachet\Data\Requests\Metric\CreateMetricPointRequestData; use Cachet\Models\Metric; use Cachet\Models\MetricPoint; +use Illuminate\Support\Carbon; class CreateMetricPoint { + public function __construct(private readonly RecordsMetricObservations $recorder) {} + /** * Handle the action. + * + * The submitted value is now added to the metric's bucket for that + * moment rather than replacing it, so an observation arriving inside + * the metric's threshold is no longer thrown away. */ public function handle(Metric $metric, ?CreateMetricPointRequestData $data = null): MetricPoint { - $lastPoint = $metric->metricPoints()->latest()->first(); - - // If the last point was created within the threshold, increment the counter. - if ($lastPoint && $lastPoint->withinThreshold($metric->threshold, $data->timestamp ?? null)) { - $lastPoint->increment('counter'); - - return $lastPoint; - } + $timestamp = $data->timestamp ?? null; - return $metric->metricPoints()->create([ - 'value' => $data->value ?? $metric->default_value, - 'counter' => 1, - 'created_at' => $data->timestamp ?? now(), - ]); + return $this->recorder->record($metric, new MetricObservation( + value: (float) ($data->value ?? $metric->default_value ?? 0), + recordedAt: $timestamp === null ? null : Carbon::parse($timestamp), + )); } } diff --git a/src/Actions/Metric/RecordMetricObservations.php b/src/Actions/Metric/RecordMetricObservations.php new file mode 100644 index 00000000..360ce304 --- /dev/null +++ b/src/Actions/Metric/RecordMetricObservations.php @@ -0,0 +1,260 @@ +recordMany($metric, [$observation])->firstOrFail(); + } + + /** + * Record a batch of observations against a metric. + * + * Observations landing in the same bucket are combined before they are + * written, so a batch costs one statement per bucket it touches. + * + * @param iterable $observations + * @return Collection + */ + public function recordMany(Metric $metric, iterable $observations): Collection + { + $buckets = $this->bucket($metric, $observations); + + if ($buckets->isEmpty()) { + return new Collection; + } + + foreach ($buckets as $bucket) { + $this->accumulate($metric, $bucket); + } + + return $this->points($metric, $buckets->keys()->all()); + } + + /** + * Combine the observations into the buckets they belong to, keyed by the + * bucket's timestamp. The last observation to land in a bucket sets its + * "value", which is the most recent reading rather than a total. + * + * @param iterable $observations + * @return Collection + */ + private function bucket(Metric $metric, iterable $observations): Collection + { + $width = $this->bucketSeconds($metric); + + /** @var Collection $buckets */ + $buckets = new Collection; + + foreach ($observations as $observation) { + $at = $this->bucketFor($observation->timestamp(), $width); + $key = $at->toDateTimeString(); + + $buckets->put($key, [ + 'at' => $at, + 'value' => $observation->value, + 'sum' => ($buckets->get($key)['sum'] ?? 0.0) + $observation->value, + 'counter' => ($buckets->get($key)['counter'] ?? 0) + 1, + ]); + } + + return $buckets; + } + + /** + * Floor a timestamp onto the bucket boundary it falls within. + */ + private function bucketFor(CarbonInterface $timestamp, int $width): CarbonInterface + { + return Carbon::createFromTimestamp(intdiv($timestamp->getTimestamp(), $width) * $width); + } + + /** + * The bucket width, in seconds, for a metric. + */ + private function bucketSeconds(Metric $metric): int + { + $threshold = (int) $metric->threshold; + + return $threshold > 0 ? $threshold * 60 : self::DEFAULT_BUCKET_SECONDS; + } + + /** + * Add a bucket's observations to the metric's point for that bucket, + * creating the point when this is the first observation to land in it. + * + * @param array{at: CarbonInterface, value: float, sum: float, counter: int} $bucket + */ + private function accumulate(Metric $metric, array $bucket): void + { + $connection = $metric->getConnection(); + + if (($statement = $this->upsertStatement($connection)) === null) { + $this->accumulateWithoutUpsert($connection, $metric, $bucket); + + return; + } + + $connection->statement($statement, [ + $metric->getKey(), + $bucket['value'], + $bucket['sum'], + $bucket['counter'], + $bucket['at'], + Carbon::now(), + ]); + } + + /** + * Build the accumulating upsert for the connection's driver. + * + * Laravel's own upsert() overwrites the conflicting row, so the totals + * have to be added in driver specific SQL. MySQL's values() form is + * used over the newer row alias so MariaDB is covered too. + */ + private function upsertStatement(Connection $connection): ?string + { + $grammar = $connection->getQueryGrammar(); + $table = $grammar->wrapTable((new MetricPoint)->getTable()); + + $columns = []; + + foreach (['metric_id', 'value', 'sum_value', 'counter', 'created_at', 'updated_at'] as $column) { + $columns[$column] = $grammar->wrap($column); + } + + $insert = sprintf( + 'insert into %s (%s) values (?, ?, ?, ?, ?, ?)', + $table, + implode(', ', $columns), + ); + + return match ($connection->getDriverName()) { + 'mysql', 'mariadb' => $insert.sprintf( + ' on duplicate key update %s = %s + values(%s), %s = %s + values(%s), %s = values(%s), %s = values(%s)', + $columns['sum_value'], $columns['sum_value'], $columns['sum_value'], + $columns['counter'], $columns['counter'], $columns['counter'], + $columns['value'], $columns['value'], + $columns['updated_at'], $columns['updated_at'], + ), + 'pgsql', 'sqlite' => $insert.sprintf( + ' on conflict (%s, %s) do update set %s = %s.%s + excluded.%s, %s = %s.%s + excluded.%s, %s = excluded.%s, %s = excluded.%s', + $columns['metric_id'], $columns['created_at'], + $columns['sum_value'], $table, $columns['sum_value'], $columns['sum_value'], + $columns['counter'], $table, $columns['counter'], $columns['counter'], + $columns['value'], $columns['value'], + $columns['updated_at'], $columns['updated_at'], + ), + default => null, + }; + } + + /** + * Accumulate on a driver with no accumulating upsert of its own. + * + * The update is still a single atomic statement; only the first + * observation in a bucket costs a second round trip, and a writer + * that loses the race to insert simply accumulates instead. + * + * @param array{at: CarbonInterface, value: float, sum: float, counter: int} $bucket + */ + private function accumulateWithoutUpsert(Connection $connection, Metric $metric, array $bucket): void + { + if ($this->increment($connection, $metric, $bucket) > 0) { + return; + } + + try { + $this->table($connection)->insert([ + 'metric_id' => $metric->getKey(), + 'value' => $bucket['value'], + 'sum_value' => $bucket['sum'], + 'counter' => $bucket['counter'], + 'created_at' => $bucket['at'], + 'updated_at' => Carbon::now(), + ]); + } catch (UniqueConstraintViolationException) { + $this->increment($connection, $metric, $bucket); + } + } + + /** + * Add a bucket's totals to an existing point, returning the rows affected. + * + * @param array{at: CarbonInterface, value: float, sum: float, counter: int} $bucket + */ + private function increment(Connection $connection, Metric $metric, array $bucket): int + { + return $this->table($connection) + ->where('metric_id', $metric->getKey()) + ->where('created_at', $bucket['at']) + ->incrementEach( + ['sum_value' => $bucket['sum'], 'counter' => $bucket['counter']], + ['value' => $bucket['value'], 'updated_at' => Carbon::now()], + ); + } + + /** + * A query against the metric points table, without the model's events. + */ + private function table(Connection $connection): QueryBuilder + { + return $connection->table((new MetricPoint)->getTable()); + } + + /** + * Read the buckets back, with their metric already attached so callers + * can resolve each point's value without a query per point. + * + * @param list $buckets + * @return Collection + */ + private function points(Metric $metric, array $buckets): Collection + { + return MetricPoint::query() + ->where('metric_id', $metric->getKey()) + ->whereIn('created_at', $buckets) + ->orderBy('created_at') + ->get() + ->each(fn (MetricPoint $point) => $point->setRelation('metric', $metric)); + } +} diff --git a/src/CachetCoreServiceProvider.php b/src/CachetCoreServiceProvider.php index e80dbe38..330c27d9 100644 --- a/src/CachetCoreServiceProvider.php +++ b/src/CachetCoreServiceProvider.php @@ -3,6 +3,7 @@ namespace Cachet; use BladeUI\Icons\Factory; +use Cachet\Actions\Metric\RecordMetricObservations; use Cachet\Commands\AssetsCommand; use Cachet\Commands\CheckComponentsCommand; use Cachet\Commands\MakeUserCommand; @@ -11,6 +12,7 @@ use Cachet\Commands\PublishScheduledCommand; use Cachet\Commands\SendBeaconCommand; use Cachet\Commands\VersionCommand; +use Cachet\Concerns\RecordsMetricObservations; use Cachet\Database\Seeders\DatabaseSeeder; use Cachet\Database\Seeders\DemoMetricSeeder; use Cachet\Listeners\SendWebhookListener; @@ -20,6 +22,7 @@ use Cachet\Models\ComponentCheck; use Cachet\Models\ComponentGroup; use Cachet\Models\Incident; +use Cachet\Models\MetricPoint; use Cachet\Models\Schedule; use Cachet\Models\Subscriber; use Cachet\Models\WebhookAttempt; @@ -65,6 +68,8 @@ public function register(): void $this->app->singleton(ViewManager::class); $this->app->scoped(Status::class); + $this->app->bind(RecordsMetricObservations::class, RecordMetricObservations::class); + $this->configureSettingsCache(); } @@ -316,7 +321,7 @@ private function registerSchedules(): void $schedule->command('cachet:publish-scheduled')->everyMinute(); $schedule->command('model:prune', [ - '--model' => [WebhookAttempt::class, ComponentCheck::class], + '--model' => [WebhookAttempt::class, ComponentCheck::class, MetricPoint::class], ])->daily(); $schedule->command('db:seed', [ diff --git a/src/Concerns/RecordsMetricObservations.php b/src/Concerns/RecordsMetricObservations.php new file mode 100644 index 00000000..c6e8bbf6 --- /dev/null +++ b/src/Concerns/RecordsMetricObservations.php @@ -0,0 +1,31 @@ + $observations + * @return Collection + */ + public function recordMany(Metric $metric, iterable $observations): Collection; +} diff --git a/src/Data/Metrics/MetricObservation.php b/src/Data/Metrics/MetricObservation.php new file mode 100644 index 00000000..18016281 --- /dev/null +++ b/src/Data/Metrics/MetricObservation.php @@ -0,0 +1,35 @@ +recordedAt ?? Carbon::now(); + } +} diff --git a/src/Data/Requests/Metric/CreateMetricPointsRequestData.php b/src/Data/Requests/Metric/CreateMetricPointsRequestData.php new file mode 100644 index 00000000..3b6778b2 --- /dev/null +++ b/src/Data/Requests/Metric/CreateMetricPointsRequestData.php @@ -0,0 +1,38 @@ + $points + */ + public function __construct( + #[DataCollectionOf(CreateMetricPointRequestData::class)] + public readonly DataCollection $points, + ) {} + + /** + * @return array + */ + public static function rules(ValidationContext $context): array + { + return [ + /** + * The points to record. Points falling into the same bucket are + * combined, so a batch costs one write per bucket it touches. + */ + 'points' => [ + 'required', + 'array', + 'min:1', + 'max:'.(int) config('cachet.metrics.max_batch_points', 1000), + ], + ]; + } +} diff --git a/src/Data/Requests/Metric/CreateMetricRequestData.php b/src/Data/Requests/Metric/CreateMetricRequestData.php index ebcba0bf..3e32851f 100644 --- a/src/Data/Requests/Metric/CreateMetricRequestData.php +++ b/src/Data/Requests/Metric/CreateMetricRequestData.php @@ -19,6 +19,7 @@ public function __construct( public readonly ?bool $displayChart = null, public readonly ?int $threshold = null, public readonly ?int $places = null, + public readonly ?int $componentId = null, ) {} public static function rules(ValidationContext $context): array @@ -32,6 +33,11 @@ public static function rules(ValidationContext $context): array 'display_chart' => ['nullable', 'boolean'], 'threshold' => ['int', 'min:0', 'max:60', new FactorOfSixty], 'places' => ['int', 'min:0', 'max:4'], + /** + * The component to display this metric alongside. Display only: + * a metric never affects a component's status. + */ + 'component_id' => ['nullable', 'integer', 'exists:components,id'], ]; } } diff --git a/src/Data/Requests/Metric/UpdateMetricRequestData.php b/src/Data/Requests/Metric/UpdateMetricRequestData.php index 0111da37..e9d6ed4d 100644 --- a/src/Data/Requests/Metric/UpdateMetricRequestData.php +++ b/src/Data/Requests/Metric/UpdateMetricRequestData.php @@ -14,6 +14,7 @@ public function __construct( public readonly ?string $description = null, public readonly ?float $defaultValue = null, public readonly ?int $threshold = null, + public readonly ?int $componentId = null, ) {} public static function rules(ValidationContext $context): array @@ -24,6 +25,11 @@ public static function rules(ValidationContext $context): array 'description' => ['string'], 'default_value' => ['decimal:1,2'], 'threshold' => ['int', 'min:0', 'max:60', new FactorOfSixty], + /** + * The component to display this metric alongside. Display only: + * a metric never affects a component's status. + */ + 'component_id' => ['nullable', 'integer', 'exists:components,id'], ]; } } diff --git a/src/Filament/Resources/Metrics/MetricResource.php b/src/Filament/Resources/Metrics/MetricResource.php index 2d1396ed..ff544598 100644 --- a/src/Filament/Resources/Metrics/MetricResource.php +++ b/src/Filament/Resources/Metrics/MetricResource.php @@ -79,6 +79,13 @@ public static function form(Schema $schema): Schema ->numeric() ->default(5) ->columnSpan(2), + Select::make('component_id') + ->label(__('cachet::metric.form.component_label')) + ->helperText(__('cachet::metric.form.component_helper_text')) + ->relationship('component', 'name') + ->searchable() + ->preload() + ->columnSpan(4), ])->columnSpan(3), Section::make()->schema([ ToggleButtons::make('visible') @@ -142,6 +149,10 @@ public static function table(Table $table): Table ->numeric() ->sortable() ->toggleable(isToggledHiddenByDefault: true), + TextColumn::make('component.name') + ->label(__('cachet::metric.list.headers.component')) + ->sortable() + ->toggleable(isToggledHiddenByDefault: true), TextColumn::make('visible') ->label(__('cachet::metric.list.headers.visible')) ->badge() diff --git a/src/Http/Controllers/Api/MetricController.php b/src/Http/Controllers/Api/MetricController.php index 647de29c..1e8421c4 100644 --- a/src/Http/Controllers/Api/MetricController.php +++ b/src/Http/Controllers/Api/MetricController.php @@ -33,6 +33,7 @@ class MetricController extends Controller */ #[QueryParameter('filter[name]', 'Filter by name.', example: 'metric name')] #[QueryParameter('filter[calc_type]', 'Filter by calculation type.', type: MetricTypeEnum::class)] + #[QueryParameter('filter[component_id]', 'Filter by the component the metric is displayed with.', type: 'int', example: 1)] #[QueryParameter('per_page', 'How many items to show per page.', type: 'int', default: 15, example: 20)] #[QueryParameter('page', 'Which page to show.', type: 'int', example: 2)] public function index(Request $request) @@ -45,9 +46,10 @@ public function index(Request $request) $metrics = QueryBuilder::for($query) ->allowedIncludes([ - AllowedInclude::relationship('points', 'metricPoints'), + AllowedInclude::relationship('points', 'includedMetricPoints'), + AllowedInclude::relationship('component'), ]) - ->allowedFilters(['name', 'calc_type']) + ->allowedFilters(['name', 'calc_type', 'component_id']) ->allowedSorts(['name', 'order', 'id']) ->simplePaginate(Number::clamp($request->integer('per_page', 15), min: 1, max: 100)); @@ -73,7 +75,8 @@ public function show(Metric $metric) { $metricQuery = QueryBuilder::for(Metric::query()->visible($this->isAuthenticated())) ->allowedIncludes([ - AllowedInclude::relationship('points', 'metricPoints'), + AllowedInclude::relationship('points', 'includedMetricPoints'), + AllowedInclude::relationship('component'), ]) ->findOrFail($metric->id); diff --git a/src/Http/Controllers/Api/MetricPointController.php b/src/Http/Controllers/Api/MetricPointController.php index f3ce3965..29c6d75e 100644 --- a/src/Http/Controllers/Api/MetricPointController.php +++ b/src/Http/Controllers/Api/MetricPointController.php @@ -6,15 +6,20 @@ use Cachet\Actions\Metric\DeleteMetricPoint; use Cachet\Concerns\ChecksApiAuthentication; use Cachet\Concerns\GuardsApiAbilities; +use Cachet\Concerns\RecordsMetricObservations; +use Cachet\Data\Metrics\MetricObservation; use Cachet\Data\Requests\Metric\CreateMetricPointRequestData; +use Cachet\Data\Requests\Metric\CreateMetricPointsRequestData; use Cachet\Http\Resources\MetricPoint as MetricPointResource; use Cachet\Models\Metric; use Cachet\Models\MetricPoint; use Dedoc\Scramble\Attributes\Group; use Dedoc\Scramble\Attributes\QueryParameter; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Http\Request; use Illuminate\Http\Response; use Illuminate\Routing\Controller; +use Illuminate\Support\Carbon; use Illuminate\Support\Number; use Spatie\QueryBuilder\QueryBuilder; @@ -34,13 +39,20 @@ public function index(Request $request, Metric $metric) $this->ensureMetricVisible($metric); $query = MetricPoint::query() - ->where('metric_id', $metric->id); + ->where('metric_id', $metric->id) + ->when(! $request->input('sort'), function (Builder $builder) { + $builder->orderBy('id'); + }); $points = QueryBuilder::for($query) ->allowedIncludes(['metric']) ->allowedSorts(['id', 'value', 'created_at']) ->simplePaginate(Number::clamp($request->integer('per_page', 15), min: 1, max: 100)); + // Every point on this page belongs to the metric we already have, and + // each one reads its calculation type back off it. + $points->through(fn (MetricPoint $point) => $point->setRelation('metric', $metric)); + return MetricPointResource::collection($points); } @@ -58,6 +70,29 @@ public function store(CreateMetricPointRequestData $data, Metric $metric, Create ->setStatusCode(Response::HTTP_CREATED); } + /** + * Create Metric Points + * + * Record many observations against a metric in one request. Observations + * that fall into the same bucket are combined before they are written, + * so the response holds one point per bucket the batch touched. + */ + public function storeBatch(CreateMetricPointsRequestData $data, Metric $metric, RecordsMetricObservations $recorder) + { + $this->guard('metric-points.manage'); + + $points = $recorder->recordMany($metric, $data->points->toCollection()->map( + fn (CreateMetricPointRequestData $point): MetricObservation => new MetricObservation( + value: $point->value, + recordedAt: $point->timestamp === null ? null : Carbon::parse($point->timestamp), + ), + )); + + return MetricPointResource::collection($points) + ->response() + ->setStatusCode(Response::HTTP_CREATED); + } + /** * Get Metric Point */ @@ -69,6 +104,8 @@ public function show(Metric $metric, MetricPoint $metricPoint) ->allowedIncludes(['metric']) ->find($metricPoint->id); + $metricPointQuery?->setRelation('metric', $metric); + return MetricPointResource::make($metricPointQuery) ->response() ->setStatusCode(Response::HTTP_OK); diff --git a/src/Http/Resources/Metric.php b/src/Http/Resources/Metric.php index 785a16d0..187381cd 100644 --- a/src/Http/Resources/Metric.php +++ b/src/Http/Resources/Metric.php @@ -22,6 +22,7 @@ public function toAttributes(Request $request): array 'default_view' => $this->default_view, 'threshold' => $this->threshold, 'order' => $this->order, + 'component_id' => $this->component_id, 'visible' => $this->visible, 'created' => [ 'human' => $this->created_at?->diffForHumans(), @@ -37,7 +38,8 @@ public function toAttributes(Request $request): array public function toRelationships(Request $request): array { return [ - 'points' => fn () => MetricPoint::collection($this->metricPoints), + 'points' => fn () => MetricPoint::collection($this->includedMetricPoints), + 'component' => fn () => Component::make($this->component), ]; } } diff --git a/src/Http/Resources/MetricPoint.php b/src/Http/Resources/MetricPoint.php index d13c61a7..f2daceae 100644 --- a/src/Http/Resources/MetricPoint.php +++ b/src/Http/Resources/MetricPoint.php @@ -15,6 +15,7 @@ public function toAttributes(Request $request): array 'metric_id' => $this->metric_id, 'calculated_value' => $this->calculated_value, 'value' => $this->value, + 'sum_value' => $this->sum_value, 'counter' => $this->counter, 'created' => [ 'human' => $this->created_at?->diffForHumans(), diff --git a/src/Mcp/Concerns/PresentsResources.php b/src/Mcp/Concerns/PresentsResources.php index 30870203..95a8ac02 100644 --- a/src/Mcp/Concerns/PresentsResources.php +++ b/src/Mcp/Concerns/PresentsResources.php @@ -165,6 +165,7 @@ protected function presentMetric(Metric $metric): array 'threshold' => $metric->threshold, 'visible' => $this->presentEnum($metric->visible), 'order' => $metric->order, + 'component_id' => $metric->component_id, 'created_at' => $metric->created_at?->toIso8601String(), 'updated_at' => $metric->updated_at?->toIso8601String(), ], $metric->relationLoaded('metricPoints') ? [ @@ -181,6 +182,7 @@ protected function presentMetricPoint(MetricPoint $point): array 'id' => $point->id, 'metric_id' => $point->metric_id, 'value' => $point->value, + 'sum_value' => $point->sum_value, 'counter' => $point->counter, 'calculated_value' => $point->calculated_value, 'created_at' => $point->created_at?->toIso8601String(), diff --git a/src/Models/Component.php b/src/Models/Component.php index b70b479c..cbd8590a 100644 --- a/src/Models/Component.php +++ b/src/Models/Component.php @@ -147,6 +147,20 @@ public function checks(): HasMany return $this->hasMany(ComponentCheck::class); } + /** + * Get the metrics displayed alongside the component. + * + * Checks are operational and drive the component's status; metrics are + * communicative and only ever describe it. The two are deliberately + * separate, and metrics never feed back into component status. + * + * @return HasMany + */ + public function metrics(): HasMany + { + return $this->hasMany(Metric::class); + } + /** * Get the subscribers for this component. */ diff --git a/src/Models/Metric.php b/src/Models/Metric.php index 43d5e3ac..f1f8e117 100644 --- a/src/Models/Metric.php +++ b/src/Models/Metric.php @@ -14,6 +14,7 @@ use Illuminate\Database\Eloquent\Factories\Factory; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; +use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Support\Collection; @@ -31,7 +32,9 @@ * @property MetricViewEnum $default_view * @property int $threshold * @property int $order + * @property ?int $component_id * @property ResourceVisibilityEnum $visible + * @property ?Component $component * @property Collection $metricPoints * @property Collection $recentMetricPoints * @@ -76,8 +79,23 @@ class Metric extends Model 'visible', 'order', 'threshold', + 'component_id', ]; + /** + * Get the component this metric describes, if any. + * + * The link groups the metric's chart under a component on the status + * page and is display only. A metric never drives component status; + * that is what component checks and incidents are for. + * + * @return BelongsTo + */ + public function component(): BelongsTo + { + return $this->belongsTo(Component::class); + } + /** * Get the metrics points. * @@ -85,17 +103,35 @@ class Metric extends Model */ public function metricPoints(): HasMany { - return $this->hasMany(MetricPoint::class); + return $this->hasMany(MetricPoint::class)->chaperone(); } /** * Get the most recent metric points. + * + * @return HasMany */ public function recentMetricPoints(int $points = 15): HasMany { return $this->metricPoints()->latest()->limit($points); } + /** + * Get the metric points the API is willing to embed in a metric. + * + * A metric's point history is unbounded, so the relationship behind + * "?include=points" has to be capped. Walk the paginated metric + * points endpoint for anything more than the cap. + * + * @return HasMany + */ + public function includedMetricPoints(): HasMany + { + return $this->recentMetricPoints( + (int) config('cachet.metrics.max_included_points', 100) + ); + } + /** * Create a new factory instance for the model. */ diff --git a/src/Models/MetricPoint.php b/src/Models/MetricPoint.php index 43c2f29c..4c03a145 100644 --- a/src/Models/MetricPoint.php +++ b/src/Models/MetricPoint.php @@ -3,20 +3,33 @@ namespace Cachet\Models; use Cachet\Database\Factories\MetricPointFactory; +use Cachet\Enums\MetricTypeEnum; use Cachet\Events\Metrics\MetricPointCreated; use Cachet\Events\Metrics\MetricPointDeleted; use DateTime; +use DateTimeInterface; +use Illuminate\Database\Connection; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\Factory; use Illuminate\Database\Eloquent\Factories\HasFactory; +use Illuminate\Database\Eloquent\MassPrunable; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Support\Carbon; +use Illuminate\Support\Collection; /** + * A bucket of observations recorded against a metric. + * + * "value" is the most recent observation to land in the bucket, "sum_value" + * the total of every observation in it, and "counter" how many there were. + * The metric's calculation type decides which of those it reads back as. + * * @property int $id * @property int $metric_id * @property float $value + * @property float $sum_value * @property ?Carbon $created_at * @property ?Carbon $updated_at * @property int $counter @@ -30,9 +43,12 @@ class MetricPoint extends Model /** @use HasFactory */ use HasFactory; + use MassPrunable; + /** @var array */ protected $casts = [ 'value' => 'float', + 'sum_value' => 'float', ]; /** @var array */ @@ -44,17 +60,94 @@ class MetricPoint extends Model /** @var list */ protected $fillable = [ 'value', + 'sum_value', 'counter', 'created_at', ]; + /** + * Combine a bucket's totals into the value it displays as. + */ + public static function resolveValue(MetricTypeEnum $calcType, float $sum, int $counter): float + { + return match ($calcType) { + MetricTypeEnum::sum => $sum, + MetricTypeEnum::average => $counter > 0 ? $sum / $counter : $sum, + }; + } + + /** + * Roll the given metrics' points up into hourly totals. + * + * The wider chart windows do not need every bucket, and reading 30 days + * of raw points to draw a month of chart is what made the status page + * inline megabytes of JSON. Returns null when the connection's driver + * has no hour expression, leaving the caller to read raw points. + * + * @param list $metricIds + * @return ?Collection> + */ + public static function hourlyTotals(array $metricIds, DateTimeInterface $since): ?Collection + { + if ($metricIds === []) { + return new Collection; + } + + $instance = new self; + $connection = $instance->getConnection(); + + if (($hour = self::hourExpression($connection)) === null) { + return null; + } + + $grammar = $connection->getQueryGrammar(); + + /** @var Collection> $totals */ + $totals = $connection->table($instance->getTable()) + ->selectRaw(sprintf( + '%s as metric_id, %s as bucket, sum(%s) as sum_value, sum(%s) as counter', + $grammar->wrap('metric_id'), + $hour, + $grammar->wrap('sum_value'), + $grammar->wrap('counter'), + )) + ->whereIn('metric_id', $metricIds) + ->where('created_at', '>=', $since) + ->groupByRaw(sprintf('%s, %s', $grammar->wrap('metric_id'), $hour)) + ->orderByRaw($hour) + ->get() + ->groupBy(fn (object $row): int => (int) $row->metric_id); + + return $totals; + } + + /** + * The bucket's value, resolved through its metric's calculation type. + */ public function calculatedValue(): Attribute { return Attribute::make( - get: fn () => $this->value * $this->counter + get: fn (): float => self::resolveValue( + $this->calcType(), + (float) $this->sum_value, + (int) $this->counter, + ) ); } + /** + * The calculation type of the metric this point belongs to. + * + * A point orphaned from its metric reads back as a sum, which is what + * every point was before the calculation type meant anything. + */ + public function calcType(): MetricTypeEnum + { + $metric = $this->getRelationValue('metric'); + + return $metric instanceof Metric ? $metric->calc_type : MetricTypeEnum::sum; + } + /** * Override the created_at column to round the value into a 30-second interval. */ @@ -80,6 +173,8 @@ public function createdAt(): Attribute /** * Get the metric the point belongs to. + * + * @return BelongsTo */ public function metric(): BelongsTo { @@ -100,6 +195,58 @@ public function withinThreshold(int $threshold, string|int|DateTime|null $timest return $this->created_at->startOfMinute()->diffInMinutes($now->startOfMinute(), true) < $threshold; } + /** + * Get the prunable model query. + * + * A retention of null — or anything that is not a positive number of + * days — keeps every point forever, and lets the table grow without + * bound. Nothing is ever pruned by accident. + * + * @return Builder + */ + public function prunable(): Builder + { + $days = config('cachet.metrics.retention_days', 90); + + if (! is_numeric($days) || (int) $days <= 0) { + return static::query()->whereRaw('1 = 0'); + } + + return static::query()->where('created_at', '<', now()->subDays((int) $days)); + } + + /** + * Seed the running total for points created through the model, so a point + * written outside the recorder still reads back correctly. + */ + protected static function booted(): void + { + static::creating(function (MetricPoint $point): void { + if ($point->getAttribute('sum_value') === null) { + $point->setAttribute( + 'sum_value', + (float) $point->value * max(1, (int) ($point->counter ?? 1)), + ); + } + }); + } + + /** + * Truncate a point's timestamp to the hour it falls in. + */ + private static function hourExpression(Connection $connection): ?string + { + $column = $connection->getQueryGrammar()->wrap('created_at'); + + return match ($connection->getDriverName()) { + 'mysql', 'mariadb' => "date_format({$column}, '%Y-%m-%d %H:00:00')", + 'pgsql' => "to_char({$column}, 'YYYY-MM-DD HH24:00:00')", + 'sqlite' => "strftime('%Y-%m-%d %H:00:00', {$column})", + 'sqlsrv' => "format({$column}, 'yyyy-MM-dd HH:00:00')", + default => null, + }; + } + /** * Create a new factory instance for the model. */ diff --git a/src/View/Components/Metrics.php b/src/View/Components/Metrics.php index b9d3e2de..17e201ae 100644 --- a/src/View/Components/Metrics.php +++ b/src/View/Components/Metrics.php @@ -2,10 +2,13 @@ namespace Cachet\View\Components; +use Cachet\Enums\MetricTypeEnum; use Cachet\Models\Metric; +use Cachet\Models\MetricPoint; use Cachet\Settings\AppSettings; use Illuminate\Contracts\View\View; use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Eloquent\Collection as EloquentCollection; use Illuminate\Support\Carbon; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Cache; @@ -18,6 +21,16 @@ class Metrics extends Component */ private const CACHE_TTL = 30; + /** + * How far back the raw buckets behind the hour and day windows reach. + */ + private const RAW_WINDOW_HOURS = 24; + + /** + * How far back the hourly totals behind the week and month windows reach. + */ + private const HOURLY_WINDOW_DAYS = 30; + public function __construct(protected AppSettings $appSettings) { // @@ -25,7 +38,7 @@ public function __construct(protected AppSettings $appSettings) public function render(): View { - $cacheKey = 'cachet::metrics.'.(auth()->check() ? 'users' : 'guests'); + $cacheKey = $this->cacheKey(); $cached = Cache::remember($cacheKey, self::CACHE_TTL, fn (): Collection => $this->prepareMetrics()); @@ -34,6 +47,19 @@ public function render(): View ]); } + /** + * The cache key the prepared metrics are stored under. + * + * Cachet's cache keys are tenant-agnostic: they identify the audience a + * value was built for, never the installation it belongs to. Anything + * running more than one Cachet against a shared cache store has to + * separate them with the cache store's own prefix. + */ + private function cacheKey(): string + { + return 'cachet::metrics.'.(auth()->check() ? 'users' : 'guests'); + } + /** * Return the cached metrics, or rebuild them from the database when the * cached value is not a collection of metrics. A stale entry left behind @@ -52,35 +78,133 @@ private function validMetrics(mixed $cached, string $cacheKey): Collection } /** - * Build the metrics collection with each point cast to Chart.js format. + * Build the metrics collection, with the series each chart draws attached. + * + * The hour and day windows are drawn from raw buckets, and the week and + * month windows from hourly totals rolled up in the database. A month of + * raw buckets was previously inlined into the page to draw all four. */ private function prepareMetrics(): Collection { - $metrics = $this->metrics(Carbon::now()->subDays(30)); + $rawSince = Carbon::now()->subHours(self::RAW_WINDOW_HOURS); + $hourlySince = Carbon::now()->subDays(self::HOURLY_WINDOW_DAYS); + + $metrics = $this->metrics($hourlySince); + + if ($metrics->isEmpty()) { + return $metrics; + } - $metrics->each(function ($metric) { - $metric->metricPoints->transform(fn ($point) => [ - 'x' => $point->created_at->utc(), - 'y' => $point->value, - ]); - }); + $metricIds = $metrics->modelKeys(); - return $metrics; + $raw = $this->rawTotals($metricIds, $rawSince); + $hourly = $this->hourlyTotals($metricIds, $hourlySince) + ?? $this->rawTotals($metricIds, $hourlySince); + + return $metrics->each(fn (Metric $metric) => $metric->setAttribute('chart_points', [ + 'raw' => $this->chartPoints($metric, $raw->get($metric->getKey(), [])), + 'hourly' => $this->chartPoints($metric, $hourly->get($metric->getKey(), [])), + ])); } /** - * Fetch the available metrics and their points within the chart window. + * Fetch the metrics with a chart to draw. + * + * @return EloquentCollection */ - private function metrics(Carbon $startDate): Collection + private function metrics(Carbon $startDate): EloquentCollection { return Metric::query() ->visible(auth()->check()) - ->with([ - 'metricPoints' => fn ($query) => $query->where('created_at', '>=', $startDate)->orderBy('created_at'), - ]) + ->with('component') ->where('display_chart', true) ->where(fn (Builder $query) => $query->where('show_when_empty', true)->orWhereHas('metricPoints', fn (Builder $query) => $query->where('created_at', '>=', $startDate))) - ->orderBy('places') + ->orderBy('order') + ->orderBy('id') ->get(); } + + /** + * Read the raw buckets for the given metrics, keyed by metric. + * + * @param list $metricIds + * @return Collection> + */ + private function rawTotals(array $metricIds, Carbon $since): Collection + { + return MetricPoint::query() + ->whereIn('metric_id', $metricIds) + ->where('created_at', '>=', $since) + ->orderBy('created_at') + ->get(['metric_id', 'created_at', 'sum_value', 'counter']) + ->groupBy(fn (MetricPoint $point): int => (int) $point->metric_id) + ->map(function (EloquentCollection $points): array { + $totals = []; + + foreach ($points as $point) { + if (! ($at = $point->created_at) instanceof Carbon) { + continue; + } + + $totals[] = [ + 'at' => $at, + 'sum' => (float) $point->sum_value, + 'counter' => (int) $point->counter, + ]; + } + + return $totals; + }); + } + + /** + * Roll the given metrics' buckets up into hourly totals, keyed by metric. + * + * Returns null when the database cannot do the rollup, leaving the caller + * to fall back to raw buckets. + * + * @param list $metricIds + * @return ?Collection> + */ + private function hourlyTotals(array $metricIds, Carbon $since): ?Collection + { + return MetricPoint::hourlyTotals($metricIds, $since) + ?->map(function (Collection $rows): array { + $totals = []; + + foreach ($rows as $row) { + $totals[] = [ + 'at' => Carbon::parse($row->bucket), + 'sum' => (float) $row->sum_value, + 'counter' => (int) $row->counter, + ]; + } + + return $totals; + }); + } + + /** + * Cast a metric's totals into the series Chart.js draws, resolving each + * bucket through the metric's calculation type. + * + * @param list $totals + * @return list + */ + private function chartPoints(Metric $metric, array $totals): array + { + $places = $metric->places ?? 2; + + return array_map(fn (array $total): array => [ + 'x' => $total['at']->utc()->toIso8601String(), + 'y' => round( + MetricPoint::resolveValue( + $metric->calc_type ?? MetricTypeEnum::sum, + $total['sum'], + $total['counter'], + ), + $places, + ), + ], $totals); + } } diff --git a/tests/Feature/Api/MetricPointTest.php b/tests/Feature/Api/MetricPointTest.php index 93081a3a..d377f28a 100644 --- a/tests/Feature/Api/MetricPointTest.php +++ b/tests/Feature/Api/MetricPointTest.php @@ -1,5 +1,6 @@ assertNotFound(); }); + +it('cannot batch create metric points without the token ability', function () { + Sanctum::actingAs(User::factory()->create()); + + $metric = Metric::factory()->create(); + + $response = postJson('/status/api/metrics/'.$metric->id.'/points/batch', [ + 'points' => [['value' => 1]], + ]); + + $response->assertForbidden(); +}); + +it('can batch create metric points', function () { + Sanctum::actingAs(User::factory()->create(), ['metric-points.manage']); + + $metric = Metric::factory()->create(['threshold' => 5]); + + $response = postJson('/status/api/metrics/'.$metric->id.'/points/batch', [ + 'points' => [ + ['value' => 1, 'timestamp' => '2026-07-25 12:01:00'], + ['value' => 2, 'timestamp' => '2026-07-25 12:03:00'], + ['value' => 4, 'timestamp' => '2026-07-25 12:06:00'], + ], + ]); + + $response->assertCreated(); + + // Two of the three observations share a bucket, so they are combined. + $response->assertJsonCount(2, 'data'); + $response->assertJsonFragment(['sum_value' => 3, 'counter' => 2]); + $this->assertDatabaseCount('metric_points', 2); +}); + +it('rejects a batch without any points', function () { + Sanctum::actingAs(User::factory()->create(), ['metric-points.manage']); + + $metric = Metric::factory()->create(); + + $response = postJson('/status/api/metrics/'.$metric->id.'/points/batch', ['points' => []]); + + $response->assertUnprocessable(); + $response->assertJsonValidationErrors('points'); +}); + +it('rejects a batch larger than the configured maximum', function () { + config()->set('cachet.metrics.max_batch_points', 2); + + Sanctum::actingAs(User::factory()->create(), ['metric-points.manage']); + + $metric = Metric::factory()->create(); + + $response = postJson('/status/api/metrics/'.$metric->id.'/points/batch', [ + 'points' => [['value' => 1], ['value' => 2], ['value' => 3]], + ]); + + $response->assertUnprocessable(); + $response->assertJsonValidationErrors('points'); +}); + +it('rejects a batch holding an invalid point', function () { + Sanctum::actingAs(User::factory()->create(), ['metric-points.manage']); + + $metric = Metric::factory()->create(); + + $response = postJson('/status/api/metrics/'.$metric->id.'/points/batch', [ + 'points' => [['value' => 1], ['value' => 'not-a-number']], + ]); + + $response->assertUnprocessable(); + $response->assertJsonValidationErrors('points.1.value'); +}); + +it('reports a point through its metric calculation type', function () { + $metric = Metric::factory() + ->hasMetricPoints(1, ['value' => 5, 'counter' => 4]) + ->create(['calc_type' => MetricTypeEnum::average]); + + $response = getJson('/status/api/metrics/'.$metric->id.'/points'); + + $response->assertOk(); + $response->assertJsonPath('data.0.attributes.calculated_value', 5); + $response->assertJsonPath('data.0.attributes.sum_value', 20); +}); diff --git a/tests/Feature/Api/MetricTest.php b/tests/Feature/Api/MetricTest.php index 3189fff1..6c484ff2 100644 --- a/tests/Feature/Api/MetricTest.php +++ b/tests/Feature/Api/MetricTest.php @@ -2,6 +2,7 @@ use Cachet\Enums\MetricTypeEnum; use Cachet\Enums\ResourceVisibilityEnum; +use Cachet\Models\Component; use Cachet\Models\Metric; use Laravel\Sanctum\Sanctum; use Workbench\App\User; @@ -420,3 +421,68 @@ $response->assertOk(); $response->assertJsonCount(2, 'data'); }); + +it('caps the points embedded in a metric', function () { + config()->set('cachet.metrics.max_included_points', 3); + + $metric = Metric::factory()->hasMetricPoints(10)->create(); + + $response = getJson('/status/api/metrics/'.$metric->id.'?include=points'); + + $response->assertOk(); + $response->assertJsonCount(3, 'included'); +}); + +it('can filter metrics by component', function () { + $component = Component::factory()->create(); + + Metric::factory()->create(['component_id' => $component->id]); + Metric::factory()->create(); + + $response = getJson('/status/api/metrics?filter[component_id]='.$component->id); + + $response->assertOk(); + $response->assertJsonCount(1, 'data'); + $response->assertJsonPath('data.0.attributes.component_id', $component->id); +}); + +it('can include the component a metric is displayed alongside', function () { + $component = Component::factory()->create(['name' => 'API Gateway']); + $metric = Metric::factory()->create(['component_id' => $component->id]); + + $response = getJson('/status/api/metrics/'.$metric->id.'?include=component'); + + $response->assertOk(); + $response->assertJsonPath('included.0.attributes.name', 'API Gateway'); +}); + +it('can set the component a metric is displayed alongside', function () { + Sanctum::actingAs(User::factory()->create(), ['metrics.manage']); + + $component = Component::factory()->create(); + + $response = postJson('/status/api/metrics', [ + 'name' => 'Response time', + 'suffix' => 'ms', + 'component_id' => $component->id, + ]); + + $response->assertCreated(); + $this->assertDatabaseHas('metrics', [ + 'name' => 'Response time', + 'component_id' => $component->id, + ]); +}); + +it('cannot set an unknown component on a metric', function () { + Sanctum::actingAs(User::factory()->create(), ['metrics.manage']); + + $response = postJson('/status/api/metrics', [ + 'name' => 'Response time', + 'suffix' => 'ms', + 'component_id' => 12345, + ]); + + $response->assertUnprocessable(); + $response->assertJsonValidationErrors('component_id'); +}); diff --git a/tests/Unit/Actions/Metric/CreateMetricPointTest.php b/tests/Unit/Actions/Metric/CreateMetricPointTest.php index 0e51b753..950f8673 100644 --- a/tests/Unit/Actions/Metric/CreateMetricPointTest.php +++ b/tests/Unit/Actions/Metric/CreateMetricPointTest.php @@ -2,6 +2,7 @@ use Cachet\Actions\Metric\CreateMetricPoint; use Cachet\Data\Requests\Metric\CreateMetricPointRequestData; +use Cachet\Enums\MetricTypeEnum; use Cachet\Events\Metrics\MetricPointCreated; use Cachet\Models\Metric; use Cachet\Models\MetricPoint; @@ -9,8 +10,6 @@ use Illuminate\Support\Facades\Event; it('creates a metric point if it is the first point', function () { - Event::fake(); - $metric = Metric::factory()->create(); $point = app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from([ @@ -23,14 +22,13 @@ $this->assertDatabaseHas('metric_points', [ 'metric_id' => $metric->id, 'value' => 1, + 'sum_value' => 1, 'counter' => 1, ]); $this->assertDatabaseCount('metric_points', 1); - Event::assertDispatched(MetricPointCreated::class); }); it('creates a metric point with a default value', function () { - Event::fake(); $metric = Metric::factory()->create([ 'default_value' => 1234, ]); @@ -41,63 +39,90 @@ $this->assertDatabaseHas('metric_points', [ 'metric_id' => $metric->id, 'value' => 1234, + 'sum_value' => 1234, 'counter' => 1, ]); $this->assertDatabaseCount('metric_points', 1); - Event::assertDispatched(MetricPointCreated::class); }); -it('increments the counter if within the metric\'s threshold', function () { - Event::fake(); - $metric = Metric::factory()->hasMetricPoints(1, [ - 'created_at' => now()->startOfMinute(), - ])->create([ - 'threshold' => 1, - ]); +it('accumulates into the bucket if within the metric\'s threshold', function () { + Carbon::setTestNow('2026-07-25 12:01:00'); - $point = app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from([ - 'value' => 1, - ])); + $metric = Metric::factory()->create(['threshold' => 5]); - expect($point)->toBeInstanceOf(MetricPoint::class); - $this->assertDatabaseHas('metric_points', [ - 'metric_id' => $metric->id, - 'value' => 1, - 'counter' => 2, - ]); + app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from(['value' => 4])); + + Carbon::setTestNow('2026-07-25 12:03:00'); + + $point = app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from(['value' => 6])); + + // The second value is added to the bucket rather than thrown away, and + // "value" holds the reading that arrived most recently. + expect($point->value)->toBe(6.0) + ->and($point->sum_value)->toBe(10.0) + ->and($point->counter)->toBe(2) + ->and($point->created_at->toDateTimeString())->toBe('2026-07-25 12:00:00'); $this->assertDatabaseCount('metric_points', 1); - Event::assertDispatched(MetricPointCreated::class); }); it('creates a metric point if it is outside of the metric\'s threshold', function () { - Event::fake(); - $metric = Metric::factory()->hasMetricPoints(1, [ - 'created_at' => now()->addMinutes(5), - ])->create([ - 'threshold' => 1, + Carbon::setTestNow('2026-07-25 12:01:00'); + + $metric = Metric::factory()->create(['threshold' => 5]); + + app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from(['value' => 4])); + + Carbon::setTestNow('2026-07-25 12:06:00'); + + $point = app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from(['value' => 6])); + + expect($point->sum_value)->toBe(6.0) + ->and($point->counter)->toBe(1) + ->and($point->created_at->toDateTimeString())->toBe('2026-07-25 12:05:00'); + $this->assertDatabaseCount('metric_points', 2); +}); + +it('resolves a sum metric to the total of its observations', function () { + $metric = Metric::factory()->create([ + 'calc_type' => MetricTypeEnum::sum, + 'threshold' => 5, ]); - $point = app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from([ - 'value' => 1, - ])); + Carbon::setTestNow('2026-07-25 12:01:00'); + app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from(['value' => 4])); + $point = app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from(['value' => 6])); - expect($point)->toBeInstanceOf(MetricPoint::class); - $this->assertDatabaseHas('metric_points', [ - 'metric_id' => $metric->id, - 'value' => 1, - 'counter' => 1, + expect($point->calculated_value)->toBe(10.0); +}); + +it('resolves an average metric to the mean of its observations', function () { + $metric = Metric::factory()->create([ + 'calc_type' => MetricTypeEnum::average, + 'threshold' => 5, ]); - $this->assertDatabaseCount('metric_points', 2); - Event::assertDispatched(MetricPointCreated::class); + + Carbon::setTestNow('2026-07-25 12:01:00'); + app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from(['value' => 4])); + $point = app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from(['value' => 6])); + + expect($point->calculated_value)->toBe(5.0); }); -it('creates a metric point for a given timestamp', function ($timestamp) { +it('no longer dispatches a created event per observation', function () { Event::fake(); - $metric = Metric::factory()->hasMetricPoints(1, [ - 'created_at' => now()->subHour()->startOfMinute(), - ])->create([ - 'threshold' => 1, - ]); + + $metric = Metric::factory()->create(); + + app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from(['value' => 1])); + + // Points are written with an accumulating upsert, which loads no model + // and so fires no model events. Counter increments never fired one + // either, so consumers only ever saw an arbitrary subsample. + Event::assertNotDispatched(MetricPointCreated::class); +}); + +it('creates a metric point for a given timestamp', function ($timestamp) { + $metric = Metric::factory()->create(['threshold' => 1]); $point = app(CreateMetricPoint::class)->handle($metric, CreateMetricPointRequestData::from([ 'value' => 1, diff --git a/tests/Unit/Actions/Metric/RecordMetricObservationsTest.php b/tests/Unit/Actions/Metric/RecordMetricObservationsTest.php new file mode 100644 index 00000000..e00275d3 --- /dev/null +++ b/tests/Unit/Actions/Metric/RecordMetricObservationsTest.php @@ -0,0 +1,175 @@ +toBeInstanceOf(RecordMetricObservations::class); +}); + +it('can be rebound by a monitoring package', function () { + $fake = new class implements RecordsMetricObservations + { + public function record(Metric $metric, MetricObservation $observation): MetricPoint + { + return $this->recordMany($metric, [$observation])->firstOrFail(); + } + + public function recordMany(Metric $metric, iterable $observations): Collection + { + return new Collection([new MetricPoint(['value' => 99])]); + } + }; + + app()->instance(RecordsMetricObservations::class, $fake); + + $point = app(RecordsMetricObservations::class)->record( + Metric::factory()->create(), + new MetricObservation(value: 1), + ); + + expect($point->value)->toBe(99.0); + $this->assertDatabaseCount('metric_points', 0); +}); + +it('folds a batch into one point per bucket', function () { + $metric = Metric::factory()->create(['threshold' => 5]); + + $points = app(RecordsMetricObservations::class)->recordMany($metric, [ + new MetricObservation(value: 1, recordedAt: Carbon::parse('2026-07-25 12:01:00')), + new MetricObservation(value: 2, recordedAt: Carbon::parse('2026-07-25 12:03:00')), + new MetricObservation(value: 4, recordedAt: Carbon::parse('2026-07-25 12:04:59')), + new MetricObservation(value: 8, recordedAt: Carbon::parse('2026-07-25 12:05:00')), + ]); + + expect($points)->toHaveCount(2); + $this->assertDatabaseCount('metric_points', 2); + + expect($points->first()->created_at->toDateTimeString())->toBe('2026-07-25 12:00:00') + ->and($points->first()->sum_value)->toBe(7.0) + ->and($points->first()->counter)->toBe(3); + + expect($points->last()->created_at->toDateTimeString())->toBe('2026-07-25 12:05:00') + ->and($points->last()->sum_value)->toBe(8.0) + ->and($points->last()->counter)->toBe(1); +}); + +it('accumulates a batch onto buckets that already exist', function () { + $metric = Metric::factory()->create(['threshold' => 5]); + $recorder = app(RecordsMetricObservations::class); + + $recorder->record($metric, new MetricObservation(value: 3, recordedAt: Carbon::parse('2026-07-25 12:02:00'))); + $recorder->recordMany($metric, [ + new MetricObservation(value: 5, recordedAt: Carbon::parse('2026-07-25 12:03:00')), + new MetricObservation(value: 7, recordedAt: Carbon::parse('2026-07-25 12:04:00')), + ]); + + $this->assertDatabaseCount('metric_points', 1); + $point = MetricPoint::query()->sole(); + + expect($point->sum_value)->toBe(15.0) + ->and($point->counter)->toBe(3) + ->and($point->value)->toBe(7.0); +}); + +it('returns an empty collection for an empty batch', function () { + $points = app(RecordsMetricObservations::class)->recordMany(Metric::factory()->create(), []); + + expect($points)->toBeEmpty(); + $this->assertDatabaseCount('metric_points', 0); +}); + +it('freezes the bucket width at write time', function () { + $metric = Metric::factory()->create(['threshold' => 60]); + $recorder = app(RecordsMetricObservations::class); + + $recorder->record($metric, new MetricObservation(value: 1, recordedAt: Carbon::parse('2026-07-25 12:30:00'))); + + // Narrowing the threshold buckets later observations differently and + // leaves everything already written alone. + $metric->update(['threshold' => 5]); + + $recorder->record($metric, new MetricObservation(value: 2, recordedAt: Carbon::parse('2026-07-25 12:47:00'))); + + expect(MetricPoint::query()->orderBy('created_at')->pluck('created_at')->map->toDateTimeString()->all()) + ->toBe(['2026-07-25 12:00:00', '2026-07-25 12:45:00']); +}); + +it('buckets to thirty seconds when a metric has no threshold', function () { + $metric = Metric::factory()->create(['threshold' => 0]); + + $point = app(RecordsMetricObservations::class)->record( + $metric, + new MetricObservation(value: 1, recordedAt: Carbon::parse('2026-07-25 12:00:44')), + ); + + expect($point->created_at->toDateTimeString())->toBe('2026-07-25 12:00:30'); +}); + +it('defaults an observation without a timestamp to now', function () { + Carbon::setTestNow('2026-07-25 12:07:00'); + + $metric = Metric::factory()->create(['threshold' => 5]); + + $point = app(RecordsMetricObservations::class)->record($metric, new MetricObservation(value: 1)); + + expect($point->created_at->toDateTimeString())->toBe('2026-07-25 12:05:00'); +}); + +it('attaches the metric to the points it returns', function () { + $metric = Metric::factory()->create(); + + $point = app(RecordsMetricObservations::class)->record($metric, new MetricObservation(value: 1)); + + expect($point->relationLoaded('metric'))->toBeTrue() + ->and($point->metric->is($metric))->toBeTrue(); +}); + +it('builds an accumulating upsert for each driver', function (string $connection, string $driver, ?string $expected) { + $connection = new $connection(fn () => null, 'testing', '', ['driver' => $driver]); + + // Only SQLite runs in the test suite, so the statement the other drivers + // would be sent is pinned here rather than left unverified. + $statement = (new ReflectionMethod(RecordMetricObservations::class, 'upsertStatement')) + ->invoke(new RecordMetricObservations, $connection); + + expect($statement)->toBe($expected); +})->with([ + [ + MySqlConnection::class, + 'mysql', + 'insert into `metric_points` (`metric_id`, `value`, `sum_value`, `counter`, `created_at`, `updated_at`) values (?, ?, ?, ?, ?, ?)' + .' on duplicate key update `sum_value` = `sum_value` + values(`sum_value`), `counter` = `counter` + values(`counter`),' + .' `value` = values(`value`), `updated_at` = values(`updated_at`)', + ], + [ + MariaDbConnection::class, + 'mariadb', + 'insert into `metric_points` (`metric_id`, `value`, `sum_value`, `counter`, `created_at`, `updated_at`) values (?, ?, ?, ?, ?, ?)' + .' on duplicate key update `sum_value` = `sum_value` + values(`sum_value`), `counter` = `counter` + values(`counter`),' + .' `value` = values(`value`), `updated_at` = values(`updated_at`)', + ], + [ + PostgresConnection::class, + 'pgsql', + 'insert into "metric_points" ("metric_id", "value", "sum_value", "counter", "created_at", "updated_at") values (?, ?, ?, ?, ?, ?)' + .' on conflict ("metric_id", "created_at") do update set "sum_value" = "metric_points"."sum_value" + excluded."sum_value",' + .' "counter" = "metric_points"."counter" + excluded."counter", "value" = excluded."value", "updated_at" = excluded."updated_at"', + ], + [ + // No accumulating upsert: the recorder falls back to an atomic + // increment, and an insert only when the bucket does not exist. + SqlServerConnection::class, + 'sqlsrv', + null, + ], +]); diff --git a/tests/Unit/Database/MetricPointBucketMigrationTest.php b/tests/Unit/Database/MetricPointBucketMigrationTest.php new file mode 100644 index 00000000..9c7e8c9d --- /dev/null +++ b/tests/Unit/Database/MetricPointBucketMigrationTest.php @@ -0,0 +1,79 @@ +dropUnique(['metric_id', 'created_at']); + }); + + DB::table('metric_points')->insert([ + ['metric_id' => $metric->id, 'value' => 1, 'sum_value' => 1, 'counter' => 1, 'created_at' => '2026-07-25 12:00:00', 'updated_at' => '2026-07-25 12:00:00'], + ['metric_id' => $metric->id, 'value' => 3, 'sum_value' => 6, 'counter' => 2, 'created_at' => '2026-07-25 12:00:00', 'updated_at' => '2026-07-25 12:00:00'], + ['metric_id' => $metric->id, 'value' => 9, 'sum_value' => 9, 'counter' => 1, 'created_at' => '2026-07-25 12:05:00', 'updated_at' => '2026-07-25 12:05:00'], + ['metric_id' => $metric->id, 'value' => 4, 'sum_value' => 4, 'counter' => 1, 'created_at' => null, 'updated_at' => null], + ['metric_id' => $metric->id, 'value' => 5, 'sum_value' => 5, 'counter' => 1, 'created_at' => null, 'updated_at' => null], + ]); +} + +function bucketMigration(): object +{ + return require dirname(__DIR__, 3).'/database/migrations/2026_07_25_000002_add_bucket_index_to_metric_points_table.php'; +} + +it('collapses duplicate buckets into one point', function () { + $metric = Metric::factory()->create(); + + seedDuplicateBuckets($metric); + + bucketMigration()->up(); + + // The duplicated noon bucket keeps the oldest row, carries the totals of + // both, and reads back the value that arrived most recently. + $noon = DB::table('metric_points')->where('created_at', '2026-07-25 12:00:00')->get(); + + expect($noon)->toHaveCount(1) + ->and((float) $noon->first()->sum_value)->toBe(7.0) + ->and((int) $noon->first()->counter)->toBe(3) + ->and((float) $noon->first()->value)->toBe(3.0); + + expect(DB::table('metric_points')->where('created_at', '2026-07-25 12:05:00')->count())->toBe(1); +}); + +it('leaves points without a timestamp alone', function () { + $metric = Metric::factory()->create(); + + seedDuplicateBuckets($metric); + + bucketMigration()->up(); + + // A unique index treats nulls as distinct, so these were never in the way + // and merging them would have destroyed data. + expect(DB::table('metric_points')->whereNull('created_at')->count())->toBe(2); +}); + +it('leaves a metric only one point per timestamp', function () { + $metric = Metric::factory()->create(); + + seedDuplicateBuckets($metric); + + bucketMigration()->up(); + + DB::table('metric_points')->insert([ + 'metric_id' => $metric->id, + 'value' => 1, + 'sum_value' => 1, + 'counter' => 1, + 'created_at' => '2026-07-25 12:00:00', + 'updated_at' => '2026-07-25 12:00:00', + ]); +})->throws(UniqueConstraintViolationException::class); diff --git a/tests/Unit/Models/MetricPointTest.php b/tests/Unit/Models/MetricPointTest.php new file mode 100644 index 00000000..aee40727 --- /dev/null +++ b/tests/Unit/Models/MetricPointTest.php @@ -0,0 +1,89 @@ +create(); + + $point = $metric->metricPoints()->create(['value' => 7, 'counter' => 3]); + + expect($point->sum_value)->toBe(21.0); +}); + +it('resolves a bucket through its metric\'s calculation type', function (MetricTypeEnum $calcType, float $expected) { + expect(MetricPoint::resolveValue($calcType, 10.0, 4))->toBe($expected); +})->with([ + [MetricTypeEnum::sum, 10.0], + [MetricTypeEnum::average, 2.5], +]); + +it('resolves an empty bucket without dividing by zero', function () { + expect(MetricPoint::resolveValue(MetricTypeEnum::average, 0.0, 0))->toBe(0.0); +}); + +it('rolls points up into hourly totals', function () { + $metric = Metric::factory()->create(['calc_type' => MetricTypeEnum::average]); + + $metric->metricPoints()->createMany([ + ['value' => 2, 'counter' => 1, 'created_at' => '2026-07-25 12:05:00'], + ['value' => 4, 'counter' => 1, 'created_at' => '2026-07-25 12:35:00'], + ['value' => 9, 'counter' => 1, 'created_at' => '2026-07-25 13:05:00'], + ]); + + $totals = MetricPoint::hourlyTotals([$metric->id], Carbon::parse('2026-07-25 00:00:00')); + + expect($totals->get($metric->id))->toHaveCount(2); + + [$noon, $onePm] = $totals->get($metric->id)->all(); + + expect((float) $noon->sum_value)->toBe(6.0) + ->and((int) $noon->counter)->toBe(2) + ->and((float) $onePm->sum_value)->toBe(9.0) + ->and((int) $onePm->counter)->toBe(1); +}); + +it('keeps hourly totals of different metrics apart', function () { + $first = Metric::factory()->create(); + $second = Metric::factory()->create(); + + $first->metricPoints()->create(['value' => 1, 'created_at' => '2026-07-25 12:05:00']); + $second->metricPoints()->create(['value' => 5, 'created_at' => '2026-07-25 12:05:00']); + + $totals = MetricPoint::hourlyTotals([$first->id, $second->id], Carbon::parse('2026-07-25 00:00:00')); + + expect((float) $totals->get($first->id)->first()->sum_value)->toBe(1.0) + ->and((float) $totals->get($second->id)->first()->sum_value)->toBe(5.0); +}); + +it('prunes points past the retention window', function () { + config()->set('cachet.metrics.retention_days', 90); + + $metric = Metric::factory()->create(); + $metric->metricPoints()->createMany([ + ['value' => 1, 'created_at' => now()->subDays(91)], + ['value' => 2, 'created_at' => now()->subDays(89)], + ]); + + Artisan::call('model:prune', ['--model' => [MetricPoint::class]]); + + expect(MetricPoint::query()->count())->toBe(1) + ->and(MetricPoint::query()->sole()->value)->toBe(2.0); +}); + +it('keeps every point when retention is disabled', function (mixed $retention) { + config()->set('cachet.metrics.retention_days', $retention); + + $metric = Metric::factory()->create(); + $metric->metricPoints()->createMany([ + ['value' => 1, 'created_at' => now()->subYears(5)], + ['value' => 2, 'created_at' => now()->subDays(1)], + ]); + + Artisan::call('model:prune', ['--model' => [MetricPoint::class]]); + + expect(MetricPoint::query()->count())->toBe(2); +})->with([null, '', 0]); diff --git a/tests/Unit/Models/MetricTest.php b/tests/Unit/Models/MetricTest.php index 08d81a96..03179982 100644 --- a/tests/Unit/Models/MetricTest.php +++ b/tests/Unit/Models/MetricTest.php @@ -1,6 +1,9 @@ hasMetricPoints(2)->create(); @@ -14,6 +17,14 @@ expect($metric->recentMetricPoints(5)->get())->toHaveCount(5); }); +it('caps the points the API is willing to embed', function () { + config()->set('cachet.metrics.max_included_points', 3); + + $metric = Metric::factory()->hasMetricPoints(10)->create(); + + expect($metric->includedMetricPoints)->toHaveCount(3); +}); + it('calculates counted values', function ($value, $counter, $expected) { $metric = Metric::factory()->hasMetricPoints(1, ['value' => $value, 'counter' => $counter])->create(); @@ -24,3 +35,25 @@ [2, 5, 10.0], [5, 5, 25.0], ]); + +it('averages counted values for an average metric', function () { + $metric = Metric::factory() + ->hasMetricPoints(1, ['value' => 5, 'counter' => 4]) + ->create(['calc_type' => MetricTypeEnum::average]); + + expect($metric->metricPoints->first()->calculated_value)->toBe(5.0); +}); + +it('reads the calculation type back without a query per point', function () { + $metric = Metric::factory()->hasMetricPoints(3)->create(); + + expect($metric->metricPoints->every(fn (MetricPoint $point) => $point->relationLoaded('metric')))->toBeTrue(); +}); + +it('can be displayed alongside a component', function () { + $component = Component::factory()->create(); + $metric = Metric::factory()->create(['component_id' => $component->id]); + + expect($metric->component->is($component))->toBeTrue() + ->and($component->metrics->first()->is($metric))->toBeTrue(); +}); diff --git a/tests/Unit/Views/MetricsTest.php b/tests/Unit/Views/MetricsTest.php index f0bc4842..a492a20d 100644 --- a/tests/Unit/Views/MetricsTest.php +++ b/tests/Unit/Views/MetricsTest.php @@ -1,8 +1,11 @@ toBeInstanceOf(Collection::class) ->first()->toBeInstanceOf(Metric::class); }); + +it('orders the charts by the metric order, not its decimal places', function () { + Metric::factory()->create([ + 'name' => 'second', + 'order' => 2, + 'places' => 0, + 'visible' => ResourceVisibilityEnum::guest, + 'display_chart' => true, + 'show_when_empty' => true, + ]); + Metric::factory()->create([ + 'name' => 'first', + 'order' => 1, + 'places' => 4, + 'visible' => ResourceVisibilityEnum::guest, + 'display_chart' => true, + 'show_when_empty' => true, + ]); + + $html = Blade::render(''); + + expect(strpos($html, 'first'))->toBeLessThan(strpos($html, 'second')); +}); + +it('draws the wider windows from hourly totals', function () { + $metric = Metric::factory()->create([ + 'calc_type' => MetricTypeEnum::sum, + 'places' => 2, + 'visible' => ResourceVisibilityEnum::guest, + 'display_chart' => true, + 'show_when_empty' => true, + ]); + + // Two buckets inside the same hour, well outside the raw window. + $metric->metricPoints()->createMany([ + ['value' => 2, 'created_at' => now()->subDays(3)->startOfHour()], + ['value' => 3, 'created_at' => now()->subDays(3)->startOfHour()->addMinutes(30)], + ]); + + $chartPoints = app(Metrics::class) + ->render() + ->getData()['metrics'] + ->first() + ->chart_points; + + expect($chartPoints['raw'])->toBeEmpty() + ->and($chartPoints['hourly'])->toHaveCount(1) + ->and($chartPoints['hourly'][0]['y'])->toBe(5.0); +}); + +it('resolves the chart series through the metric calculation type', function () { + $metric = Metric::factory()->create([ + 'calc_type' => MetricTypeEnum::average, + 'places' => 2, + 'visible' => ResourceVisibilityEnum::guest, + 'display_chart' => true, + 'show_when_empty' => true, + ]); + + $metric->metricPoints()->create([ + 'value' => 4, + 'counter' => 4, + 'created_at' => now()->subMinutes(10), + ]); + + $chartPoints = app(Metrics::class) + ->render() + ->getData()['metrics'] + ->first() + ->chart_points; + + expect($chartPoints['raw'])->toHaveCount(1) + ->and($chartPoints['raw'][0]['y'])->toBe(4.0); +}); + +it('labels a chart with the component it is displayed alongside', function () { + $component = Component::factory()->create(['name' => 'API Gateway']); + + Metric::factory()->create([ + 'component_id' => $component->id, + 'visible' => ResourceVisibilityEnum::guest, + 'display_chart' => true, + 'show_when_empty' => true, + ]); + + $html = Blade::render(''); + + expect($html)->toContain('API Gateway'); +}); From b2b3b08ed5a8699a3e7b80bd0505fac1be2cc14d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Jul 2026 09:56:29 +0000 Subject: [PATCH 2/2] Key the chart series by hand instead of through Collection::groupBy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Static analysis failed on the dependency floor. Collection generics are invariant, and the two supported Laravel versions disagree about what groupBy() returns: on 11.x it propagates the callback's int, on 13.x it widens to array-key. No single annotation satisfies both, so rawTotals() was correct on one and wrong on the other. Both helpers now build a plain array keyed by metric id with a foreach, which types identically on every supported version. hourlyTotals() moves with them, returning the resolved bucket shape rather than raw rows, so the view component no longer needs to normalise what it reads back. Verified against the failing matrix cell — PHP 8.3, Laravel 11.x, prefer-lowest — as well as the ceiling. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BP5RBEiV6NoeQsGPQLC1s1 --- src/Models/MetricPoint.php | 25 +++++---- src/View/Components/Metrics.php | 77 +++++++++------------------ tests/Unit/Models/MetricPointTest.php | 17 +++--- 3 files changed, 49 insertions(+), 70 deletions(-) diff --git a/src/Models/MetricPoint.php b/src/Models/MetricPoint.php index 4c03a145..b27abc0b 100644 --- a/src/Models/MetricPoint.php +++ b/src/Models/MetricPoint.php @@ -17,7 +17,6 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Support\Carbon; -use Illuminate\Support\Collection; /** * A bucket of observations recorded against a metric. @@ -84,13 +83,13 @@ public static function resolveValue(MetricTypeEnum $calcType, float $sum, int $c * inline megabytes of JSON. Returns null when the connection's driver * has no hour expression, leaving the caller to read raw points. * - * @param list $metricIds - * @return ?Collection> + * @param array $metricIds + * @return ?array> */ - public static function hourlyTotals(array $metricIds, DateTimeInterface $since): ?Collection + public static function hourlyTotals(array $metricIds, DateTimeInterface $since): ?array { if ($metricIds === []) { - return new Collection; + return []; } $instance = new self; @@ -102,8 +101,7 @@ public static function hourlyTotals(array $metricIds, DateTimeInterface $since): $grammar = $connection->getQueryGrammar(); - /** @var Collection> $totals */ - $totals = $connection->table($instance->getTable()) + $rows = $connection->table($instance->getTable()) ->selectRaw(sprintf( '%s as metric_id, %s as bucket, sum(%s) as sum_value, sum(%s) as counter', $grammar->wrap('metric_id'), @@ -115,8 +113,17 @@ public static function hourlyTotals(array $metricIds, DateTimeInterface $since): ->where('created_at', '>=', $since) ->groupByRaw(sprintf('%s, %s', $grammar->wrap('metric_id'), $hour)) ->orderByRaw($hour) - ->get() - ->groupBy(fn (object $row): int => (int) $row->metric_id); + ->get(); + + $totals = []; + + foreach ($rows as $row) { + $totals[(int) $row->metric_id][] = [ + 'at' => Carbon::parse($row->bucket), + 'sum' => (float) $row->sum_value, + 'counter' => (int) $row->counter, + ]; + } return $totals; } diff --git a/src/View/Components/Metrics.php b/src/View/Components/Metrics.php index 17e201ae..f805e724 100644 --- a/src/View/Components/Metrics.php +++ b/src/View/Components/Metrics.php @@ -95,15 +95,15 @@ private function prepareMetrics(): Collection return $metrics; } - $metricIds = $metrics->modelKeys(); + $metricIds = $metrics->map(fn (Metric $metric): int => $metric->id)->all(); $raw = $this->rawTotals($metricIds, $rawSince); - $hourly = $this->hourlyTotals($metricIds, $hourlySince) + $hourly = MetricPoint::hourlyTotals($metricIds, $hourlySince) ?? $this->rawTotals($metricIds, $hourlySince); return $metrics->each(fn (Metric $metric) => $metric->setAttribute('chart_points', [ - 'raw' => $this->chartPoints($metric, $raw->get($metric->getKey(), [])), - 'hourly' => $this->chartPoints($metric, $hourly->get($metric->getKey(), [])), + 'raw' => $this->chartPoints($metric, $raw[$metric->id] ?? []), + 'hourly' => $this->chartPoints($metric, $hourly[$metric->id] ?? []), ])); } @@ -127,61 +127,32 @@ private function metrics(Carbon $startDate): EloquentCollection /** * Read the raw buckets for the given metrics, keyed by metric. * - * @param list $metricIds - * @return Collection> + * @param array $metricIds + * @return array> */ - private function rawTotals(array $metricIds, Carbon $since): Collection + private function rawTotals(array $metricIds, Carbon $since): array { - return MetricPoint::query() + $points = MetricPoint::query() ->whereIn('metric_id', $metricIds) ->where('created_at', '>=', $since) ->orderBy('created_at') - ->get(['metric_id', 'created_at', 'sum_value', 'counter']) - ->groupBy(fn (MetricPoint $point): int => (int) $point->metric_id) - ->map(function (EloquentCollection $points): array { - $totals = []; - - foreach ($points as $point) { - if (! ($at = $point->created_at) instanceof Carbon) { - continue; - } - - $totals[] = [ - 'at' => $at, - 'sum' => (float) $point->sum_value, - 'counter' => (int) $point->counter, - ]; - } - - return $totals; - }); - } + ->get(['metric_id', 'created_at', 'sum_value', 'counter']); - /** - * Roll the given metrics' buckets up into hourly totals, keyed by metric. - * - * Returns null when the database cannot do the rollup, leaving the caller - * to fall back to raw buckets. - * - * @param list $metricIds - * @return ?Collection> - */ - private function hourlyTotals(array $metricIds, Carbon $since): ?Collection - { - return MetricPoint::hourlyTotals($metricIds, $since) - ?->map(function (Collection $rows): array { - $totals = []; - - foreach ($rows as $row) { - $totals[] = [ - 'at' => Carbon::parse($row->bucket), - 'sum' => (float) $row->sum_value, - 'counter' => (int) $row->counter, - ]; - } - - return $totals; - }); + $totals = []; + + foreach ($points as $point) { + if (! ($at = $point->created_at) instanceof Carbon) { + continue; + } + + $totals[(int) $point->metric_id][] = [ + 'at' => $at, + 'sum' => (float) $point->sum_value, + 'counter' => (int) $point->counter, + ]; + } + + return $totals; } /** diff --git a/tests/Unit/Models/MetricPointTest.php b/tests/Unit/Models/MetricPointTest.php index aee40727..3e40c52b 100644 --- a/tests/Unit/Models/MetricPointTest.php +++ b/tests/Unit/Models/MetricPointTest.php @@ -36,14 +36,15 @@ $totals = MetricPoint::hourlyTotals([$metric->id], Carbon::parse('2026-07-25 00:00:00')); - expect($totals->get($metric->id))->toHaveCount(2); + expect($totals[$metric->id])->toHaveCount(2); - [$noon, $onePm] = $totals->get($metric->id)->all(); + [$noon, $onePm] = $totals[$metric->id]; - expect((float) $noon->sum_value)->toBe(6.0) - ->and((int) $noon->counter)->toBe(2) - ->and((float) $onePm->sum_value)->toBe(9.0) - ->and((int) $onePm->counter)->toBe(1); + expect($noon['at']->toDateTimeString())->toBe('2026-07-25 12:00:00') + ->and($noon['sum'])->toBe(6.0) + ->and($noon['counter'])->toBe(2) + ->and($onePm['sum'])->toBe(9.0) + ->and($onePm['counter'])->toBe(1); }); it('keeps hourly totals of different metrics apart', function () { @@ -55,8 +56,8 @@ $totals = MetricPoint::hourlyTotals([$first->id, $second->id], Carbon::parse('2026-07-25 00:00:00')); - expect((float) $totals->get($first->id)->first()->sum_value)->toBe(1.0) - ->and((float) $totals->get($second->id)->first()->sum_value)->toBe(5.0); + expect($totals[$first->id][0]['sum'])->toBe(1.0) + ->and($totals[$second->id][0]['sum'])->toBe(5.0); }); it('prunes points past the retention window', function () {