FEATURE: add metrics logging + log retention after content release is deleted

Classes/BackendUi/BackendUiDataService.php

@@ -6,6 +6,8 @@
 
 use Flowpack\DecoupledContentStore\BackendUi\Dto\ContentReleaseDetails;
 use Flowpack\DecoupledContentStore\BackendUi\Dto\ContentReleaseOverviewRow;
+use Flowpack\DecoupledContentStore\BackendUi\Dto\RemovedContentReleaseOverviewRow;
+use Flowpack\DecoupledContentStore\ContentReleaseManager;
 use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
 use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\PrunnerJobId;
 use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\RedisInstanceIdentifier;
@@ -16,6 +18,7 @@
 use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata;
 use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
 use Flowpack\DecoupledContentStore\ReleaseSwitch\Infrastructure\RedisReleaseSwitchService;
+use Flowpack\Prunner\Dto\Job;
 use Flowpack\Prunner\PrunnerApiService;
 use Neos\Flow\Annotations as Flow;
 
@@ -121,6 +124,80 @@ public function loadBackendOverviewData(RedisInstanceIdentifier $redisInstanceId
         return $result;
     }
 
+    /**
+     * The releases which Redis no longer holds but whose prunner job - and so its logs - is still there, newest first.
+     *
+     * Only the primary content store builds releases, so on any other one a job without a registered release is just
+     * a release which was never transferred there. Jobs which never started are left out: with queue_strategy
+     * "replace", most of them were replaced while waiting and have no logs at all.
+     *
+     * @return RemovedContentReleaseOverviewRow[]|null NULL when the prunner API could not be reached
+     */
+    public function loadRemovedReleasesOverviewData(RedisInstanceIdentifier $redisInstanceIdentifier): ?array
+    {
+        if (!$redisInstanceIdentifier->isPrimary()) {
+            return [];
+        }
+
+        try {
+            $jobs = $this->prunnerApiService->loadPipelinesAndJobs()->getJobs();
+        } catch (\Throwable $throwable) {
+            // the overview also holds the controls to pause releases, which must keep working while prunner is down
+            return null;
+        }
+
+        $registeredReleaseIds = array_map(
+            static fn(ContentReleaseIdentifier $id): string => $id->getIdentifier(),
+            $this->redisContentReleaseService->fetchAllReleaseIds($redisInstanceIdentifier),
+        );
+        $releasePipelines = [
+            ContentReleaseManager::CONTENT_RELEASE_PIPELINE_NAME,
+            ContentReleaseManager::QUICK_CONTENT_RELEASE_PIPELINE_NAME,
+        ];
+        $removedReleaseJobs = $jobs->filter(
+            static function (Job $job) use ($releasePipelines, $registeredReleaseIds): bool {
+                $contentReleaseId = $job->getVariables()['contentReleaseId'] ?? null;
+                return (
+                    in_array($job->getPipeline(), $releasePipelines, true)
+                    && $job->getStart() !== null
+                    && is_string($contentReleaseId)
+                    && !in_array($contentReleaseId, $registeredReleaseIds, true)
+                );
+            },
+        )->getArray();
+        usort($removedReleaseJobs, static fn(Job $a, Job $b): int => $b->getStart() <=> $a->getStart());
+
+        $contentReleaseIds = array_map(
+            static fn(Job $job): ContentReleaseIdentifier => ContentReleaseIdentifier::fromString(
+                $job->getVariables()['contentReleaseId'],
+            ),
+            $removedReleaseJobs,
+        );
+        $metadata = $this->redisContentReleaseService->fetchMetadataForContentReleases(
+            $redisInstanceIdentifier,
+            ...$contentReleaseIds,
+        );
+        $errorCounts = $this->redisRenderingErrorManager->countMultipleErrors(
+            $redisInstanceIdentifier,
+            ...$contentReleaseIds,
+        );
+
+        $result = [];
+        foreach ($removedReleaseJobs as $index => $job) {
+            $contentReleaseId = $contentReleaseIds[$index];
+            $metadataForContentRelease = $metadata->getResultForContentRelease($contentReleaseId);
+            $errorCountForContentRelease = $errorCounts->getResultForContentRelease($contentReleaseId);
+            $result[] = new RemovedContentReleaseOverviewRow(
+                $contentReleaseId,
+                $job,
+                $metadataForContentRelease instanceof ContentReleaseMetadata ? $metadataForContentRelease : null,
+                is_int($errorCountForContentRelease) ? $errorCountForContentRelease : 0,
+            );
+        }
+
+        return $result;
+    }
+
     public function loadDetailsData(
         ContentReleaseIdentifier $contentReleaseIdentifier,
         RedisInstanceIdentifier $redisInstanceIdentifier,
@@ -170,6 +247,7 @@ function (string $item) {
             $contentReleaseIdentifier->equals($currentReleaseIdentifier),
             $manualTransferJobs,
             $contentReleaseMetadata->getContentReleaseSize(),
+            !$this->redisContentReleaseService->isRegistered($contentReleaseIdentifier, $redisInstanceIdentifier),
         );
     }
 }

Classes/BackendUi/Dto/ContentReleaseDetails.php

@@ -20,6 +20,7 @@ class ContentReleaseDetails
     private int $renderingErrorCount;
     private bool $isActive;
     private ?float $contentReleaseSize;
+    private bool $isRemoved;
 
     /**
      * @var Job[]
@@ -40,6 +41,7 @@ public function __construct(
         bool $isActive,
         array $manualTransferJobIds,
         ?float $contentReleaseSize = null,
+        bool $isRemoved = false,
     ) {
         $this->contentReleaseIdentifier = $contentReleaseIdentifier;
         $this->job = $job;
@@ -49,6 +51,7 @@ public function __construct(
         $this->isActive = $isActive;
         $this->manualTransferJobs = $manualTransferJobIds;
         $this->contentReleaseSize = $contentReleaseSize;
+        $this->isRemoved = $isRemoved;
     }
 
     /**
@@ -114,4 +117,13 @@ public function getManualTransferJobs(): array
     {
         return $this->manualTransferJobs;
     }
+
+    /**
+     * A removed release is no longer registered, so it can neither be switched to nor transferred - only the keys it
+     * keeps after removal are left, and its enumeration is gone.
+     */
+    public function isRemoved(): bool
+    {
+        return $this->isRemoved;
+    }
 }

Classes/BackendUi/Dto/RemovedContentReleaseOverviewRow.php

@@ -0,0 +1,61 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flowpack\DecoupledContentStore\BackendUi\Dto;
+
+use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
+use Flowpack\DecoupledContentStore\PrepareContentRelease\Dto\ContentReleaseMetadata;
+use Flowpack\Prunner\Dto\Job;
+use Neos\Flow\Annotations as Flow;
+
+/**
+ * A release which Redis no longer holds, listed through the prunner job which built it.
+ *
+ * The metadata is only there while the keys a removed release keeps have not expired yet; without it, the job is all
+ * that is known about the release.
+ */
+#[Flow\Proxy(false)]
+final class RemovedContentReleaseOverviewRow
+{
+    public function __construct(
+        private readonly ContentReleaseIdentifier $contentReleaseIdentifier,
+        private readonly Job $job,
+        private readonly ?ContentReleaseMetadata $metadata,
+        private readonly int $errorCount,
+    ) {}
+
+    public function getContentReleaseIdentifier(): ContentReleaseIdentifier
+    {
+        return $this->contentReleaseIdentifier;
+    }
+
+    public function getJob(): Job
+    {
+        return $this->job;
+    }
+
+    public function getMetadata(): ?ContentReleaseMetadata
+    {
+        return $this->metadata;
+    }
+
+    public function getErrorCount(): int
+    {
+        return $this->errorCount;
+    }
+
+    public function getStatus(): string
+    {
+        if ($this->metadata !== null) {
+            return $this->metadata->getStatus()->getDisplayName();
+        }
+        if ($this->job->isCanceled()) {
+            return 'canceled';
+        }
+        if ($this->job->isErrored()) {
+            return 'failed';
+        }
+        return $this->job->isCompleted() ? 'done' : 'running';
+    }
+}

Classes/Controller/BackendController.php

@@ -136,6 +136,10 @@ public function indexAction(?string $contentStore = null)
 
         $this->view->assign('contentStore', $contentStore->getIdentifier());
         $this->view->assign('overviewData', $this->backendUiDataService->loadBackendOverviewData($contentStore));
+        $this->view->assign(
+            'removedReleases',
+            $this->backendUiDataService->loadRemovedReleasesOverviewData($contentStore),
+        );
         $this->view->assign('redisContentStores', array_keys($this->redisContentStores));
         $this->view->assign('storeSize', $storeSize);
         $this->view->assign('toggleFromConfigEpoch', $configEpochRedis);
@@ -174,14 +178,24 @@ public function detailsAction(
         $this->view->assign('redisContentStores', array_keys($this->redisContentStores));
         $this->view->assign('isPrimary', $contentStore->isPrimary());
 
+        // once the keys a removed release keeps have expired, its prunner job is all that is left of it
+        $removedReleaseJob = null;
+        if (is_string($prunnerJobId) && $prunnerJobId !== '' && $detailsData === null) {
+            $removedReleaseJob = $this->prunnerApiService->loadJobDetail(
+                PrunnerJobId::fromString($prunnerJobId)->toJobId(),
+            );
+        }
+        $this->view->assign('removedReleaseJob', $removedReleaseJob);
+        $job = $detailsData?->getJob() ?? $removedReleaseJob;
+
         if ($detailTaskName !== '') {
             $this->view->assign('detailTaskName', $detailTaskName);
             $this->view->assign('jobLogs', $this->prunnerApiService->loadJobLogs(
                 $prunnerJobId ? PrunnerJobId::fromString($prunnerJobId)->toJobId() : $detailsData->getJob()->getId(),
                 $detailTaskName,
             ));
-        } elseif ($showAllRenderingErrors && $detailsData->getJob() !== null) {
-            $this->view->assign('workerErrorLogs', $this->workerErrorLogAggregator->aggregate($detailsData->getJob()));
+        } elseif ($showAllRenderingErrors && $job !== null) {
+            $this->view->assign('workerErrorLogs', $this->workerErrorLogAggregator->aggregate($job));
         }
     }
 

Classes/Core/Domain/ValueObject/AutomaticReleasePauseState.php

@@ -4,8 +4,8 @@
 
 namespace Flowpack\DecoupledContentStore\Core\Domain\ValueObject;
 
-use DateMalformedStringException;
 use DateTimeImmutable;
+use Exception;
 use InvalidArgumentException;
 use Neos\Flow\Annotations as Flow;
 
@@ -34,7 +34,7 @@ private function __construct(DateTimeImmutable $pausedAt, ?string $accountId, in
      * hide the inconsistency behind a banner claiming the pause started just now.
      *
      * @param array<string, string> $redisHash
-     * @throws DateMalformedStringException
+     * @throws Exception
      */
     public static function fromRedisHash(array $redisHash): self
     {

Classes/Core/Infrastructure/RedisClientManager.php

@@ -104,4 +104,13 @@ public function getRetentionCount(RedisInstanceIdentifier $redisInstanceIdentifi
         }
         return $this->configuration[$redisInstanceIdentifier->getIdentifier()]['contentReleaseRetentionCount'];
     }
+
+    /**
+     * 0 when the content store keeps nothing of a removed release.
+     */
+    public function getRemovedReleaseRetentionSeconds(RedisInstanceIdentifier $redisInstanceIdentifier): int
+    {
+        $instanceConfiguration = $this->configuration[$redisInstanceIdentifier->getIdentifier()];
+        return (int) ($instanceConfiguration['removedReleaseRetentionSeconds'] ?? 0);
+    }
 }

Classes/Core/RedisPruneService.php

@@ -15,7 +15,8 @@ class RedisPruneService
 {
     // go through all keys in the selected content store
     // check whether they are reserved keys or currently active or contain one of the currently registered releases ids
-    // if not: delete
+    // if not: delete - unless the key expires on its own, which is how a removed release keeps its details for the
+    // Backend UI (see removedReleaseRetentionSeconds)
     const PRUNE_LUA_SCRIPT = '
         local contentStoreCurrent = redis.call("GET", "contentStore:current")
         local contentStoreAllKeys = redis.call("KEYS", "*")
@@ -36,7 +37,8 @@ class RedisPruneService
             and contentStoreKey ~= "contentStore:registeredReleases"
             and contentStoreKey ~= "contentStore:configEpoch"
             and string.sub(contentStoreKey, 1, string.len(currentContentStoreStart)) ~= currentContentStoreStart
-            and not table_contains_value(contentStoreRegisteredReleases, contentStoreKey) then
+            and not table_contains_value(contentStoreRegisteredReleases, contentStoreKey)
+            and redis.call("TTL", contentStoreKey) < 0 then
                 redis.call("DEL", contentStoreKey)
             end
         end