feat(server): add two admin endpoints for queue and environment concurrency debugging and repairing (#2559)

Author: ericallam

apps/webapp/app/routes/admin.api.v1.environments.$environmentId.engine.repair-queues.ts

@@ -0,0 +1,95 @@
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import pMap from "p-map";
+import { z } from "zod";
+import { $replica, prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { determineEngineVersion } from "~/v3/engineVersion.server";
+import { engine } from "~/v3/runEngine.server";
+
+const ParamsSchema = z.object({
+  environmentId: z.string(),
+});
+
+const BodySchema = z.object({
+  dryRun: z.boolean().default(true),
+  queues: z.array(z.string()).default([]),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: {
+      id: authenticationResult.userId,
+    },
+  });
+
+  if (!user) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!user.admin) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  const parsedParams = ParamsSchema.parse(params);
+
+  const environment = await prisma.runtimeEnvironment.findFirst({
+    where: {
+      id: parsedParams.environmentId,
+    },
+    include: {
+      organization: true,
+      project: true,
+      orgMember: true,
+    },
+  });
+
+  if (!environment) {
+    return json({ error: "Environment not found" }, { status: 404 });
+  }
+
+  const engineVersion = await determineEngineVersion({ environment });
+
+  if (engineVersion === "V1") {
+    return json({ error: "Engine version is V1" }, { status: 400 });
+  }
+
+  const body = await request.json();
+  const parsedBody = BodySchema.parse(body);
+
+  const queues = await $replica.taskQueue.findMany({
+    where: {
+      runtimeEnvironmentId: environment.id,
+      version: "V2",
+      name: parsedBody.queues.length > 0 ? { in: parsedBody.queues } : undefined,
+    },
+    select: {
+      friendlyId: true,
+      name: true,
+      concurrencyLimit: true,
+      type: true,
+      paused: true,
+    },
+    orderBy: {
+      orderableName: "asc",
+    },
+  });
+
+  const repairEnvironmentResults = await engine.repairEnvironment(environment, parsedBody.dryRun);
+
+  const repairResults = await pMap(
+    queues,
+    async (queue) => {
+      return engine.repairQueue(environment, queue.name, parsedBody.dryRun);
+    },
+    { concurrency: 5 }
+  );
+
+  return json({ environment: repairEnvironmentResults, queues: repairResults });
+}

apps/webapp/app/routes/admin.api.v1.environments.$environmentId.engine.report.ts

@@ -0,0 +1,95 @@
+import { json, LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { $replica, prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { determineEngineVersion } from "~/v3/engineVersion.server";
+import { engine } from "~/v3/runEngine.server";
+
+const ParamsSchema = z.object({
+  environmentId: z.string(),
+});
+
+const SearchParamsSchema = z.object({
+  verbose: z.string().default("0"),
+  page: z.coerce.number().optional(),
+  per_page: z.coerce.number().optional(),
+});
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: {
+      id: authenticationResult.userId,
+    },
+  });
+
+  if (!user) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!user.admin) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  const parsedParams = ParamsSchema.parse(params);
+
+  const environment = await prisma.runtimeEnvironment.findFirst({
+    where: {
+      id: parsedParams.environmentId,
+    },
+    include: {
+      organization: true,
+      project: true,
+      orgMember: true,
+    },
+  });
+
+  if (!environment) {
+    return json({ error: "Environment not found" }, { status: 404 });
+  }
+
+  const engineVersion = await determineEngineVersion({ environment });
+
+  if (engineVersion === "V1") {
+    return json({ error: "Engine version is V1" }, { status: 400 });
+  }
+
+  const url = new URL(request.url);
+  const searchParams = SearchParamsSchema.parse(Object.fromEntries(url.searchParams));
+
+  const page = searchParams.page ?? 1;
+  const perPage = searchParams.per_page ?? 50;
+
+  const queues = await $replica.taskQueue.findMany({
+    where: {
+      runtimeEnvironmentId: environment.id,
+      version: "V2",
+    },
+    select: {
+      friendlyId: true,
+      name: true,
+      concurrencyLimit: true,
+      type: true,
+      paused: true,
+    },
+    orderBy: {
+      orderableName: "asc",
+    },
+    skip: (page - 1) * perPage,
+    take: perPage,
+  });
+
+  const report = await engine.generateEnvironmentReport(
+    environment,
+    queues,
+    searchParams.verbose === "1"
+  );
+
+  return json(report);
+}

internal-packages/run-engine/package.json

@@ -30,7 +30,8 @@
     "nanoid": "3.3.8",
     "redlock": "5.0.0-beta.2",
     "seedrandom": "^3.0.5",
-    "zod": "3.25.76"
+    "zod": "3.25.76",
+    "p-map": "^6.0.0"
   },
   "devDependencies": {
     "@internal/testcontainers": "workspace:*",