PAV-127 — [Front Admin] Implementar dashboard Scraper Health e métricas de classificação

.github/dependabot.yml

@@ -7,6 +7,10 @@ updates:
     schedule:
       interval: "monthly"
     open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: "typescript"
+        update-types:
+          - "version-update:semver-major"
     groups:
       npm-minor-patch:
         update-types:

backend/package.json

@@ -35,40 +35,40 @@
   "license": "ISC",
   "dependencies": {
     "@react-email/render": "^2.1.0",
-    "argon2": "^0.44.0",
-    "axios": "^1.17.0",
-    "bullmq": "^5.81.2",
+    "argon2": "^0.45.1",
+    "axios": "^1.20.0",
+    "bullmq": "^6.3.11",
     "cheerio": "^1.2.0",
     "cors": "^2.8.6",
     "dotenv": "^17.4.2",
-    "drizzle-orm": "^0.45.2",
+    "drizzle-orm": "^0.45.4",
     "express": "^5.2.1",
     "ioredis": "^5.11.1",
     "iron-session": "^8.0.4",
-    "pdfkit": "^0.18.0",
+    "pdfkit": "^0.20.2",
     "prom-client": "^15.1.3",
     "react": "^19.2.8",
     "react-dom": "^19.2.8",
-    "redis": "^5.12.1",
+    "redis": "^6.3.0",
     "resend": "^6.18.0",
     "swagger-jsdoc": "^6.3.0",
     "swagger-ui-express": "^5.0.1",
     "xlsx": "^0.18.5",
-    "zod": "^4.4.3"
+    "zod": "^4.6.5"
   },
   "engines": {
     "node": ">=22.0.0"
   },
   "devDependencies": {
-    "@types/node": "^25.9.1",
+    "@types/node": "^26.6.4",
     "@types/pg": "^8.20.0",
     "@types/react": "^19.2.17",
     "@types/supertest": "^7.2.0",
-    "@vitest/coverage-v8": "^4.1.8",
+    "@vitest/coverage-v8": "5.0.3",
     "drizzle-kit": "^0.31.10",
     "supertest": "^7.2.2",
     "tsx": "^4.22.4",
-    "typescript": "^6.0.3",
-    "vitest": "^4.1.8"
+    "typescript": "~5.9.3",
+    "vitest": "5.0.3"
   }
 }

backend/src/modules/admin/observability/observability.types.ts

@@ -88,10 +88,27 @@ export type ObservabilityOverview = z.infer<typeof ObservabilityOverviewSchema>;
 // Processor-owned runtime state. Unknown fields are stripped before forwarding,
 // so an internal response cannot expose credentials or incidental payloads.
 const nullableTimestamp = z.string().datetime().nullable();
+const runCount = z.number().int().nonnegative();
+const ProviderStateSchema = z.object({
+  id: z.enum(["linkedin", "adzuna", "themuse", "gupy", "inhire", "jooble", "greenhouse", "lever", "unknown"]),
+  configuredConcurrency: runCount,
+  active: runCount,
+  waiting: runCount,
+  tasksTotal: runCount,
+  tasksCompleted: runCount,
+  tasksFailed: runCount,
+  tasksCanceled: runCount,
+  jobsCollected: runCount,
+  lastTaskStatus: z.enum(["success", "failed", "canceled", "timeout"]).nullable(),
+  lastTaskFinishedAt: nullableTimestamp,
+  lastTaskDurationSeconds: z.number().nonnegative().nullable(),
+  lastErrorType: z.enum(["network", "timeout", "canceled", "rate_limit", "authentication", "parsing", "validation", "unknown"]).nullable(),
+});
 export const ProcessorSnapshotSchema = z.object({
   status: z.enum(["ok", "partial"]),
   timestamp: z.string().datetime(),
   execution: z.object({
+    runId: z.string().min(1).max(128).nullable().optional().default(null),
     status: z.enum(["idle", "running", "canceling", "failed", "completed"]),
     source: z.enum(["manual", "cron"]).nullable(),
     startedAt: nullableTimestamp,
@@ -108,6 +125,11 @@ export const ProcessorSnapshotSchema = z.object({
     applicationVersion: z.string().max(128),
     taxonomyVersion: z.string().max(128),
   }),
+  providers: z.array(ProviderStateSchema).max(9)
+    .refine((providers) => new Set(providers.map((p) => p.id)).size === providers.length, "Providers duplicados")
+    .nullable().optional().default(null),
+  classification: z.object({ received: runCount, classified: runCount, approved: runCount, rejected: runCount, invalid: runCount }).nullable().optional().default(null),
+  persistence: z.object({ received: runCount, inserted: runCount, updated: runCount, skipped: runCount, failed: runCount, failedBatches: runCount }).nullable().optional().default(null),
   lock: z.object({ held: z.boolean(), ttlSeconds: z.number().nonnegative() }),
   concurrency: z.object({
     configured: z.number().int().nonnegative(),

backend/src/modules/admin/scrapers/scraperClient.ts

@@ -1,3 +1,4 @@
+import { JobsCountResultSchema } from "./scrapers.types";
 import { config } from "../../../config";
 import type {
   GetJobsResult,
@@ -67,8 +68,8 @@ export const scraperClient = {
     return request<GetJobsResult>(`/admin/jobs${suffix}`);
   },
 
-  getJobsCount(): Promise<JobsCountResult> {
-    return request<JobsCountResult>("/admin/jobs/count");
+  async getJobsCount(): Promise<JobsCountResult> {
+    return JobsCountResultSchema.parse(await request<unknown>("/admin/jobs/count"));
   },
 
   reprocessJobs(): Promise<ReprocessScrapeResult> {

backend/src/modules/admin/scrapers/scrapers.types.ts

@@ -68,7 +68,13 @@ export const GetJobsResultSchema = z.object({
 export type GetJobsResult = z.infer<typeof GetJobsResultSchema>;
 
 // --- JobsCountResult ---
+const providerId = z.enum(["linkedin", "adzuna", "themuse", "gupy", "inhire", "jooble", "greenhouse", "lever", "unknown"]);
 export const JobsCountResultSchema = z.object({
-  total: z.number().int().min(0),
+  total: z.number().int().nonnegative(),
+  sources: z.array(z.object({ source: z.string().nullable(), provider: providerId, total: z.number().int().nonnegative() })).nullable().optional(),
+  configuredProviders: z.array(providerId).max(9).refine((ids) => new Set(ids).size === ids.length, "Providers duplicados").nullable().optional(),
+}).superRefine((value, ctx) => {
+  if (value.sources != null && value.sources.reduce((sum, source) => sum + source.total, 0) !== value.total)
+    ctx.addIssue({ code: "custom", message: "Distribuição incompatível com total do catálogo", path: ["sources"] });
 });
 export type JobsCountResult = z.infer<typeof JobsCountResultSchema>;

backend/tests/unit/modules/admin/catalogInventory.test.ts

@@ -0,0 +1,13 @@
+import { describe,expect,it,vi,afterEach } from "vitest";
+import { JobsCountResultSchema } from "../../../../src/modules/admin/scrapers/scrapers.types";
+import { scraperClient } from "../../../../src/modules/admin/scrapers/scraperClient";
+const fixture={total:205,sources:[{source:"Green House",provider:"greenhouse",total:200},{source:"LinkedIn",provider:"linkedin",total:4},{source:null,provider:"unknown",total:1}],configuredProviders:["greenhouse","themuse"]};
+afterEach(()=>vi.unstubAllGlobals());
+describe("complete catalog attribution",()=>{
+ it("forwards global totals independently of preview and keeps configured absent provider",async()=>{
+  vi.stubGlobal("fetch",vi.fn().mockResolvedValue({ok:true,status:200,json:async()=>({...fixture,token:"secret"})}));
+  const result=await scraperClient.getJobsCount();expect(result).toEqual(fixture);expect(result.configuredProviders).toContain("themuse");expect(result.sources?.reduce((n,s)=>n+s.total,0)).toBe(result.total);
+ });
+ it.each([{total:0,sources:[],configuredProviders:["themuse"]},{total:205,sources:null,configuredProviders:["themuse"]},{total:205,sources:fixture.sources,configuredProviders:null},{total:205}])("accepts zero, partial and old contract: %j",payload=>{expect(JobsCountResultSchema.parse(payload)).toEqual(payload)});
+ it.each([{...fixture,total:206},{...fixture,sources:[{source:"x",provider:"made-up",total:205}]},{...fixture,configuredProviders:["gupy","gupy"]},{...fixture,sources:[{source:null,provider:"unknown",total:-1}]}])("rejects incompatible counts %j",payload=>expect(JobsCountResultSchema.safeParse(payload).success).toBe(false));
+});

backend/tests/unit/modules/admin/operationalSnapshot.test.ts

@@ -179,3 +179,35 @@ it("honors the internal deadline signal and returns partial after abort", async
   timeout.mockRestore();
   vi.unstubAllGlobals();
 });
+
+const classification = { received: 3, classified: 2, approved: 1, rejected: 1, invalid: 1 };
+const persistence = { received: 1, inserted: 1, updated: 0, skipped: 0, failed: 0, failedBatches: 0 };
+const provider = { id: "greenhouse", configuredConcurrency: 2, active: 0, waiting: 0, tasksTotal: 2, tasksCompleted: 1, tasksFailed: 0, tasksCanceled: 0, jobsCollected: 3, lastTaskStatus: "success", lastTaskFinishedAt: now, lastTaskDurationSeconds: 1, lastErrorType: null };
+describe("per-execution additive contract", () => {
+  it("accepts the old Processor and preserves absence instead of inventing zero", () => {
+    const result = ProcessorSnapshotSchema.parse(fixture);
+    expect(result).toMatchObject({ providers: null, classification: null, persistence: null, execution: { runId: null } });
+  });
+  it.each(["running", "completed", "failed"])("forwards real totals and identity for %s", async (status) => {
+    vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true, json: async () => ({ ...fixture, execution: { ...fixture.execution, status, runId: "real-lease-id", startedAt: now, finishedAt: status === "running" ? null : now }, providers: [{ ...provider, token: "private-secret", error: "private-secret" }], classification, persistence }) }));
+    const result = await new ObservabilityService({} as any, {} as any).getOperationalSnapshot();
+    expect(result.processor).toMatchObject({ execution: { runId: "real-lease-id", status }, classification, persistence, providers: [provider] });
+    expect(JSON.stringify(result)).not.toContain("private-secret");
+  });
+  it("keeps explicit zero, known empty plan and unavailable totals distinct", () => {
+    const result = ProcessorSnapshotSchema.parse({ ...fixture, providers: [], classification: { received: 0, classified: 0, approved: 0, rejected: 0, invalid: 0 }, persistence: null });
+    expect(result.providers).toEqual([]);
+    expect(result.classification?.classified).toBe(0);
+    expect(result.persistence).toBeNull();
+  });
+  it.each([
+    { providers: [{ ...provider, id: "company-secret" }] },
+    { providers: Array(10).fill(provider) },
+    { providers: [provider, provider] },
+    { providers: [{ ...provider, lastErrorType: "password-secret" }] },
+    { classification: { ...classification, received: -1 } },
+    { persistence: { ...persistence, failed: 0.5 } },
+  ])("rejects unbounded or invalid new state %j", (patch) => {
+    expect(ProcessorSnapshotSchema.safeParse({ ...fixture, ...patch }).success).toBe(false);
+  });
+});

backend/tests/unit/modules/admin/scrapers.test.ts

@@ -119,11 +119,7 @@ describe("scraperClient", () => {
       jobs: [],
       total: 0,
     });
-    await expect(scraperClient.getJobsCount()).resolves.toEqual({
-      ok: true,
-      jobs: [],
-      total: 0,
-    });
+    await expect(scraperClient.getJobsCount()).resolves.toEqual({ total: 0 });
     await expect(scraperClient.reprocessJobs()).resolves.toEqual({
       ok: true,
       jobs: [],

docker-compose.yml

@@ -1,120 +1,11 @@
-services:
-  scraper-go:
-    build:
-      context: ./scraper-go
-      dockerfile: Dockerfile
-    container_name: vagas-scraper-go
-    cpus: 1.5
-    mem_limit: 2g
-    env_file:
-      - ./.env
-    volumes:
-      - ./.env:/app/.env:ro
-    environment:
-      - GO_SCRAPER_ADDR=:8081
-      - VALKEY_URL=redis://valkey:6379/0
-      - SCRAPER_MAX_CONCURRENCY=${SCRAPER_MAX_CONCURRENCY-12}
-      - SCRAPER_PROVIDER_MAX_CONCURRENCY=${SCRAPER_PROVIDER_MAX_CONCURRENCY-2}
-      - SCRAPER_PROVIDER_CONCURRENCY_OVERRIDES=${SCRAPER_PROVIDER_CONCURRENCY_OVERRIDES-}
-      - SCRAPER_RUN_LOCK_TTL=${SCRAPER_RUN_LOCK_TTL-120s}
-      - SCRAPER_RUN_LOCK_RENEW_INTERVAL=${SCRAPER_RUN_LOCK_RENEW_INTERVAL-30s}
-      - SCRAPER_CLASSIFICATION_BATCH_SIZE=${SCRAPER_CLASSIFICATION_BATCH_SIZE-100}
-      - SCRAPER_PERSIST_BATCH_SIZE=${SCRAPER_PERSIST_BATCH_SIZE-100}
-      - SCRAPER_INDEX_BATCH_SIZE=${SCRAPER_INDEX_BATCH_SIZE-250}
-      - SCRAPER_CATALOG_LIFETIME=${SCRAPER_CATALOG_LIFETIME-216h}
-      - APPLICATION_VERSION=${APPLICATION_VERSION-unknown}
-      - GOMAXPROCS=${GOMAXPROCS:-2}
-      - GOMEMLIMIT=${GOMEMLIMIT:-1500MiB}
-      - GUPY_ENABLED=${GUPY_ENABLED:-true}
-      - GUPY_RAW_DISCOVERY_ENABLED=${GUPY_RAW_DISCOVERY_ENABLED:-true}
-      - GUPY_FULL_SWEEP_ENABLED=${GUPY_FULL_SWEEP_ENABLED:-true}
-      - GUPY_FULL_REMOTE_SWEEP_ENABLED=${GUPY_FULL_REMOTE_SWEEP_ENABLED:-true}
-      - INHIRE_ENABLED=${INHIRE_ENABLED:-true}
-      - INHIRE_TENANTS_FILE=/app/internal/interfaces/inhireTenants.json
-      - INHIRE_ENRICH_DETAILS=${INHIRE_ENRICH_DETAILS:-false}
-      - INHIRE_DETAILS_MODE=${INHIRE_DETAILS_MODE:-ambiguous}
-      - INHIRE_DETAILS_TIMEOUT_MS=${INHIRE_DETAILS_TIMEOUT_MS:-10000}
-      - GREENHOUSE_ENABLED=${GREENHOUSE_ENABLED:-true}
-      - GREENHOUSE_COMPANIES_FILE=/app/internal/interfaces/greenhouseCompanies.json
-      - LEVER_ENABLED=${LEVER_ENABLED:-false}
-      - LEVER_COMPANIES_FILE=/app/internal/interfaces/leverCompanies.json
-      - LEVER_INCLUDE_ALL_JOBS=${LEVER_INCLUDE_ALL_JOBS:-true}
-    expose:
-      - "8081"
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "http://localhost:8081/health"]
-      interval: 5s
-      timeout: 3s
-      retries: 5
-      start_period: 5s
-    networks:
-      - vagas-net
-    restart: unless-stopped
+name: candidate
 
-  backend:
-    build:
-      context: .
-      dockerfile: docker/node.Dockerfile
-      target: backend
-    container_name: vagas-backend
-    env_file:
-      - ./.env
-    environment:
-      - PORT=3001
-      - JOB_SEARCH_CACHE_TTL_SECONDS=${JOB_SEARCH_CACHE_TTL_SECONDS-120}
-      - PROMETHEUS_URL=http://prometheus:9090
-      - SCRAPER_URL=http://scraper-go:8081
-      - GO_SCRAPER_URL=http://scraper-go:8081
-      - KEYWORDS_STORAGE_MODE=${KEYWORDS_STORAGE_MODE:-env}
-      - CACHE_TTL_MS=${CACHE_TTL_MS:-600000}
-      - REDIS_KEY_PREFIX=${REDIS_KEY_PREFIX:-vagas-full}
-    volumes:
-      - ./backend/output:/app/output
-    ports:
-      - "3001:3001"
-    depends_on:
-      scraper-go:
-        condition: service_healthy
-    networks:
-      - vagas-net
-    restart: unless-stopped
+include:
+  - path: ./docker/docker-compose.app.yml
+    project_directory: .
 
-  frontend:
-    build:
-      context: .
-      dockerfile: docker/node.Dockerfile
-      target: frontend-runtime
-      args:
-        VITE_API_BASE_URL: ${VITE_API_BASE_URL}
-        VITE_API_URL: ${VITE_API_URL}
-        VITE_APP_ENV: ${VITE_APP_ENV:-production}
-    container_name: vagas-frontend
-    ports:
-      - "5173:80"
-    depends_on:
-      - backend
-    networks:
-      - vagas-net
-    restart: unless-stopped
+  - path: ./docker/docker-compose.infra.yml
+    project_directory: .
 
-  front_admin:
-    build:
-      context: .
-      dockerfile: docker/node.Dockerfile
-      target: front-admin-runtime
-      args:
-        VITE_API_BASE_URL: ${VITE_API_BASE_URL}
-        VITE_API_URL: ${VITE_API_URL}
-        VITE_APP_ENV: ${VITE_APP_ENV:-production}
-    container_name: vagas-front-admin
-    ports:
-      - "5174:80"
-    depends_on:
-      - backend
-    networks:
-      - vagas-net
-    restart: unless-stopped
-
-networks:
-  vagas-net:
-    external: true
+  - path: ./docker/docker-compose.migrate.yml
+    project_directory: .

docker-full.docker-compose.yml

@@ -0,0 +1,14 @@
+name: candidate-full
+
+include:
+  - path: ./docker/docker-compose.infra.yml
+    project_directory: .
+
+  - path: ./docker/docker-compose.migrate.yml
+    project_directory: .
+
+  - path: ./docker/docker-compose.app.yml
+    project_directory: .
+
+  - path: ./docker/docker-compose.observability.yml
+    project_directory: .