feat(research): Add research module with manual retry, resume, and lifecycle states

packages/db/src/migrations/0094_smart_orphan.sql

@@ -0,0 +1,173 @@
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'research_depth') THEN CREATE TYPE "public"."research_depth" AS ENUM('shallow', 'medium', 'deep'); END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'research_finding_confidence') THEN CREATE TYPE "public"."research_finding_confidence" AS ENUM('high', 'medium', 'low'); END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'research_session_status') THEN CREATE TYPE "public"."research_session_status" AS ENUM('planning', 'running', 'cancelling', 'paused', 'completed', 'failed', 'cancelled'); END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'research_task_status') THEN CREATE TYPE "public"."research_task_status" AS ENUM('pending', 'running', 'completed', 'failed', 'skipped'); END IF; END $$;--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "cloud_upstream_connections" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"company_id" uuid NOT NULL,
+	"remote_url" text NOT NULL,
+	"source_instance_id" text NOT NULL,
+	"source_instance_fingerprint" text NOT NULL,
+	"source_public_key" text NOT NULL,
+	"private_key_pem" text NOT NULL,
+	"token_status" text NOT NULL,
+	"scopes" text[] DEFAULT '{}' NOT NULL,
+	"authorized_global_user_id" text,
+	"access_token" text,
+	"token_id" text,
+	"token_expires_at" timestamp with time zone,
+	"target_stack_id" text NOT NULL,
+	"target_stack_slug" text,
+	"target_stack_display_name" text,
+	"target_company_id" text NOT NULL,
+	"target_origin" text NOT NULL,
+	"target_primary_host" text NOT NULL,
+	"target_product" text NOT NULL,
+	"target_schema_major" integer NOT NULL,
+	"target_max_chunk_bytes" integer NOT NULL,
+	"pending_state" text,
+	"pending_code_verifier" text,
+	"pending_redirect_uri" text,
+	"pending_token_url" text,
+	"last_run_id" uuid,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "cloud_upstream_runs" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"connection_id" uuid NOT NULL,
+	"company_id" uuid NOT NULL,
+	"remote_run_id" text,
+	"status" text NOT NULL,
+	"active_step" text NOT NULL,
+	"progress_percent" integer DEFAULT 0 NOT NULL,
+	"dry_run" boolean DEFAULT false NOT NULL,
+	"retry_of_run_id" uuid,
+	"summary" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	"warnings" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	"conflicts" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	"events" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	"report" jsonb DEFAULT '{}'::jsonb NOT NULL,
+	"idempotency_key" text NOT NULL,
+	"manifest_hash" text NOT NULL,
+	"target_url" text,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"completed_at" timestamp with time zone
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "research_findings" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"task_id" uuid NOT NULL,
+	"session_id" uuid NOT NULL,
+	"company_id" uuid NOT NULL,
+	"content" text NOT NULL,
+	"source_url" text,
+	"source_title" text,
+	"source_domain" text,
+	"confidence" "research_finding_confidence" DEFAULT 'medium',
+	"reliability_score" integer,
+	"category" text,
+	"is_duplicate" boolean DEFAULT false,
+	"duplicate_of_id" uuid,
+	"metadata" jsonb DEFAULT '{}'::jsonb,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "research_memory" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"company_id" uuid NOT NULL,
+	"key" text NOT NULL,
+	"value" jsonb NOT NULL,
+	"session_id" uuid,
+	"source_finding_id" uuid,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "research_sessions" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"company_id" uuid NOT NULL,
+	"title" text NOT NULL,
+	"query" text NOT NULL,
+	"status" "research_session_status" DEFAULT 'planning' NOT NULL,
+	"plan" jsonb,
+	"report" text,
+	"original_report" text,
+	"is_edited" boolean DEFAULT false NOT NULL,
+	"progress_percent" integer DEFAULT 0 NOT NULL,
+	"depth" "research_depth" DEFAULT 'medium' NOT NULL,
+	"max_subtopics" integer DEFAULT 5 NOT NULL,
+	"created_by" text NOT NULL,
+	"started_at" timestamp with time zone,
+	"completed_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "research_sources" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"session_id" uuid NOT NULL,
+	"company_id" uuid NOT NULL,
+	"url" text NOT NULL,
+	"title" text,
+	"domain" text,
+	"reliability_score" integer,
+	"access_count" integer DEFAULT 1 NOT NULL,
+	"last_accessed_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "research_tasks" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"session_id" uuid NOT NULL,
+	"company_id" uuid NOT NULL,
+	"title" text NOT NULL,
+	"status" "research_task_status" DEFAULT 'pending' NOT NULL,
+	"findings_summary" text,
+	"sources" jsonb DEFAULT '[]'::jsonb,
+	"reliability_score" integer,
+	"started_at" timestamp with time zone,
+	"completed_at" timestamp with time zone,
+	"sequence_order" integer DEFAULT 0 NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'cloud_upstream_connections_company_id_companies_id_fk') THEN ALTER TABLE "cloud_upstream_connections" ADD CONSTRAINT "cloud_upstream_connections_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'cloud_upstream_runs_connection_id_cloud_upstream_connections_id_fk') THEN ALTER TABLE "cloud_upstream_runs" ADD CONSTRAINT "cloud_upstream_runs_connection_id_cloud_upstream_connections_id_fk" FOREIGN KEY ("connection_id") REFERENCES "public"."cloud_upstream_connections"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'cloud_upstream_runs_company_id_companies_id_fk') THEN ALTER TABLE "cloud_upstream_runs" ADD CONSTRAINT "cloud_upstream_runs_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_findings_task_id_research_tasks_id_fk') THEN ALTER TABLE "research_findings" ADD CONSTRAINT "research_findings_task_id_research_tasks_id_fk" FOREIGN KEY ("task_id") REFERENCES "public"."research_tasks"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_findings_session_id_research_sessions_id_fk') THEN ALTER TABLE "research_findings" ADD CONSTRAINT "research_findings_session_id_research_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."research_sessions"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_findings_company_id_companies_id_fk') THEN ALTER TABLE "research_findings" ADD CONSTRAINT "research_findings_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_findings_duplicate_of_id_research_findings_id_fk') THEN ALTER TABLE "research_findings" ADD CONSTRAINT "research_findings_duplicate_of_id_research_findings_id_fk" FOREIGN KEY ("duplicate_of_id") REFERENCES "public"."research_findings"("id") ON DELETE no action ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_memory_company_id_companies_id_fk') THEN ALTER TABLE "research_memory" ADD CONSTRAINT "research_memory_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_memory_session_id_research_sessions_id_fk') THEN ALTER TABLE "research_memory" ADD CONSTRAINT "research_memory_session_id_research_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."research_sessions"("id") ON DELETE set null ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_memory_source_finding_id_research_findings_id_fk') THEN ALTER TABLE "research_memory" ADD CONSTRAINT "research_memory_source_finding_id_research_findings_id_fk" FOREIGN KEY ("source_finding_id") REFERENCES "public"."research_findings"("id") ON DELETE set null ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_sessions_company_id_companies_id_fk') THEN ALTER TABLE "research_sessions" ADD CONSTRAINT "research_sessions_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_sources_session_id_research_sessions_id_fk') THEN ALTER TABLE "research_sources" ADD CONSTRAINT "research_sources_session_id_research_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."research_sessions"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_sources_company_id_companies_id_fk') THEN ALTER TABLE "research_sources" ADD CONSTRAINT "research_sources_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_tasks_session_id_research_sessions_id_fk') THEN ALTER TABLE "research_tasks" ADD CONSTRAINT "research_tasks_session_id_research_sessions_id_fk" FOREIGN KEY ("session_id") REFERENCES "public"."research_sessions"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'research_tasks_company_id_companies_id_fk') THEN ALTER TABLE "research_tasks" ADD CONSTRAINT "research_tasks_company_id_companies_id_fk" FOREIGN KEY ("company_id") REFERENCES "public"."companies"("id") ON DELETE cascade ON UPDATE no action; END IF; END $$;--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "cloud_upstream_connections_company_idx" ON "cloud_upstream_connections" USING btree ("company_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "cloud_upstream_runs_company_created_idx" ON "cloud_upstream_runs" USING btree ("company_id","created_at");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "cloud_upstream_runs_connection_idx" ON "cloud_upstream_runs" USING btree ("connection_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_findings_task_idx" ON "research_findings" USING btree ("task_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_findings_session_idx" ON "research_findings" USING btree ("session_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_findings_company_idx" ON "research_findings" USING btree ("company_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_findings_duplicate_idx" ON "research_findings" USING btree ("is_duplicate","duplicate_of_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_findings_category_idx" ON "research_findings" USING btree ("category");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_findings_company_session_created_idx" ON "research_findings" USING btree ("company_id","session_id","created_at");--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "research_memory_company_key_idx" ON "research_memory" USING btree ("company_id","key");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_memory_session_idx" ON "research_memory" USING btree ("session_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_sessions_company_idx" ON "research_sessions" USING btree ("company_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_sessions_status_idx" ON "research_sessions" USING btree ("status");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_sessions_created_idx" ON "research_sessions" USING btree ("company_id","created_at");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_sources_session_idx" ON "research_sources" USING btree ("session_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_sources_url_idx" ON "research_sources" USING btree ("url");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_sources_domain_idx" ON "research_sources" USING btree ("domain");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_tasks_session_idx" ON "research_tasks" USING btree ("session_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_tasks_company_idx" ON "research_tasks" USING btree ("company_id");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_tasks_status_idx" ON "research_tasks" USING btree ("status");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "research_tasks_session_order_idx" ON "research_tasks" USING btree ("session_id","sequence_order");

packages/db/src/migrations/0095_research_cancelled_status.sql

@@ -0,0 +1,12 @@
+-- Add missing enum values for research session status
+DO $$ BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_enum WHERE enumlabel = 'cancelling' AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'research_session_status')) THEN
+        ALTER TYPE "research_session_status" ADD VALUE 'cancelling';
+    END IF;
+END $$;
+
+DO $$ BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_enum WHERE enumlabel = 'cancelled' AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'research_session_status')) THEN
+        ALTER TYPE "research_session_status" ADD VALUE 'cancelled';
+    END IF;
+END $$;

packages/db/src/migrations/meta/_journal.json

@@ -656,9 +656,23 @@
     {
       "idx": 93,
       "version": "7",
-      "when": 1780040470886,
+      "when": 1779408000000,
       "tag": "0093_giant_green_goblin",
       "breakpoints": true
+    },
+    {
+      "idx": 94,
+      "version": "7",
+      "when": 1779580800000,
+      "tag": "0094_smart_orphan",
+      "breakpoints": true
+    },
+    {
+      "idx": 95,
+      "version": "7",
+      "when": 1779667200000,
+      "tag": "0095_research_cancelled_status",
+      "breakpoints": true
     }
   ]
 }

packages/db/src/schema/index.ts

@@ -83,3 +83,8 @@ export { pluginDatabaseNamespaces, pluginMigrations } from "./plugin_database.js
 export { pluginJobs, pluginJobRuns } from "./plugin_jobs.js";
 export { pluginWebhookDeliveries } from "./plugin_webhooks.js";
 export { pluginLogs } from "./plugin_logs.js";
+export { researchSessions } from "./research_sessions.js";
+export { researchTasks } from "./research_tasks.js";
+export { researchFindings } from "./research_findings.js";
+export { researchSources } from "./research_sources.js";
+export { researchMemory } from "./research_memory.js";

packages/db/src/schema/research_findings.ts

@@ -0,0 +1,45 @@
+import { pgTable, uuid, text, integer, timestamp, boolean, jsonb, pgEnum, index } from "drizzle-orm/pg-core";
+import { companies } from "./companies.js";
+import { researchSessions } from "./research_sessions.js";
+import { researchTasks } from "./research_tasks.js";
+
+export const researchFindingConfidenceEnum = pgEnum("research_finding_confidence", [
+  "high",
+  "medium",
+  "low",
+]);
+
+export const researchFindings = pgTable(
+  "research_findings",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    taskId: uuid("task_id")
+      .notNull()
+      .references(() => researchTasks.id, { onDelete: "cascade" }),
+    sessionId: uuid("session_id")
+      .notNull()
+      .references(() => researchSessions.id, { onDelete: "cascade" }),
+    companyId: uuid("company_id")
+      .notNull()
+      .references(() => companies.id, { onDelete: "cascade" }),
+    content: text("content").notNull(),
+    sourceUrl: text("source_url"),
+    sourceTitle: text("source_title"),
+    sourceDomain: text("source_domain"),
+    confidence: researchFindingConfidenceEnum("confidence").default("medium"),
+    reliabilityScore: integer("reliability_score"),
+    category: text("category"),
+    isDuplicate: boolean("is_duplicate").default(false),
+    duplicateOfId: uuid("duplicate_of_id").references((): any => researchFindings.id),
+    metadata: jsonb("metadata").default({}),
+    createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  },
+  (table) => ({
+    taskIdx: index("research_findings_task_idx").on(table.taskId),
+    sessionIdx: index("research_findings_session_idx").on(table.sessionId),
+    companyIdx: index("research_findings_company_idx").on(table.companyId),
+    duplicateIdx: index("research_findings_duplicate_idx").on(table.isDuplicate, table.duplicateOfId),
+    categoryIdx: index("research_findings_category_idx").on(table.category),
+    companySessionCreatedIdx: index("research_findings_company_session_created_idx").on(table.companyId, table.sessionId, table.createdAt),
+  }),
+);

packages/db/src/schema/research_memory.ts

@@ -0,0 +1,24 @@
+import { pgTable, uuid, text, timestamp, jsonb, uniqueIndex, index } from "drizzle-orm/pg-core";
+import { companies } from "./companies.js";
+import { researchSessions } from "./research_sessions.js";
+import { researchFindings } from "./research_findings.js";
+
+export const researchMemory = pgTable(
+  "research_memory",
+  {
+    id: uuid("id").primaryKey().defaultRandom(),
+    companyId: uuid("company_id")
+      .notNull()
+      .references(() => companies.id, { onDelete: "cascade" }),
+    key: text("key").notNull(),
+    value: jsonb("value").notNull(),
+    sessionId: uuid("session_id").references(() => researchSessions.id, { onDelete: "set null" }),
+    sourceFindingId: uuid("source_finding_id").references(() => researchFindings.id, { onDelete: "set null" }),
+    createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+    updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(),
+  },
+  (table) => ({
+    companyKeyIdx: uniqueIndex("research_memory_company_key_idx").on(table.companyId, table.key),
+    sessionIdx: index("research_memory_session_idx").on(table.sessionId),
+  }),
+);