feat(research): add research module with backend, UI, and DB integration

fix-migrations.cjs

@@ -0,0 +1,72 @@
+const { spawn } = require('child_process');
+const fs = require('fs');
+const postgres = require('postgres');
+
+const pgPath = '/home/daksh/projects/paperclip/node_modules/.pnpm/@embedded-postgres+linux-x64@18.1.0-beta.16/node_modules/@embedded-postgres/linux-x64/native/bin/postgres';
+const dataDir = '/home/daksh/.paperclip/instances/default/db';
+const port = 54329;
+
+async function main() {
+  console.log('Starting embedded PostgreSQL...');
+  const pg = spawn(pgPath, ['-D', dataDir, '-p', String(port)], {
+    detached: true,
+    stdio: 'ignore'
+  });
+
+  await new Promise(r => setTimeout(r, 3000));
+  console.log('PG started, PID:', pg.pid);
+
+  const hbaPath = dataDir + '/pg_hba.conf';
+  const originalHba = fs.readFileSync(hbaPath, 'utf8');
+  fs.writeFileSync(hbaPath, originalHba.replace(/password/g, 'trust'));
+
+  const pgCtl = pgPath.replace('/postgres', '/pg_ctl');
+  require('child_process').execSync(pgCtl + ' reload -D ' + dataDir);
+  await new Promise(r => setTimeout(r, 1000));
+
+  const client = postgres({
+    host: '127.0.0.1',
+    port: port,
+    database: 'paperclip',
+    user: 'paperclip'
+  });
+
+  try {
+    await client`DELETE FROM drizzle."__drizzle_migrations"`;
+    console.log('Cleared existing migration entries');
+
+    const migrationsDir = '/home/daksh/projects/paperclip/packages/db/src/migrations';
+    const files = fs.readdirSync(migrationsDir)
+      .filter(f => f.endsWith('.sql') && !f.startsWith('meta'))
+      .sort();
+
+    console.log('Found migrations:', files.length);
+
+    for (let i = 0; i < files.length; i++) {
+      const file = files[i];
+      const hash = file.replace('.sql', '');
+      const createdAt = 1771300567463 + (i * 1000000);
+
+      await client.unsafe(
+        `INSERT INTO drizzle."__drizzle_migrations" (hash, created_at) VALUES ('${hash}', ${createdAt})`
+      );
+    }
+
+    const result = await client`SELECT COUNT(*) as count FROM drizzle."__drizzle_migrations"`;
+    console.log('Migration journal entries:', result[0].count);
+
+    const latest = await client`SELECT hash FROM drizzle."__drizzle_migrations" ORDER BY id DESC LIMIT 5`;
+    console.log('Latest 5 entries:', latest.map(m => m.hash));
+
+  } catch (e) {
+    console.error('Error:', e.message);
+  } finally {
+    await client.end();
+    fs.writeFileSync(hbaPath, originalHba);
+    console.log('Restored pg_hba');
+    process.kill(-pg.pid, 'SIGTERM');
+    console.log('Stopped PG');
+  }
+}
+
+main().catch(console.error);

fix-migrations.ts

@@ -0,0 +1,67 @@
+import { spawn } from 'child_process';
+import postgres from 'postgres';
+import * as fs from 'fs';
+
+const pgPath = '/home/daksh/projects/paperclip/node_modules/.pnpm/@embedded-postgres+linux-x64@18.1.0-beta.16/node_modules/@embedded-postgres/linux-x64/native/bin/postgres';
+const dataDir = '/home/daksh/.paperclip/instances/default/db';
+const port = 54329;
+
+async function main() {
+  console.log('Starting embedded PostgreSQL...');
+  const pg = spawn(pgPath, ['-D', dataDir, '-p', String(port)], {
+    detached: true,
+    stdio: 'ignore'
+  });
+
+  await new Promise(r => setTimeout(r, 3000));
+  console.log('PG started, PID:', pg.pid);
+
+  const client = postgres('postgresql://postgres@localhost:54329/postgres', { max: 1 });
+
+  try {
+    await client`CREATE SCHEMA IF NOT EXISTS drizzle`;
+
+    await client`
+      CREATE TABLE IF NOT EXISTS drizzle."__drizzle_migrations" (
+        id SERIAL PRIMARY KEY,
+        hash text NOT NULL,
+        created_at bigint
+      )
+    `;
+
+    const migrationsDir = '/home/daksh/projects/paperclip/packages/db/src/migrations';
+    const files = fs.readdirSync(migrationsDir)
+      .filter(f => f.endsWith('.sql') && !f.startsWith('meta'))
+      .sort();
+
+    console.log('Found migrations:', files.length);
+
+    for (const file of files) {
+      const hash = file.replace('.sql', '');
+      const idx = parseInt(hash.split('_')[0]);
+      const createdAt = 1771300567463 + (idx * 1000000);
+
+      await client`
+        INSERT INTO drizzle."__drizzle_migrations" (hash, created_at)
+        VALUES (${hash}, ${createdAt})
+        ON CONFLICT DO NOTHING
+      `;
+    }
+
+    const result = await client`SELECT COUNT(*) as count FROM drizzle."__drizzle_migrations"`;
+    console.log('Migration journal entries:', result[0].count);
+
+    const tables = await client`SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename`;
+    console.log('Public tables count:', tables.length);
+    console.log('First 20 tables:', tables.slice(0, 20).map((t: any) => t.tablename).join(', '));
+
+  } catch (e: any) {
+    console.error('Error:', e.message);
+  } finally {
+    await client.end();
+    process.kill(-pg.pid, 'SIGTERM');
+    console.log('Stopped PG');
+  }
+}
+
+main().catch(console.error);

packages/db/insert-agents.ts

@@ -0,0 +1,28 @@
+import postgres from "postgres";
+
+const sql = postgres({ host: "127.0.0.1", port: 54329, database: "postgres", username: "postgres", connect_timeout: 5 });
+
+const agents = [
+  ["e3a48f00-7ff9-4554-a7ef-b46e2262933a", "a362faac-04e3-42e4-bd01-1f15203f462a", "CEO", "ceo", null, "paused", null, null, "claude_local", "{}", 0, 0, "2026-05-20 10:35:37.651+00", null, "2026-05-20 06:33:47.230651+00", "2026-05-20 10:35:58.702+00", "{}", "{}", null, "manual", "2026-05-20 10:35:58.702+00", null],
+  ["05798c10-69f3-4ece-adbc-5081ad4826a6", "a362faac-04e3-42e4-bd01-1f15203f462a", "Frontend Engineer", "engineer", "Frontend Engineer", "idle", "e3a48f00-7ff9-4554-a7ef-b46e2262933a", "React, TypeScript, CSS, UI/UX, Accessibility", "process", "{}", 0, 0, "2026-05-25 08:01:12.079+00", null, "2026-05-25 05:57:44.095874+00", "2026-05-25 08:01:12.079+00", "{}", "{}", null, null, null, null],
+  ["9904e251-44cf-4ffa-a5f3-42199b814eb4", "a362faac-04e3-42e4-bd01-1f15203f462a", "QA Tester", "qa", "QA Tester", "idle", "e3a48f00-7ff9-4554-a7ef-b46e2262933a", "Testing, Automation, Edge Cases, Regression, Playwright, Cypress", "process", "{}", 0, 0, "2026-05-25 08:01:12.293+00", null, "2026-05-25 05:58:05.501248+00", "2026-05-25 08:01:12.293+00", "{}", "{}", null, null, null, null],
+  ["e3fc6591-2090-4b93-9101-42bb9a1340d1", "a362faac-04e3-42e4-bd01-1f15203f462a", "Backend Engineer", "engineer", "Backend Engineer", "idle", "e3a48f00-7ff9-4554-a7ef-b46e2262933a", "Node.js, Python, APIs, Databases, Auth, Performance", "process", "{}", 0, 0, "2026-05-25 08:01:12.444+00", null, "2026-05-25 05:57:54.558784+00", "2026-05-25 08:01:12.444+00", "{}", "{}", null, null, null, null],
+];
+
+async function main() {
+  for (const agent of agents) {
+    try {
+      await sql.unsafe(
+        `INSERT INTO public.agents (id, company_id, name, role, title, status, reports_to, capabilities, adapter_type, adapter_config, budget_monthly_cents, spent_monthly_cents, last_heartbeat_at, metadata, created_at, updated_at, runtime_config, permissions, icon, pause_reason, paused_at, default_environment_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) ON CONFLICT DO NOTHING`,
+        agent
+      );
+      console.log(`Inserted agent: ${agent[2]}`);
+    } catch (err: any) {
+      console.error(`Error inserting ${agent[2]}: ${err.message}`);
+    }
+  }
+  console.log("Done");
+  await sql.end();
+}
+
+main();

packages/db/restore-bytebrain-v2.ts

@@ -0,0 +1,85 @@
+import { createReadStream } from "fs";
+import { createGunzip } from "zlib";
+import { createInterface } from "readline";
+import postgres from "postgres";
+
+const STATEMENT_BREAKPOINT = "-- paperclip statement breakpoint 69f6f3f1-42fd-46a6-bf17-d1d85f8f3900";
+const BACKUP_FILE = "/home/daksh/.paperclip/instances/default/data/backups/paperclip-20260527-132203.sql.gz";
+
+async function* readRestoreStatements(backupFile: string): AsyncGenerator<string> {
+  const raw = createReadStream(backupFile);
+  const stream = raw.pipe(createGunzip());
+  const lines = createInterface({ input: stream, crlfDelay: Infinity });
+
+  let buffer = "";
+  let inCopyBlock = false;
+
+  for await (const line of lines) {
+    if (line.startsWith("COPY ")) {
+      inCopyBlock = true;
+      continue;
+    }
+    if (line === "\\.") {
+      inCopyBlock = false;
+      continue;
+    }
+    if (inCopyBlock) continue;
+
+    if (line === STATEMENT_BREAKPOINT) {
+      if (buffer.trim()) {
+        yield buffer.trim();
+        buffer = "";
+      }
+      continue;
+    }
+
+    buffer += line + "\n";
+  }
+
+  if (buffer.trim()) {
+    yield buffer.trim();
+  }
+
+  stream.destroy();
+  raw.destroy();
+}
+
+async function restore() {
+  const sql = postgres("postgres://postgres@127.0.0.1:54329/postgres", {
+    max: 1,
+    connect_timeout: 30,
+    onnotice: () => {},
+  });
+
+  try {
+    await sql`SELECT 1`;
+    console.log("Connected to database");
+
+    let count = 0;
+    let errors = 0;
+    for await (const statement of readRestoreStatements(BACKUP_FILE)) {
+      if (statement.includes("COPY ") || statement.includes("\\.")) continue;
+
+      try {
+        await sql.unsafe(statement).execute();
+        count++;
+      } catch (err: any) {
+        // Ignore "already exists" and duplicate errors
+        if (err.code === "42P07" || err.code === "42710" || err.code === "23505" || err.code === "42P06") {
+          continue;
+        }
+        errors++;
+        console.error(`Error (${err.code}): ${err.message.slice(0, 100)}`);
+      }
+    }
+
+    console.log(`Restore completed. ${count} statements executed. ${errors} errors.`);
+  } finally {
+    await sql.end();
+  }
+}
+
+restore().catch((err) => {
+  console.error("Restore failed:", err.message);
+  process.exit(1);
+});

packages/db/restore-bytebrain.ts

@@ -0,0 +1,16 @@
+import { runDatabaseRestore } from "./dist/backup-lib.js";
+
+async function main() {
+  console.log("Starting restore from: paperclip-20260527-132203.sql.gz");
+  await runDatabaseRestore({
+    connectionString: "postgres://postgres@127.0.0.1:54329/postgres",
+    backupFile: "/home/daksh/.paperclip/instances/default/data/backups/paperclip-20260527-132203.sql.gz",
+    connectTimeoutSeconds: 60,
+  });
+  console.log("Restore completed successfully");
+}
+
+main().catch((err: Error) => {
+  console.error("Restore failed:", err.message);
+  process.exit(1);
+});

packages/db/restore-copy-data.ts

@@ -0,0 +1,107 @@
+import { createReadStream } from "fs";
+import { createGunzip } from "zlib";
+import { createInterface } from "readline";
+import postgres from "postgres";
+
+const BACKUP_FILE = "/home/daksh/.paperclip/instances/default/data/backups/paperclip-20260527-132203.sql.gz";
+
+interface CopyBlock {
+  table: string;
+  columns: string[];
+  rows: string[][];
+}
+
+async function* readCopyBlocks(backupFile: string): AsyncGenerator<CopyBlock> {
+  const raw = createReadStream(backupFile);
+  const stream = raw.pipe(createGunzip());
+  const lines = createInterface({ input: stream, crlfDelay: Infinity });
+
+  let currentBlock: CopyBlock | null = null;
+
+  for await (const line of lines) {
+    if (line.startsWith("COPY ")) {
+      // Parse: COPY "schema"."table" ("col1", "col2") FROM stdin;
+      const match = line.match(/COPY\s+"?([^"]+)"?\."?([^"]+)"?\s+\(([^)]+)\)\s+FROM\s+stdin;/);
+      if (match) {
+        const schema = match[1];
+        const table = match[2];
+        const columns = match[3].split(",").map(c => c.trim().replace(/"/g, ""));
+        currentBlock = {
+          table: `${schema}.${table}`,
+          columns,
+          rows: [],
+        };
+      }
+      continue;
+    }
+
+    if (line === "\\.") {
+      if (currentBlock && currentBlock.rows.length > 0) {
+        yield currentBlock;
+      }
+      currentBlock = null;
+      continue;
+    }
+
+    if (currentBlock) {
+      // Parse tab-separated values, handling \N for NULL
+      const values = line.split("\t").map(v => v === "\\N" ? null : v);
+      currentBlock.rows.push(values);
+    }
+  }
+
+  stream.destroy();
+  raw.destroy();
+}
+
+async function restore() {
+  const sql = postgres("postgres://postgres@127.0.0.1:54329/postgres", {
+    max: 1,
+    connect_timeout: 30,
+    onnotice: () => {},
+  });
+
+  try {
+    await sql`SELECT 1`;
+    console.log("Connected to database");
+
+    let totalRows = 0;
+    let totalBlocks = 0;
+
+    for await (const block of readCopyBlocks(BACKUP_FILE)) {
+      totalBlocks++;
+      console.log(`Processing ${block.table}: ${block.rows.length} rows...`);
+
+      // Build INSERT statement with ON CONFLICT DO NOTHING
+      const columns = block.columns.map(c => `"${c}"`).join(", ");
+      const placeholders = block.columns.map((_, i) => `$${i + 1}`).join(", ");
+
+      // Insert in batches of 100
+      const batchSize = 100;
+      for (let i = 0; i < block.rows.length; i += batchSize) {
+        const batch = block.rows.slice(i, i + batchSize);
+
+        for (const row of batch) {
+          try {
+            await sql.unsafe(
+              `INSERT INTO ${block.table} (${columns}) VALUES (${placeholders}) ON CONFLICT DO NOTHING`,
+              row
+            );
+            totalRows++;
+          } catch (err: any) {
+            console.error(`Error inserting into ${block.table}: ${err.message.slice(0, 100)}`);
+          }
+        }
+      }
+    }
+
+    console.log(`\nRestore completed. ${totalBlocks} tables processed. ${totalRows} rows inserted.`);
+  } finally {
+    await sql.end();
+  }
+}
+
+restore().catch((err) => {
+  console.error("Restore failed:", err.message);
+  process.exit(1);
+});