refactor how skills are saved in db

.cursor/rules/alpha-migrations-in-0001.mdc

@@ -0,0 +1,10 @@
+---
+description: During alpha, put every database schema change in migration 0001
+alwaysApply: true
+---
+
+# Alpha database migrations
+
+Derrick is in alpha and is not deployed. Until it leaves alpha, put every database schema change in `packages/DBRepository/Sources/DBRepository/Resources/Migrations/0001_initial_schema.up.sql` and the matching `0001_initial_schema.down.sql`.
+
+Do not add `0002` or any later migration. Keep `DatabaseSchema.latestVersion` at 1. A local database is wiped and recreated instead of upgraded in place.

.cursor/rules/worktrees-inside-derrick-project.mdc

@@ -0,0 +1,12 @@
+---
+description: Create Derrick git worktrees inside derrick_project
+alwaysApply: true
+---
+
+# Worktree location
+
+Create every Derrick git worktree inside `/Users/davidchoi/Work/derrick_project`, next to the main checkout.
+
+Example: `/Users/davidchoi/Work/derrick_project/derrick-plugin-skills`.
+
+Do not create worktrees in `/Users/davidchoi/Work`.

packages/DBRepository/Sources/DBRepository/DBRepositoryPluginFactory.swift

@@ -21,35 +21,35 @@ public extension DBRepository {
             }
             return
         }
-        let skillsData = try JSONEncoder().encode(release.skillFiles)
-        let skillsJSON = String(decoding: skillsData, as: UTF8.self)
         let artifact = release.compiledArtifact.base64EncodedString()
         try withDatabaseHandle { handle in
-            try Self.execute("""
-            INSERT INTO plugin_factory_releases (
-                plugin_id, version, content_hash, manifest_json, runtime_json,
-                guest_source, artifact_base64, skill_files_json, review_summary, created_at
-            ) VALUES (
-                \(quoted(release.pluginID)),
-                \(quoted(release.version)),
-                \(quoted(release.contentHash.rawValue)),
-                \(quoted(release.manifestJSON)),
-                \(quoted(release.runtimeJSON)),
-                \(quoted(release.guestSource)),
-                \(quoted(artifact)),
-                \(quoted(skillsJSON)),
-                \(quoted(release.reviewSummary)),
-                \(quoted(Self.iso8601Formatter().string(from: .now)))
-            );
-            """, on: handle)
+            try Self.withImmediateTransaction(on: handle) {
+                try Self.execute("""
+                INSERT INTO plugin_factory_releases (
+                    plugin_id, version, content_hash, manifest_json, runtime_json,
+                    guest_source, artifact_base64, review_summary, created_at
+                ) VALUES (
+                    \(quoted(release.pluginID)),
+                    \(quoted(release.version)),
+                    \(quoted(release.contentHash.rawValue)),
+                    \(quoted(release.manifestJSON)),
+                    \(quoted(release.runtimeJSON)),
+                    \(quoted(release.guestSource)),
+                    \(quoted(artifact)),
+                    \(quoted(release.reviewSummary)),
+                    \(quoted(Self.iso8601Formatter().string(from: .now)))
+                );
+                """, on: handle)
+                try self.insertSkillFiles(release, on: handle)
+            }
         }
     }
 
     func pluginFactoryRelease(pluginID: String, version: String) throws -> PluginFactoryRelease? {
         try withDatabaseHandle { handle in
             let sql = """
             SELECT plugin_id, version, content_hash, manifest_json, runtime_json,
-                   guest_source, artifact_base64, skill_files_json, review_summary
+                   guest_source, artifact_base64, review_summary
             FROM plugin_factory_releases
             WHERE plugin_id = \(quoted(pluginID))
               AND version = \(quoted(version))
@@ -69,11 +69,8 @@ public extension DBRepository {
                 let runtimeC = sqlite3_column_text(statement, 4),
                 let sourceC = sqlite3_column_text(statement, 5),
                 let artifactC = sqlite3_column_text(statement, 6),
-                let skillsC = sqlite3_column_text(statement, 7),
-                let summaryC = sqlite3_column_text(statement, 8),
+                let summaryC = sqlite3_column_text(statement, 7),
                 let artifact = Data(base64Encoded: String(cString: artifactC)),
-                let skillsData = String(cString: skillsC).data(using: .utf8),
-                let skillFiles = try? JSONDecoder().decode([String: String].self, from: skillsData),
                 let contentHash = try? PluginContentHash(hex: String(cString: hashC))
             else {
                 throw DBRepositoryError.sqliteOperationFailed("Stored plugin factory release is corrupt.")
@@ -85,7 +82,7 @@ public extension DBRepository {
                 runtimeJSON: String(cString: runtimeC),
                 guestSource: String(cString: sourceC),
                 compiledArtifact: artifact,
-                skillFiles: skillFiles,
+                skillFiles: try skillFiles(pluginID: pluginID, version: version, on: handle),
                 contentHash: contentHash,
                 reviewSummary: String(cString: summaryC)
             )
@@ -183,17 +180,202 @@ public extension DBRepository {
         }
         // Version-only row replace — do not cascade associated plugin data.
         try withDatabaseHandle { handle in
+            let pluginID = quoted(release.pluginID)
+            let version = quoted(release.version)
+            try Self.execute(
+                "DELETE FROM plugin_skill_references WHERE plugin_id = \(pluginID) AND version = \(version);",
+                on: handle
+            )
             try Self.execute(
-                """
-                DELETE FROM plugin_factory_releases
-                WHERE plugin_id = \(quoted(release.pluginID))
-                  AND version = \(quoted(release.version));
-                """,
+                "DELETE FROM plugin_skills WHERE plugin_id = \(pluginID) AND version = \(version);",
+                on: handle
+            )
+            try Self.execute(
+                "DELETE FROM plugin_factory_releases WHERE plugin_id = \(pluginID) AND version = \(version);",
                 on: handle
             )
         }
         try savePluginFactoryRelease(release)
     }
+
+    func listPluginSkillIndex() throws -> [PluginSkillDisclosure.IndexEntry] {
+        try withDatabaseHandle { handle in
+            let sql = """
+            SELECT s.plugin_id, s.path, s.skill_name, s.skill_description
+            FROM plugin_skills s
+            INNER JOIN plugin_factory_releases r
+              ON r.plugin_id = s.plugin_id AND r.version = s.version
+            INNER JOIN (
+                SELECT plugin_id, MAX(created_at) AS created_at
+                FROM plugin_factory_releases
+                GROUP BY plugin_id
+            ) latest
+              ON latest.plugin_id = r.plugin_id AND latest.created_at = r.created_at
+            ORDER BY s.plugin_id ASC, s.skill_name ASC;
+            """
+            var statement: OpaquePointer?
+            guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK, let statement else {
+                throw Self.sqliteError(handle: handle, fallback: "Failed to list plugin skills.")
+            }
+            defer { sqlite3_finalize(statement) }
+            var entries: [PluginSkillDisclosure.IndexEntry] = []
+            while sqlite3_step(statement) == SQLITE_ROW {
+                guard
+                    let pluginIDC = sqlite3_column_text(statement, 0),
+                    let pathC = sqlite3_column_text(statement, 1),
+                    let nameC = sqlite3_column_text(statement, 2),
+                    let descriptionC = sqlite3_column_text(statement, 3)
+                else { continue }
+                entries.append(
+                    PluginSkillDisclosure.IndexEntry(
+                        pluginID: String(cString: pluginIDC),
+                        skillName: String(cString: nameC),
+                        description: String(cString: descriptionC),
+                        skillMarkdownPath: String(cString: pathC)
+                    )
+                )
+            }
+            return entries
+        }
+    }
+
+    func pluginSkillBody(pluginID: String, version: String, skill: String) throws -> String? {
+        let needle = skill.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !needle.isEmpty else { return nil }
+        return try withDatabaseHandle { handle in
+            let sql = """
+            SELECT body FROM plugin_skills
+            WHERE plugin_id = \(quoted(pluginID))
+              AND version = \(quoted(version))
+              AND (skill_name = \(quoted(needle)) OR path = \(quoted(needle)))
+            LIMIT 1;
+            """
+            var statement: OpaquePointer?
+            guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK, let statement else {
+                throw Self.sqliteError(handle: handle, fallback: "Failed to load a plugin skill.")
+            }
+            defer { sqlite3_finalize(statement) }
+            guard sqlite3_step(statement) == SQLITE_ROW, let bodyC = sqlite3_column_text(statement, 0) else {
+                return nil
+            }
+            return String(cString: bodyC)
+        }
+    }
+
+    func pluginSkillReference(
+        pluginID: String,
+        version: String,
+        requested: String
+    ) throws -> (path: String, body: String)? {
+        let needle = requested.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !needle.isEmpty else { return nil }
+        let filename = (needle as NSString).lastPathComponent
+        return try withDatabaseHandle { handle in
+            let sql = """
+            SELECT path, body FROM plugin_skill_references
+            WHERE plugin_id = \(quoted(pluginID))
+              AND version = \(quoted(version))
+              AND (
+                path = \(quoted(needle))
+                OR path LIKE \(quoted("%/references/\(filename)"))
+              )
+            ORDER BY CASE WHEN path = \(quoted(needle)) THEN 0 ELSE 1 END
+            LIMIT 1;
+            """
+            var statement: OpaquePointer?
+            guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK, let statement else {
+                throw Self.sqliteError(handle: handle, fallback: "Failed to load a plugin skill reference.")
+            }
+            defer { sqlite3_finalize(statement) }
+            guard sqlite3_step(statement) == SQLITE_ROW,
+                  let pathC = sqlite3_column_text(statement, 0),
+                  let bodyC = sqlite3_column_text(statement, 1)
+            else { return nil }
+            return (String(cString: pathC), String(cString: bodyC))
+        }
+    }
+
+    private func skillFiles(
+        pluginID: String,
+        version: String,
+        on handle: OpaquePointer
+    ) throws -> [String: String] {
+        var files: [String: String] = [:]
+        try appendSkillFiles(
+            """
+            SELECT path, body FROM plugin_skills
+            WHERE plugin_id = \(quoted(pluginID)) AND version = \(quoted(version));
+            """,
+            into: &files,
+            on: handle
+        )
+        try appendSkillFiles(
+            """
+            SELECT path, body FROM plugin_skill_references
+            WHERE plugin_id = \(quoted(pluginID)) AND version = \(quoted(version));
+            """,
+            into: &files,
+            on: handle
+        )
+        return files
+    }
+
+    private func appendSkillFiles(
+        _ sql: String,
+        into files: inout [String: String],
+        on handle: OpaquePointer
+    ) throws {
+        var statement: OpaquePointer?
+        guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK, let statement else {
+            throw Self.sqliteError(handle: handle, fallback: "Failed to load plugin skill files.")
+        }
+        defer { sqlite3_finalize(statement) }
+        while sqlite3_step(statement) == SQLITE_ROW {
+            guard
+                let pathC = sqlite3_column_text(statement, 0),
+                let bodyC = sqlite3_column_text(statement, 1)
+            else { continue }
+            files[String(cString: pathC)] = String(cString: bodyC)
+        }
+    }
+
+    private func insertSkillFiles(_ release: PluginFactoryRelease, on handle: OpaquePointer) throws {
+        for (path, body) in release.skillFiles.sorted(by: { $0.key < $1.key }) {
+            let catalog = PluginSkillDisclosure.catalogFields(path: path, body: body)
+            if let name = catalog.name {
+                let description = catalog.description ?? release.reviewSummary
+                try Self.execute(
+                    """
+                    INSERT INTO plugin_skills (
+                        plugin_id, version, path, skill_name, skill_description, body
+                    ) VALUES (
+                        \(quoted(release.pluginID)),
+                        \(quoted(release.version)),
+                        \(quoted(path)),
+                        \(quoted(name)),
+                        \(quoted(description)),
+                        \(quoted(body))
+                    );
+                    """,
+                    on: handle
+                )
+            } else if PluginFactorySkillFile.isSkillReferencePath(path) {
+                try Self.execute(
+                    """
+                    INSERT INTO plugin_skill_references (
+                        plugin_id, version, path, body
+                    ) VALUES (
+                        \(quoted(release.pluginID)),
+                        \(quoted(release.version)),
+                        \(quoted(path)),
+                        \(quoted(body))
+                    );
+                    """,
+                    on: handle
+                )
+            }
+        }
+    }
 }
 
 extension DBRepository: PluginFactoryManifestCatalog {}

packages/DBRepository/Sources/DBRepository/DBRepositoryPluginPurge.swift

@@ -171,6 +171,20 @@ private extension DBRepository {
 
         let versionClause = version.map { " AND version = \(quoted($0))" } ?? ""
         if beforeCount > 0 {
+            try execute(
+                """
+                DELETE FROM plugin_skill_references
+                WHERE plugin_id = \(quoted(pluginID))\(versionClause);
+                """,
+                on: handle
+            )
+            try execute(
+                """
+                DELETE FROM plugin_skills
+                WHERE plugin_id = \(quoted(pluginID))\(versionClause);
+                """,
+                on: handle
+            )
             try execute(
                 """
                 DELETE FROM plugin_factory_releases

packages/DBRepository/Sources/DBRepository/DatabaseSchema.swift

@@ -2,7 +2,7 @@ import Foundation
 import Structure
 
 public enum DatabaseSchema {
-    public static let latestVersion = 13
+    public static let latestVersion = 1
 
     public static func migrationSQL(version: Int, isUp: Bool) throws -> String {
         let migrationName = String(format: "%04d_%@", version, migrationFileBaseName(for: version))
@@ -25,30 +25,6 @@ public enum DatabaseSchema {
         switch version {
         case 1:
             return "initial_schema"
-        case 2:
-            return "schema_refresh"
-        case 3:
-            return "messaging_listening_since"
-        case 4:
-            return "messaging_reply_threads"
-        case 5:
-            return "news_readers"
-        case 6:
-            return "agent_profiles"
-        case 7:
-            return "messaging_agent_handled"
-        case 8:
-            return "messaging_thread_default_profile"
-        case 9:
-            return "drop_news_readers"
-        case 10:
-            return "plugin_host_ui"
-        case 11:
-            return "messaging_agent_work"
-        case 12:
-            return "agent_profile_capabilities"
-        case 13:
-            return "agent_profile_alias"
         default:
             return "unknown"
         }

packages/DBRepository/Sources/DBRepository/Resources/Migrations/0001_initial_schema.down.sql

@@ -6,7 +6,13 @@ DROP TABLE IF EXISTS tool_runs;
 DROP TABLE IF EXISTS messaging_messages;
 DROP TABLE IF EXISTS messaging_threads;
 DROP TABLE IF EXISTS messaging_connectors;
+DROP TABLE IF EXISTS messaging_agent_work;
+DROP TABLE IF EXISTS messaging_agent_handled;
+DROP TABLE IF EXISTS plugin_host_ui;
+DROP TABLE IF EXISTS plugin_skill_references;
+DROP TABLE IF EXISTS plugin_skills;
 DROP TABLE IF EXISTS plugin_factory_releases;
+DROP TABLE IF EXISTS agent_profiles;
 DROP TABLE IF EXISTS egress_blacklist_exceptions;
 DROP TABLE IF EXISTS egress_blacklist;
 DROP TABLE IF EXISTS agent_turns;