diff --git a/bun.lock b/bun.lock index cf85716..35a8eaf 100644 --- a/bun.lock +++ b/bun.lock @@ -23,14 +23,34 @@ }, "packages/core": { "name": "lossless-core", - "version": "0.1.1", + "version": "0.1.2", "dependencies": { "typebox": ">=1.4.3 <2", }, }, + "packages/deepseek": { + "name": "deepseek-lossless", + "version": "0.1.0", + "dependencies": { + "@deepseek-ai/cordis": "4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-commands": "0.2.1-rc.2", + "@deepseek-ai/dsh-compaction": "0.2.1-rc.2", + "@deepseek-ai/dsh-compaction-basic": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-tools": "0.2.1-rc.2", + "lossless-core": "workspace:*", + }, + "devDependencies": { + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", + "@deepseek-ai/dsh-token-meter": "0.2.1-rc.2", + }, + }, "packages/opencode": { "name": "opencode-lossless", - "version": "0.1.1", + "version": "0.1.2", "devDependencies": { "@opencode/ai": "2.1.4", "@opencode/plugin": "2.1.4", @@ -39,7 +59,7 @@ }, "packages/pi": { "name": "pi-lossless", - "version": "0.1.1", + "version": "0.1.2", "devDependencies": { "@earendil-works/pi-agent-core": "1.4.0", "@earendil-works/pi-ai": "1.4.0", @@ -267,6 +287,62 @@ "@clack/prompts": ["@clack/prompts@1.8.0", "", { "dependencies": { "@clack/core": "1.5.0", "fast-string-width": "^3.0.2", "fast-wrap-ansi": "^0.2.0", "sisteransi": "^1.0.5" } }, "sha512-PXzLZ8N34rxmuo4dJg3xtOXhcBse94qGjDqsteoEYrFrrZ5FSjIGwMAuOcv64ln8rHVBBD06XeVGr+/JX+plcA=="], + "@deepseek-ai/cordis": ["@deepseek-ai/cordis@4.0.5", "", { "dependencies": { "@deepseek-ai/cosmokit": "~1.8.6", "@standard-schema/spec": "^1.1.0" }, "peerDependencies": { "@deepseek-ai/cordis-plugin-include": "~1.0.10", "@deepseek-ai/cordis-plugin-loader": "~1.0.6" }, "optionalPeers": ["@deepseek-ai/cordis-plugin-include", "@deepseek-ai/cordis-plugin-loader"], "bin": { "cordis": "bin.js" } }, "sha512-Sktq5axy0owk9PdnivBRNOJjHEVEibCt0+TDDZcv0+UH2KS1M910fX4kn5y45YFsrpZIi1mJf/1s5VzywMUnyg=="], + + "@deepseek-ai/cosmokit": ["@deepseek-ai/cosmokit@1.8.6", "", {}, "sha512-ZSD/fN52L498LBIOYwjIOlaLv5veY6ZEwI2yzvZUKQtctn0QAPqu8tKdJ+OF5+Tv6NBq1U+wJWH+QI6OGy5EvQ=="], + + "@deepseek-ai/dsh-agent": ["@deepseek-ai/dsh-agent@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-scope": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2", "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", "@deepseek-ai/dsh-workspace": "0.2.1-rc.2" }, "optionalPeers": ["@deepseek-ai/dsh-workspace"] }, "sha512-e96Tnq8g/RNCDDUW7kAypY/4cho5QxjLHMhk86BGGIEu9b31XEkSFqwqQIwtYXQez3cDR5z7LZpAM8Uqx9Y8KA=="], + + "@deepseek-ai/dsh-attachment": ["@deepseek-ai/dsh-attachment@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-brand": "0.2.1-rc.2" } }, "sha512-VtSSPYq6cOTZ3kLUAHkjIRq6SOjecfZ4p3lzn1dBU1EcUFJ5fOcN1pM2CLVec4ZYrvMxiB5XwtMeFDQPQ7jgpw=="], + + "@deepseek-ai/dsh-brand": ["@deepseek-ai/dsh-brand@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5" } }, "sha512-T2kcygikRS8rv8cKjgho2x381EgGprv5yQmvHl4/iYbuHE19OW+SLGtHWbV27PGdSHz5U8CJxasYzFosKW4e2A=="], + + "@deepseek-ai/dsh-commands": ["@deepseek-ai/dsh-commands@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/dsh-util-crypto": "0.2.1-rc.2", "zod": "4.4.3" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-agent": "0.2.1-rc.2", "@deepseek-ai/dsh-attachment": "0.2.1-rc.2", "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-scope": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2", "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2" } }, "sha512-wLeEVXHuTULr0WYVHcCf1djQTEql2oxNSkbckbyfzw/r381ki1y39CPHw3Xtt6D9eXKulxTzgZzDPnHdh29QkQ=="], + + "@deepseek-ai/dsh-compaction": ["@deepseek-ai/dsh-compaction@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "@deepseek-ai/dsh-commands": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "sha512-rag79zVeoMWXqKfIB0uRyx/1RRXSvucXJVUMOxM+iJxBgHObCn61RTLiFrI8M5DabHnmy+ugCGAeL5HwD/JPVw=="], + + "@deepseek-ai/dsh-compaction-basic": ["@deepseek-ai/dsh-compaction-basic@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-agent": "0.2.1-rc.2", "@deepseek-ai/dsh-commands": "0.2.1-rc.2", "@deepseek-ai/dsh-compaction": "0.2.1-rc.2", "@deepseek-ai/dsh-compaction-tool-result-pruner": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2", "@deepseek-ai/dsh-token-meter": "0.2.1-rc.2" }, "optionalPeers": ["@deepseek-ai/dsh-compaction-tool-result-pruner"] }, "sha512-nx3uY0auC4JP7dEQDlQ6bQAtsDgrcCu/gPPDqKy3iM9ZpJTi2BE7pObuqhIeRIUpIodByX4QHU0N9p7kFxi4Ng=="], + + "@deepseek-ai/dsh-compaction-image-offload": ["@deepseek-ai/dsh-compaction-image-offload@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/dsh-util-values": "0.2.1-rc.2" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-agent": "0.2.1-rc.2", "@deepseek-ai/dsh-compaction": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "sha512-mofQQ5sI09pt3m4y4FPGEkhhNtlwz/0Ffyr4jMqwr4UMYfy1Jb2+6HawovmdEY3Xcm64/ZX0Eitgo5MiHFf9+w=="], + + "@deepseek-ai/dsh-fs": ["@deepseek-ai/dsh-fs@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2" } }, "sha512-Z6d2ZKsAiJc0FaiES8wC66vuUQMGohl9tCc2p3bAsPFYkJ0+L2zTViQ9Y5BArtRe9Br8qhNjNgLlN0jb/uwv1g=="], + + "@deepseek-ai/dsh-llm": ["@deepseek-ai/dsh-llm@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "@deepseek-ai/dsh-timeout": "0.2.1-rc.2", "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", "@deepseek-ai/dsh-util-crypto": "0.2.1-rc.2", "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5" } }, "sha512-3xfoMDHEpUJGcH2Lu+lNXUJYRBclYfBkdCG9Au02VzIjYAm9BrFmc/MNuaVTg0vwgvPpTsd6Fx1IeGtfmQFvlg=="], + + "@deepseek-ai/dsh-llm-retry": ["@deepseek-ai/dsh-llm-retry@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-agent": "0.2.1-rc.2", "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2", "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", "@deepseek-ai/dsh-timeout": "0.2.1-rc.2" } }, "sha512-6nnM6y2eTz2gHPu9MVG5aABafMqQLPQ4u3K/csQG5WLr91ABOlAZomM0IN9BdmDBIyf+Kr7F9cQCYQUbADl3SA=="], + + "@deepseek-ai/dsh-ptc-runtime": ["@deepseek-ai/dsh-ptc-runtime@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2" } }, "sha512-e81rbNwp3YNIpzIbkFI4NsWQzTD3uAP8zXU16diL9wLmITUWjMS+kvrWEXtrkWAWp5cnK2Pgn4xCbqVZ6n4I8w=="], + + "@deepseek-ai/dsh-sandbox": ["@deepseek-ai/dsh-sandbox@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/dsh-util-values": "0.2.1-rc.2" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "sha512-nnxNIkgi4A4tkPrTV8sbVMse3c32OT0U+mwYhLqDz/+cIm5tUGufQD+Ax0zqVeClyzXsxZo0/pVtcqDJrsp+7w=="], + + "@deepseek-ai/dsh-sandbox-policy": ["@deepseek-ai/dsh-sandbox-policy@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-agent": "0.2.1-rc.2", "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2", "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "sha512-fBb8zQ13CLKM60ynl3EHvFmO8F0GARofQYf7CXUkqJEuBcjxXUG14rzDKRS/PhE8oJ023lEvd5HJjE6qUjslhw=="], + + "@deepseek-ai/dsh-scope": ["@deepseek-ai/dsh-scope@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5" } }, "sha512-CVGPI4lzmUy7O3zg8X1LavUSmD3JKFkUdykfHN7VkyWXG5oAXSvig9s5PLnIb+sxRItmSpiqPXGnT/48X1T1qg=="], + + "@deepseek-ai/dsh-session": ["@deepseek-ai/dsh-session@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-util-values": "0.2.1-rc.2" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-scope": "0.2.1-rc.2" } }, "sha512-WDA4lIrXtG5nf1dn2wrA4UzEr7D6e8T3AdPpVSZ7eY4N4ICMDZ0RYuD4GkM+iSb7WkLojdO8ljr0etiIMEbqJg=="], + + "@deepseek-ai/dsh-session-projection": ["@deepseek-ai/dsh-session-projection@0.2.1-rc.2", "", { "dependencies": { "zod": "4.4.3" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "sha512-GBGgIAaC+Cw1rV7tpyGy9aaENUOkQCosP+LJHcmzqPWg64Ufts36mDEU3f0lC/nyGb0z66Ov+RbyaLLsCnZMwA=="], + + "@deepseek-ai/dsh-system-prompt": ["@deepseek-ai/dsh-system-prompt@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-scope": "0.2.1-rc.2" } }, "sha512-X1WG2FMyfZapAkcCYyTl0UO5vVqDcCsA9wECJY7eeDTLHJDsqrilQJg2DRgHQzM+2/7T01QyzFZhKd7hfALJxg=="], + + "@deepseek-ai/dsh-timeout": ["@deepseek-ai/dsh-timeout@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5" } }, "sha512-R75r+GkYP+ZhtIsIfn8MMuRXPkugJr0OyQ1Y+rauoTae2sdVfnE8v5zvQORRCHdh7MiOcmh10dQN70KjHh7qhQ=="], + + "@deepseek-ai/dsh-token-meter": ["@deepseek-ai/dsh-token-meter@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-compaction": "0.2.1-rc.2", "@deepseek-ai/dsh-compaction-image-offload": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-llm-retry": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2", "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2" } }, "sha512-ObJQBb8fbFtw4oAPBTpAcbEdYH6Yqf2cnG5V2RShKmFQeMkuKv9uYCRqflnmimIhgslWO2kdOeg1ACDBYHemvQ=="], + + "@deepseek-ai/dsh-tools": ["@deepseek-ai/dsh-tools@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-agent": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-ptc-runtime": "0.2.1-rc.2", "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", "@deepseek-ai/dsh-sandbox-policy": "0.2.1-rc.2", "@deepseek-ai/dsh-scope": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2", "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", "@deepseek-ai/dsh-user-approval": "0.2.1-rc.2", "@deepseek-ai/dsh-working-directory": "0.2.1-rc.2" } }, "sha512-K3gET3VPbtxM66pkeRsEGrZ+bQKiBqrTbNXiIreBKu0hP+qPvlSwhiSI5MTnFp2tMFK5rpuYMZosRIdbfrbJ/g=="], + + "@deepseek-ai/dsh-typert-protocol": ["@deepseek-ai/dsh-typert-protocol@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "zod": "^4.4.3" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5" } }, "sha512-rtsHJHGE9SWX+zOgtDbPUqz3h7r+qFm3h1WASzRJxDnULqo7JaBdIiGshNTFPmetsX+r1T69CKFdTZAJmHwMug=="], + + "@deepseek-ai/dsh-user-approval": ["@deepseek-ai/dsh-user-approval@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-agent": "0.2.1-rc.2", "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-scope": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2", "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "sha512-pwhw1U1IZZm5tKyPayj2APZBq5I7/I0C0ad+ohd4vKuNb6twDIltt1GEHdCNt7I19IGHLRiUY+/Q9S/8+q5K0A=="], + + "@deepseek-ai/dsh-util-crypto": ["@deepseek-ai/dsh-util-crypto@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5" } }, "sha512-V4SxvXL4ZbY5RYJckVtmo3qz4YsA52pYKvDF13+8hsDCEmQIv9YfmLRMA61pjK/i05pGMe3p1pPhAB6L+NOIEA=="], + + "@deepseek-ai/dsh-util-values": ["@deepseek-ai/dsh-util-values@0.2.1-rc.2", "", { "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5" } }, "sha512-mgnByPadOgZtoBxbwMI184li494O6sNn66/NvdqN0RIrzAJmFGpFxpjFkQtQN/yetwWhlx7Ic17+6p7eKl8DNg=="], + + "@deepseek-ai/dsh-working-directory": ["@deepseek-ai/dsh-working-directory@0.2.1-rc.2", "", { "dependencies": { "@deepseek-ai/schemastery": "~3.18.5", "zod": "^4.4.3" }, "peerDependencies": { "@deepseek-ai/cordis": "~4.0.5", "@deepseek-ai/dsh-agent": "0.2.1-rc.2", "@deepseek-ai/dsh-fs": "0.2.1-rc.2", "@deepseek-ai/dsh-llm": "0.2.1-rc.2", "@deepseek-ai/dsh-session": "0.2.1-rc.2", "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "sha512-zhPtvAaCxa9U0WUN9OEG37zVBq7GOXXXwnDx2Jds4DFksGY5Lmpl3ZK1QMGz+F/Aye4VHt9zoISI7m4wDA8DEg=="], + + "@deepseek-ai/schemastery": ["@deepseek-ai/schemastery@3.18.5", "", { "dependencies": { "@deepseek-ai/cosmokit": "~1.8.6", "@standard-schema/spec": "^1.1.0" } }, "sha512-Mnobvqf9rHsZYv4g5edj5ddxS1oF1kIdSlz0DunUgnY3LWG5F26MqvH3YOOsSaodCgyzJ16OMN9O99/s5Crtdw=="], + "@earendil-works/pi-agent-core": ["@earendil-works/pi-agent-core@1.4.0", "", { "dependencies": { "@earendil-works/pi-ai": "1.4.0", "typebox": "1.3.27" } }, "sha512-PoKY84SKUwnt7S5QBZaGUqfNO1kjP3KHpHUA9QpFTj4cOjaVWMYIcBPJ6FDRCdE8Zen9OSECQUd0uUfVgtSmgA=="], "@earendil-works/pi-ai": ["@earendil-works/pi-ai@1.4.0", "", { "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", "@earendil-works/pi-telemetry": "1.4.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", "partial-json": "0.1.7", "typebox": "1.3.27" }, "bin": { "pi-ai": "dist/cli.js" } }, "sha512-hQM/C4+1w8rHYACzDDf+0vQMnEnpzIop1tptIERVgR3ZeT/hHSzBNG8PaWPgxWQjqSeKCyBZSyp8uTT6eKK1pQ=="], @@ -1387,6 +1463,8 @@ "decode-named-character-reference": ["decode-named-character-reference@1.3.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q=="], + "deepseek-lossless": ["deepseek-lossless@workspace:packages/deepseek"], + "default-browser": ["default-browser@5.5.1", "", { "dependencies": { "bundle-name": "^4.1.0", "default-browser-id": "^5.0.0" } }, "sha512-m1pAzaJgZ/gssEqlOhJkPJp8Xly7QyW6xcrkUa2KKcDeDSEMP7X8xipU3snUcfisTQx0w1AGae+9UtJSfVnXGw=="], "default-browser-id": ["default-browser-id@5.0.1", "", {}, "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q=="], @@ -2527,6 +2605,18 @@ "@babel/code-frame/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + "@deepseek-ai/dsh-commands/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + + "@deepseek-ai/dsh-llm/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + + "@deepseek-ai/dsh-llm-retry/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + + "@deepseek-ai/dsh-sandbox-policy/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + + "@deepseek-ai/dsh-session-projection/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + + "@deepseek-ai/dsh-token-meter/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "@earendil-works/pi-agent-core/typebox": ["typebox@1.3.27", "", {}, "sha512-zu+jc1pcy4UiNThxikUr36f0Rybk9PEeCg/NE6adeWr/SKsdNO4EzZHYRDlv2YCVAfj3Odq3dESSo/jNyoBXzA=="], "@earendil-works/pi-ai/typebox": ["typebox@1.3.27", "", {}, "sha512-zu+jc1pcy4UiNThxikUr36f0Rybk9PEeCg/NE6adeWr/SKsdNO4EzZHYRDlv2YCVAfj3Odq3dESSo/jNyoBXzA=="], diff --git a/docs/deepseek-plan.md b/docs/deepseek-plan.md new file mode 100644 index 0000000..2a4a73f --- /dev/null +++ b/docs/deepseek-plan.md @@ -0,0 +1,450 @@ +# DeepSeek plan + +Status: active spike. The code lives on `feat/deepseek`, extracted from +`spike/second-consumer`. OpenCode v2 already shipped alongside Pi; this plan +tracks the DeepSeek adapter. Roadmap item 10 describes the live-proof gate. + +Goal: answer whether `lossless-core` can serve the DeepSeek harness as one +more adapter, without weakening the engine boundary or the Pi adapter. + +## Constraints the design must honor + +These come from the tree, not from preference. + +- The engine boundary holds. `packages/core/src/host.ts` is the only host + surface (`ModelHost` plus `Reader`), and the boundary check fails any module + under `packages/core/src/` that imports a host SDK. Another adapter proves + the boundary instead of widening it. +- The target harness owns surface mutation. Its session is an append-only log, + `deriveMessages()` renders the model history, and loop-built requests arrive + frozen. There is no equivalent of the Pi `context` event that rewrites the + outgoing request in flight. Model-visible change must be a logged event, normally + a `user/message` with a `replace` operation inside the `compaction/start` to + `compaction/end` bracket. +- The target's compaction seam is `compactIfNeeded`, `compactNow`, and + `compactRegion` on one `CompactionEngine` per composition, with tool-pairing + balance checks on both edges and a shrink check that rejects a summary no + smaller than its span. +- Lossless behavior worth keeping: verbatim ingest, the leaf plus condensed DAG + with escalation to a deterministic floor, addressable summary ids, a budgeted + reader loop over `lcm_grep`, `lcm_describe`, and `lcm_expand`, per-model + calibration, and the metrics rows that make a pass auditable. +- Names stay unscoped until a collision forces the one-commit migration that + `docs/roadmap.md` already describes. The working name below is a placeholder + the owner replaces. +- No comparison claims ship with this work. `apps/docs/COMPARISON.md` stays + Pi-only per roadmap item 10. + +## Candidates considered + +Three shapes, structurally distinct. + +### A. Recall-only sidecar + +Keep the shipped basic compactor. Add a plugin that mirrors session history +into a sidecar store and registers the three recall tools plus one command. + +```yaml +- name: basic-compaction-backend +- name: lossless-recall +``` + +Smallest diff. It proves ingest mapping, the `ModelHost` mapping over +streaming calls, and tool registration. It concedes compaction policy entirely, +so the DAG, the swap timing, and the pointer projection buy nothing yet. + +### B. Summarizer subclass + +Subclass the shipped `BasicCompactionEngine` and override only `summarize()`. +Route that hook through lossless escalation and the sidecar store. Keep the +target's transaction, meter, retention, and lock behavior untouched. + +```ts +class LosslessSummarizerEngine extends BasicCompactionEngine { + protected summarize( + input: SummarizationInput, + agent: Agent, + signal?: AbortSignal, + ): Promise { + // not implemented: build the lossless prompt from input.messages, + // call through the host port, frame the checkpoint + } +} +``` + +Medium diff. It changes summary quality without owning range selection or +surface writes. It still concedes the DAG shape, because the seam expects one +checkpoint per transaction, not a frontier of nodes. + +### C. Full engine backend + +Implement `CompactionEngine` directly. Map lossless retention onto region +selection, lossless `runCompaction` onto the bracket transaction, and the +frontier onto the replacement checkpoint. + +Largest diff and largest risk. It must reimplement lock handling, stability +checks for whole-surface versus selected-span calls, and the busy taxonomy +correctly. It is the only shape that carries the full lossless policy across. + +## Synthesis decision + +Order the work A, then B, then C only on evidence. Each phase is usable on its +own, and each is the prerequisite of the next. This follows Foundational +thinking (scaffold before features) and Sequence verifiable units (one checkable +state per phase). + +## Candidate C plan (DAG parity slices) + +Verified against the published tarball at the pinned release: the package +root exposes config types and `BasicCompactionEngine` only. No region +internals, no range selection, and not even the `summarize()` contract cross +the published boundary. So any C built on published code stays inside the +subclass seam, and the risky machinery stays the target's without any +reimplementation: the lock is a durable unmatched `compaction/start` marker +read from the log, and stability checks, the busy taxonomy, the shrink check, +and the commit framing all run behind `super.compactRegion`. The earlier +warning that C must reimplement them applied only to a C that bypasses the +engine; that C is not planned. + +The DAG already runs in the sidecar under B: `runCompaction` stores leaf and +condensed nodes with coverage, proven by the engine tests. The remaining gap +to voltropy parity is three slices, each independently testable: + +- C1, checkpoint pointers. The ladder outcome already returns leaf and + condensed summary ids. Include `[lcm:summary #N]` refs in the checkpoint + text so the model and the recall tools can cite and resolve DAG nodes. + Evidence: a real compaction whose derived checkpoint names ids that + resolve in the sidecar. +- C2, previous summary. Feed the prior checkpoint as `previousSummary` so + the ladder merges instead of re-summarizing. The plugin tracks the last + checkpoint text per session (projection state), which avoids parsing + checkpoint tags. Evidence: a second compaction over checkpointed history + whose ladder input carries the previous summary. +- C3, already-memory mapping. Map a ladder `nothing` outcome (span rows all + covered) to a graceful `null` instead of a failed compaction. The planned + coverage-aware narrowing was dropped after analysis: replace semantics keep + the surface head always uncovered, so a covered prefix cannot occur, and + covered-but-live rows (range expansion catching retained rows older than a + checkpoint) stay truly covered under shadowing, so compacting over them is + already correct. Evidence: forcibly compacting an already-covered live span + rejects with `SpanAlreadyMemoryError` instead of the seam's generic + could-not-summarize failure. Null is not the answer there: the forcible + seam always compacts or throws by contract, and the auto path never selects + a fully covered span because replace semantics keep the head uncovered. + +Seam constraint, accepted: one checkpoint per transaction. The DAG lives in +the sidecar and the checkpoint is its projection, which is sufficient for +recall because the reader resolves pointers. Optional later upstream ask: a +small PR exporting the region functions and the summarize contract from the +package root, which removes the structural-copy drift rule. Not blocking. + +## Custom transaction flow (DAG-wins assumption, design sketch) + +The C slices are done, so this section designs what the confirmed +assumption commits us to: the DAG itself living in the DeepSeek log +instead of projected from the sidecar into one checkpoint per +transaction. If the item 1 numbers, once runnable, show the DAG does not +beat structured checkpoints, this flow was unnecessary and candidate B +stays the ceiling. + +### Problem: what a transaction is, and what constrains a custom one + +Traced from the pinned `dsh-compaction-basic` lib. Config types and +`BasicCompactionEngine` are the whole published boundary. + +- One transaction is `compaction/start`, then `compaction/summary` (a log + record with shadowed range, seqs, token count, provider, model, usage) + plus `user/message` with a `replace` surface op, then + `compaction/end`. Failure still appends `compaction/end` carrying the + error, which is what releases the durable lock (an unmatched + `compaction/start` read from the log). +- The shrink check rejects a framed checkpoint at or above the shadowed + route token count. Near-verbatim leaf nodes cannot pass it as replaces. +- Stability is span-scoped or whole-surface; the busy taxonomy separates + manual-idle (`owner: null`, no open turn) from auto (open turn + required). +- Checkpoint framing (preamble, `` tags, + `compactCheckpointSource`) is reader and tooling surface: custom commits + must reuse it byte for byte. +- No live-model verification exists on this branch: every slice proves + itself through scripted adapters only. + +### Candidate A: orchestrated multi-compact through the published seam + +Run the ladder once, then commit one DAG node per `super.compactRegion` +call, staging each node text for the per-transaction `summarize()` hook. +No reimplementation, no upstream PR, every bracket independently valid to +their tooling. + +It fails on the shrink check: leaf text is near-verbatim, so leaf replaces +are rejected, and the seam always pairs a summary record with a replace. +Through the published seam only condensed nodes can be committed, which is +a chain the C2 carryover already yields across compactions, not a DAG in +the log. A is status quo with orchestration, not parity. + +### Candidate B: one bracket, custom commit body + +Reimplement the bracket in the adapter: open, run the ladder once outside +the seam, commit one condensed replace plus one log-only +`compaction/summary` record per leaf (no surface op, linked by +`sourceEventSeqs`), close. The DAG lives in the log while the surface +pays only for the condensed node. + +Caller usage first: + +```ts +// B1 implements this shape, with one amendment recorded below +const result = await commitDagTransaction(deps, agent, selection, { + owner: null, // or a non-null marker for the auto path + stability: "whole-surface", // or "selected-span" + signal, +}); +// result: { compactionId, startSeq, summarySeqs, leafSeqs, endSeq } +``` + +Types derived from it: `DagCommitResult` as above (no `DagNodeCommit`: +the ladder runs inside the bracket, see the amendment). Module map: a new +`packages/deepseek/src/dag-commit.ts` holds the bracket; `engine.ts` gives +up a `runLadder(ctx, lossless, target, sessionId, span, signal)` helper +that `summarize()` keeps using for the B path, so the ladder runs +identically inside and outside the seam. Framing, lock check, stability +predicate, the shrink check on the condensed replace, and the manual/auto +error taxonomy are reimplemented against the pinned lib line for line, +each marked with the source line it mirrors so a pin move has a checklist. + +Amendment (B1): the sketch passed precomputed `nodes` in. The bracket +owns the ladder instead, mirroring how `compactSurfaceRegion` owns its +summarizer. Passing nodes in would split open/commit across the caller +boundary and leave ladder errors outside the failure taxonomy, which is +the half the spike exists to prove. Two deliberate divergences from the +lib, both marked B1 in code: single ladder attempt (no recover retry +loop; arrives with production wiring) and one log-only summary record +per stored DAG node, appended before the condensed record so the seam's +shadow-price adjacency contract still holds. + +### Synthesis decision + +B is the only shape that puts the DAG in the log; A cannot carry leaves +through the seam. Build B in slices: B0 this sketch, B1 bracket spike +against scripted models (open, commit, close, plus the failure path +releasing the lock), B2 production wiring through the plugin paths, with +convergence onto exported region primitives as a later option if the +upstream PR ever lands. Each slice re-verifies the B path has not +regressed, because the custom flow and the subclass seam now share the +ladder helper. + +B1 landed: `packages/deepseek/src/dag-commit.ts` (`commitDagTransaction`), +`packages/deepseek/test/dag-commit.test.ts` (3 tests: condensed replace +plus leaf records in one bracket, ladder failure closes with error and +releases the lock, manual taxonomy on idle), and the `runLadder` / +`resolveSummarizationTarget` / `withSummaryRefs` extraction in +`engine.ts` that `summarize()` keeps using. Gates green: `vp check`, 584 +tests, typecheck, pack. + +B2 landed (manual production path through the bracket): + +- `packages/deepseek/src/ladder.ts` holds the shared ladder + (`LosslessEngineDeps`, `runLadder`, `resolveSummarizationTarget`, + `withSummaryRefs`, `SpanAlreadyMemoryError`, `checkpointMemory`); + `engine.ts` re-exports the pre-split names so no consumer changed. + The split keeps the module graph acyclic (engine references the + bracket, the bracket references the ladder). +- `DagCommitResult` is a superset of the seam's `CompactionResult` + (branded ids plus `summarySeqs`/`leafSeqs`), so the engine override + and `/lcm compact` read it unchanged. +- `selectDagRange` mirrors `selectCompactableRange`; the engine + `compactNow` override mirrors the seam's idle admission, selection, + and flush, with the bracket as the transaction. +- Auto pressure (`compactRegion`) stays on the B seam, so the B-path + regression suite keeps pinning it. Routing auto through the bracket + waits until the manual path has proven itself live. + +B3 landed (flag-gated auto routing): `bracketAuto` on `LosslessEngineDeps` +and `LosslessPluginConfig` routes `compactRegion` through the bracket +(owner current-turn, whole-surface stability) while the default stays the +seam. The flag rides our own deps because the seam's `resolveConfig` +rejects unknown keys, so it cannot travel on `BasicCompactionConfig`. +Tests pin the bracket commit on an open turn and the raw (unwrapped) +ladder failure on the auto path. Gates green: `vp check`, 594 tests, +typecheck, pack. + +- Tests: `selectDagRange` (null, range shape, drift fails closed), + `compactNow` (bracket commit, null, busy on open turn, busy on + refused maintenance, cancelled), and the `/lcm compact` production + test now pins the DAG shape (condensed plus leaf records under one + id). Gates green: `vp check`, 592 tests, typecheck, pack. + +## Comparison docs direction (staged, waits on live proof) + +- The shipped docs keep per-harness reference pages (`pi`, `opencode`, + and `deepseek` when it ships): host-specific facts only. A root page + with shared guarantees stays a staged idea until the adapter proves + itself live; nothing is ever copied between pages. +- The docs-governance skill gains a lane for the root-plus-pages structure + only if the root page happens. +- Package READMEs stay as they are. Only the docs site changes. + +## Quality bar for this workstream + +Every slice ships production grade: clean, efficient, performant, composable, +and extensible, with keep-it-stupid-simple winning wherever it does not cost +one of the others. Concretely: no new machinery where the seam already owns +it, no speculative abstraction, no dead config, per-session state instead of +globals, fail-closed errors the engine's recovery path already handles, and +green gates (`vp check`, `tsc`, `bun run test`, `bun run pack`) before every +commit. + +## Module map + +One new package holds every host-aware line. Core stays host-free. + +```text +packages/deepseek/ + src/ + index.ts plugin entry: engine registration, tools, command + model-host.ts ModelHost over the streaming call API + reader.ts Reader over the recall tools + ingest.ts log observation mapped to ingestible entries + tools.ts recall tool definitions + engine.ts summarizer subclass over the lossless ladder + command.ts operator command and config verbs + plugin.ts composition wiring behind the enabled flag +``` + +The caller configures it the way the target composes everything else, by +selecting the capability in configuration: + +```yaml +- name: lossless-recall +- name: lossless-compaction + config: + keepRecentTokens: 20000 + summaryTokens: 1500 +``` + +## Phases and evidence gates + +Run `vp check` after every phase. Add `bun run test` and `bun run typecheck` +for any phase that touches `packages/*/src/**`. + +1. Port proof in a scratch composition. Implement the host port over one + streaming call and run one summary plus one retrieval against the + mechanical path. Evidence: the test log showing text assembled from chunks + and mapped to the `ModelAnswer` union, including the capped case. +2. Ingest mirror. Observe the live log and the persisted copy, write both to + a throwaway store, and compare entry counts against the derived message + count. Evidence: counts match across a session that includes tool calls, + a steering injection, and a resume. +3. Recall tools. Register the three tools in agent scope, ask a question whose + answer lives only in aged history, and require the answer to cite an entry + id or a summary id. Evidence: cited answer plus the retrieval budget rows. +4. Engine hook behind a flag. Ship candidate A or B with the engine disabled + by default, then enable it on one composition and replay a compacted + session deterministically. Evidence: identical derived messages on replay + and the target's own compaction tests still green. +5. Policy, command, and config. Wire thresholds, per-model overrides, the + operator command, and the metrics rows. Evidence: a pressure run that swaps + before the window fills and a quiet run that leaves the log untouched. + +Stop after any phase whose evidence fails. A failed phase is a finding for +roadmap item 10, not a reason to start the next phase. + +## Owner decisions (recorded) + +- Package name: `deepseek-lossless`, in `packages/deepseek/` to mirror the + `packages/pi` holding `pi-lossless` convention (internal only; the shipped + name is what matters). +- Comparison docs: restructure toward a root page with per-package sub-pages + (see below). Implementation waits on live proof, per roadmap item 10. +- Candidate C: proceeded as DAG-parity work in slices (done: C1 refs, C2 + carryover, C3 already-memory). The item 1 measurement is blocked (no + DeepSeek account, so no live-model DeepSeek arm); the work continues on + the confirmed DAG-wins assumption, with the numbers as a stop/confirm + gate when a key exists. Roadmap item 10 tracks this work until then. + +## Evidence (spike branch `spike/second-consumer`) + +Target checkout `38286a6cf9` (release `dsh-0.1.7-alpha.2`, vendor `4.0.4`). +Pins match exactly: `dsh-*@0.1.7-alpha.2`, `cordis@4.0.4`. + +- Target baseline: `pnpm vitest run packages/compaction` in the checkout, + 13 files and 263 tests green. +- Lossless suite with the new package: 578 tests green (`vp check`, `tsc`, + `bun run test`, `bun run pack` all pass; the package packs to 26KB with + the harness left external). +- Phase 1: scripted adapter through the real escalation ladder and one + `runRetrieval` in two steps against a real store. +- Phase 2: every derived text block lands in the sidecar, re-mirroring + inserts nothing, and a replace checkpoint shrinks derived history while + shadowed entries stay stored. +- Phase 3: three tools registered on a real tools composition; answers read + from a real store. +- Phase 4: one real `compactRegion` through the ladder with leaves in the + sidecar and the envelope on the summary event; fail-closed on unmapped + history; the engine shrink check fires on an irreducible span. +- Phase 5: disabled-by-default plugin; live-feed mirror with `/lcm status`; + one full manual compaction through the `/lcm compact` handler; one quiet + pressure run returning null with the log untouched. +- C1: the derived checkpoint names `[lcm:summary #N]` refs that all resolve + in the sidecar. +- C2: a second compaction over checkpointed history produces a checkpoint + starting with the first checkpoint's exact text, through the live feed + across two manual compactions. +- C3: spans order by sidecar rowid (core reads a span as a storage range); + forcibly compacting an already-covered live span rejects with + `SpanAlreadyMemoryError` while the surface stays untouched. +- Suite with the C slices: 581 tests green. +- B1: `commitDagTransaction` commits one condensed replace plus log-only + leaf records in one bracket; ladder failure closes with error and + releases the lock; manual taxonomy on idle. `runLadder` is shared with + the seam `summarize()` so the summarizer cannot drift. +- B2: the production manual path runs the bracket (`compactNow` override + mirrors idle admission, selection, flush); `DagCommitResult` stays a + superset of the seam result so `/lcm compact` reads it unchanged; the + `/lcm compact` test pins the production DAG shape. +- B3: flag-gated auto routing (`bracketAuto`, default off) through the + bracket; the flag rides our own deps because the seam's `resolveConfig` + rejects unknown keys. +- Suite with the B slices: 594 tests green (`vp check` clean, `tsc`, + `bun run test`, `bun run pack`). +- Memory hydration: `checkpointMemory` takes a sidecar lookup and rebuilds + the last checkpoint from the newest stored summary plus its children + when the process has no noted text, so a resumed session compacts with + continuity. Exact for leaf-plus-condensed runs; a single-leaf run + replaces its row's text on condensation, so the rebuild carries the + stored form. Suite: 596 tests green. +- Finding for a future pin move: the package root omits the `summarize()` + contract members, so the override repeats their shape (drift rule in + `packages/deepseek/src/engine.ts`). + +## Open questions for the owner + +- Whether the single-prompt `complete()` shape is acceptable when the target + reuses provider prefix cache only for verbatim replays of system prompt, + tools, and messages. +- Whether the sidecar store stays a file beside the session or moves onto the + target storage backend later. +- The package name, the config key spellings, and the compat pins against a + harness that warns of breaking changes. +- Whether item 10 ever advances past tracking, and whether the extracted + branch merges to main. The item 1 evaluation stays unrunnable until a + DeepSeek key exists; its numbers are a stop/confirm gate on the + transaction flow, not a precondition. + +## What this plan does not change + +No source, no config, and no version beyond what `feat/deepseek` already +carries. No changeset, because nothing ships. No edit to the docs site or +any package README beyond this file. The B1-B3 bracket spike was proven on +scripted models; roadmap item 10 stays a tracking item, so no spike log +attaches to it beyond this plan. + +Spike verdict: the custom transaction flow is proven on scripted models +across all three production paths (seam auto, bracket manual, bracket auto +behind a flag), with the B path's regression suite pinning the default. +What scripted models cannot prove is live-harness behavior (real pressure +timing, real flush backends, real prefix-cache effects), so merging the +spike to main and reopening item 10 stay owner decisions. The recommended +next work is roadmap item 1, the long-context evaluation: it is the only +source of measured evidence for the rest of the list, and its numbers are +the stop/confirm gate on this transaction flow. diff --git a/package.json b/package.json index dc07798..21ac005 100644 --- a/package.json +++ b/package.json @@ -20,9 +20,9 @@ "opencode:check": "bun scripts/opencode-live.ts", "opencode:smoke": "bun scripts/opencode-smoke.ts", "opencode:compact": "bun scripts/opencode-compact.ts", - "pack": "vp run --filter ./packages/core prepack && vp run --filter ./packages/pi prepack && vp run --filter ./packages/opencode prepack", + "pack": "vp run --filter ./packages/core prepack && vp run --filter ./packages/pi prepack && vp run --filter ./packages/deepseek prepack && vp run --filter ./packages/opencode prepack", "changeset": "changeset", - "version": "changeset version && bun scripts/sync-skill-versions.ts && vp fmt packages/*/CHANGELOG.md", + "version": "changeset version && rm -f bun.lock && bun install && bun scripts/sync-skill-versions.ts && vp fmt packages/*/CHANGELOG.md", "release": "changeset publish", "prepare": "vp config || true", "eval": "bun scripts/eval-long-context.ts" diff --git a/packages/deepseek/LICENSE b/packages/deepseek/LICENSE new file mode 100644 index 0000000..0e8c8a8 --- /dev/null +++ b/packages/deepseek/LICENSE @@ -0,0 +1,13 @@ +MIT License + +This software is an original implementation of the Lossless Context Management +(LCM) architecture, specified in the paper "Lossless Context Management" by +Clint Ehrlich and Theodore Blackman, Voltropy PBC +(https://papers.voltropy.com/LCM). The paper is credited as the algorithmic +specification; all code in this repository was written from that spec. + +The "smart zone" framing (quality degradation at a token budget, not the +window limit) is credited to Matt Pocock, AI Coding Dictionary +(https://www.aihero.dev/ai-coding-dictionary/smart-zone). + +Copyright (c) 2026 stainless-code diff --git a/packages/deepseek/README.md b/packages/deepseek/README.md new file mode 100644 index 0000000..2596ad8 --- /dev/null +++ b/packages/deepseek/README.md @@ -0,0 +1,27 @@ +# `deepseek-lossless` (working name) + +Lossless Context Management for the DeepSeek harness, built on `lossless-core`. +Status: evaluation spike on a branch. It implements the plan in +`docs/deepseek-plan.md` phase by phase, and `docs/roadmap.md` item 10 +tracks this work until the live-proof gate is met. + +## Pins + +The adapter compiles against these exact published versions, which match the +pinned local checkout (release `dsh-0.1.7-alpha.2`, vendor `4.0.4`): + +- `@deepseek-ai/dsh-agent@0.1.7-alpha.2` +- `@deepseek-ai/dsh-commands@0.1.7-alpha.2` +- `@deepseek-ai/dsh-compaction@0.1.7-alpha.2` +- `@deepseek-ai/dsh-compaction-basic@0.1.7-alpha.2` +- `@deepseek-ai/dsh-llm@0.1.7-alpha.2` +- `@deepseek-ai/dsh-session@0.1.7-alpha.2` +- `@deepseek-ai/dsh-session-projection@0.1.7-alpha.2` (tests only) +- `@deepseek-ai/dsh-system-prompt@0.1.7-alpha.2` (tests only) +- `@deepseek-ai/dsh-token-meter@0.1.7-alpha.2` (tests only) +- `@deepseek-ai/dsh-tools@0.1.7-alpha.2` +- `@deepseek-ai/cordis@4.0.4` + +The checkout lives outside this repo and is never vendored here. Every phase +log names the commit it ran against, because the harness warns of breaking +changes. diff --git a/packages/deepseek/package.json b/packages/deepseek/package.json new file mode 100644 index 0000000..abe5ae4 --- /dev/null +++ b/packages/deepseek/package.json @@ -0,0 +1,60 @@ +{ + "name": "deepseek-lossless", + "version": "0.1.0", + "description": "Lossless Context Management for the DeepSeek harness: long sessions keep working past the point where the model would forget, and every verbatim message stays searchable and readable.", + "keywords": [ + "coding-agent", + "context", + "deepseek", + "dsh-plugin", + "lcm", + "memory" + ], + "homepage": "https://github.com/stainless-code/lossless#readme", + "bugs": { + "url": "https://github.com/stainless-code/lossless/issues" + }, + "license": "MIT", + "author": "stainless-code", + "repository": { + "type": "git", + "url": "git+https://github.com/stainless-code/lossless.git", + "directory": "packages/deepseek" + }, + "files": [ + "dist", + "CHANGELOG.md" + ], + "type": "module", + "main": "./dist/index.mjs", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "scripts": { + "pack:watch": "vp pack --watch", + "prepack": "vp pack", + "prepublishOnly": "vp check && vp test" + }, + "dependencies": { + "@deepseek-ai/cordis": "4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-commands": "0.2.1-rc.2", + "@deepseek-ai/dsh-compaction": "0.2.1-rc.2", + "@deepseek-ai/dsh-compaction-basic": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-tools": "0.2.1-rc.2", + "lossless-core": "workspace:*" + }, + "devDependencies": { + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", + "@deepseek-ai/dsh-token-meter": "0.2.1-rc.2" + }, + "engines": { + "node": ">=22.19.0" + } +} diff --git a/packages/deepseek/src/command.ts b/packages/deepseek/src/command.ts new file mode 100644 index 0000000..26a1331 --- /dev/null +++ b/packages/deepseek/src/command.ts @@ -0,0 +1,103 @@ +import type { Context } from "@deepseek-ai/cordis"; +import type { CommandInvocation, CommandResult } from "@deepseek-ai/dsh-commands"; +import { CommandDefinitionId } from "@deepseek-ai/dsh-commands/brand"; +import { ManualCompactionError } from "@deepseek-ai/dsh-compaction"; +import type { LcmStore } from "lossless-core"; + +export interface LcmCommandDeps { + readonly stores: () => ReadonlyMap; +} + +const USAGE = "Usage: /lcm [status|compact]"; + +/** Fail loudly if a locally closed union gains an unhandled member. */ +function assertNever(value: never): never { + throw new Error(`unknown manual compaction error code: ${String(value)}`); +} + +/** Convert expected capability failures into concise human-only outcomes. */ +function expectedFailure(error: ManualCompactionError): CommandResult { + switch (error.code) { + case "busy": + return { + kind: "error", + text: "Compaction is unavailable because this process has an active compaction, or the agent is not idle.", + }; + case "cancelled": + return { kind: "error", text: "Compaction cancelled." }; + case "changed": + return { + kind: "error", + text: "The history selected for compaction changed before it could be replaced. The attempt is recorded in the session log.", + }; + case "summary": + return { + kind: "error", + text: "Compaction could not produce a useful summary. The attempt is recorded in the session log.", + }; + case "commit": + return { + kind: "error", + text: "Compaction did not finish cleanly; some session history may have changed. Inspect the current session state before retrying.", + }; + case "persistence": + return { + kind: "error", + text: "Compaction finished, but the session could not be saved.", + }; + default: + return assertNever(error.code); + } +} + +async function handleCompact(ctx: Context, invocation: CommandInvocation): Promise { + try { + const result = await ctx.compaction.compactNow( + invocation.agent, + invocation.signal, + invocation.commandId, + ); + if (result === null) return { kind: "success", text: "No compactable history yet." }; + return { + kind: "success", + text: `Compacted ${result.shadowedSeqs.length} history items (~${result.shadowedTokenCount} tokens).`, + sourceEventSeq: result.summarySeq, + }; + } catch (error: unknown) { + if (invocation.signal.aborted) return { kind: "error", text: "Compaction cancelled." }; + if (error instanceof ManualCompactionError) return expectedFailure(error); + throw error; + } +} + +function handleStatus(deps: LcmCommandDeps): CommandResult { + const stores = deps.stores(); + if (stores.size === 0) return { kind: "success", text: "LCM has mirrored no sessions yet." }; + const lines = [...stores].map(([id, store]) => { + const messages = store.allMessages().length; + const summaries = store.allSummaries().length; + return `${id}: ${messages} messages, ${summaries} summaries`; + }); + return { kind: "success", text: lines.join("\n") }; +} + +/** The `/lcm` command: `status` reports the mirror, `compact` forces one + * manual compaction through whichever backend the composition mounted. */ +export function lcmCommandDefinition(ctx: Context, deps: LcmCommandDeps) { + return { + definitionId: CommandDefinitionId("deepseek-lossless"), + name: "lcm", + description: "Lossless session memory: status and manual compaction", + handler: (invocation: CommandInvocation): Promise => { + const verb = invocation.rawInput.trim().split(/\s+/, 1)[0] ?? ""; + if (verb === "" || verb === "status") return Promise.resolve(handleStatus(deps)); + if (verb === "compact") return handleCompact(ctx, invocation); + return Promise.resolve({ kind: "error", text: USAGE }); + }, + }; +} + +/** Register `/lcm` on a composition with a commands service. Returns its disposer. */ +export function registerLcmCommand(ctx: Context, deps: LcmCommandDeps): () => void { + return ctx.commands.register(lcmCommandDefinition(ctx, deps)); +} diff --git a/packages/deepseek/src/dag-commit.ts b/packages/deepseek/src/dag-commit.ts new file mode 100644 index 0000000..23ca88a --- /dev/null +++ b/packages/deepseek/src/dag-commit.ts @@ -0,0 +1,601 @@ +import { randomUUID } from "node:crypto"; +import { isDeepStrictEqual } from "node:util"; + +import type { Context } from "@deepseek-ai/cordis"; +import type { Agent } from "@deepseek-ai/dsh-agent"; +import type { CommandId } from "@deepseek-ai/dsh-commands/brand"; +import { + CompactionId, + ManualCompactionError, + compactCheckpointSource, + toolPairingBalancedAfter, + toolPairingBalancedBefore, +} from "@deepseek-ai/dsh-compaction"; +import { createUserMessage, errorChain } from "@deepseek-ai/dsh-llm"; +import type { ContentBlock } from "@deepseek-ai/dsh-llm"; +import { SessionSeq } from "@deepseek-ai/dsh-session"; +import type { Session, SessionEvent } from "@deepseek-ai/dsh-session"; +import type { TokenMeasurement, TokenMeter, TokenSurfaceNode } from "@deepseek-ai/dsh-token-meter"; +import type { CompactionSpanEntry } from "lossless-core"; + +import type { LadderOutcome, LosslessEngineDeps } from "./ladder.ts"; +import { resolveSummarizationTarget, runLadder, withSummaryRefs } from "./ladder.ts"; +import type { DshRoute } from "./model-host.ts"; + +/** Deps for the DAG bracket: the ladder deps, the composition the ladder + * runs inside, and the summarization fallback the target resolves from. The + * token meter is read off the composition, which the host harness provides; + * the import is type-only, so the runtime boundary stays honest. */ +export interface DagCommitDeps extends LosslessEngineDeps { + readonly ctx: Context; + /** Empty means unset: resolution falls through to the routed request, then + * the agent options, exactly like the seam. */ + readonly summarizationProvider: string; + readonly summarizationModel: string; +} + +/** Inclusive surface-node span to condense, mirroring the seam's start/end. */ +export interface DagCommitSelection { + readonly start: SessionSeq; + readonly end: SessionSeq; +} + +export interface DagCommitOptions { + /** Null compacts an idle session (manual taxonomy); any other value + * requires an open turn and reports that turn as the lifecycle owner. */ + readonly owner: string | null; + readonly stability: "whole-surface" | "selected-span"; + readonly sourceCommandId?: CommandId; + readonly flush?: () => Promise; +} + +/** Bracket result: the seam's CompactionResult fields plus the DAG node + * records. The singular summary fields name the condensed metering record + * (the event directly before the checkpoint); summarySeqs lists it first, + * then one log-only record per stored leaf. The seam shape is kept so the + * engine override and the /lcm command read this result unchanged. */ +export interface DagCommitResult { + readonly compactionId: CompactionId; + readonly sourceCommandId?: CommandId; + readonly startSeq: SessionSeq; + readonly summarySeq: SessionSeq; + readonly endSeq: SessionSeq; + readonly summary: ContentBlock[]; + readonly shadowedRange: { readonly start: SessionSeq; readonly end: SessionSeq }; + readonly shadowedSeqs: SessionSeq[]; + readonly shadowedTokenCount: number; + /** Condensed record first, then one log-only record per stored leaf. */ + readonly summarySeqs: readonly SessionSeq[]; + readonly leafSeqs: readonly SessionSeq[]; +} + +/** Stability failure inside the bracket: the span moved under the ladder. + * Plays the seam's internal SurfaceChangedError role in the taxonomy. */ +export class DagStabilityError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "DagStabilityError"; + } +} + +/** Checkpoint framing copied from the seam so a DAG checkpoint reads like + * one: dsh-compaction-basic@0.1.7-alpha.2 lib/index.js:234-235, 280. */ +const DAG_SUMMARY_OPEN_TAG = ""; +const DAG_SUMMARY_CLOSE_TAG = ""; +const DAG_CHECKPOINT_PREAMBLE = + "This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint."; + +type StartEvent = Extract; + +interface DagValidated { + readonly start: SessionSeq; + readonly end: SessionSeq; + readonly startIdx: number; + readonly endIdx: number; + readonly shadowedSeqs: SessionSeq[]; +} + +interface DagPrepared extends DagValidated { + readonly sessionId: string; + readonly target: DshRoute; + readonly measurement: TokenMeasurement; + readonly selectedNodes: TokenSurfaceNode[]; + readonly shadowedTokenCount: number; + readonly shadowedRouteTokenCount: number; + readonly span: CompactionSpanEntry[]; +} + +type DagStage = "summary" | "commit"; + +/** One custom compaction transaction: the seam's bracket (lock check, open, + * stability, shrink check, commit, close, failure taxonomy) with the + * lossless ladder as the summarizer and one log-only summary record per + * stored DAG node. Structure mirrors compactSurfaceRegion line for line + * (dsh-compaction-basic@0.1.7-alpha.2 lib/index.js:449-521); the two + * deliberate divergences are marked B1 below. */ +export async function commitDagTransaction( + deps: DagCommitDeps, + agent: Agent, + selection: DagCommitSelection, + options: DagCommitOptions, + signal?: AbortSignal, +): Promise { + const session = agent.session; + if (options.owner === null) signal?.throwIfAborted(); + const validated = validateDagRegion(session, selection.start, selection.end); + const entryState = inspectDagEntryState(session); + assertDagInactive(entryState.unmatchedStart, entryState.latestEndSeedSeq, "compaction"); + let owner: number | null; + if (options.owner === null) { + if (entryState.openTurn !== null) { + throw new ManualCompactionError( + "busy", + "manual compaction: the session already has an open turn", + ); + } + owner = null; + } else { + if (entryState.openTurn === null) { + throw new Error( + "compactRegion: no open turn — automatic compaction events must be enclosed in a turn", + ); + } + owner = entryState.openTurn; + } + const compactionId = CompactionId(randomUUID()); + const lifecycle = { + compactionId, + ...(options.sourceCommandId === undefined ? {} : { sourceCommandId: options.sourceCommandId }), + turn: owner, + }; + const startEvent = session.append("compaction/start", lifecycle); + const assertStable = + options.stability === "whole-surface" ? assertWholeSurfaceUnchanged : assertSelectedSpanStable; + let failure: { error: unknown; stage: DagStage } | undefined; + let flushFailure: unknown; + let result: DagCommitResult | undefined; + let closed = false; + let closing = false; + let stage: DagStage = "summary"; + try { + const prepared = prepareDagCommit(deps, agent, session, validated); + signal?.throwIfAborted(); + // B1 runs one ladder attempt. The seam retries through its recover hook + // with a reprepared span; that loop is an auto-path production concern + // and arrives with the production wiring, not the spike. + const ladder = await runLadder( + deps.ctx, + deps, + prepared.target, + prepared.sessionId, + prepared.span, + signal, + ); + const condensed = frameCondensed(deps, prepared, ladder, compactionId, options.sourceCommandId); + if (options.owner === null) signal?.throwIfAborted(); + assertStable(deps.ctx.tokenMeter, session, prepared); + stage = "commit"; + const pending = commitDagBody(session, startEvent, prepared, ladder, condensed); + closing = true; + const endEvent = session.append("compaction/end", lifecycle); + closed = true; + result = { ...pending, endSeq: endEvent.seq }; + } catch (error) { + failure = { error, stage: closing ? "commit" : stage }; + if (!closing) { + closing = true; + try { + session.append("compaction/end", { ...lifecycle, error: errorChain(error) }); + closed = true; + } catch (closeError) { + failure = { error: closeError, stage: "commit" }; + } + } + } + if (closed && options.flush !== undefined) { + try { + await options.flush(); + } catch (error) { + flushFailure = error; + } + } + if (options.owner === null) signal?.throwIfAborted(); + if (failure !== undefined) { + if (options.owner === null) throwDagFailure(failure); + throw failure.error; + } + if (flushFailure !== undefined) { + throw new ManualCompactionError( + "persistence", + "manual compaction durability checkpoint failed", + { + cause: flushFailure, + }, + ); + } + if (result === undefined) throw new Error("compaction committed without a result"); + return result; +} + +/** Classify one closed manual attempt without weakening cancellation + * precedence. Mirrors throwManualFailure (lib/index.js:523-528). */ +function throwDagFailure(failure: { error: unknown; stage: DagStage }): never { + if (failure.stage === "commit") { + throw new ManualCompactionError("commit", "manual compaction did not commit cleanly", { + cause: failure.error, + }); + } + if (failure.error instanceof DagStabilityError) { + throw new ManualCompactionError( + "changed", + "the compacted history changed during manual compaction", + { + cause: failure.error, + }, + ); + } + throw new ManualCompactionError( + "summary", + "manual compaction could not produce a smaller summary", + { + cause: failure.error, + }, + ); +} + +/** Reject a durable unmatched compaction marker unless a later + * constructor-seed boundary proves its owner belongs to an earlier session + * lifecycle. Mirrors assertCompactionInactive (lib/index.js:535-542). */ +function assertDagInactive( + unmatchedStart: StartEvent | undefined, + latestEndSeedSeq: SessionSeq | undefined, + stage: string, +): void { + if (unmatchedStart === undefined) return; + if (latestEndSeedSeq !== undefined && latestEndSeedSeq > unmatchedStart.seq) return; + throw new ManualCompactionError( + "busy", + `${stage}: compaction already in progress; the session compaction lock is already active`, + ); +} + +/** Resolve the next range starting at the first non-system surface node + * while retaining a priced recent tail and never splitting an assistant + * tool-call/result pair. Mirrors selectCompactableRange + * (lib/index.js:409-448). */ +export function selectDagRange( + session: Session, + measurement: TokenMeasurement, + retainTokens: number, +): DagCommitSelection | null { + const pricedNodes = measurement.nodes; + if (pricedNodes.length === 0) return null; + const surfaceNodes = session.surface.nodes; + if ( + surfaceNodes.length !== pricedNodes.length || + surfaceNodes.some((seq, index) => seq !== pricedNodes[index]?.seq) + ) { + throw new Error( + "lossless dag commit: token-meter surface does not match the current session surface", + ); + } + const head = surfaceNodes[0]; + if (head === undefined) return null; + // A system/message at surface node 0 is the retained head, never inside + // the range; without one the range starts at node 0. + const firstIdx = dagSystemHead(session, head) === undefined ? 0 : 1; + let accumulated = 0; + let keepFromIdx = pricedNodes.length; + for (let index = pricedNodes.length - 1; index >= 0; index -= 1) { + accumulated += pricedNodes[index]?.tokens ?? 0; + keepFromIdx = index; + if (accumulated >= retainTokens) break; + } + if (keepFromIdx <= firstIdx) return null; + while (keepFromIdx > firstIdx) { + const node = surfaceNodes[keepFromIdx]; + if (node !== undefined && toolPairingBalancedBefore(session, node)) break; + keepFromIdx -= 1; + } + if (keepFromIdx <= firstIdx) return null; + const start = surfaceNodes[firstIdx]; + const end = surfaceNodes[keepFromIdx - 1]; + if (start === undefined || end === undefined) return null; + return { start, end }; +} + +function dagSystemHead(session: Session, headSeq: SessionSeq) { + const head = session.eventAt(headSeq); + return head?.type === "system/message" ? head : undefined; +} + +/** Validate one requested surface-position span before the bracket opens. + * Mirrors validateSurfaceRegion (lib/index.js:549-565). */ +function validateDagRegion(session: Session, start: SessionSeq, end: SessionSeq): DagValidated { + const nodes = session.surface.nodes; + const startIdx = nodes.indexOf(start); + const endIdx = nodes.indexOf(end); + const startNode = nodes[startIdx]; + const endNode = nodes[endIdx]; + if (startNode === undefined) + throw new Error(`dag commit: start seq ${start} not found in surface`); + if (endNode === undefined) throw new Error(`dag commit: end seq ${end} not found in surface`); + if (startIdx > endIdx) { + throw new Error( + `dag commit: start seq ${start} (position ${startIdx}) is after end seq ${end} (position ${endIdx}) on the surface`, + ); + } + if (!toolPairingBalancedBefore(session, startNode)) { + throw new Error( + `dag commit: start seq ${start} is not a balanced boundary (would split a step's tool-call/result pair)`, + ); + } + if (!toolPairingBalancedAfter(session, endNode)) { + throw new Error( + `dag commit: end seq ${end} is not a balanced boundary (would split a step, or the step is still open)`, + ); + } + return { start, end, startIdx, endIdx, shadowedSeqs: nodes.slice(startIdx, endIdx + 1) }; +} + +/** Snapshot pricing, resolve the summarization target, and map the span + * through the sidecar entry ids. Mirrors prepareCompaction + * (lib/index.js:567-579); the span mapping follows the seam's spanOf through + * entry ids the mirror records, never through text matching. */ +function prepareDagCommit( + deps: DagCommitDeps, + agent: Agent, + session: Session, + validated: DagValidated, +): DagPrepared { + const meter = deps.ctx.tokenMeter; + const measurement = meter.measure(session); + const selectedNodes = measurement.nodes.slice(validated.startIdx, validated.endIdx + 1); + if ( + selectedNodes.length !== validated.shadowedSeqs.length || + selectedNodes.some((node, index) => node.seq !== validated.shadowedSeqs[index]) + ) { + throw new DagStabilityError( + "lossless dag commit: selected surface changed before summarization began", + ); + } + const sessionId = String(session.id); + const target = resolveSummarizationTarget( + { + summarizationProvider: deps.summarizationProvider, + summarizationModel: deps.summarizationModel, + }, + agent, + ); + if (!target) { + throw new Error( + "lossless compaction: no provider/model available for summarization: set both engine summarization fields, route one request, or set both agent options fields", + ); + } + const store = deps.store(sessionId); + if (!store) throw new Error(`lossless compaction: no sidecar store for session ${sessionId}`); + const rowsByEntry = new Map(store.allMessages().map((row) => [row.entryId, row])); + // Core reads a span as a storage range, so frontierFrom must name the + // storage minimum. Surface order diverges from storage order after a + // replace (the checkpoint row is newest), so order by rowid here. + const items: Array<{ rowId: number; entry: CompactionSpanEntry }> = []; + for (const seq of validated.shadowedSeqs) { + // Entry ids mirror the ingest projection (ingest.ts projectedEntry), so + // the seq maps to its row without touching the live message index. + const entryId = `${session.id}:${seq}`; + const row = rowsByEntry.get(entryId); + if (!row) throw new Error(`lossless dag commit: sidecar holds no row for ${entryId}`); + items.push({ rowId: row.id, entry: { entryId, role: row.role, text: row.text } }); + } + items.sort((a, b) => a.rowId - b.rowId); + return { + ...validated, + sessionId, + target, + measurement, + selectedNodes: [...selectedNodes], + shadowedTokenCount: selectedNodes.reduce((total, node) => total + node.heuristicTokens, 0), + shadowedRouteTokenCount: selectedNodes.reduce((total, node) => total + node.tokens, 0), + span: items.map((item) => item.entry), + }; +} + +interface CondensedCommit { + readonly checkpointMessage: ReturnType; + readonly leafTexts: readonly string[]; + readonly condensedBlocks: ContentBlock[]; +} + +/** Frame the condensed checkpoint, resolve every stored leaf text, and run + * the shrink check. Mirrors the framing half of summarizeCompaction + * (lib/index.js:581-606). */ +function frameCondensed( + deps: DagCommitDeps, + prepared: DagPrepared, + ladder: LadderOutcome, + compactionId: ReturnType, + sourceCommandId: CommandId | undefined, +): CondensedCommit { + const condensedText = withSummaryRefs(ladder.summaryText, [ + ...ladder.leafSummaryIds, + ...ladder.condensedSummaryIds, + ]); + deps.memory?.note(prepared.sessionId, condensedText); + const store = deps.store(prepared.sessionId); + const leafTexts = ladder.leafSummaryIds.map((id) => { + const text = store?.getSummary(id)?.text; + if (text === undefined) throw new Error(`lossless dag commit: sidecar holds no summary #${id}`); + return text; + }); + const condensedBlocks: ContentBlock[] = [{ type: "text", text: condensedText }]; + const checkpointMessage = createUserMessage({ + content: [ + { type: "text", text: `${DAG_CHECKPOINT_PREAMBLE}\n\n${DAG_SUMMARY_OPEN_TAG}` }, + { type: "text", text: condensedText }, + { type: "text", text: DAG_SUMMARY_CLOSE_TAG }, + ], + source: compactCheckpointSource(compactionId, sourceCommandId), + }); + const framedSummaryTokenCount = deps.ctx.tokenMeter.estimateMessage(checkpointMessage); + if (framedSummaryTokenCount >= prepared.shadowedRouteTokenCount) { + throw new Error( + `summary is not smaller than the shadowed content (${framedSummaryTokenCount} estimated framed tokens >= ${prepared.shadowedRouteTokenCount})`, + ); + } + return { checkpointMessage, leafTexts, condensedBlocks }; +} + +/** Append the condensed summary record, one log-only record per stored + * leaf, and the replacement body without yielding. The leaf records come + * first so the condensed metering event sits directly before the checkpoint, + * preserving the seam's shadow-price adjacency contract + * (lib/index.js:627-675 and the compaction/summary doc at types/types.d.ts). + * B1 is the second deliberate divergence: the seam commits one summary + * record; the bracket commits the stored leaves alongside it. */ +function commitDagBody( + session: Session, + startEvent: StartEvent, + prepared: DagPrepared, + ladder: LadderOutcome, + condensed: CondensedCommit, +): Omit { + const compactionId = startEvent.data.compactionId; + const sourceCommandId = + startEvent.data.sourceCommandId === undefined + ? {} + : { sourceCommandId: startEvent.data.sourceCommandId }; + const leafEvents = condensed.leafTexts.map((text) => + session.append("compaction/summary", { + compactionId, + ...sourceCommandId, + summary: [{ type: "text", text }], + shadowedRange: { start: prepared.start, end: prepared.end }, + shadowedSeqs: [...prepared.shadowedSeqs], + shadowedTokenCount: prepared.shadowedTokenCount, + provider: ladder.provider, + model: ladder.model, + }), + ); + const summaryEvent = session.append("compaction/summary", { + compactionId, + ...sourceCommandId, + summary: condensed.condensedBlocks, + rawOutput: condensed.condensedBlocks, + shadowedRange: { start: prepared.start, end: prepared.end }, + shadowedSeqs: [...prepared.shadowedSeqs], + shadowedTokenCount: prepared.shadowedTokenCount, + provider: ladder.provider, + model: ladder.model, + ...(ladder.usage === undefined ? {} : { usage: ladder.usage }), + }); + session.append("user/message", condensed.checkpointMessage, { + surfaceOp: { op: "replace", startSeq: prepared.start, endSeq: prepared.end }, + sourceEventSeqs: [ + startEvent.seq, + summaryEvent.seq, + ...leafEvents.map((event) => event.seq), + ...prepared.shadowedSeqs, + ], + }); + const leafSeqs = leafEvents.map((event) => event.seq); + return { + compactionId, + ...sourceCommandId, + startSeq: startEvent.seq, + summarySeq: summaryEvent.seq, + summary: condensed.condensedBlocks, + shadowedRange: { start: prepared.start, end: prepared.end }, + shadowedSeqs: [...prepared.shadowedSeqs], + shadowedTokenCount: prepared.shadowedTokenCount, + summarySeqs: [summaryEvent.seq, ...leafSeqs], + leafSeqs, + }; +} + +/** Reject a summary prepared against any earlier surface generation. + * Mirrors assertWholeSurfaceUnchanged (lib/index.js:608-610). */ +function assertWholeSurfaceUnchanged( + meter: TokenMeter, + session: Session, + prepared: DagPrepared, +): void { + if (!isDeepStrictEqual(meter.measure(session).nodes, prepared.measurement.nodes)) { + throw new DagStabilityError( + "lossless dag commit: session surface changed during summarization", + ); + } +} + +/** Require only that the selected span remain the same present, contiguous, + * equally priced, balanced replacement target. Mirrors + * assertSelectedSpanStable (lib/index.js:616-626). */ +function assertSelectedSpanStable( + meter: TokenMeter, + session: Session, + prepared: DagPrepared, +): void { + let current: DagValidated; + try { + current = validateDagRegion(session, prepared.start, prepared.end); + } catch (error) { + throw new DagStabilityError( + "lossless dag commit: the selected span is no longer a valid replacement target", + { cause: error }, + ); + } + if (!isDeepStrictEqual([...current.shadowedSeqs], [...prepared.shadowedSeqs])) { + throw new DagStabilityError( + "lossless dag commit: the selected span changed during summarization", + ); + } + if ( + !isDeepStrictEqual( + meter.measure(session).nodes.slice(current.startIdx, current.endIdx + 1), + prepared.selectedNodes, + ) + ) { + throw new DagStabilityError( + "lossless dag commit: the selected span was rewritten during summarization", + ); + } +} + +/** Latest bracket and turn state walking the log tailward. Mirrors + * inspectCompactionEntryState (lib/index.js:705-741). */ +function inspectDagEntryState(session: Session): { + readonly openTurn: number | null; + readonly unmatchedStart: StartEvent | undefined; + readonly latestEndSeedSeq: SessionSeq | undefined; +} { + let openTurn: number | null = null; + let openTurnStateKnown = false; + let unmatchedStart: StartEvent | undefined; + let startKnown = false; + let latestEndSeedSeq: SessionSeq | undefined; + for (let seq = session.seq - 1; seq >= 0; seq -= 1) { + const event = session.eventAt(SessionSeq(seq)); + if (event === undefined) + throw new Error(`lossless dag commit: session log has no event at seq ${seq}`); + if (latestEndSeedSeq === undefined && event.type === "session/end-seed") { + latestEndSeedSeq = event.seq; + } + if (!startKnown) { + if (event.type === "compaction/start") { + unmatchedStart = event; + startKnown = true; + } else if (event.type === "compaction/end") { + startKnown = true; + } + } + if (!openTurnStateKnown) { + if (event.type === "turn/start") { + openTurn = event.data.turn; + openTurnStateKnown = true; + } else if (event.type === "turn/end") { + openTurnStateKnown = true; + } + } + if (openTurnStateKnown && startKnown && latestEndSeedSeq !== undefined) break; + } + return { openTurn, unmatchedStart, latestEndSeedSeq }; +} diff --git a/packages/deepseek/src/engine.ts b/packages/deepseek/src/engine.ts new file mode 100644 index 0000000..b97c6f7 --- /dev/null +++ b/packages/deepseek/src/engine.ts @@ -0,0 +1,238 @@ +import type { Context } from "@deepseek-ai/cordis"; +import type { Agent } from "@deepseek-ai/dsh-agent"; +import type { CommandId } from "@deepseek-ai/dsh-commands/brand"; +import { ManualCompactionError } from "@deepseek-ai/dsh-compaction"; +import BasicCompactionEngine from "@deepseek-ai/dsh-compaction-basic"; +import type { BasicCompactionConfig } from "@deepseek-ai/dsh-compaction-basic"; +import type { ContentBlock, Message, TokenUsage, ToolSchema } from "@deepseek-ai/dsh-llm"; +import type { CompactionSpanEntry, LcmStore } from "lossless-core"; + +import { commitDagTransaction, selectDagRange } from "./dag-commit.ts"; +import type { DagCommitResult } from "./dag-commit.ts"; +import { MessageIndex } from "./ingest.ts"; +import type { CheckpointMemory, LadderOutcome, LosslessEngineDeps } from "./ladder.ts"; +import { + SpanAlreadyMemoryError, + checkpointMemory, + resolveSummarizationTarget, + runLadder, + withSummaryRefs, +} from "./ladder.ts"; + +/** Import surface this module keeps stable for the package root. The ladder + * lives in ladder.ts; these re-exports hold the pre-split import paths so + * the move changes no consumer. */ +export { SpanAlreadyMemoryError, checkpointMemory }; +export type { CheckpointMemory, LosslessEngineDeps }; + +/** The pinned `summarize()` contract, repeated here because the package root + * does not export these members at this pin. The override check below fails + * the build on renames and removals; additions need a look when the pin + * moves. */ +interface SpanInput { + readonly tools?: readonly ToolSchema[]; + readonly messages: readonly Message[]; +} + +interface SpanSummary { + summary: ContentBlock[]; + provider: string; + model: string; + maxTokens?: number; + usage?: TokenUsage; + rawOutput?: ContentBlock[]; +} + +/** Candidate B: the shipped backend keeps range selection, the transaction, + * the meter, and the shrink check. Only the summarizer runs through the + * lossless ladder and the sidecar, and the checkpoint text it returns is + * what the engine frames and commits. Manual compaction (`compactNow`) + * commits through the DAG bracket instead; automatic pressure compaction + * (`compactRegion`) stays on this seam unless `bracketAuto` routes it + * through the bracket too, so the B path keeps its regression suite. */ +export class LosslessSummarizerEngine extends BasicCompactionEngine { + constructor( + ctx: Context, + config: BasicCompactionConfig = {}, + private readonly lossless: LosslessEngineDeps, + ) { + super(ctx, config); + } + + override async compactRegion( + ...args: Parameters + ): Promise>> { + if (this.lossless.bracketAuto === true) { + const [start, end, agent, signal] = args; + return commitDagTransaction( + { + ctx: this.ctx, + summarizationProvider: this.config.summarizationProvider, + summarizationModel: this.config.summarizationModel, + ...this.lossless, + }, + agent, + { start, end }, + { owner: "current-turn", stability: "whole-surface" }, + signal, + ); + } + try { + return await super.compactRegion(...args); + } catch (error) { + // The seam wraps summarize failures in a generic could-not-summarize + // error; unwrap ours so the caller hears already-memory instead. + // Matched structurally (name plus code) so a second copy of the root + // package cannot break the recognition, and without importing it. + const cause = isManualSummaryError(error) ? error.cause : undefined; + if (cause instanceof SpanAlreadyMemoryError) throw cause; + throw error; + } + } + + /** Manual compaction through the DAG bracket: idle-agent admission, + * range selection, and the durability flush mirror the seam's compactNow + * (dsh-compaction-basic@0.1.7-alpha.2 lib/index.js:974-1009); the + * transaction itself is commitDagTransaction. Returns null when no safe + * useful range exists. */ + override compactNow( + agent: Agent, + signal: AbortSignal, + sourceCommandId?: CommandId, + ): Promise { + signal.throwIfAborted(); + try { + // Bare return, not await: only a synchronous admission refusal maps + // to busy here. A task failure rejects the returned promise past this + // catch, preserving the seam's cancellation precedence. + return agent.runMaintenance(async (agentSignal) => { + const operationSignal = AbortSignal.any([agentSignal, signal]); + try { + operationSignal.throwIfAborted(); + const range = selectDagRange( + agent.session, + this.ctx.tokenMeter.measure(agent.session), + 0, + ); + if (range === null) return null; + return await commitDagTransaction( + { + ctx: this.ctx, + summarizationProvider: this.config.summarizationProvider, + summarizationModel: this.config.summarizationModel, + ...this.lossless, + }, + agent, + range, + { + owner: null, + stability: "selected-span", + ...(sourceCommandId === undefined ? {} : { sourceCommandId }), + flush: async () => { + await this.ctx.sessions.flush(agent.session); + }, + }, + operationSignal, + ); + } catch (error) { + if (agentSignal.aborted && operationSignal.reason === agentSignal.reason) { + throw new ManualCompactionError("cancelled", "manual compaction was cancelled", { + cause: error, + }); + } + operationSignal.throwIfAborted(); + throw error; + } + }); + } catch (error) { + throw new ManualCompactionError( + "busy", + "manual compaction requires an idle agent with no waking queued work", + { cause: error }, + ); + } + } + + protected override async summarize( + input: SpanInput, + agent: Agent, + signal?: AbortSignal, + ): Promise { + const sessionId = String(agent.session.id); + const store = this.lossless.store(sessionId); + if (!store) throw new Error(`lossless compaction: no sidecar store for session ${sessionId}`); + const index = this.lossless.index(sessionId); + if (!index) { + throw new Error(`lossless compaction: session ${sessionId} has no message map`); + } + const target = resolveSummarizationTarget(this.config, agent); + if (!target) { + throw new Error( + "lossless compaction: no provider/model available for summarization: set both engine summarization fields, route one request, or set both agent options fields", + ); + } + const span = this.spanOf(input, store, index); + const ladder = await runLadder(this.ctx, this.lossless, target, sessionId, span, signal); + const checkpoint = withSummaryRefs(ladder.summaryText, [ + ...ladder.leafSummaryIds, + ...ladder.condensedSummaryIds, + ]); + this.lossless.memory?.note(sessionId, checkpoint); + return toSummaryResult(checkpoint, ladder, this.config.maxTokens); + } + + private spanOf(input: SpanInput, store: LcmStore, index: MessageIndex): CompactionSpanEntry[] { + const rowsByEntry = new Map(store.allMessages().map((row) => [row.entryId, row])); + // A leading system message is the retained head, not shadowed content. + const startAt = input.messages[0]?.role === "system" ? 1 : 0; + const items: Array<{ rowId: number; entry: CompactionSpanEntry }> = []; + for (const message of input.messages.slice(startAt)) { + const entryId = index.entryId(message.id); + if (entryId === undefined) { + throw new Error( + `lossless compaction: message ${message.id} has no sidecar row; the span is not fully mirrored`, + ); + } + const row = rowsByEntry.get(entryId); + if (!row) { + throw new Error(`lossless compaction: sidecar holds no row for ${entryId}`); + } + items.push({ rowId: row.id, entry: { entryId, role: row.role, text: row.text } }); + } + if (items.length === 0) + throw new Error("lossless compaction: no mirrorable messages in the span"); + // Core reads a span as a storage range, so frontierFrom must name the + // storage minimum. Surface order diverges from storage order after a + // replace (the checkpoint row is newest), so order by rowid here. + items.sort((a, b) => a.rowId - b.rowId); + return items.map((item) => item.entry); + } +} + +function toSummaryResult( + checkpoint: string, + ladder: Pick, + maxTokens: number, +): SpanSummary { + if (checkpoint.trim().length === 0) { + throw new Error("lossless compaction: ladder produced no text"); + } + return { + summary: [{ type: "text", text: checkpoint }], + rawOutput: [{ type: "text", text: checkpoint }], + provider: ladder.provider, + model: ladder.model, + maxTokens, + ...(ladder.usage === undefined ? {} : { usage: ladder.usage }), + }; +} + +/** True for the seam's generic summary failure, which wraps whatever + * `summarize` threw as its cause. */ +function isManualSummaryError(error: unknown): error is Error & { cause?: unknown } { + return ( + error instanceof Error && + error.name === "ManualCompactionError" && + (error as { code?: unknown }).code === "summary" + ); +} diff --git a/packages/deepseek/src/index.ts b/packages/deepseek/src/index.ts new file mode 100644 index 0000000..07e76d1 --- /dev/null +++ b/packages/deepseek/src/index.ts @@ -0,0 +1,13 @@ +export { dshModelHost, dshRef, usageOf } from "./model-host.ts"; +export type { DshHostOptions, DshRoute } from "./model-host.ts"; +export { dshReader } from "./reader.ts"; +export type { DshReaderOptions } from "./reader.ts"; +export { entryFor, mirrorEvent, observeSession, MessageIndex } from "./ingest.ts"; +export { LosslessSummarizerEngine, SpanAlreadyMemoryError, checkpointMemory } from "./engine.ts"; +export type { CheckpointMemory, LosslessEngineDeps } from "./engine.ts"; +export { lcmCommandDefinition, registerLcmCommand } from "./command.ts"; +export type { LcmCommandDeps } from "./command.ts"; +export { applyLossless } from "./plugin.ts"; +export type { LosslessPluginConfig } from "./plugin.ts"; +export { describeAnswer, expandAnswer, grepAnswer, registerRecallTools } from "./tools.ts"; +export type { RecallDeps } from "./tools.ts"; diff --git a/packages/deepseek/src/ingest.ts b/packages/deepseek/src/ingest.ts new file mode 100644 index 0000000..4bfa837 --- /dev/null +++ b/packages/deepseek/src/ingest.ts @@ -0,0 +1,112 @@ +import type { Context } from "@deepseek-ai/cordis"; +import type { Message } from "@deepseek-ai/dsh-llm"; +import type { Session, SessionEvent } from "@deepseek-ai/dsh-session"; +import { ingestEntries } from "lossless-core"; +import type { IngestStats, IngestibleEntry, LcmRole, LcmStore } from "lossless-core"; + +/** Live projection state: derived message id to sidecar entry id. The engine + * hook resolves its span through this map instead of matching text, so resume + * without it fails closed rather than guessing rows. */ +export class MessageIndex { + private readonly ids = new Map(); + + record(messageId: string, entryId: string): void { + this.ids.set(messageId, entryId); + } + + entryId(messageId: string): string | undefined { + return this.ids.get(messageId); + } +} + +/** One sidecar row per message-producing event, keyed by session and sequence. + * A compaction checkpoint ingests like any other model-visible message, and + * the entries it shadows stay stored: the log moves on, the sidecar keeps + * every address the reader can name. */ +export function entryFor(session: Session, event: SessionEvent): IngestibleEntry | undefined { + return projectedEntry(session, event)?.entry; +} + +/** Mirror one committed event into the sidecar. Ingest dedups on the entry + * id, so a re-delivered event inserts nothing. */ +export function mirrorEvent( + session: Session, + event: SessionEvent, + store: LcmStore, + index?: MessageIndex, +): IngestStats { + const projected = projectedEntry(session, event); + if (projected === undefined) { + return { inserted: 0, skipped: 1, piCompactions: [], files: [], largeInline: [] }; + } + const stats = ingestEntries(store, [projected.entry]); + if (stats.inserted > 0) index?.record(projected.messageId, projected.entry.entryId); + return stats; +} + +/** Follow one session's live feed into its sidecar. The feed starts at + * subscription time: synchronous history scans are prohibited for new + * production code in this harness, so there is no backfill here. A reopened + * sidecar is the resume path, and a session that predates the plugin keeps + * its earlier history out of reach until the async pagination path exists. */ +export function observeSession( + ctx: Context, + session: Session, + store: LcmStore, + index?: MessageIndex, +): () => void { + return ctx.on("session/event", (live, event) => { + if (live.id !== session.id) return; + mirrorEvent(live, event, store, index); + }); +} + +function projectedEntry( + session: Session, + event: SessionEvent, +): { entry: IngestibleEntry; messageId: string } | undefined { + const message = session.deriveEventMessage(event); + if (message === null) return undefined; + const text = renderText(message); + if (text === undefined) return undefined; + return { + entry: { + entryId: `${session.id}:${event.seq}`, + role: dshRole(message.role), + text, + timestamp: event.time, + }, + messageId: message.id, + }; +} + +function dshRole(role: Message["role"]): LcmRole { + switch (role) { + case "user": + return "user"; + case "assistant": + return "assistant"; + case "tool": + return "toolResult"; + case "system": + case "developer": + return "custom"; + default: { + const _exhaustive: never = role; + throw new Error(`unhandled message role: ${JSON.stringify(_exhaustive)}`); + } + } +} + +/** Verbatim text plus the calls the transcript needs to stay replayable. + * Reasoning, image, file, and tool-change blocks stay out: the mirror keeps + * what the model saw as text, not every modality the log can carry. */ +function renderText(message: Message): string | undefined { + const parts: string[] = []; + for (const block of message.content) { + if (block.type === "text") parts.push(block.text); + else if (block.type === "tool-call") parts.push(`tool-call: ${block.name} ${block.arguments}`); + } + const text = parts.join("\n").trim(); + return text.length > 0 ? text : undefined; +} diff --git a/packages/deepseek/src/ladder.ts b/packages/deepseek/src/ladder.ts new file mode 100644 index 0000000..344f031 --- /dev/null +++ b/packages/deepseek/src/ladder.ts @@ -0,0 +1,244 @@ +import type { Context } from "@deepseek-ai/cordis"; +import type { Agent } from "@deepseek-ai/dsh-agent"; +import type { TokenUsage } from "@deepseek-ai/dsh-llm"; +import { makeLlm, runCompaction } from "lossless-core"; +import type { + CompactionSpanEntry, + CompletionRequest, + LcmStore, + ModelAnswer, + ModelHost, + ModelRef, +} from "lossless-core"; + +import type { MessageIndex } from "./ingest.ts"; +import { dshModelHost } from "./model-host.ts"; +import type { DshRoute } from "./model-host.ts"; + +export interface LosslessEngineDeps { + readonly store: (sessionId: string) => LcmStore | undefined; + /** Live message-id to entry-id map, fed by the mirror. Spans resolve through it, never through text matching. */ + readonly index: (sessionId: string) => MessageIndex | undefined; + /** Last checkpoint text per session, fed back as the next ladder's previous + * summary. Absent until the engine stores its first checkpoint here. */ + readonly memory?: CheckpointMemory; + readonly redact: (text: string) => string; + readonly summaryTokens?: number; + readonly leafChunkTokens?: number; + /** Route automatic pressure compaction through the DAG bracket instead of + * the seam. Default false: the B path stays the observable auto behavior + * while the bracket proves itself on the manual path. */ + readonly bracketAuto?: boolean; +} + +/** Per-session checkpoint memory: the plugin owns one, the engine reads the + * previous summary from it and notes every checkpoint it stores. Absent + * until the session stores its first checkpoint, in memory or in the + * sidecar. */ +export interface CheckpointMemory { + previous(sessionId: string): string | undefined; + note(sessionId: string, text: string): void; +} + +const DEFAULT_SUMMARY_TOKENS = 1500; +const DEFAULT_LEAF_CHUNK_TOKENS = 3000; + +export function checkpointMemory( + stores?: (sessionId: string) => LcmStore | undefined, +): CheckpointMemory { + const texts = new Map(); + return { + previous: (sessionId) => { + const cached = texts.get(sessionId); + if (cached !== undefined) return cached; + const hydrated = hydrateCheckpoint(stores?.(sessionId)); + if (hydrated !== undefined) texts.set(sessionId, hydrated); + return hydrated; + }, + note: (sessionId, text) => { + texts.set(sessionId, text); + }, + }; +} + +/** Rebuild the last checkpoint from the sidecar: the newest stored summary + * plus its stored children, cited the same way the ladder cites them. The + * newest row is the last successful run's final node. Exact when the run + * stored leaves plus a fresh condensed row; a single-leaf run replaces its + * row's text on condensation, so the rebuild carries the stored form; with + * deeper condensation every cited id still resolves. Either way a resumed + * session compacts with continuity instead of from scratch. */ +function hydrateCheckpoint(store: LcmStore | undefined): string | undefined { + if (store === undefined) return undefined; + const summaries = store.allSummaries(); + const latest = summaries[summaries.length - 1]; + if (latest === undefined) return undefined; + return withSummaryRefs(latest.text, [...store.childSummaryIds(latest.id), latest.id]); +} + +/** The ladder stored nothing because every row in the span is already covered: + * the span is memory, not history. Thrown instead of the seam's generic + * summary failure so the caller hears the true state. */ +export class SpanAlreadyMemoryError extends Error { + constructor(span: string) { + super(`lossless compaction: span ${span} is already memory; every row is covered`); + this.name = "SpanAlreadyMemoryError"; + } +} + +/** Shared provider/model fallback chain (explicit config, latest routed + * request, agent options) for the seam path and the DAG bracket alike. */ +export function resolveSummarizationTarget( + config: { summarizationProvider: string; summarizationModel: string }, + agent: Agent, +): DshRoute | undefined { + const configured = + config.summarizationProvider.length > 0 + ? { provider: config.summarizationProvider, model: config.summarizationModel } + : undefined; + const header = agent.session.requestHeader()?.config; + const latest = + header && header.provider.length > 0 && header.model.length > 0 + ? { provider: header.provider, model: header.model } + : undefined; + const options = agent.options; + const agentTarget = + options.provider !== undefined && + options.provider.length > 0 && + options.model !== undefined && + options.model.length > 0 + ? { provider: options.provider, model: options.model } + : undefined; + return configured ?? latest ?? agentTarget; +} + +/** What one ladder run stored: the condensed text plus the sidecar ids and + * the model call facts for the checkpoint record. */ +export interface LadderOutcome { + readonly summaryText: string; + readonly leafSummaryIds: readonly number[]; + readonly condensedSummaryIds: readonly number[]; + readonly provider: string; + readonly model: string; + readonly usage?: TokenUsage; +} + +/** One lossless ladder run against a resolved span: leaf chunking, then + * condensation, sharing one usage tracker. The seam `summarize()` and the + * DAG bracket commit through here, so the summarizer cannot drift between + * the two paths. Throws SpanAlreadyMemoryError when every row in the span + * is already covered. */ +export async function runLadder( + ctx: Context, + lossless: LosslessEngineDeps, + target: DshRoute, + sessionId: string, + span: CompactionSpanEntry[], + signal?: AbortSignal, +): Promise { + const first = span[0]; + const last = span[span.length - 1]; + if (!first || !last) throw new Error("lossless compaction: span mapping came back empty"); + const host = dshModelHost(ctx, { + defaultRoute: target, + ...(signal === undefined ? {} : { signal }), + }); + const tracker = new UsageTracker(); + const llm = makeLlm( + tracker.wrap(host), + {}, + { + session: sessionId, + stage: "blocking", + span: { firstEntryId: first.entryId, lastEntryId: last.entryId }, + }, + ); + if (!llm) throw new Error("lossless compaction: no summarizer model resolves"); + const store = lossless.store(sessionId); + if (!store) throw new Error(`lossless compaction: no sidecar store for session ${sessionId}`); + const { token } = store.openRun({ session: sessionId }); + try { + const outcome = await runCompaction( + store.forRun(token), + { + span, + targetTokens: lossless.summaryTokens ?? DEFAULT_SUMMARY_TOKENS, + frontierFrom: first.entryId, + previousSummary: lossless.memory?.previous(sessionId), + }, + llm, + { + leafChunkTokens: lossless.leafChunkTokens ?? DEFAULT_LEAF_CHUNK_TOKENS, + redact: lossless.redact, + session: sessionId, + stage: "blocking", + }, + ); + store.commitRun(token); + if (outcome.kind === "nothing") { + throw new SpanAlreadyMemoryError(`${first.entryId}..${last.entryId} (${outcome.reason})`); + } + const route = tracker.route ?? target; + const usage = tracker.tokenUsage(); + return { + summaryText: outcome.summaryText, + leafSummaryIds: outcome.leafSummaryIds, + condensedSummaryIds: outcome.condensedSummaryIds, + provider: route.provider, + model: route.model, + ...(usage === undefined ? {} : { usage }), + }; + } finally { + store.abandonRun(token); + } +} + +/** Name the stored DAG nodes in the checkpoint using the same refs the + * recall tools resolve, so a future reader pulls full nodes from a short + * citation instead of re-deriving them. Shared with the DAG bracket so its + * checkpoint cites the same nodes. */ +export function withSummaryRefs(text: string, ids: readonly number[]): string { + if (ids.length === 0) return text; + return `${text}\n\nStored summaries: ${ids.map((id) => `[lcm:summary #${id}]`).join(" ")}`; +} + +/** Sums provider counts across every ladder call into one envelope. The route reported is the one that produced the stored text. */ +class UsageTracker { + private input = 0; + private output = 0; + private cacheRead = 0; + private cacheWrite = 0; + route: DshRoute | undefined = undefined; + + wrap(host: ModelHost): ModelHost { + return { + ...host, + complete: async ( + model: ModelRef, + request: CompletionRequest, + ): Promise => { + const answer = await host.complete(model, request); + this.route = model.model; + const usage = answer.usage; + if (usage) { + this.input += usage.input; + this.output += usage.output; + this.cacheRead += usage.cacheRead; + this.cacheWrite += usage.cacheWrite; + } + return answer; + }, + }; + } + + tokenUsage(): TokenUsage | undefined { + if (this.input === 0 && this.output === 0) return undefined; + return { + inputTokens: this.input, + outputTokens: this.output, + totalTokens: this.input + this.output, + cacheReadTokens: this.cacheRead, + cacheWriteTokens: this.cacheWrite, + }; + } +} diff --git a/packages/deepseek/src/model-host.ts b/packages/deepseek/src/model-host.ts new file mode 100644 index 0000000..3498f27 --- /dev/null +++ b/packages/deepseek/src/model-host.ts @@ -0,0 +1,147 @@ +import type { Context } from "@deepseek-ai/cordis"; +import { BlockAssembler } from "@deepseek-ai/dsh-llm"; +import type { FinishReason, GenerateOptions, TokenUsage } from "@deepseek-ai/dsh-llm"; +import type { + CompletionRequest, + ModelAnswer, + ModelHost, + ModelRef, + Notices, + Usage, +} from "lossless-core"; + +/** One model route on the composition: the provider key that selects the + * adapter instance, and the model id that adapter serves. */ +export interface DshRoute { + readonly provider: string; + readonly model: string; +} + +export interface DshHostOptions { + /** The route `session()` answers with. Absent until the plugin wires config. */ + readonly defaultRoute?: DshRoute; + /** Abort for every call this host makes. The reader owns its own signal. */ + readonly signal?: AbortSignal; +} + +/** One ref for one route. `reasoning` stays false until the host resolves + * model info, so a reasoning model never silently spends the output cap. */ +export function dshRef(route: DshRoute): ModelRef { + return { key: `${route.provider}/${route.model}`, reasoning: false, model: route }; +} + +/** The core port over one streaming call. `session()` answers the configured + * default route, `find()` builds a ref for any well-formed request and lets + * the stream fail `NO_ADAPTER` for routes nothing registered, and `complete()` + * sends the pre-joined prompt as one user message with `purpose: compaction`. */ +export function dshModelHost(ctx: Context, options?: DshHostOptions): ModelHost { + const signal = options?.signal; + const fallback = options?.defaultRoute; + return { + notices: { kind: "none" } satisfies Notices, + session: () => (fallback === undefined ? undefined : dshRef(fallback)), + find: (requested) => + requested.provider.length > 0 && requested.id.length > 0 + ? dshRef({ provider: requested.provider, model: requested.id }) + : undefined, + complete: (model, request) => complete(ctx, { signal, ref: model, request }), + }; +} + +interface CompleteArgs { + readonly signal: AbortSignal | undefined; + readonly ref: ModelRef; + readonly request: CompletionRequest; +} + +async function complete(ctx: Context, args: CompleteArgs): Promise { + const assembler = new BlockAssembler(); + const options: GenerateOptions = { + provider: args.ref.model.provider, + model: args.ref.model.model, + messages: [{ role: "user", content: [{ type: "text", text: args.request.prompt }] }], + maxTokens: args.request.maxTokens, + purpose: "compaction", + ...(args.signal === undefined ? {} : { signal: args.signal }), + }; + try { + for await (const chunk of ctx.llm.stream(options)) assembler.push(chunk); + } catch (error) { + return { + outcome: "failed", + reason: message(error), + capped: false, + stop: "threw", + usage: usageOf(assembler.usage), + }; + } + return answerOf(assembler); +} + +/** The terminal chunk decides, with the same words Pi uses: `stop` answers, + * `max-tokens` fails capped so the ladder stores nothing partial, and anything + * else fails open with the provider's own code in the reason. */ +function answerOf(assembler: BlockAssembler): ModelAnswer { + const usage = usageOf(assembler.usage); + const finish: FinishReason = assembler.finish; + switch (finish.kind) { + case "stop": + return { outcome: "answer", text: joinedText(assembler), stop: finish.kind, usage }; + case "max-tokens": + return { + outcome: "failed", + reason: "generation hit the token cap and the summary is incomplete", + capped: true, + stop: finish.kind, + usage, + }; + case "tool-calls": + return { + outcome: "failed", + reason: "the model returned tool calls for a request that sent no tools", + capped: false, + stop: finish.kind, + usage, + }; + case "aborted": + case "error": + return { + outcome: "failed", + reason: `${finish.failure.code}: ${finish.failure.message}`, + capped: false, + stop: finish.kind, + usage, + }; + default: { + const _exhaustive: never = finish; + throw new Error(`unhandled finish kind: ${JSON.stringify(_exhaustive)}`); + } + } +} + +function joinedText(assembler: BlockAssembler): string { + return assembler + .blocks() + .flatMap((block) => (block.type === "text" ? [block.text] : [])) + .join("\n"); +} + +/** The provider's counts with no prices attached: nothing offline knows what a + * token costs here, so every cost leg is zero and the totals stay exact. */ +export function usageOf(usage: TokenUsage | undefined): Usage | undefined { + if (!usage) return undefined; + const input = usage.inputTokens; + const output = usage.outputTokens; + return { + input, + output, + cacheRead: usage.cacheReadTokens ?? 0, + cacheWrite: usage.cacheWriteTokens ?? 0, + totalTokens: usage.totalTokens ?? input + output, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +function message(error: unknown): string { + return error instanceof Error ? error.message.slice(0, 300) : String(error).slice(0, 300); +} diff --git a/packages/deepseek/src/plugin.ts b/packages/deepseek/src/plugin.ts new file mode 100644 index 0000000..ed4919e --- /dev/null +++ b/packages/deepseek/src/plugin.ts @@ -0,0 +1,97 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; + +import type { Context } from "@deepseek-ai/cordis"; +import { LcmStore, redactSecrets } from "lossless-core"; + +import { registerLcmCommand } from "./command.ts"; +import { LosslessSummarizerEngine, checkpointMemory } from "./engine.ts"; +import { MessageIndex, mirrorEvent } from "./ingest.ts"; +import { registerRecallTools } from "./tools.ts"; + +export interface LosslessPluginConfig { + /** Master flag. Nothing registers, mirrors, or mounts until true. */ + readonly enabled?: boolean; + /** Automatic pressure compaction inside the turn. Default false. */ + readonly auto?: boolean; + /** Route automatic pressure compaction through the DAG bracket instead of + * the seam. Default false. */ + readonly bracketAuto?: boolean; + /** Directory for one sidecar file per session. Created when missing. */ + readonly sidecarDir?: string; + readonly summaryTokens?: number; + readonly leafChunkTokens?: number; + readonly summarizationProvider?: string; + readonly summarizationModel?: string; +} + +const DEFAULT_SIDECAR_DIR = "./.lossless"; + +function sanitize(sessionId: string): string { + return sessionId.replace(/[^a-zA-Z0-9_-]/g, "_"); +} + +/** Compose lossless memory into a harness context: mirror every session + * into its own sidecar, register the recall tools against the calling + * session, mount the summarizer engine as the compaction backend, and + * register `/lcm`. Disabled by default; enable on one composition at a + * time. The returned disposer stops the mirror, the tools, and the command. + * The engine follows the composition lifetime. */ +export function applyLossless(ctx: Context, config: LosslessPluginConfig = {}): () => void { + if (config.enabled !== true) return () => undefined; + const dir = config.sidecarDir ?? DEFAULT_SIDECAR_DIR; + mkdirSync(dir, { recursive: true }); + const sidecars = new Map(); + const sidecarFor = (sessionId: string): { store: LcmStore; index: MessageIndex } => { + let entry = sidecars.get(sessionId); + if (!entry) { + entry = { + store: new LcmStore(join(dir, `${sanitize(sessionId)}.sqlite`)), + index: new MessageIndex(), + }; + sidecars.set(sessionId, entry); + } + return entry; + }; + const stopMirror = ctx.on("session/event", (live, event) => { + const entry = sidecarFor(String(live.id)); + mirrorEvent(live, event, entry.store, entry.index); + }); + const stopTools = registerRecallTools(ctx, { + store: (sessionId) => (sessionId === undefined ? undefined : sidecarFor(sessionId).store), + redact: redactSecrets, + }); + // Hydration fallback: a resumed session rebuilds its last checkpoint + // from its sidecar until it compacts again. Reads only; a lookup miss + // creates no store. + const memory = checkpointMemory((sessionId) => sidecars.get(sessionId)?.store); + void new LosslessSummarizerEngine( + ctx, + { + auto: config.auto ?? false, + ...(config.summarizationProvider === undefined + ? {} + : { summarizationProvider: config.summarizationProvider }), + ...(config.summarizationModel === undefined + ? {} + : { summarizationModel: config.summarizationModel }), + }, + { + store: (sessionId) => sidecars.get(sessionId)?.store, + index: (sessionId) => sidecars.get(sessionId)?.index, + memory, + redact: redactSecrets, + ...(config.summaryTokens === undefined ? {} : { summaryTokens: config.summaryTokens }), + ...(config.leafChunkTokens === undefined ? {} : { leafChunkTokens: config.leafChunkTokens }), + ...(config.bracketAuto === undefined ? {} : { bracketAuto: config.bracketAuto }), + }, + ); + const stopCommand = registerLcmCommand(ctx, { + stores: () => new Map([...sidecars].map(([id, entry]) => [id, entry.store])), + }); + return () => { + stopMirror(); + stopTools(); + stopCommand(); + }; +} diff --git a/packages/deepseek/src/reader.ts b/packages/deepseek/src/reader.ts new file mode 100644 index 0000000..f14da86 --- /dev/null +++ b/packages/deepseek/src/reader.ts @@ -0,0 +1,166 @@ +import type { Context } from "@deepseek-ai/cordis"; +import { + BlockAssembler, + ToolCallId, + createAssistantMessage, + createToolResultMessage, +} from "@deepseek-ai/dsh-llm"; +import type { + ContentBlock, + FinishReason, + GenerateOptions, + RequestMessage, + ToolSchema, +} from "@deepseek-ai/dsh-llm"; +import type { + ModelRef, + Reader, + ReaderCall, + ReaderReply, + ReaderRequest, + ReaderResult, + Reading, +} from "lossless-core"; + +import type { DshRoute } from "./model-host.ts"; +import { usageOf } from "./model-host.ts"; + +export interface DshReaderOptions { + readonly ref: ModelRef; + readonly signal?: AbortSignal; +} + +/** The core reader port over one streaming call per step. The loop owns the + * transcript the way Pi does: each step sends the brief plus every answered + * call so far, and a step that asks for reads appends its assistant message + * before the results land. Tool schemas pass through untouched. */ +export function dshReader(ctx: Context, options: DshReaderOptions): Reader { + return { begin: (request) => beginReading(ctx, options, request) }; +} + +function beginReading(ctx: Context, options: DshReaderOptions, request: ReaderRequest): Reading { + const messages: RequestMessage[] = [ + { role: "user", content: [{ type: "text", text: request.brief }] }, + ]; + let opened = false; + return { + async next(results: readonly ReaderResult[]): Promise { + if (opened) for (const result of results) messages.push(toolResultMessage(result)); + opened = true; + const assembler = new BlockAssembler(); + const stream: GenerateOptions = { + provider: options.ref.model.provider, + model: options.ref.model.model, + system: request.systemPrompt, + messages, + tools: request.tools.map(toolSchema), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }; + try { + for await (const chunk of ctx.llm.stream(stream)) assembler.push(chunk); + } catch (error) { + return { outcome: "failed", reason: message(error), usage: usageOf(assembler.usage) }; + } + const usage = usageOf(assembler.usage); + let blocks: ContentBlock[]; + try { + blocks = assembler.blocks(); + } catch (error) { + return { outcome: "failed", reason: message(error), usage }; + } + const reply = replyOf(assembler.finish, blocks, usage); + if (reply.outcome === "reads") { + messages.push(assistantMessage(options.ref, blocks)); + } + return reply; + }, + }; +} + +/** What one reader step means, with the same words Pi uses: error and aborted + * fail, calls ask for execution, and anything else answers, including a + * max-tokens stop whose partial text stays usable. */ +function replyOf( + finish: FinishReason, + blocks: ContentBlock[], + usage: ReaderReply["usage"], +): ReaderReply { + switch (finish.kind) { + case "error": + case "aborted": + return { + outcome: "failed", + reason: `${finish.failure.code}: ${finish.failure.message}`, + usage, + }; + case "stop": + case "max-tokens": + case "tool-calls": + break; + default: { + const _exhaustive: never = finish; + throw new Error(`unhandled finish kind: ${JSON.stringify(_exhaustive)}`); + } + } + const calls: ReaderCall[] = []; + for (const block of blocks) { + if (block.type !== "tool-call") continue; + const parsed: unknown = safeParse(block.arguments); + if (!isRecord(parsed)) { + return { + outcome: "failed", + reason: `tool call ${block.id} carries arguments that are not a JSON object`, + usage, + }; + } + calls.push({ id: block.id, name: block.name, arguments: parsed }); + } + if (calls.length > 0) return { outcome: "reads", calls, text: joinedText(blocks), usage }; + if (finish.kind === "tool-calls") { + return { outcome: "failed", reason: "the model finished with tool-calls but sent none", usage }; + } + return { outcome: "answered", text: joinedText(blocks), usage }; +} + +function toolSchema(tool: { name: string; description: string; parameters: unknown }): ToolSchema { + if (!isRecord(tool.parameters)) throw new Error(`tool ${tool.name} has no parameters object`); + return { name: tool.name, description: tool.description, parameters: tool.parameters }; +} + +function assistantMessage(ref: ModelRef, blocks: ContentBlock[]) { + return createAssistantMessage({ + content: blocks, + source: { provider: ref.model.provider, model: ref.model.model }, + }); +} + +function toolResultMessage(result: ReaderResult) { + return createToolResultMessage({ + callId: ToolCallId(result.call.id), + content: [{ type: "text", text: result.text }], + isError: false, + }); +} + +function joinedText(blocks: ContentBlock[]): string { + return blocks + .flatMap((block) => (block.type === "text" ? [block.text] : [])) + .join("\n") + .trim(); +} + +function safeParse(json: string): unknown { + try { + return JSON.parse(json) as unknown; + } catch { + return undefined; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function message(error: unknown): string { + return error instanceof Error ? error.message.slice(0, 300) : String(error).slice(0, 300); +} diff --git a/packages/deepseek/src/tools.ts b/packages/deepseek/src/tools.ts new file mode 100644 index 0000000..7caccb1 --- /dev/null +++ b/packages/deepseek/src/tools.ts @@ -0,0 +1,153 @@ +import type { Context } from "@deepseek-ai/cordis"; +import { defineTool } from "@deepseek-ai/dsh-tools"; +import type { ToolRunContext } from "@deepseek-ai/dsh-tools"; +import { MAX_EXPAND_CHARS, describeView, expandView, grepView } from "lossless-core"; +import type { LcmStore } from "lossless-core"; + +export interface RecallDeps { + /** Sidecar by calling-agent session. Undefined answers the no-history + * text, so a call outside any agent fails explained rather than empty. */ + readonly store: (sessionId: string | undefined) => LcmStore | undefined; + readonly redact: (text: string) => string; +} + +/** Register the three recall tools on a composition with a tools service. + * Returns one disposer for all three. The descriptions match the loop tools + * core hands the reader, so both seams present the same surface. */ +export function registerRecallTools(ctx: Context, deps: RecallDeps): () => void { + const stop = [ + ctx.tools.register(lcmGrepTool(deps)), + ctx.tools.register(lcmDescribeTool(deps)), + ctx.tools.register(lcmExpandTool(deps)), + ]; + return () => { + for (const dispose of stop) dispose(); + }; +} + +const NO_STORE = + "LCM: no stored history for this session yet (nothing has been ingested, or the session is ephemeral)."; + +export function grepAnswer( + store: LcmStore | undefined, + redact: (text: string) => string, + args: { query: string; limit?: number }, +): string { + if (!store) return NO_STORE; + return grepView(store, redact, { query: args.query, limit: args.limit }).text; +} + +export function describeAnswer( + store: LcmStore | undefined, + redact: (text: string) => string, + args: { id: number }, +): string { + if (!store) return NO_STORE; + return describeView(store, redact, args.id).text; +} + +export function expandAnswer( + store: LcmStore | undefined, + redact: (text: string) => string, + args: { + id?: number; + entry_id?: string; + offset?: number; + limit?: number; + char_offset?: number; + max_chars?: number; + }, +): string { + if (!store) return NO_STORE; + const maxChars = Math.min(args.max_chars ?? MAX_EXPAND_CHARS, MAX_EXPAND_CHARS); + if (args.entry_id !== undefined) { + return expandView(store, redact, { + mode: "message", + ...(args.id === undefined ? {} : { id: args.id }), + entryId: args.entry_id, + charOffset: args.char_offset ?? 0, + maxChars, + }).text; + } + if (args.id === undefined) return "lcm_expand needs an id (a summary to list) or an entry_id."; + return expandView(store, redact, { + mode: "list", + id: args.id, + offset: args.offset ?? 0, + limit: args.limit ?? 25, + maxChars, + }).text; +} + +function sessionOf(exec: ToolRunContext): string | undefined { + const id = exec.agent?.session.id; + return id === undefined ? undefined : String(id); +} + +function lcmGrepTool(deps: RecallDeps) { + return defineTool({ + name: "lcm_grep", + description: + "Full-text search over the stored session history. Each hit names the summary (#N) that covers it, which lcm_expand reads.", + parameters: { + query: { + type: "string", + required: true, + description: "FTS5 query: quoted phrases, AND/OR, prefix*", + }, + limit: { type: "integer", description: "Hits to return" }, + }, + output: { + schema: { type: "string" }, + render: (_args, value) => [{ type: "text", text: value }], + }, + async execute(args, exec) { + return grepAnswer(deps.store(sessionOf(exec)), deps.redact, args); + }, + }); +} + +function lcmDescribeTool(deps: RecallDeps) { + return defineTool({ + name: "lcm_describe", + description: + "Inspect a summary without expanding it: kind, depth, own and source token costs, children with their costs, parents, and full text.", + parameters: { + id: { + type: "integer", + required: true, + description: "Summary id, the N in [lcm:summary #N ...]", + }, + }, + output: { + schema: { type: "string" }, + render: (_args, value) => [{ type: "text", text: value }], + }, + async execute(args, exec) { + return describeAnswer(deps.store(sessionOf(exec)), deps.redact, args); + }, + }); +} + +function lcmExpandTool(deps: RecallDeps) { + return defineTool({ + name: "lcm_expand", + description: + "Read the original messages a summary covers, or one named entry. Without entry_id, list `limit` messages from message index `offset`. With entry_id, read that one message from `char_offset` for up to `max_chars` characters; continue from the char_offset the result reports. `id` is the summary to list, or to check that the entry is covered; omit it to read an entry that no summary covers yet.", + parameters: { + id: { type: "integer", description: "Summary id, the N in [lcm:summary #N ...]" }, + entry_id: { type: "string", description: "Read this one message by entry id" }, + offset: { type: "integer", description: "Message index to start listing from" }, + limit: { type: "integer", description: "Messages to list" }, + char_offset: { type: "integer", description: "Character offset inside entry_id" }, + max_chars: { type: "integer", description: "Characters to read from entry_id" }, + }, + output: { + schema: { type: "string" }, + render: (_args, value) => [{ type: "text", text: value }], + }, + async execute(args, exec) { + return expandAnswer(deps.store(sessionOf(exec)), deps.redact, args); + }, + }); +} diff --git a/packages/deepseek/test/dag-commit.test.ts b/packages/deepseek/test/dag-commit.test.ts new file mode 100644 index 0000000..9388ce5 --- /dev/null +++ b/packages/deepseek/test/dag-commit.test.ts @@ -0,0 +1,206 @@ +import { Context } from "@deepseek-ai/cordis"; +import { isCompactCheckpointSource, ManualCompactionError } from "@deepseek-ai/dsh-compaction"; +import { createSystemMessage, createUserMessage } from "@deepseek-ai/dsh-llm"; +import { Session, SessionId } from "@deepseek-ai/dsh-session"; +import type { SessionEvent } from "@deepseek-ai/dsh-session"; +import SessionProjectionRegistry from "@deepseek-ai/dsh-session-projection"; +import TokenMeter from "@deepseek-ai/dsh-token-meter"; +import { LcmStore } from "lossless-core"; +import { describe, expect, it } from "vite-plus/test"; + +import { commitDagTransaction, selectDagRange } from "../src/dag-commit.ts"; +import type { DagCommitDeps } from "../src/dag-commit.ts"; +import { MessageIndex, mirrorEvent } from "../src/ingest.ts"; +import type { DshRoute } from "../src/model-host.ts"; +import { fakeAgent } from "./support/agent-fake.ts"; +import { appendConversation } from "./support/conversation.ts"; +import { errorChunks, setupContext, textChunks } from "./support/script-adapter.ts"; + +const ROUTE: DshRoute = { provider: "test-provider", model: "test-model" }; +const SUMMARY = "Checkpoint: engine work started and the config was read."; + +function buildSession(text: string): Session { + const session = Session.create(SessionId("dag-commit-1")); + appendConversation(session, ROUTE, text); + // The bracket commits into an open turn, like the seam's auto path. + session.append("turn/start", { turn: 3 }); + return session; +} + +function mountMeter(ctx: Context): void { + new SessionProjectionRegistry(ctx); + void new TokenMeter(ctx); +} + +/** Filtered events keep their payload types, so data access typechecks. */ +function ofType( + events: readonly SessionEvent[], + type: T, +): Extract[] { + return events.filter((event): event is Extract => event.type === type); +} + +function deps(ctx: Context, store: LcmStore, index: MessageIndex): DagCommitDeps { + return { + ctx, + summarizationProvider: "", + summarizationModel: "", + store: () => store, + index: () => index, + redact: (text) => text, + }; +} + +describe("selectDagRange", () => { + it("returns null when only the retained system head exists", () => { + const session = Session.create(SessionId("dag-range-1")); + session.append( + "system/message", + { turn: 1, step: 1, message: createSystemMessage("You are a test.") }, + { surfaceOp: "append" }, + ); + const { ctx } = setupContext(ROUTE, []); + mountMeter(ctx); + expect(selectDagRange(session, ctx.tokenMeter.measure(session), 0)).toBeNull(); + }); + + it("starts after the system head and leaves the tail edge", () => { + const session = buildSession("ranged notes ".repeat(20)); + const { ctx } = setupContext(ROUTE, []); + mountMeter(ctx); + const nodes = session.surface.nodes; + const range = selectDagRange(session, ctx.tokenMeter.measure(session), 0); + expect(range).not.toBeNull(); + expect(range?.start).toBe(nodes[1]); + expect(range?.end).toBe(nodes[nodes.length - 2]); + }); + + it("fails closed when the surface moved under the measurement", () => { + const session = buildSession("drift notes ".repeat(20)); + const { ctx } = setupContext(ROUTE, []); + mountMeter(ctx); + const measurement = ctx.tokenMeter.measure(session); + session.append( + "user/message", + createUserMessage({ + content: [{ type: "text", text: "drift intruder" }], + source: { kind: "user" }, + }), + { surfaceOp: "append" }, + ); + expect(() => selectDagRange(session, measurement, 0)).toThrow(/does not match/); + }); +}); + +describe("commitDagTransaction", () => { + it("commits one condensed replace plus log-only leaf records in a single bracket", async () => { + const session = buildSession("working notes ".repeat(20)); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const { ctx } = setupContext(ROUTE, [ + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + ]); + mountMeter(ctx); + const agent = fakeAgent(session, ROUTE); + const nodes = session.surface.nodes; + const before = session.deriveMessages().length; + const result = await commitDagTransaction( + deps(ctx, store, index), + agent, + { start: nodes[1]!, end: nodes[nodes.length - 1]! }, + { owner: "current-turn", stability: "whole-surface" }, + ); + expect(result.compactionId.length).toBeGreaterThan(0); + // One condensed record plus at least one leaf record. + expect(result.summarySeqs).toHaveLength(result.leafSeqs.length + 1); + expect(result.leafSeqs.length).toBeGreaterThan(0); + expect(result.startSeq).toBeLessThan(result.summarySeqs[0]!); + expect(result.summarySeqs[result.summarySeqs.length - 1]!).toBeLessThan(result.endSeq); + const events = session.snapshotEvents(); + const starts = ofType(events, "compaction/start"); + const ends = ofType(events, "compaction/end"); + expect(starts).toHaveLength(1); + expect(ends).toHaveLength(1); + expect(starts[0]?.data.compactionId).toBe(result.compactionId); + expect(ends[0]?.data.compactionId).toBe(result.compactionId); + expect(ends[0]?.data.error).toBeUndefined(); + const summaries = ofType(events, "compaction/summary"); + expect(summaries).toHaveLength(result.summarySeqs.length); + for (const summary of summaries) { + expect(summary.data.compactionId).toBe(result.compactionId); + } + const checkpoints = ofType(events, "user/message").filter((event) => + isCompactCheckpointSource(event.data.source), + ); + expect(checkpoints).toHaveLength(1); + // The condensed replace shrinks the derived surface. + expect(session.deriveMessages().length).toBeLessThan(before); + // No unmatched start remains: the lock is released. + let depth = 0; + for (const event of events) { + if (event.type === "compaction/start") depth += 1; + if (event.type === "compaction/end") depth -= 1; + } + expect(depth).toBe(0); + }); + + it("closes the bracket with the error and releases the lock when the ladder fails", async () => { + const session = buildSession("failing notes ".repeat(20)); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const { ctx } = setupContext(ROUTE, [errorChunks()]); + mountMeter(ctx); + const agent = fakeAgent(session, ROUTE); + const nodes = session.surface.nodes; + await expect( + commitDagTransaction( + deps(ctx, store, index), + agent, + { start: nodes[1]!, end: nodes[nodes.length - 1]! }, + { owner: "current-turn", stability: "whole-surface" }, + ), + ).rejects.toThrow(); + const events = session.snapshotEvents(); + const ends = ofType(events, "compaction/end"); + expect(ends).toHaveLength(1); + expect(typeof ends[0]?.data.error).toBe("string"); + // The lock is released: a second attempt opens a new bracket instead of + // reporting busy. Scripts are exhausted, so it fails past the lock. + await expect( + commitDagTransaction( + deps(ctx, store, index), + agent, + { start: nodes[1]!, end: nodes[nodes.length - 1]! }, + { owner: "current-turn", stability: "whole-surface" }, + ), + ).rejects.toThrow(/^(?!.*already in progress).*$/s); + }); + + it("maps a ladder failure to the manual summary taxonomy on an idle session", async () => { + const session = buildSession("manual notes ".repeat(20)); + session.append("turn/end", { turn: 3, reason: { kind: "completed" } }); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const { ctx } = setupContext(ROUTE, [errorChunks()]); + mountMeter(ctx); + const agent = fakeAgent(session, ROUTE); + const nodes = session.surface.nodes; + const failure = await commitDagTransaction( + deps(ctx, store, index), + agent, + { start: nodes[1]!, end: nodes[nodes.length - 1]! }, + { owner: null, stability: "selected-span" }, + ).catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ManualCompactionError); + expect((failure as ManualCompactionError).code).toBe("summary"); + const ends = ofType(session.snapshotEvents(), "compaction/end"); + expect(ends).toHaveLength(1); + expect(ends[0]?.data.turn).toBeNull(); + }); +}); diff --git a/packages/deepseek/test/engine.test.ts b/packages/deepseek/test/engine.test.ts new file mode 100644 index 0000000..5f61b3e --- /dev/null +++ b/packages/deepseek/test/engine.test.ts @@ -0,0 +1,468 @@ +import { Context } from "@deepseek-ai/cordis"; +import { ManualCompactionError } from "@deepseek-ai/dsh-compaction"; +import { createMessage, createSystemMessage, createUserMessage } from "@deepseek-ai/dsh-llm"; +import SessionStore, { Session, SessionId } from "@deepseek-ai/dsh-session"; +import SessionProjectionRegistry from "@deepseek-ai/dsh-session-projection"; +import TokenMeter from "@deepseek-ai/dsh-token-meter"; +import { LcmStore } from "lossless-core"; +import { describe, expect, it } from "vite-plus/test"; + +import { + LosslessSummarizerEngine, + SpanAlreadyMemoryError, + checkpointMemory, +} from "../src/engine.ts"; +import type { LosslessEngineDeps } from "../src/engine.ts"; +import { MessageIndex, mirrorEvent } from "../src/ingest.ts"; +import type { DshRoute } from "../src/model-host.ts"; +import { fakeAgent } from "./support/agent-fake.ts"; +import { appendConversation } from "./support/conversation.ts"; +import { setupContext, textChunks } from "./support/script-adapter.ts"; + +const ROUTE: DshRoute = { provider: "test-provider", model: "test-model" }; +const SUMMARY = "Checkpoint: engine work started and the config was read."; + +function buildSession(text: string): Session { + const session = Session.create(SessionId("engine-1")); + appendConversation(session, ROUTE, text); + // The engine writes automatic compaction events into an open turn. + session.append("turn/start", { turn: 3 }); + return session; +} + +function mountMeter(ctx: Context): void { + new SessionProjectionRegistry(ctx); + void new TokenMeter(ctx); +} + +const identity = (text: string): string => text; + +describe("LosslessSummarizerEngine", () => { + function engineFor( + ctx: Context, + store: LcmStore, + index: MessageIndex, + extra?: Pick, + ): LosslessSummarizerEngine { + return new LosslessSummarizerEngine( + ctx, + { auto: false }, + { store: () => store, index: () => index, redact: identity, ...extra }, + ); + } + + it("compacts a region through the ladder and stores leaves in the sidecar", async () => { + const session = buildSession("working notes ".repeat(20)); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const { ctx } = setupContext(ROUTE, [ + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + ]); + mountMeter(ctx); + const agent = fakeAgent(session, ROUTE); + const engine = new LosslessSummarizerEngine( + ctx, + { auto: false }, + { store: () => store, index: () => index, redact: identity }, + ); + const nodes = session.surface.nodes; + const before = session.deriveMessages().length; + const result = await engine.compactRegion(nodes[1]!, nodes[nodes.length - 1]!, agent); + expect(result.summary).toHaveLength(1); + expect(result.summary[0]?.type).toBe("text"); + const summaryEvent = session + .snapshotEvents() + .find((event) => event.type === "compaction/summary"); + expect(summaryEvent?.data.provider).toBe(ROUTE.provider); + expect(summaryEvent?.data.model).toBe(ROUTE.model); + expect(summaryEvent?.data.usage?.outputTokens).toBeGreaterThan(0); + expect(result.shadowedSeqs.length).toBeGreaterThan(0); + const derived = session.deriveMessages(); + expect(derived.length).toBeLessThan(before); + expect( + derived.some((message) => + message.content.some((block) => block.type === "text" && block.text.includes(SUMMARY)), + ), + ).toBe(true); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + expect(store.allSummaries().length).toBeGreaterThan(0); + expect(store.allMessages().some((row) => row.text.includes(SUMMARY))).toBe(true); + const checkpoint = derived.find((message) => + message.content.some((block) => block.type === "text" && block.text.includes(SUMMARY)), + ); + const refs = checkpoint + ? checkpoint.content.flatMap((block) => + block.type === "text" ? [...block.text.matchAll(/\[lcm:summary #(\d+)\]/g)] : [], + ) + : []; + expect(refs.length).toBeGreaterThan(0); + for (const ref of refs) { + expect(store.getSummary(Number(ref[1]))).toBeDefined(); + } + }); + + it("carries the previous checkpoint into the next ladder", async () => { + const session = buildSession("carryover notes ".repeat(20)); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const FIRST = "First checkpoint alpha."; + const SECOND = "Second checkpoint beta."; + const { ctx } = setupContext(ROUTE, [ + textChunks(FIRST), + textChunks(SECOND), + textChunks(SECOND), + ]); + mountMeter(ctx); + const memory = checkpointMemory(); + const agent = fakeAgent(session, ROUTE); + const engine = new LosslessSummarizerEngine( + ctx, + { auto: false }, + { store: () => store, index: () => index, memory, redact: identity, summaryTokens: 40 }, + ); + const nodes = session.surface.nodes; + const first = await engine.compactRegion(nodes[1]!, nodes[2]!, agent); + const firstBlock = first.summary[0]; + if (!firstBlock || firstBlock.type !== "text") throw new Error("first summary is not text"); + expect(firstBlock.text).toContain(FIRST); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const afterFirst = session.surface.nodes; + const second = await engine.compactRegion( + afterFirst[1]!, + afterFirst[afterFirst.length - 1]!, + agent, + ); + const secondBlock = second.summary[0]; + if (!secondBlock || secondBlock.type !== "text") throw new Error("second summary is not text"); + expect(secondBlock.text).toContain(SECOND); + expect(secondBlock.text.startsWith(firstBlock.text)).toBe(true); + }); + + it("rejects an already-covered span as memory instead of failing the summary", async () => { + const session = buildSession("covered live ".repeat(30)); + // A third turn inside the open turn 3: rows the later checkpoint lands + // after, which is what makes them covered-but-live intruders. + session.append( + "user/message", + createUserMessage({ + content: [{ type: "text", text: `${"covered live ".repeat(30)} third request` }], + source: { kind: "user" }, + }), + { surfaceOp: "append" }, + ); + session.append("step/start", { turn: 3, step: 1 }); + session.append("request/header", { + header: { config: { provider: ROUTE.provider, model: ROUTE.model } }, + reason: "initial", + }); + session.append( + "assistant/message", + { + stream: [], + turn: 3, + step: 1, + message: createMessage({ + role: "assistant", + content: [{ type: "text", text: `${"covered live ".repeat(30)} third response` }], + source: { kind: "model", provider: ROUTE.provider, model: ROUTE.model }, + }), + }, + { surfaceOp: "append" }, + ); + session.append("step/end", { turn: 3, step: 1 }); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const { ctx } = setupContext(ROUTE, [ + textChunks("First span."), + textChunks("Second span."), + textChunks("Third span."), + ]); + mountMeter(ctx); + const agent = fakeAgent(session, ROUTE); + const engine = new LosslessSummarizerEngine( + ctx, + { auto: false }, + { + store: () => store, + index: () => index, + memory: checkpointMemory(), + redact: identity, + summaryTokens: 40, + }, + ); + const nodes = session.surface.nodes; + await engine.compactRegion(nodes[1]!, nodes[2]!, agent); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const afterFirst = session.surface.nodes; + await engine.compactRegion(afterFirst[1]!, afterFirst[3]!, agent); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const covered = session.surface.nodes; + const before = session.deriveMessages().length; + await expect( + engine.compactRegion(covered[covered.length - 2]!, covered[covered.length - 1]!, agent), + ).rejects.toThrow(SpanAlreadyMemoryError); + expect(session.deriveMessages().length).toBe(before); + }); + + it("fails closed when the span is not mirrored", async () => { + const session = buildSession("unmirrored notes ".repeat(20)); + const store = new LcmStore(":memory:"); + const { ctx } = setupContext(ROUTE, [textChunks(SUMMARY)]); + mountMeter(ctx); + const agent = fakeAgent(session, ROUTE); + const engine = new LosslessSummarizerEngine( + ctx, + { auto: false }, + { store: () => store, index: () => new MessageIndex(), redact: identity }, + ); + const nodes = session.surface.nodes; + await expect(engine.compactRegion(nodes[1]!, nodes[nodes.length - 1]!, agent)).rejects.toThrow( + "has no sidecar row", + ); + }); + + describe("compactNow", () => { + // The durability flush needs a live session, so these build on + // ctx.sessions instead of the detached buildSession helper. + function idleSession(ctx: Context, id: string, text: string): Session { + const session = ctx.sessions.create(SessionId(id)); + appendConversation(session, ROUTE, text); + session.append("turn/start", { turn: 3 }); + session.append("turn/end", { turn: 3, reason: { kind: "completed" } }); + return session; + } + + it("commits the manual path through the DAG bracket", async () => { + const { ctx } = setupContext(ROUTE, [ + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + ]); + mountMeter(ctx); + void new SessionStore(ctx); + const session = idleSession(ctx, "compact-now-1", "manual notes ".repeat(20)); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const before = session.deriveMessages().length; + const result = await engineFor(ctx, store, index).compactNow( + fakeAgent(session, ROUTE), + new AbortController().signal, + ); + expect(result).not.toBeNull(); + expect(result?.shadowedSeqs.length).toBeGreaterThan(0); + expect(result?.summarySeqs).toHaveLength((result?.leafSeqs.length ?? 0) + 1); + expect(result?.summarySeq).toBe(result?.summarySeqs[0]); + expect(session.deriveMessages().length).toBeLessThan(before); + const events = session.snapshotEvents(); + expect(events.filter((event) => event.type === "compaction/start")).toHaveLength(1); + const ends = events.filter((event) => event.type === "compaction/end"); + expect(ends).toHaveLength(1); + expect(ends[0]?.type).toBe("compaction/end"); + }); + + it("returns null when no safe range exists", async () => { + const session = Session.create(SessionId("compact-now-empty")); + session.append( + "system/message", + { turn: 1, step: 1, message: createSystemMessage("You are a test.") }, + { surfaceOp: "append" }, + ); + const { ctx } = setupContext(ROUTE, []); + mountMeter(ctx); + void new SessionStore(ctx); + const result = await engineFor(ctx, new LcmStore(":memory:"), new MessageIndex()).compactNow( + fakeAgent(session, ROUTE), + new AbortController().signal, + ); + expect(result).toBeNull(); + }); + + it("reports busy when the session still has an open turn", async () => { + const session = buildSession("open turn ".repeat(20)); + const { ctx } = setupContext(ROUTE, []); + mountMeter(ctx); + void new SessionStore(ctx); + const failure = await engineFor(ctx, new LcmStore(":memory:"), new MessageIndex()) + .compactNow(fakeAgent(session, ROUTE), new AbortController().signal) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ManualCompactionError); + expect((failure as ManualCompactionError).code).toBe("busy"); + }); + + it("reports busy when the agent refuses maintenance", async () => { + const { ctx } = setupContext(ROUTE, []); + mountMeter(ctx); + void new SessionStore(ctx); + const session = idleSession(ctx, "compact-now-4", "refused notes ".repeat(20)); + const agent = { + ...fakeAgent(session, ROUTE), + runMaintenance(): Promise { + throw new Error("waking work queued"); + }, + }; + // A synchronous admission refusal throws before any promise exists, + // so .catch cannot see it; take it with try/catch instead. + let failure: unknown; + try { + await engineFor(ctx, new LcmStore(":memory:"), new MessageIndex()).compactNow( + agent, + new AbortController().signal, + ); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(ManualCompactionError); + expect((failure as ManualCompactionError).code).toBe("busy"); + }); + + describe("checkpointMemory", () => { + it("hydrates the last checkpoint from the sidecar on a fresh instance", async () => { + const { ctx } = setupContext(ROUTE, [ + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + ]); + mountMeter(ctx); + void new SessionStore(ctx); + const sessionId = "compact-memory-1"; + const session = idleSession(ctx, sessionId, "hydrated notes ".repeat(20)); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const first = checkpointMemory(() => store); + const engine = engineFor(ctx, store, index, { memory: first }); + const result = await engine.compactNow( + fakeAgent(session, ROUTE), + new AbortController().signal, + ); + expect(result).not.toBeNull(); + const noted = first.previous(sessionId); + expect(noted).toBeDefined(); + // A fresh instance over the same sidecar rebuilds continuity: the + // narrative plus refs that all resolve. (A single-leaf run replaces + // its row's text on condensation, so the rebuilt text is the stored + // form, not byte-identical to the noted form.) + const resumed = checkpointMemory(() => store); + const rebuilt = resumed.previous(sessionId); + expect(rebuilt).toBeDefined(); + expect(rebuilt).toContain("Checkpoint: engine work started"); + const ids = [...rebuilt!.matchAll(/\[lcm:summary #(\d+)\]/g)].map((match) => + Number(match[1]), + ); + expect(ids.length).toBeGreaterThan(0); + for (const id of ids) expect(store.getSummary(id)).toBeDefined(); + // And a new checkpoint overwrites the hydrated text. + resumed.note(sessionId, "fresh checkpoint"); + expect(resumed.previous(sessionId)).toBe("fresh checkpoint"); + }); + + it("stays absent without a store or without a condensed summary", () => { + expect(checkpointMemory().previous("ghost")).toBeUndefined(); + expect(checkpointMemory(() => new LcmStore(":memory:")).previous("ghost")).toBeUndefined(); + }); + }); + + it("reports cancelled when the maintenance signal aborts", async () => { + // No scripts: the first ladder call fails past the aborted signal. + const { ctx } = setupContext(ROUTE, []); + mountMeter(ctx); + void new SessionStore(ctx); + const session = idleSession(ctx, "compact-now-5", "cancelled notes ".repeat(20)); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const controller = new AbortController(); + controller.abort(new Error("waking")); + const agent = { + ...fakeAgent(session, ROUTE), + runMaintenance(task: (signal: AbortSignal) => Promise): Promise { + return task(controller.signal); + }, + }; + const failure = await engineFor(ctx, store, index) + .compactNow(agent, new AbortController().signal) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(ManualCompactionError); + expect((failure as ManualCompactionError).code).toBe("cancelled"); + }); + }); + + describe("compactRegion with bracketAuto", () => { + it("commits the auto path through the bracket on an open turn", async () => { + const session = buildSession("auto bracket ".repeat(20)); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const { ctx } = setupContext(ROUTE, [ + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + ]); + mountMeter(ctx); + const agent = fakeAgent(session, ROUTE); + const engine = engineFor(ctx, store, index, { bracketAuto: true }); + const nodes = session.surface.nodes; + const before = session.deriveMessages().length; + const result = await engine.compactRegion(nodes[1]!, nodes[nodes.length - 1]!, agent); + expect(result.shadowedSeqs.length).toBeGreaterThan(0); + expect(result.summarySeq).toBeDefined(); + expect(session.deriveMessages().length).toBeLessThan(before); + const summaries = session + .snapshotEvents() + .filter((event) => event.type === "compaction/summary"); + expect(summaries.length).toBeGreaterThan(1); + }); + + it("lets a ladder failure through unwrapped on the auto path", async () => { + const session = buildSession("unmirrored auto ".repeat(20)); + const { ctx } = setupContext(ROUTE, [textChunks(SUMMARY)]); + mountMeter(ctx); + const agent = fakeAgent(session, ROUTE); + const engine = engineFor(ctx, new LcmStore(":memory:"), new MessageIndex(), { + bracketAuto: true, + }); + const nodes = session.surface.nodes; + const failure = await engine + .compactRegion(nodes[1]!, nodes[nodes.length - 1]!, agent) + .catch((error: unknown) => error); + expect(failure).toBeInstanceOf(Error); + expect(failure).not.toBeInstanceOf(ManualCompactionError); + expect(String((failure as Error).message)).toMatch(/sidecar holds no row/); + }); + }); + + it("lets the engine shrink check fire on a span the ladder cannot reduce", async () => { + const session = buildSession("hi"); + const store = new LcmStore(":memory:"); + const index = new MessageIndex(); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store, index); + const { ctx } = setupContext(ROUTE, [ + textChunks(SUMMARY), + textChunks(SUMMARY), + textChunks(SUMMARY), + ]); + mountMeter(ctx); + const agent = fakeAgent(session, ROUTE); + const engine = new LosslessSummarizerEngine( + ctx, + { auto: false }, + { store: () => store, index: () => index, redact: identity }, + ); + const nodes = session.surface.nodes; + await expect(engine.compactRegion(nodes[1]!, nodes[nodes.length - 1]!, agent)).rejects.toThrow( + "not smaller than the shadowed content", + ); + }); +}); diff --git a/packages/deepseek/test/ingest.test.ts b/packages/deepseek/test/ingest.test.ts new file mode 100644 index 0000000..8398048 --- /dev/null +++ b/packages/deepseek/test/ingest.test.ts @@ -0,0 +1,172 @@ +import { + ToolCallId, + createMessage, + createSystemMessage, + createToolResultMessage, + createUserMessage, +} from "@deepseek-ai/dsh-llm"; +import { Session, SessionId } from "@deepseek-ai/dsh-session"; +import { LcmStore } from "lossless-core"; +import { describe, expect, it } from "vite-plus/test"; + +import { entryFor, mirrorEvent } from "../src/ingest.ts"; + +const MODEL = "test-model"; + +function buildSession(): Session { + const session = Session.create(SessionId("mirror-1")); + session.append("turn/start", { turn: 1 }); + session.append( + "system/message", + { turn: 1, step: 1, message: createSystemMessage("You are a test.") }, + { surfaceOp: "append" }, + ); + session.append( + "user/message", + createUserMessage({ + content: [{ type: "text", text: "start the compaction engine work" }], + source: { kind: "user" }, + }), + { surfaceOp: "append" }, + ); + session.append("step/start", { turn: 1, step: 1 }); + session.append("request/header", { + header: { config: { provider: MODEL, model: MODEL } }, + reason: "initial", + }); + session.append( + "assistant/message", + { + stream: [], + turn: 1, + step: 1, + message: createMessage({ + role: "assistant", + content: [{ type: "text", text: "engine work started" }], + source: { kind: "model", provider: MODEL, model: MODEL }, + }), + }, + { surfaceOp: "append" }, + ); + session.append("step/end", { turn: 1, step: 1 }); + session.append("turn/end", { turn: 1, reason: { kind: "completed" } }); + session.append("turn/start", { turn: 2 }); + session.append( + "user/message", + createUserMessage({ + content: [{ type: "text", text: "read the engine config" }], + source: { kind: "user" }, + }), + { surfaceOp: "append" }, + ); + session.append("step/start", { turn: 2, step: 1 }); + const callId = ToolCallId("call-1"); + session.append( + "assistant/message", + { + stream: [], + turn: 2, + step: 1, + message: createMessage({ + role: "assistant", + content: [ + { type: "text", text: "reading now" }, + { type: "tool-call", id: callId, name: "read", arguments: '{"path":"engine.conf"}' }, + ], + source: { kind: "model", provider: MODEL, model: MODEL }, + }), + }, + { surfaceOp: "append" }, + ); + session.append("tool/call", { + turn: 2, + step: 1, + callId, + name: "read", + arguments: '{"path":"engine.conf"}', + }); + session.append( + "tool/result", + { + turn: 2, + step: 1, + message: createToolResultMessage({ + callId, + content: [{ type: "text", text: "config contents here" }], + isError: false, + }), + }, + { surfaceOp: "append" }, + ); + session.append("step/end", { turn: 2, step: 1 }); + session.append("turn/end", { turn: 2, reason: { kind: "completed" } }); + return session; +} + +describe("ingest mirror", () => { + it("mirrors every derived message and nothing else, idempotently", () => { + const session = buildSession(); + const store = new LcmStore(":memory:"); + // The snapshot walk is test-only: production code mirrors the live feed. + const events = session.snapshotEvents(); + let inserted = 0; + for (const event of events) inserted += mirrorEvent(session, event, store).inserted; + const derived = session.deriveMessages(); + expect(inserted).toBe(derived.length); + expect(inserted).toBe(6); + const stored = new Map(store.allMessages().map((row) => [row.entryId, row])); + expect(stored.size).toBe(6); + for (const message of derived) { + for (const block of message.content) { + if (block.type !== "text" || block.text.trim().length === 0) continue; + const kept = [...stored.values()].some((row) => row.text.includes(block.text)); + expect(kept).toBe(true); + } + } + const turnStart = events.find((event) => event.type === "turn/start"); + expect(turnStart).toBeDefined(); + if (turnStart) expect(entryFor(session, turnStart)).toBeUndefined(); + let second = 0; + let skipped = 0; + for (const event of events) { + const stats = mirrorEvent(session, event, store); + second += stats.inserted; + skipped += stats.skipped; + } + expect(second).toBe(0); + expect(skipped).toBe(events.length); + }); + + it("keeps shadowed entries when a checkpoint replaces surface nodes", () => { + const session = buildSession(); + const store = new LcmStore(":memory:"); + for (const event of session.snapshotEvents()) mirrorEvent(session, event, store); + const before = session.deriveMessages().length; + const firstUser = session.snapshotEvents().find((event) => event.type === "user/message"); + const firstAssistant = session + .snapshotEvents() + .find((event) => event.type === "assistant/message"); + expect(firstUser).toBeDefined(); + expect(firstAssistant).toBeDefined(); + if (!firstUser || !firstAssistant) throw new Error("conversation is missing its first turn"); + const checkpoint = session.append( + "user/message", + createUserMessage({ + content: [{ type: "text", text: "checkpoint: engine work started, config read next" }], + source: { kind: "user" }, + }), + { + surfaceOp: { op: "replace", startSeq: firstUser.seq, endSeq: firstAssistant.seq }, + sourceEventSeqs: [firstUser.seq, firstAssistant.seq], + }, + ); + const stats = mirrorEvent(session, checkpoint, store); + expect(stats.inserted).toBe(1); + expect(session.deriveMessages().length).toBe(before - 1); + expect(store.allMessages()).toHaveLength(before + 1); + const texts = store.allMessages().map((row) => row.text); + expect(texts.some((text) => text.includes("start the compaction engine work"))).toBe(true); + expect(texts.some((text) => text.includes("engine work started"))).toBe(true); + expect(texts.some((text) => text.includes("checkpoint: engine work started"))).toBe(true); + }); +}); diff --git a/packages/deepseek/test/model-host.test.ts b/packages/deepseek/test/model-host.test.ts new file mode 100644 index 0000000..66689b8 --- /dev/null +++ b/packages/deepseek/test/model-host.test.ts @@ -0,0 +1,84 @@ +import { summarizeWithEscalation } from "lossless-core"; +import type { LlmComplete } from "lossless-core"; +import { describe, expect, it } from "vite-plus/test"; + +import { dshModelHost } from "../src/model-host.ts"; +import type { DshRoute } from "../src/model-host.ts"; +import { cappedChunks, errorChunks, setupContext, textChunks } from "./support/script-adapter.ts"; + +const ROUTE: DshRoute = { provider: "test-provider", model: "test-model" }; + +describe("dshModelHost.complete", () => { + it("answers with text assembled from the chunk stream", async () => { + const { ctx, adapter } = setupContext(ROUTE, [textChunks("hello summary")]); + const host = dshModelHost(ctx, { defaultRoute: ROUTE }); + const ref = host.session(); + expect(ref?.key).toBe("test-provider/test-model"); + if (!ref) throw new Error("session route did not resolve"); + const answer = await host.complete(ref, { prompt: "summarize this", maxTokens: 100 }); + expect(answer).toMatchObject({ + outcome: "answer", + text: "hello summary", + stop: "stop", + usage: { + input: 10, + output: 13, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 23, + }, + }); + expect(adapter.requests).toHaveLength(1); + expect(adapter.requests[0]).toMatchObject({ + provider: "test-provider", + model: "test-model", + maxTokens: 100, + purpose: "compaction", + }); + }); + + it("fails capped when the provider stops at its token cap", async () => { + const { ctx } = setupContext(ROUTE, [cappedChunks("partial")]); + const host = dshModelHost(ctx, { defaultRoute: ROUTE }); + const ref = host.session(); + if (!ref) throw new Error("session route did not resolve"); + const answer = await host.complete(ref, { prompt: "summarize this", maxTokens: 4 }); + expect(answer).toMatchObject({ outcome: "failed", capped: true, stop: "max-tokens" }); + }); + + it("fails open with the provider code on an error finish", async () => { + const { ctx } = setupContext(ROUTE, [errorChunks()]); + const host = dshModelHost(ctx, { defaultRoute: ROUTE }); + const ref = host.session(); + if (!ref) throw new Error("session route did not resolve"); + const answer = await host.complete(ref, { prompt: "summarize this", maxTokens: 100 }); + expect(answer.outcome).toBe("failed"); + if (answer.outcome !== "failed") throw new Error("expected a failed answer"); + expect(answer.capped).toBe(false); + expect(answer.stop).toBe("error"); + expect(answer.reason).toContain("AUTH"); + }); + + it("summarizes one span through the real escalation ladder", async () => { + const { ctx } = setupContext(ROUTE, [textChunks("Decisions: ship it. File: src/a.ts.")]); + const host = dshModelHost(ctx, { defaultRoute: ROUTE }); + const ref = host.session(); + if (!ref) throw new Error("session route did not resolve"); + const llm: LlmComplete = async (systemPrompt, userText, maxTokens) => { + const answer = await host.complete(ref, { + prompt: `${systemPrompt}\n\n${userText}`, + maxTokens, + }); + if (answer.outcome !== "answer") throw new Error(answer.reason); + return answer.text; + }; + const result = await summarizeWithEscalation( + [{ role: "user", text: "working notes ".repeat(50) }], + 200, + llm, + ); + expect(result.level).toBe(1); + expect(result.retried).toBe(false); + expect(result.text).toContain("ship it"); + }); +}); diff --git a/packages/deepseek/test/plugin.test.ts b/packages/deepseek/test/plugin.test.ts new file mode 100644 index 0000000..3735b32 --- /dev/null +++ b/packages/deepseek/test/plugin.test.ts @@ -0,0 +1,217 @@ +import { existsSync, mkdtempSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Context } from "@deepseek-ai/cordis"; +import CommandRuntime from "@deepseek-ai/dsh-commands"; +import { CommandId } from "@deepseek-ai/dsh-commands/brand"; +import { createUserMessage } from "@deepseek-ai/dsh-llm"; +import SessionStore, { SessionId } from "@deepseek-ai/dsh-session"; +import type { SessionEvent } from "@deepseek-ai/dsh-session"; +import SessionProjectionRegistry from "@deepseek-ai/dsh-session-projection"; +import SystemPrompt from "@deepseek-ai/dsh-system-prompt"; +import TokenMeter from "@deepseek-ai/dsh-token-meter"; +import ToolRuntime from "@deepseek-ai/dsh-tools"; +import { LcmStore } from "lossless-core"; +import { describe, expect, it } from "vite-plus/test"; + +import { lcmCommandDefinition } from "../src/command.ts"; +import { LosslessSummarizerEngine } from "../src/engine.ts"; +import type { DshRoute } from "../src/model-host.ts"; +import { applyLossless } from "../src/plugin.ts"; +import { fakeAgent } from "./support/agent-fake.ts"; +import { appendConversation } from "./support/conversation.ts"; +import { setupContext, textChunks } from "./support/script-adapter.ts"; + +const ROUTE: DshRoute = { provider: "test-provider", model: "test-model" }; +const SUMMARY = "Checkpoint: engine work started and the config was read."; + +function tempDir(): string { + return mkdtempSync(join(tmpdir(), "deepseek-lossless-")); +} + +async function mountBase(ctx: Context): Promise { + await ctx.plugin(SystemPrompt, {}); + void new ToolRuntime(ctx); + void new CommandRuntime(ctx); + void new SessionStore(ctx); + new SessionProjectionRegistry(ctx); + void new TokenMeter(ctx); +} + +function sidecarMap(dir: string): Map { + const map = new Map(); + for (const file of readdirSync(dir)) { + if (!file.endsWith(".sqlite")) continue; + map.set(file.slice(0, -".sqlite".length), new LcmStore(join(dir, file))); + } + return map; +} + +describe("applyLossless", () => { + it("registers nothing until enabled", async () => { + const ctx = new Context(); + await ctx.plugin(SystemPrompt, {}); + void new ToolRuntime(ctx); + const dir = join(tempDir(), "never-created"); + const dispose = applyLossless(ctx, { sidecarDir: dir }); + expect(ctx.tools.schemas().map((schema) => schema.name)).not.toContain("lcm_grep"); + expect(existsSync(dir)).toBe(false); + dispose(); + }); + + it("mounts the engine, the tools, and the live mirror", async () => { + const { ctx } = setupContext(ROUTE, [textChunks(SUMMARY)], { contextWindow: 1_000_000 }); + await mountBase(ctx); + const dir = tempDir(); + const dispose = applyLossless(ctx, { enabled: true, sidecarDir: dir }); + try { + expect(ctx.compaction).toBeInstanceOf(LosslessSummarizerEngine); + expect(ctx.tools.schemas().map((schema) => schema.name)).toContain("lcm_grep"); + const session = ctx.sessions.create(SessionId("live-1"), { meta: { cwd: dir } }); + appendConversation(session, ROUTE, "live notes ".repeat(20)); + session.append("turn/start", { turn: 3 }); + const map = sidecarMap(dir); + expect(map.has("live-1")).toBe(true); + expect(map.get("live-1")?.allMessages().length).toBe(session.deriveMessages().length); + const definition = lcmCommandDefinition(ctx, { stores: () => map }); + const status = await definition.handler({ + agent: fakeAgent(session, ROUTE), + signal: new AbortController().signal, + commandId: CommandId("cmd-status"), + rawInput: "status", + attachments: [], + }); + expect(status.kind).toBe("success"); + if (status.kind !== "success") throw new Error("status did not succeed"); + expect(status.text).toContain("live-1"); + } finally { + dispose(); + } + }); + + it("forces a manual compaction through /lcm compact", async () => { + const { ctx } = setupContext( + ROUTE, + [textChunks(SUMMARY), textChunks(SUMMARY), textChunks(SUMMARY)], + { contextWindow: 1_000_000 }, + ); + await mountBase(ctx); + const dir = tempDir(); + const dispose = applyLossless(ctx, { enabled: true, sidecarDir: dir }); + try { + const session = ctx.sessions.create(SessionId("live-2"), { meta: { cwd: dir } }); + appendConversation(session, ROUTE, "compact me ".repeat(30)); + session.append("turn/start", { turn: 3 }); + // Manual compaction runs on an idle session: close the turn first. + session.append("turn/end", { turn: 3, reason: { kind: "completed" } }); + const before = session.deriveMessages().length; + const definition = lcmCommandDefinition(ctx, { stores: () => sidecarMap(dir) }); + const result = await definition.handler({ + agent: fakeAgent(session, ROUTE), + signal: new AbortController().signal, + commandId: CommandId("cmd-compact"), + rawInput: "compact", + attachments: [], + }); + if (result.kind !== "success") throw new Error(`compact failed: ${result.text}`); + expect(result.text).toMatch(/Compacted \d+ history items/); + expect(session.deriveMessages().length).toBeLessThan(before); + // The production manual path commits the DAG: one condensed record + // plus at least one log-only leaf record under one compaction id. + const logSummaries = session + .snapshotEvents() + .filter( + (event): event is Extract => + event.type === "compaction/summary", + ); + expect(logSummaries.length).toBeGreaterThan(1); + expect(new Set(logSummaries.map((summary) => summary.data.compactionId)).size).toBe(1); + const summaries = sidecarMap(dir).get("live-2")?.allSummaries() ?? []; + expect(summaries.length).toBeGreaterThan(0); + } finally { + dispose(); + } + }); + + it("carries the checkpoint across two manual compactions", async () => { + const { ctx } = setupContext( + ROUTE, + [textChunks(SUMMARY), textChunks(SUMMARY), textChunks(SUMMARY)], + { contextWindow: 1_000_000 }, + ); + await mountBase(ctx); + const dir = tempDir(); + const dispose = applyLossless(ctx, { enabled: true, sidecarDir: dir, summaryTokens: 40 }); + try { + const session = ctx.sessions.create(SessionId("live-4"), { meta: { cwd: dir } }); + appendConversation(session, ROUTE, "carryover work ".repeat(30)); + session.append("turn/start", { turn: 3 }); + session.append("turn/end", { turn: 3, reason: { kind: "completed" } }); + const definition = lcmCommandDefinition(ctx, { stores: () => sidecarMap(dir) }); + const compact = (id: string) => + definition.handler({ + agent: fakeAgent(session, ROUTE), + signal: new AbortController().signal, + commandId: CommandId(id), + rawInput: "compact", + attachments: [], + }); + const first = await compact("cmd-first"); + if (first.kind !== "success") throw new Error(`first compact failed: ${first.text}`); + session.append("turn/start", { turn: 4 }); + session.append( + "user/message", + createUserMessage({ + content: [{ type: "text", text: `more work ${".".repeat(600)}` }], + source: { kind: "user" }, + }), + { surfaceOp: "append" }, + ); + session.append("turn/end", { turn: 4, reason: { kind: "completed" } }); + const second = await compact("cmd-second"); + if (second.kind !== "success") throw new Error(`second compact failed: ${second.text}`); + const texts = (sidecarMap(dir).get("live-4")?.allMessages() ?? []) + .map((row) => row.text) + .filter((text) => text.includes(SUMMARY)); + // The sidecar rows carry the engine's frame tags; compare the bodies. + const bodyOf = (text: string): string => { + const open = ""; + const close = ""; + const start = text.indexOf(open); + const end = text.indexOf(close); + if (start === -1 || end <= start) throw new Error("checkpoint frame not found"); + return text.slice(start + open.length, end).trim(); + }; + const bodies = texts.map(bodyOf); + const carryover = bodies.some((outer) => + bodies.some((inner) => outer !== inner && outer.includes(inner)), + ); + expect(carryover).toBe(true); + } finally { + dispose(); + } + }); + + it("leaves a quiet session untouched on pressure", async () => { + const { ctx } = setupContext(ROUTE, [textChunks(SUMMARY)], { contextWindow: 1_000_000 }); + await mountBase(ctx); + const dir = tempDir(); + const dispose = applyLossless(ctx, { enabled: true, sidecarDir: dir }); + try { + const session = ctx.sessions.create(SessionId("live-3"), { meta: { cwd: dir } }); + appendConversation(session, ROUTE, "quiet notes"); + session.append("turn/start", { turn: 3 }); + const eventsBefore = session.snapshotEvents().length; + const result = await ctx.compaction.compactIfNeeded( + fakeAgent(session, ROUTE), + "pressure", + new AbortController().signal, + ); + expect(result).toBeNull(); + expect(session.snapshotEvents().length).toBe(eventsBefore); + } finally { + dispose(); + } + }); +}); diff --git a/packages/deepseek/test/reader.test.ts b/packages/deepseek/test/reader.test.ts new file mode 100644 index 0000000..ce85ac2 --- /dev/null +++ b/packages/deepseek/test/reader.test.ts @@ -0,0 +1,57 @@ +import { LcmStore, ingestEntries, runRetrieval } from "lossless-core"; +import { describe, expect, it } from "vite-plus/test"; + +import { dshRef } from "../src/model-host.ts"; +import type { DshRoute } from "../src/model-host.ts"; +import { dshReader } from "../src/reader.ts"; +import { setupContext, textChunks, toolCallChunks } from "./support/script-adapter.ts"; + +const ROUTE: DshRoute = { provider: "test-provider", model: "test-model" }; + +function seedStore(): LcmStore { + const store = new LcmStore(":memory:"); + ingestEntries( + store, + Array.from({ length: 8 }, (_, i) => ({ + entryId: `e${String(i).padStart(2, "0")}`, + role: (i % 2 === 0 ? "user" : "assistant") as "user" | "assistant", + text: `message ${i}: deciding the ${i % 2 === 0 ? "compaction engine" : "recall pipeline"} rollout ${"detail ".repeat(20)}`, + timestamp: i, + })), + ); + return store; +} + +describe("dshReader retrieval", () => { + it("answers one query through a real grep call and a real store", async () => { + const store = seedStore(); + const { ctx, adapter } = setupContext(ROUTE, [ + toolCallChunks("call-1", "lcm_grep", { query: "compaction" }), + textChunks("The rollout decision is recorded under the compaction engine entries."), + ]); + const run = await runRetrieval( + store, + { + query: "compaction engine rollout", + prompt: "What did we decide about the compaction engine rollout?", + budgetTokens: 10000, + maxSteps: 6, + redact: (text) => text, + }, + dshReader(ctx, { ref: dshRef(ROUTE) }), + ); + expect(run.kind).toBe("answered"); + if (run.kind !== "answered") throw new Error(`retrieval did not answer: ${run.kind}`); + expect(run.steps).toBe(2); + expect(run.text).toContain("compaction engine entries"); + expect(adapter.requests).toHaveLength(2); + const first = adapter.requests[0]; + const tools = first?.tools?.map((tool) => tool.name) ?? []; + expect(tools).toContain("lcm_grep"); + expect(tools).toContain("lcm_describe"); + expect(tools).toContain("lcm_expand"); + const second = adapter.requests[1]; + const roles = second?.messages.map((message) => message.role) ?? []; + expect(roles).toEqual(["user", "assistant", "tool"]); + }); +}); diff --git a/packages/deepseek/test/support/agent-fake.ts b/packages/deepseek/test/support/agent-fake.ts new file mode 100644 index 0000000..4e6c50f --- /dev/null +++ b/packages/deepseek/test/support/agent-fake.ts @@ -0,0 +1,17 @@ +import type { Agent } from "@deepseek-ai/dsh-agent"; +import type { Session } from "@deepseek-ai/dsh-session"; + +import type { DshRoute } from "../../src/model-host.ts"; + +/** compactNow and compactRegion read session, options, and runMaintenance. A + * real Agent needs the full runtime (inbox, dispatcher); the harness compacts + * its own suite the same way, with a minimal fake cast to the runtime type. */ +export function fakeAgent(session: Session, route: DshRoute): Agent { + return { + session, + options: { provider: route.provider, model: route.model }, + runMaintenance(task: (signal: AbortSignal) => Promise): Promise { + return task(new AbortController().signal); + }, + } as unknown as Agent; +} diff --git a/packages/deepseek/test/support/conversation.ts b/packages/deepseek/test/support/conversation.ts new file mode 100644 index 0000000..fce5fdc --- /dev/null +++ b/packages/deepseek/test/support/conversation.ts @@ -0,0 +1,71 @@ +import { createMessage, createSystemMessage, createUserMessage } from "@deepseek-ai/dsh-llm"; +import type { Session } from "@deepseek-ai/dsh-session"; + +import type { DshRoute } from "../../src/model-host.ts"; + +/** Two closed turns with a system head and a routed request header. Works on + * detached and live sessions alike; the caller opens the turn the engine + * writes into. */ +export function appendConversation(session: Session, route: DshRoute, text: string): void { + session.append("turn/start", { turn: 1 }); + session.append( + "system/message", + { turn: 1, step: 1, message: createSystemMessage("You are a test.") }, + { surfaceOp: "append" }, + ); + session.append( + "user/message", + createUserMessage({ + content: [{ type: "text", text: `${text} first request` }], + source: { kind: "user" }, + }), + { surfaceOp: "append" }, + ); + session.append("step/start", { turn: 1, step: 1 }); + session.append("request/header", { + header: { config: { provider: route.provider, model: route.model } }, + reason: "initial", + }); + session.append( + "assistant/message", + { + stream: [], + turn: 1, + step: 1, + message: createMessage({ + role: "assistant", + content: [{ type: "text", text: `${text} first response` }], + source: { kind: "model", provider: route.provider, model: route.model }, + }), + }, + { surfaceOp: "append" }, + ); + session.append("step/end", { turn: 1, step: 1 }); + session.append("turn/end", { turn: 1, reason: { kind: "completed" } }); + session.append("turn/start", { turn: 2 }); + session.append( + "user/message", + createUserMessage({ + content: [{ type: "text", text: `${text} second request` }], + source: { kind: "user" }, + }), + { surfaceOp: "append" }, + ); + session.append("step/start", { turn: 2, step: 1 }); + session.append( + "assistant/message", + { + stream: [], + turn: 2, + step: 1, + message: createMessage({ + role: "assistant", + content: [{ type: "text", text: `${text} second response` }], + source: { kind: "model", provider: route.provider, model: route.model }, + }), + }, + { surfaceOp: "append" }, + ); + session.append("step/end", { turn: 2, step: 1 }); + session.append("turn/end", { turn: 2, reason: { kind: "completed" } }); +} diff --git a/packages/deepseek/test/support/script-adapter.ts b/packages/deepseek/test/support/script-adapter.ts new file mode 100644 index 0000000..b506fb8 --- /dev/null +++ b/packages/deepseek/test/support/script-adapter.ts @@ -0,0 +1,89 @@ +import { Context } from "@deepseek-ai/cordis"; +import LlmRuntime, { LlmAdapter, ToolCallId } from "@deepseek-ai/dsh-llm"; +import type { GenerateOptions, StreamChunk } from "@deepseek-ai/dsh-llm"; + +import type { DshRoute } from "../../src/model-host.ts"; + +export function textChunks(text: string): StreamChunk[] { + return [ + { type: "block-start", index: 0, blockType: "text" }, + { type: "text-delta", index: 0, text }, + { type: "block-end", index: 0, block: { type: "text", text } }, + { type: "usage", usage: { inputTokens: 10, outputTokens: text.length } }, + { type: "finish", reason: { kind: "stop" } }, + ]; +} + +export function cappedChunks(text: string): StreamChunk[] { + return [ + { type: "block-start", index: 0, blockType: "text" }, + { type: "text-delta", index: 0, text }, + { type: "block-end", index: 0, block: { type: "text", text } }, + { type: "usage", usage: { inputTokens: 10, outputTokens: text.length } }, + { type: "finish", reason: { kind: "max-tokens" } }, + ]; +} + +export function errorChunks(): StreamChunk[] { + return [ + { + type: "finish", + reason: { kind: "error", failure: { message: "bad key", code: "AUTH" } }, + }, + ]; +} + +export function toolCallChunks(callId: string, name: string, args: unknown): StreamChunk[] { + const id = ToolCallId(callId); + const json = JSON.stringify(args); + return [ + { type: "block-start", index: 0, blockType: "tool-call" }, + { type: "tool-call-delta", index: 0, id, name, argumentsDelta: json }, + { type: "block-end", index: 0, block: { type: "tool-call", id, name, arguments: json } }, + { type: "usage", usage: { inputTokens: 10, outputTokens: 5 } }, + { type: "finish", reason: { kind: "tool-calls" } }, + ]; +} + +/** One scripted provider route: each stream consumes the next entry and + * records its request, so a test asserts what the host actually sent. */ +export class ScriptAdapter extends LlmAdapter { + readonly requests: GenerateOptions[] = []; + + constructor( + private readonly scripts: StreamChunk[][], + private readonly contextWindow?: number, + ) { + super(); + } + + override resolveModel(provider: string, model: string) { + return Promise.resolve({ + provider, + id: model, + name: model, + ...(this.contextWindow === undefined + ? {} + : { context: { contextWindow: this.contextWindow } }), + }); + } + + override async *stream(options: GenerateOptions): AsyncIterable { + this.requests.push(options); + const chunks = this.scripts.shift(); + if (!chunks) throw new Error("ScriptAdapter: script exhausted"); + yield* chunks; + } +} + +export function setupContext( + route: DshRoute, + scripts: StreamChunk[][], + opts?: { contextWindow?: number }, +): { ctx: Context; adapter: ScriptAdapter } { + const ctx = new Context(); + void new LlmRuntime(ctx); + const adapter = new ScriptAdapter(scripts, opts?.contextWindow); + ctx.llm.registerAdapter([route.provider], adapter); + return { ctx, adapter }; +} diff --git a/packages/deepseek/test/tools.test.ts b/packages/deepseek/test/tools.test.ts new file mode 100644 index 0000000..eb8517a --- /dev/null +++ b/packages/deepseek/test/tools.test.ts @@ -0,0 +1,53 @@ +import { Context } from "@deepseek-ai/cordis"; +import SystemPrompt from "@deepseek-ai/dsh-system-prompt"; +import ToolRuntime from "@deepseek-ai/dsh-tools"; +import { LcmStore, ingestEntries } from "lossless-core"; +import { describe, expect, it } from "vite-plus/test"; + +import { describeAnswer, expandAnswer, grepAnswer, registerRecallTools } from "../src/tools.ts"; + +function seedStore(): LcmStore { + const store = new LcmStore(":memory:"); + ingestEntries( + store, + Array.from({ length: 4 }, (_, i) => ({ + entryId: `e${i}`, + role: "user" as const, + text: `message ${i}: the ${i % 2 === 0 ? "compaction engine" : "recall pipeline"} decision ${"detail ".repeat(10)}`, + timestamp: i, + })), + ); + return store; +} + +const identity = (text: string): string => text; + +describe("recall tools", () => { + it("registers three tools whose schemas reach the wire projection", async () => { + const ctx = new Context(); + await ctx.plugin(SystemPrompt, {}); + void new ToolRuntime(ctx); + const store = seedStore(); + const dispose = registerRecallTools(ctx, { store: () => store, redact: identity }); + const names = ctx.tools.schemas().map((schema) => schema.name); + expect(names).toContain("lcm_grep"); + expect(names).toContain("lcm_describe"); + expect(names).toContain("lcm_expand"); + const grep = ctx.tools.schemas().find((schema) => schema.name === "lcm_grep"); + expect(Object.keys(grep?.parameters.properties ?? {})).toContain("query"); + dispose(); + expect(ctx.tools.schemas().map((schema) => schema.name)).not.toContain("lcm_grep"); + }); + + it("answers grep, describe, and expand from a real store", () => { + const store = seedStore(); + const grep = grepAnswer(store, identity, { query: "compaction" }); + expect(grep).toContain("Found 2 match(es):"); + expect(grep).toContain("[e0]"); + expect(grepAnswer(undefined, identity, { query: "compaction" })).toContain("no stored history"); + expect(describeAnswer(store, identity, { id: 999 })).toBe("No summary with id 999."); + const expanded = expandAnswer(store, identity, { entry_id: "e1" }); + expect(expanded).toContain("recall pipeline"); + expect(expandAnswer(store, identity, {})).toContain("needs an id"); + }); +}); diff --git a/packages/deepseek/tsconfig.build.json b/packages/deepseek/tsconfig.build.json new file mode 100644 index 0000000..85acb7a --- /dev/null +++ b/packages/deepseek/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "paths": {} + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/deepseek/tsconfig.json b/packages/deepseek/tsconfig.json new file mode 100644 index 0000000..e8b02ca --- /dev/null +++ b/packages/deepseek/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/packages/deepseek/vite.config.ts b/packages/deepseek/vite.config.ts new file mode 100644 index 0000000..2f3f6c6 --- /dev/null +++ b/packages/deepseek/vite.config.ts @@ -0,0 +1,23 @@ +import { fileURLToPath } from "node:url"; + +import { defineConfig } from "vite-plus"; + +/** The engine read as source for the package-local test run; `pack` resolves it + * through node_modules, so nothing here reaches the tarball. */ +const engine = fileURLToPath(new URL("../core/src/index.ts", import.meta.url)); + +export default defineConfig({ + pack: { + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + outDir: "dist", + target: "node22.19.0", + tsconfig: "tsconfig.build.json", + }, + test: { + alias: { "lossless-core": engine }, + setupFiles: ["../core/test/setup.ts"], + }, +});