diff --git a/src/index.mjs b/src/index.mjs index 2c8118c..49c5126 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -10,7 +10,9 @@ export function estimateTokens(text) { if (!s) return 0; const chars = s.length; const words = (s.match(/\S+/g) || []).length; - return Math.max(Math.ceil(chars / 4), Math.ceil(words * 1.3)); + const cjkChars = (s.match(/[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\u3040-\u30FF\uAC00-\uD7AF]/g) || []).length; + const charEstimate = Math.ceil((chars - cjkChars) / 4) + cjkChars; + return Math.max(charEstimate, Math.ceil(words * 1.3)); } // Flatten any supported payload into text units: {role, kind, text}. diff --git a/test/basic.test.mjs b/test/basic.test.mjs index 1a2c66c..8380f2f 100644 --- a/test/basic.test.mjs +++ b/test/basic.test.mjs @@ -173,6 +173,17 @@ test("units rejects non-array payload.messages with TypeError", () => { ); }); + +test("estimateTokens keeps CJK text within the documented reference tolerance", () => { + const text = "漢".repeat(200); + const referenceTokens = 200; + const tolerance = 0.15; + const estimated = estimateTokens(text); + assert.ok( + Math.abs(estimated - referenceTokens) <= referenceTokens * tolerance, + `estimated ${estimated}, expected within 15% of ${referenceTokens}`, + ); +}); test("compact does not leave tool_result without its tool_use when trimming", () => { const payload = { messages: [