From e5dc8d3c0b907bf703d0f65b1c6d6c381aa1578a Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 17 Sep 2026 19:12:50 +0800 Subject: [PATCH] Require a letter to keep a bare skill token A resume or JD lists skills and requirements as loose text, and this folds four fixes to how a candidate becomes a skill token: keep a single-letter skill like C or R that a too-strict length check used to drop, strip a numbered-list marker only when digits are followed by punctuation and whitespace so a digit-led skill like 5G survives, drop a split fragment that is pure punctuation instead of a real value, and require every letterless token to carry a letter, with no exception even for a slash or percent. A bare number like 27001 or 2015 split from a shared prefix like ISO cannot prove it still belongs to that prefix, so it is dropped rather than guessed at. --- tests/browser/document-grounding.test.js | 148 +++++++++++++++++++++++ web/document-grounding.js | 34 +++++- 2 files changed, 180 insertions(+), 2 deletions(-) diff --git a/tests/browser/document-grounding.test.js b/tests/browser/document-grounding.test.js index 85b7ac6f..d3ca7f7c 100644 --- a/tests/browser/document-grounding.test.js +++ b/tests/browser/document-grounding.test.js @@ -24,6 +24,154 @@ test("rejects unsupported, spoofed, empty, oversized, and invalid UTF-8 files", } }); +test("single-character skills like C and R survive extraction", async () => { + // unique() used to filter tokens shorter than two characters, which was meant + // to drop stray punctuation left over from a bad split but also silently + // dropped one-letter language names -- exactly the ones a systems-programming + // resume is most likely to list. + const resume = await parseGroundingFile(txt("Skills: C, Go, Python, R, Rust"), "resume"); + assert.deepEqual(resume.skills, ["C", "Go", "Python", "R", "Rust"]); +}); + +test("digit-led skills are not mistaken for a numbered-list marker", async () => { + // clean()'s leading-marker strip is meant for real list prefixes like "1. " + // or "2) ", not for a bare digit run: without the "then punctuation" check, + // "5G" loses its "5" and survives as the fabricated skill "G". + const resume = await parseGroundingFile(txt("Skills: C, 5G, 3D, 4K"), "resume"); + assert.deepEqual(resume.skills, ["C", "5G", "3D", "4K"]); +}); + +test("a numbered-list marker is still stripped from a requirement line", async () => { + const jd = await parseGroundingFile(txt("1. Must know Rust\n2) Should know Go"), "jd"); + assert.deepEqual(jd.requirements, ["Must know Rust", "Should know Go"]); +}); + +test("a numbered or dashed marker with no space after it is still stripped", async () => { + // Requiring whitespace after every marker dropped the marker's punctuation + // but not the marker itself for these common copy-paste shapes, so the + // digit case relies on "." or ")" following the digits instead: that still + // protects "5G" and "3D", which no "." or ")" ever follows. + const jd = await parseGroundingFile(txt("1.Must know Rust\n1.) Should know Go\n-- Must know Python"), "jd"); + assert.deepEqual(jd.requirements, ["Must know Rust", "Should know Go", "Must know Python"]); +}); + +test("a split fragment that is pure punctuation is dropped, not kept as a skill", async () => { + // A stray delimiter or copy-paste artifact landing as its own comma/semicolon + // fragment must not survive filter(Boolean) just because clean() doesn't + // happen to strip that particular symbol. + const resume = await parseGroundingFile(txt("Skills: C, /, Go, #, &, Java"), "resume"); + assert.deepEqual(resume.skills, ["C", "Go", "Java"]); +}); + +test("lone numeric fragments are not kept as skills", async () => { + const resume = await parseGroundingFile(txt("Skills: Python, 1, 1., Rust"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Rust"]); +}); + +test("a single isolated skill with no delimiter still survives extraction", async () => { + const resume = await parseGroundingFile(txt("Skills: Python"), "resume"); + assert.deepEqual(resume.skills, ["Python"]); +}); + +test("a skills header missing its colon is not treated as a skills line", async () => { + // parseResume only recognizes "skills/technologies/stack" followed by ":", + // so a header that drops the colon must yield no skills at all rather than + // matching loosely on the leading word. + const resume = await parseGroundingFile(txt("Skills Python, Go"), "resume"); + assert.deepEqual(resume.skills, []); +}); + +test("header casing, synonyms, and stray whitespace around the colon are tolerated", async () => { + const resume = await parseGroundingFile(txt("TECHNOLOGIES : Python, Go"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Go"]); +}); + +test("empty segments from doubled-up delimiters are dropped, not kept as blank skills", async () => { + const resume = await parseGroundingFile(txt("Skills: Python,, Go;;Rust||C++"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Go", "Rust", "C++"]); +}); + +test("a slash or percent no longer exempts a digit-only token", async () => { + // "24/7" and "100%" are the same digit-plus-symbol shape as "-50" or + // "1-2" -- there is no principled reason to carry a letter's exemption + // for these two symbols and not others, so only a letter keeps a token. + const resume = await parseGroundingFile(txt("Skills: C++11, 24/7, 100%, v2, 5, -5"), "resume"); + assert.deepEqual(resume.skills, ["C++11", "v2"]); +}); + +test("a bare digit.digit shape is dropped as an orphaned version or GPA fragment", async () => { + // "3.14", "5.2", and "802.11" can't be told apart from a GPA or a version + // number split off its software name -- there is no letter left to say + // which one it is, so the whole shape is dropped, standard or not. + const resume = await parseGroundingFile(txt("Skills: Python, 3.14, 5.2, 802.11, Go"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Go"]); +}); + +test("a bare multi-digit integer gets no special treatment either", async () => { + // A plain digit run has no dot to make it read as a split version number + // or GPA, but that shape isn't what decides this: whether "27001" and + // "2015" arrived split off "ISO 9001" by parseResume's "," / ";" / "|" + // split, or had been typed alone, there is no letter left to say whether + // either one is still part of that standard, a separate one, or a year + // with nothing to do with it. Without a letter to carry a value's scope + // through the split, a bare number carries none of its own -- so "27001" + // and "2015" are dropped exactly like "3.14" is, and only "ISO 9001" + // keeps its meaning. + const resume = await parseGroundingFile(txt("Skills: ISO 9001, 27001, 2015"), "resume"); + assert.deepEqual(resume.skills, ["ISO 9001"]); +}); + +test("a generation suffix or org prefix carries a standard's number through", async () => { + // Real-world listings almost always attach a generation letter ("ac", "ax") + // or an org name ("IEEE", "Wi-Fi") to a standard's number, which is exactly + // the letter that lets it survive as its own token. + const resume = await parseGroundingFile(txt("Skills: 802.11ac, 802.11ax, IEEE 802.11, Wi-Fi 802.11"), "resume"); + assert.deepEqual(resume.skills, ["802.11ac", "802.11ax", "IEEE 802.11", "Wi-Fi 802.11"]); +}); + +test("a standard survives named but not split off as a bare number", async () => { + // "IEEE 754" and "ISO 27001" keep their org name, so the letter carries + // them through same as any other skill. Once "754" is split off from + // "IEEE" it is just a bare digit run with no letter left to scope it, and + // is dropped the same way "802.3" is -- both are real standards, but + // neither token carries anything to say so on its own. + const resume = await parseGroundingFile(txt("Skills: IEEE 754, ISO 27001, IEEE, 754, 802.3"), "resume"); + assert.deepEqual(resume.skills, ["IEEE 754", "ISO 27001", "IEEE"]); +}); + +test("a non-ASCII decimal digit dotted fragment is dropped like its ASCII equivalent", async () => { + // \p{N} covers any numeral script, not just ASCII 0-9, so a GPA or version + // fragment spelled in full-width or Arabic-Indic digits carries no letter + // either and is dropped the same way "3.14" is. + const resume = await parseGroundingFile(txt("Skills: Python, ３.１４, ٣.١٤, Go"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Go"]); +}); + +test("digits elsewhere in a token do not earn it a letter's exemption", async () => { + // None of these carry a letter, so none of them get to survive as a + // negative number, a parenthesized GPA, a digit range, or a year range -- + // the same rule that drops a bare "27001" drops these too. + const resume = await parseGroundingFile(txt("Skills: Python, -50, (3.14), 1-2, 2020-2024, Rust"), "resume"); + assert.deepEqual(resume.skills, ["Python", "Rust"]); +}); + +test("a shared prefix does not carry over to the values after it", async () => { + // ISO does not scope "124141" or "2015" just because it appeared earlier + // on the line -- parseResume splits Skills: ISO 27001, 124141, ISO 8981, + // 2015 into four independent candidates, and each one is judged only on + // what it itself carries. "ISO 27001" and "ISO 8981" keep their own "ISO", + // but "124141" and "2015" reached this filter with no letter of their own + // and are dropped, even though a human reader might guess they belong to + // the same certification family. + const resume = await parseGroundingFile(txt("Skills: ISO 27001, 124141, ISO 8981, 2015"), "resume"); + assert.deepEqual(resume.skills, ["ISO 27001", "ISO 8981"]); +}); + +test("stacked list markers on one line are stripped in full, not just the first", async () => { + const jd = await parseGroundingFile(txt("1. - Must know Rust\n* 2) Should know Go"), "jd"); + assert.deepEqual(jd.requirements, ["Must know Rust", "Should know Go"]); +}); + test("selection requires consent and storage is one-time", () => { const extracted = { requirements: ["Must know Rust"], skills: ["Rust"], anchors: ["Built a parser"] }; const selected = { requirements: [0], skills: [], anchors: [0] }; diff --git a/web/document-grounding.js b/web/document-grounding.js index 372ea2a9..fe2c3650 100644 --- a/web/document-grounding.js +++ b/web/document-grounding.js @@ -115,12 +115,42 @@ function normalizeLines(text) { .split(/\r?\n/).map((line) => line.replace(/\s+/g, " ").trim()).filter(Boolean); } +// A bullet glyph is stripped on its own, because it never spells anything +// else and needs no space after it to say so. A dash run or an asterisk +// stripped that freely would eat into real content, so those still only +// count as a marker once whitespace after them confirms it. +// +// A numbered marker does not get that same whitespace requirement, because +// requiring it drops the marker's punctuation but not the marker itself: +// "1.Must know Rust" and "1.) Must know Rust" are common enough copy-paste +// shapes that a candidate line must not reach the interviewer still wearing +// its "1." or "1.)". What still has to hold is the reason the whitespace +// check existed at all -- keeping "5G" and "3D" intact -- and that only +// needs the digits to be followed by "." or ")": neither of those two ever +// follows a digit inside an alphanumeric skill, so requiring one before +// the marker can be dropped is enough on its own, whitespace or not. function clean(line) { - return line.replace(/^[-*•\d.)\s]+/, "").slice(0, textLimit).trim(); + return line.trimStart() + .replace(/^(?:(?:-+|\*)\s+|\d+[.)]+(?=\s|\p{L})\s*|•\s*)+/u, "") + .slice(0, textLimit).trim(); } +// A token with no letter is a number, or nothing but punctuation, wearing a +// list item's clothes, and neither one is a skill on its own. +// +// A letter is what makes a token legible as a named thing: "ISO 27001" and +// "IEEE 754" keep the org name that scopes their number, "5G" and "3D" carry +// their own label, and a plain "5" or "27001" or "2015" carries no such +// scope, whether it arrived alone -- "Skills: 2025" -- or split off a shared +// prefix by parseResume's "," / ";" / "|" split -- "Skills: ISO 27001, +// 124141, 2015". Either way, there is nothing left to tell whether it is +// still part of a standard, a separate one, or an unrelated year. Rather +// than guess, every letterless token is dropped, with no exception: "24/7" +// and "100%" are the same digit-plus-symbol shape as "-50", "1-2", and +// "2020-2024", and none of them carry a letter to claim a meaning others +// would have to guess at. function unique(values, max) { - return [...new Set(values.map(clean).filter((value) => value.length >= 2))].slice(0, max); + return [...new Set(values.map(clean).filter((value) => /\p{L}/u.test(value)))].slice(0, max); } function parseJd(lines) {