From ac7c94173248b95c1950e83799574d043abc0127 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Wed, 30 Sep 2026 23:47:34 -0400 Subject: [PATCH 1/2] Read a binary dictionary's XML with a reader built for it Reading a compressed-binary .sldd was 94% XML parsing: 3396 ms of a 3804 ms ingest, 22 MB/s over the 74.7 MB that 2.75 MB of zip inflates to. The cost was fast-xml-parser being a general engine -- namespaces, DOCTYPE, CDATA, stop nodes, path matching, entity-expansion limits -- paid on every one of 6.9 M elements. A dictionary's XML has a closed ELEMENT vocabulary. Across all 32 zipped dictionaries available (469.4 MB) there are five element names and nothing else: P, Element, Object, Field, DataSource. DictionaryXmlFast reads exactly that subset in one pass and returns null for anything else, which the seam sends to the old engine -- so every lenient and defensive behaviour the XmlReader contract pins is inherited by falling back rather than reimplemented. A bail is never wrong, only slower. Whole-file ingest 3804 ms -> 1062 ms (3.58x), parse alone 5.5-6.0x at 120-129 MB/s, peak RSS 1237 MB -> 889 MB. Both arms measured in separate cold processes with the compiled seam patched between them, not inferred by subtracting a delta. XML parsing is no longer what reading a dictionary costs: unzip and the data model are now the majority. Correctness is not argued. The two readers' output is compared structurally over every binary dictionary available -- 32 files, 469.4 MB, 18,505,864 keys, 6,910,760 objects -- and is identical, key insertion order included, with zero declines. Key order is in the claim because compress() builds children, then #text, then attributes, and Object.keys exposes that downstream. Coercion is not reimplemented: strnum does it, with the same options fast-xml-parser passes, so 007, 0x1F, 1.0 and an integer too large for a double land where they already landed. It is now a declared dependency rather than one borrowed through fast-xml-parser, which moduleBoundaries caught. Numeric character references are read as the literal text the engine also leaves standing -- it does not decode them either, since numericAllowed follows htmlEntities and that is off. Only a reference to a codepoint XML 1.0 prohibits declines, because the engine DELETES those from the value and that table belongs to the engine. MATLAB cannot write one: asked to store a character XML forbids, it re-spells the whole property as Format="decimal" rather than emitting a reference it is not allowed to emit. Six new MATLAB-written fixtures cover what the corpus could not: subclasses of Simulink.Parameter and Simulink.Signal, a twenty-property custom MCOS class, a custom enumeration, and the type-defining MathWorks classes as MATLAB actually writes them in binary rather than as a hand-authored file claims it does. Each is a twin pair -- the same values written both compressed-binary and JSON text -- so a difference between the two readers cannot be confused with a difference between the two formats. Those fixtures earned themselves immediately. A census over 469.4 MB had concluded that no dictionary contains a numeric character reference; it was a true statement about 32 files that was false about the format, because not one of them held a multi-line char value and MATLAB writes CR as . The first generated fixture had one, the differential test went red, and the reader now reads them. The decline had been correct -- the output was never wrong -- but a decline costs the whole file, so one carriage return in 74.7 MB would have paid the full 3358 ms. Tests: 100 new (76 unit, 24 fixture/fuzz differential over all 19 binary fixtures plus 6,000 seeded generated documents per run), suite 4,999 -> 5,099. Every accepted case asserts the value, the live engine's agreement with it, and key insertion order; every declined case asserts null AND that the engine still answers what it answered before this reader existed. The reader was then broken on purpose 30 ways and the suite re-run: 26 behaviour-changing mutations all caught, 3 provably equivalent, 1 control that must survive. That run found four real holes in the tests, all closed. --- dist/datamodel/parser/DictionaryXmlFast.d.ts | 17 + .../parser/DictionaryXmlFast.d.ts.map | 1 + dist/datamodel/parser/DictionaryXmlFast.js | 611 ++++++++++++++++ .../datamodel/parser/DictionaryXmlFast.js.map | 1 + dist/datamodel/parser/XmlReader.d.ts | 12 + dist/datamodel/parser/XmlReader.d.ts.map | 2 +- dist/datamodel/parser/XmlReader.js | 27 + dist/datamodel/parser/XmlReader.js.map | 2 +- package-lock.json | 3 +- package.json | 3 +- src/datamodel/parser/DictionaryXmlFast.ts | 598 +++++++++++++++ src/datamodel/parser/XmlReader.ts | 29 + src/datamodel/parser/strnum.d.ts | 52 ++ test/dictionaryXmlFast.test.ts | 680 ++++++++++++++++++ test/dictionaryXmlFastDifferential.test.ts | 363 ++++++++++ test/fixtures/bus_types_binary.sldd | Bin 0 -> 3710 bytes test/fixtures/bus_types_text.sldd | 246 +++++++ test/fixtures/custom_object_binary.sldd | Bin 0 -> 3832 bytes test/fixtures/custom_object_text.sldd | 322 +++++++++ test/fixtures/derived_class_binary.sldd | Bin 0 -> 3617 bytes test/fixtures/derived_class_text.sldd | 209 ++++++ .../+dexdata/@DerivedParam/DerivedParam.m | 36 + .../+dexdata/@DerivedSignal/DerivedSignal.m | 15 + test/fixtures/mcos/GearMode.m | 19 + test/fixtures/mcos/Widget.m | 101 +++ test/fixtures/mcos/WidgetPart.m | 14 + test/fixtures/mcos/make_class_fixtures.m | 344 +++++++++ test/fixtures/mcos/verify_class_fixtures.m | 219 ++++++ 28 files changed, 3922 insertions(+), 4 deletions(-) create mode 100644 dist/datamodel/parser/DictionaryXmlFast.d.ts create mode 100644 dist/datamodel/parser/DictionaryXmlFast.d.ts.map create mode 100644 dist/datamodel/parser/DictionaryXmlFast.js create mode 100644 dist/datamodel/parser/DictionaryXmlFast.js.map create mode 100644 src/datamodel/parser/DictionaryXmlFast.ts create mode 100644 src/datamodel/parser/strnum.d.ts create mode 100644 test/dictionaryXmlFast.test.ts create mode 100644 test/dictionaryXmlFastDifferential.test.ts create mode 100644 test/fixtures/bus_types_binary.sldd create mode 100644 test/fixtures/bus_types_text.sldd create mode 100644 test/fixtures/custom_object_binary.sldd create mode 100644 test/fixtures/custom_object_text.sldd create mode 100644 test/fixtures/derived_class_binary.sldd create mode 100644 test/fixtures/derived_class_text.sldd create mode 100644 test/fixtures/mcos/+dexdata/@DerivedParam/DerivedParam.m create mode 100644 test/fixtures/mcos/+dexdata/@DerivedSignal/DerivedSignal.m create mode 100644 test/fixtures/mcos/GearMode.m create mode 100644 test/fixtures/mcos/Widget.m create mode 100644 test/fixtures/mcos/WidgetPart.m create mode 100644 test/fixtures/mcos/make_class_fixtures.m create mode 100644 test/fixtures/mcos/verify_class_fixtures.m diff --git a/dist/datamodel/parser/DictionaryXmlFast.d.ts b/dist/datamodel/parser/DictionaryXmlFast.d.ts new file mode 100644 index 00000000..d0af9f03 --- /dev/null +++ b/dist/datamodel/parser/DictionaryXmlFast.d.ts @@ -0,0 +1,17 @@ +/** + * Read a dictionary XML part, or decline. + * + * `null` means "I did not read this" — never "this is broken". The caller must fall back to the + * general engine, which owns every judgement about damaged input. + * + * HOW THE STATE IS HELD, and why it is seven arrays instead of a stack of objects. + * + * The obvious shape is a `Frame` object per open element, pushed and popped. It works and it + * was the first version, and it cost one object plus one attribute array for every element in + * the file — 13.8 M allocations on the corpus's largest dictionary, all of them dying + * immediately. Dictionaries nest shallowly (single digits), so the arrays below are indexed by + * DEPTH and reused: a `

` at depth 3 writes over the state of the previous `

` at depth 3. + * The element's own object (`kids[d]`) is of course fresh each time, because it is the output. + */ +export declare function readDictionaryXmlFast(text: string): Record | null; +//# sourceMappingURL=DictionaryXmlFast.d.ts.map \ No newline at end of file diff --git a/dist/datamodel/parser/DictionaryXmlFast.d.ts.map b/dist/datamodel/parser/DictionaryXmlFast.d.ts.map new file mode 100644 index 00000000..0fe13cc1 --- /dev/null +++ b/dist/datamodel/parser/DictionaryXmlFast.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"DictionaryXmlFast.d.ts","sourceRoot":"","sources":["../../../src/datamodel/parser/DictionaryXmlFast.ts"],"names":[],"mappings":"AAqZA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAiLlF"} \ No newline at end of file diff --git a/dist/datamodel/parser/DictionaryXmlFast.js b/dist/datamodel/parser/DictionaryXmlFast.js new file mode 100644 index 00000000..25d00d41 --- /dev/null +++ b/dist/datamodel/parser/DictionaryXmlFast.js @@ -0,0 +1,611 @@ +// src/datamodel/parser/DictionaryXmlFast.ts +// Copyright 2026 The MathWorks, Inc. +// +// A reader for ONE XML vocabulary: the `data/chunk0.xml` part of a binary `.sldd`. +// +// WHY THIS EXISTS +// +// A zipped dictionary is small on disk and enormous once inflated — the corpus's +// `sldd-zip-entries` is 2.75 MB of zip and 74.7 MB of XML — and reading that XML was +// 94% of the whole ingest: +// +// unzip (native zlib under the Inflate seam) ~140 ms +// utf8 decode 74.7 MB -> JS string 26 ms +// readDictionaryXml (fast-xml-parser) 3358 ms <- 94% +// everything else, data model included ~80 ms +// +// 22 MB/s, where a bare `charCodeAt` pass over the same string is 149 ms (478 MB/s). The cost +// was the general-purpose engine, not the XML: it must cope with namespaces, DOCTYPE, CDATA, +// stop nodes, path matching and entity-expansion limits, and it pays for all of that on every +// one of the 6.9 M elements a dictionary holds. +// +// This file pays for none of it, because a dictionary's XML is a closed vocabulary. Over all +// 32 zipped dictionaries available (469.4 MB of XML, `.scratch/probe-schema.mjs`): +// +// elements

6,022,235 624,550 241,644 22,237 +// 32 — and NOTHING else +// attributes Class, Name, Dimension, Source, PropertyType, EnumerationName, +// EnumerationType, IsEnum, IsComplex, Encoding, EncodedLength, +// FormatVersion, MinRelease, Arch +// +// and `.scratch/probe-unusual.mjs` finds, across those same 32 parts: no DOCTYPE, no CDATA, +// no comments, no processing instruction other than the leading declaration, no namespaced +// tag, and no named entity outside XML's own five. That is what makes a special-purpose reader +// honest rather than a gamble. +// +// One line of that census has already proved blind, and it is the reason to distrust the rest. +// It also said "no numeric character reference" — true of those 32 files, false of dictionaries: +// MATLAB writes ` ` for a CR inside a char property, and the corpus simply held no +// multi-line value. A MATLAB-written fixture found it in minutes +// (`test/fixtures/custom_object_binary.sldd`). A census measures the files you have; what makes +// this reader safe is not the census but the bail, and the fix was to read ` ` rather than +// decline a 74.7 MB file over one carriage return. +// +// The attribute line is blind in the same way and it does not matter, which is the useful half of +// the comparison. That same fixture carries a fifteenth name, `Format="decimal"`, found in none of +// the 32 — and nothing happened: an attribute outside the interned table pays a string +// concatenation and is copied faithfully (see `ATTR_KEYS`). So the ELEMENT vocabulary is what this +// reader bets on being closed. The attribute vocabulary is open, by construction. +// +// WHY IT CAN BAIL, AND WHY THAT IS THE WHOLE SAFETY ARGUMENT +// +// `readDictionaryXmlFast` returns `null` the moment it sees anything it was not built for, and +// `XmlReader.readDictionaryXml` then hands the same text to fast-xml-parser. So this file does +// not have to be a correct XML parser — it has to be a correct reader of the subset it +// recognises, and *provably silent* on everything else. Every lenient and defensive behaviour +// the contract in `XmlReader.ts` pins — an unclosed tag that keeps its attributes, a +// mismatched close tag that is accepted, a non-`DataSource` root, input with no markup reading +// as the empty object, the entity-expansion limits — is inherited by falling back, not +// reimplemented here. A bail is never wrong, only slower. +// +// This is also why the bail conditions are deliberately coarse. A single `` is an object with no `#text` +// while `

` is `''`. +// * Text is entity-decoded, then offered to `dropLayoutWhitespace`, and only then coerced — +// and coerced ONLY IF IT EQUALS ITS OWN TRIM, because `trimValues: false` makes +// fast-xml-parser hand back any padded value raw. `' 7 '` stays three characters; `7` +// becomes the number. That asymmetry is load-bearing: it is how a MATLAB char array keeps +// its padding. +// * Coercion itself is NOT reimplemented. `strnum` does it, with the same options +// fast-xml-parser passes, so `007`, `0x1F`, `1.0` and an integer too large for a double +// all land where they landed before by construction rather than by my having read its +// source carefully enough. See `strnum.d.ts` for why that dependency is declared. +// * Attribute values are never coerced and never trimmed (`parseAttributeValue: false`). +// +// `.scratch/probe-differential.mjs` compares the two readers' output over every binary +// dictionary available — 32 files, 469.4 MB, 18,505,864 keys, identical including key order. +// That corpus run is the claim that actually matters, and it is blind in one direction: it +// proves what the writer DOES emit and says nothing about the branches it never reaches. So +// `test/dictionaryXmlFast.test.ts` pins each rule above against the engine case by case, and +// `test/dictionaryXmlFastDifferential.test.ts` asserts the contract — identical or `null` — +// over every binary fixture in the repo plus 6,000 generated documents per run. +// +// Those three are all blind to one thing: whether a passing assertion would ever fail. +// `.scratch/probe-mutants.mjs` answers that by breaking this file 30 different ways and +// checking the suite goes red; it is the check that found four of the five real holes in those +// tests, and `docs/deep-work/2026-09-30-sldd-xml-fast-reader/testing.md` says which. The fifth +// was found by a new fixture, and is the numeric-reference story above. +import toNumber from 'strnum'; +/** The options fast-xml-parser's OptionsBuilder defaults to; see `numberParseOptions`. */ +const NUMBER_OPTIONS = { hex: true, leadingZeros: true, eNotation: true, unicode: false }; +const TEXT_KEY = '#text'; +/** + * Attribute keys, pre-prefixed and interned. + * + * Not a micro-optimization: `Class` and `Name` occur 6.3 M and 6.0 M times across the corpus, + * so building `'@_' + name` per occurrence is twelve million string allocations that this + * lookup replaces with twelve million pointer copies. An attribute NOT in this table is still + * read — it just pays the concatenation — because an unknown attribute is harmless to copy + * faithfully and bailing on one would strand a whole file for nothing. + */ +const ATTR_KEYS = new Map([ + ['Class', '@_Class'], + ['Name', '@_Name'], + ['Dimension', '@_Dimension'], + ['Source', '@_Source'], + ['PropertyType', '@_PropertyType'], + ['IsComplex', '@_IsComplex'], + ['IsEnum', '@_IsEnum'], + ['EnumerationName', '@_EnumerationName'], + ['EnumerationType', '@_EnumerationType'], + ['Encoding', '@_Encoding'], + ['EncodedLength', '@_EncodedLength'], + ['FormatVersion', '@_FormatVersion'], + ['MinRelease', '@_MinRelease'], + ['Arch', '@_Arch'], + ['version', '@_version'], + ['encoding', '@_encoding'], + ['standalone', '@_standalone'], +]); +/** Character codes, named so the scanner reads as prose rather than as arithmetic. */ +const SLASH = 0x2f; // / +const BANG = 0x21; // ! +const QUESTION = 0x3f; // ? +const EQUALS = 0x3d; // = +const QUOTE = 0x22; // " +const APOS = 0x27; // ' +const SPACE = 0x20; +const TAB = 0x09; +const LF = 0x0a; +const CR = 0x0d; +const DIGIT_0 = 0x30; +const DIGIT_9 = 0x39; +const PLUS = 0x2b; +const MINUS = 0x2d; +const DOT = 0x2e; +const UPPER_P = 0x50; +const HASH = 0x23; // # +const SEMI = 0x3b; // ; +const LOWER_X = 0x78; // x +const UPPER_X = 0x58; // X +function isSpace(c) { + return c === SPACE || c === LF || c === TAB || c === CR; +} +/** + * The end of the `&...;` token starting at `at`, or `-1` when there is none. The entity decoder + * looks at most 32 characters past the `&` and gives up if it has not found a `;` by then, so a + * `;` further away does not close a reference — the `&` is just text. + */ +function entityEnd(value, at) { + let j = at + 1; + while (j < value.length && value.charCodeAt(j) !== SEMI && j - at <= 32) + j++; + return j < value.length && value.charCodeAt(j) === SEMI ? j : -1; +} +/** + * Whether the entity decoder DELETES this codepoint instead of leaving its reference standing: a + * C0 control other than tab/LF/CR — NUL included — or a surrogate half. XML 1.0 cannot carry any + * of them at all, so the decoder drops them even though it otherwise decodes no numeric reference. + * + * A codepoint it cannot parse, or one out of Unicode's range, is NOT deleted — it is left alone, + * which is the same thing that happens to an ordinary one. + * + * NUL has no case of its own even though the decoder gives it a separate, separately configurable + * rule, because every setting of that rule still deletes it and `0 <= 0x1f` already says so. It + * had one until mutation testing showed that removing it changed nothing: an untestable branch. + * + * This is deliberately a superset of what the decoder deletes, not an equal: a document declaring + * `version="1.1"` exempts the C0 block, and this does not follow it there. The consequence of the + * difference is only ever an extra bail, which costs speed and not fidelity — the direction every + * disagreement in this file has to fall. + */ +function isProhibitedCodePoint(cp) { + if (Number.isNaN(cp) || cp < 0 || cp > 0x10ffff) + return false; + if (cp <= 0x1f) + return cp !== TAB && cp !== LF && cp !== CR; + return cp >= 0xd800 && cp <= 0xdfff; +} +/** + * The five entities XML defines, and the only five a dictionary uses. `htmlEntities` is off and + * `numericAllowed` follows it, so fast-xml-parser decodes exactly these five and nothing else; + * ` ` and a bare `&` are a bail, so this never has to be the authority on what the other + * engine would have done with them. + * + * A numeric character reference is the one case worth more than a bail. The engine does not + * decode ` ` — it leaves it standing as literal text — and MATLAB writes exactly that for a + * CR inside a char property, which `test/fixtures/custom_object_binary.sldd` now contains. The + * 32-dictionary corpus had none and said these never occur; it was simply blind, since none of + * those files held a multi-line value. Declining would have been correct and would have cost the + * whole file: one CR anywhere in 74 MB sends every byte of it back to the slow engine. + * + * Callers check for `&` before calling, so the no-entity case costs one `indexOf` and no call. + */ +function decodeEntities(value) { + let out = ''; + let from = 0; + let at = value.indexOf('&'); + while (at !== -1) { + out += value.slice(from, at); + if (value.startsWith('amp;', at + 1)) { + out += '&'; + from = at + 5; + } + else if (value.startsWith('lt;', at + 1)) { + out += '<'; + from = at + 4; + } + else if (value.startsWith('gt;', at + 1)) { + out += '>'; + from = at + 4; + } + else if (value.startsWith('quot;', at + 1)) { + out += '"'; + from = at + 6; + } + else if (value.startsWith('apos;', at + 1)) { + out += "'"; + from = at + 6; + } + else if (value.charCodeAt(at + 1) === HASH) { + // Left standing, reference and all — EXCEPT a codepoint XML 1.0 prohibits, which the engine + // deletes from the value. That table is not reimplemented here: nothing a dictionary holds + // reaches it, so this bails and lets the engine be the one that knows. + const end = entityEnd(value, at); + if (end !== -1) { + const c = value.charCodeAt(at + 2); + const hex = c === LOWER_X || c === UPPER_X; + // The same two `parseInt` calls the decoder makes, lenient tail and all, because a token + // like `zz;` has to be classified the way it classifies it. + const cp = hex + ? parseInt(value.slice(at + 3, end), 16) + : parseInt(value.slice(at + 2, end), 10); + if (isProhibitedCodePoint(cp)) + return null; + } + out += '&'; + from = at + 1; + } + else { + return null; // not ours to interpret + } + at = value.indexOf('&', from); + } + return out + value.slice(from); +} +/** + * The text pipeline, in fast-xml-parser's order: decode, then offer to the layout-whitespace + * hook, then coerce only an unpadded value. + * + * Returns `undefined` for "add no `#text` key", which is how both the empty chunk and the + * dropped layout whitespace arrive — `saveTextToParentTag` adds the key only for a non-empty + * processed value. Returns `null` for "bail", which text can only do by carrying an entity + * this reader will not interpret. + * + * `trim` is called ONCE. It used to be called twice, which cost a second pass over every + * value in the file for a question the first pass had already answered. + */ +function processText(raw, isLeaf) { + let decoded = raw; + if (raw.indexOf('&') !== -1) { + const d = decodeEntities(raw); + if (d === null) + return null; + decoded = d; + } + const trimmed = decoded.trim(); + // `dropLayoutWhitespace`: whitespace between two tags is layout, and removing the KEY rather + // than emptying it is what the real hook achieves by returning ''. + if (trimmed === '' && !isLeaf) + return undefined; + if (trimmed !== decoded) + return decoded; // padded: handed back raw, uncoerced + if (decoded === 'true') + return true; + if (decoded === 'false') + return false; + // `strnum` can only ever return a number for a value whose first character is a digit, a + // sign or a dot — EXCEPT when it is leading whitespace, and the line above has just proved + // there is none. Verified exhaustively over all 65,535 BMP first characters: the only ones + // that coerce despite not being digit-like are the 25 that `trim` strips. So this guard + // skips the call for every name, class and enum literal in the file, which is most of them, + // without being able to change an answer. + const c0 = decoded.charCodeAt(0); + if ((c0 >= DIGIT_0 && c0 <= DIGIT_9) || c0 === PLUS || c0 === MINUS || c0 === DOT) { + return toNumber(decoded, NUMBER_OPTIONS); + } + return decoded; +} +/** `Object`, `P`, `Element` and `Field` are forced to arrays by `isArray`; nothing else is. */ +function alwaysArray(tag) { + return tag === 'P' || tag === 'Element' || tag === 'Object' || tag === 'Field'; +} +/** + * Attach a finished child value to its parent, matching `compress`'s accumulation: the + * forced-array names start as a one-element array and push; anything else is set once and only + * becomes an array if the name repeats. + */ +function attach(parent, tag, value) { + const existing = parent[tag]; + if (existing === undefined && !Object.prototype.hasOwnProperty.call(parent, tag)) { + parent[tag] = alwaysArray(tag) ? [value] : value; + return; + } + if (Array.isArray(existing)) { + existing.push(value); + return; + } + parent[tag] = [existing, value]; +} +/** + * The five element names a dictionary uses, interned — so 6.9 M elements share five strings + * instead of allocating 6.9 M. Any other name is a bail: it would mean this is not the + * document this reader was measured against. + */ +function internTag(text, from, to) { + switch (to - from) { + case 1: + return text.charCodeAt(from) === UPPER_P ? 'P' : null; + case 5: + return text.startsWith('Field', from) ? 'Field' : null; + case 6: + return text.startsWith('Object', from) ? 'Object' : null; + case 7: + return text.startsWith('Element', from) ? 'Element' : null; + case 10: + return text.startsWith('DataSource', from) ? 'DataSource' : null; + default: + return null; + } +} +/** + * Read `name="value"` pairs out of `text[from, limit)` into the two parallel buffers, and + * answer how many there were — or `-1` to bail. + * + * Quoted values only. fast-xml-parser's own attribute regex tolerates a name with no value; a + * dictionary has never written one, and guessing what the engine would make of it is exactly + * the kind of judgement this reader defers instead of imitating. + */ +function readAttributes(text, from, limit, keys, vals) { + let k = from; + let count = 0; + while (k < limit) { + if (isSpace(text.charCodeAt(k))) { + k++; + continue; + } + const nameStart = k; + while (k < limit) { + const c = text.charCodeAt(k); + if (isSpace(c) || c === EQUALS) + break; + k++; + } + if (k === nameStart) + return -1; + const name = text.slice(nameStart, k); + while (k < limit && isSpace(text.charCodeAt(k))) + k++; + if (text.charCodeAt(k) !== EQUALS) + return -1; + k++; + while (k < limit && isSpace(text.charCodeAt(k))) + k++; + const quote = text.charCodeAt(k); + if (quote !== QUOTE && quote !== APOS) + return -1; + const valueStart = k + 1; + const valueEnd = text.indexOf(quote === QUOTE ? '"' : "'", valueStart); + if (valueEnd === -1 || valueEnd > limit) + return -1; + let value = text.slice(valueStart, valueEnd); + // Attribute values are entity-decoded but never coerced and never trimmed. + if (value.indexOf('&') !== -1) { + const decoded = decodeEntities(value); + if (decoded === null) + return -1; + value = decoded; + } + keys[count] = ATTR_KEYS.get(name) ?? `@_${name}`; + vals[count] = value; + count++; + k = valueEnd + 1; + } + return count; +} +/** + * Read a dictionary XML part, or decline. + * + * `null` means "I did not read this" — never "this is broken". The caller must fall back to the + * general engine, which owns every judgement about damaged input. + * + * HOW THE STATE IS HELD, and why it is seven arrays instead of a stack of objects. + * + * The obvious shape is a `Frame` object per open element, pushed and popped. It works and it + * was the first version, and it cost one object plus one attribute array for every element in + * the file — 13.8 M allocations on the corpus's largest dictionary, all of them dying + * immediately. Dictionaries nest shallowly (single digits), so the arrays below are indexed by + * DEPTH and reused: a `

` at depth 3 writes over the state of the previous `

` at depth 3. + * The element's own object (`kids[d]`) is of course fresh each time, because it is the output. + */ +export function readDictionaryXmlFast(text) { + const n = text.length; + const root = {}; + // Per-depth element state, grown on demand and then reused. `kidCount` is how the collapse + // rules are decided without calling `Object.keys` 6.9 M times: "no child keys" is exactly + // "no child was attached", and that is the only thing those rules need to know. + const tags = []; + const kids = []; + const kidCount = []; + const textVal = []; + const textCount = []; + const attrKey = []; + const attrVal = []; + const attrCount = []; + /** Finish the element in slot `d` and answer the value its parent should hold. */ + const closeSlot = (d) => { + const val = kids[d]; + const text0 = textVal[d]; + const hasText = textCount[d] > 0 && text0 !== undefined && text0 !== ''; + if (hasText) + val[TEXT_KEY] = text0; + const ac = attrCount[d]; + if (ac > 0) { + const ks = attrKey[d]; + const vs = attrVal[d]; + for (let a = 0; a < ac; a++) + val[ks[a]] = vs[a]; + return val; // an element with attributes never collapses + } + if (kidCount[d] === 0) + return hasText ? text0 : ''; + return val; + }; + /** Hand a finished value to its parent, or to the root when there is no parent. */ + const emit = (d, tag, value) => { + if (d === 0) { + attach(root, tag, value); + return true; // a root element was seen + } + attach(kids[d - 1], tag, value); + kidCount[d - 1]++; + return false; + }; + let depth = 0; + let i = 0; + let sawRoot = false; + while (i < n) { + const lt = text.indexOf('<', i); + // --- text between tags ------------------------------------------------------------ + if (lt !== i) { + const end = lt === -1 ? n : lt; + // Find the first non-whitespace WITHOUT slicing. The gap between two pretty-printed + // tags is layout and is dropped; there are 204,373 such gaps in the corpus's largest + // dictionary, and slicing each one only to `trim` it and throw it away is two string + // allocations per gap for an answer this loop already has. + // + // Only XML's four whitespace characters are tested here, while `trim` strips the whole + // Unicode set — which is safe in this direction: "all ASCII whitespace" implies "trims + // to empty", and anything this loop calls non-whitespace still goes through + // `processText`, where the real `trim` has the final say. + let w = i; + while (w < end && isSpace(text.charCodeAt(w))) + w++; + const allSpace = w === end; + if (depth === 0) { + // Outside any element. Only whitespace may live here; anything else is content this + // reader has no place to put, so it is the general engine's problem. + if (!allSpace) + return null; + } + else { + // A close tag makes this the element's last chunk, and "leaf" means no child was ever + // attached — which is what `OrderedObjParser` asks when it flushes there. An opening + // tag follows, so it hands `isLeafNode: false` outright. + const isClose = lt !== -1 && text.charCodeAt(lt + 1) === SLASH; + const leaf = isClose && kidCount[depth - 1] === 0; + // `allSpace && !leaf` is precisely the dropped-layout case, so skipping it here is + // not a shortcut past `processText` — it is the same answer without the two strings. + if (!allSpace || leaf) { + const processed = processText(text.slice(i, end), leaf); + if (processed === null) + return null; + if (processed !== undefined) { + const d = depth - 1; + textVal[d] = + textCount[d] === 0 ? processed : `${String(textVal[d])}${String(processed)}`; + textCount[d]++; + } + } + } + if (lt === -1) + break; + i = lt; + } + const after = text.charCodeAt(i + 1); + // --- anything this reader refuses to interpret ------------------------------------- + // One `', i + 2); + if (pi === -1) + return null; + if (!text.startsWith(' 0 ? decl : ''; + i = pi + 2; + continue; + } + // --- close tag -------------------------------------------------------------------- + if (after === SLASH) { + const gt = text.indexOf('>', i + 2); + if (gt === -1 || depth === 0) + return null; + let end = gt; + while (end > i + 2 && isSpace(text.charCodeAt(end - 1))) + end--; + depth--; + const tag = tags[depth]; + // A mismatched close is not ours to forgive — the engine accepts it, and what it builds + // from there is its own documented leniency. + if (text.slice(i + 2, end) !== tag) + return null; + if (emit(depth, tag, closeSlot(depth))) + sawRoot = true; + i = gt + 1; + continue; + } + // --- open tag --------------------------------------------------------------------- + const gt = text.indexOf('>', i + 1); + if (gt === -1) + return null; // unclosed: the general engine decides what survives + const selfClosing = text.charCodeAt(gt - 1) === SLASH; + const tagEnd = selfClosing ? gt - 1 : gt; + let nameEnd = i + 1; + while (nameEnd < tagEnd && !isSpace(text.charCodeAt(nameEnd))) + nameEnd++; + const tag = internTag(text, i + 1, nameEnd); + if (tag === null) + return null; + if (depth === 0 && sawRoot) + return null; // a second root is a shape we lack + const d = depth; + let ks = attrKey[d]; + if (ks === undefined) { + ks = attrKey[d] = []; + attrVal[d] = []; + } + const count = readAttributes(text, nameEnd, tagEnd, ks, attrVal[d]); + if (count < 0) + return null; + tags[d] = tag; + kids[d] = {}; + kidCount[d] = 0; + textVal[d] = undefined; + textCount[d] = 0; + attrCount[d] = count; + if (selfClosing) { + if (emit(d, tag, closeSlot(d))) + sawRoot = true; + } + else { + depth++; + } + i = gt + 1; + } + // An unclosed element, or nothing recognised at all. Both are the fallback's to report: + // "no markup" reads as the empty object there, and this reader must not invent that. + if (depth > 0 || !sawRoot) + return null; + return root; +} +//# sourceMappingURL=DictionaryXmlFast.js.map \ No newline at end of file diff --git a/dist/datamodel/parser/DictionaryXmlFast.js.map b/dist/datamodel/parser/DictionaryXmlFast.js.map new file mode 100644 index 00000000..d7209174 --- /dev/null +++ b/dist/datamodel/parser/DictionaryXmlFast.js.map @@ -0,0 +1 @@ +{"version":3,"file":"DictionaryXmlFast.js","sourceRoot":"","sources":["../../../src/datamodel/parser/DictionaryXmlFast.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,qCAAqC;AACrC,EAAE;AACF,mFAAmF;AACnF,EAAE;AACF,kBAAkB;AAClB,EAAE;AACF,iFAAiF;AACjF,qFAAqF;AACrF,2BAA2B;AAC3B,EAAE;AACF,4DAA4D;AAC5D,4DAA4D;AAC5D,qEAAqE;AACrE,6DAA6D;AAC7D,EAAE;AACF,8FAA8F;AAC9F,6FAA6F;AAC7F,8FAA8F;AAC9F,gDAAgD;AAChD,EAAE;AACF,6FAA6F;AAC7F,mFAAmF;AACnF,EAAE;AACF,qFAAqF;AACrF,+DAA+D;AAC/D,8EAA8E;AAC9E,4EAA4E;AAC5E,+CAA+C;AAC/C,EAAE;AACF,4FAA4F;AAC5F,2FAA2F;AAC3F,+FAA+F;AAC/F,+BAA+B;AAC/B,EAAE;AACF,+FAA+F;AAC/F,iGAAiG;AACjG,uFAAuF;AACvF,iEAAiE;AACjE,gGAAgG;AAChG,+FAA+F;AAC/F,mDAAmD;AACnD,EAAE;AACF,kGAAkG;AAClG,mGAAmG;AACnG,uFAAuF;AACvF,mGAAmG;AACnG,kFAAkF;AAClF,EAAE;AACF,6DAA6D;AAC7D,EAAE;AACF,+FAA+F;AAC/F,+FAA+F;AAC/F,uFAAuF;AACvF,8FAA8F;AAC9F,qFAAqF;AACrF,+FAA+F;AAC/F,uFAAuF;AACvF,0DAA0D;AAC1D,EAAE;AACF,6FAA6F;AAC7F,6FAA6F;AAC7F,qDAAqD;AACrD,EAAE;AACF,gGAAgG;AAChG,iGAAiG;AACjG,gGAAgG;AAChG,oFAAoF;AACpF,EAAE;AACF,2CAA2C;AAC3C,EAAE;AACF,wFAAwF;AACxF,6FAA6F;AAC7F,kFAAkF;AAClF,4FAA4F;AAC5F,EAAE;AACF,6FAA6F;AAC7F,yFAAyF;AACzF,4EAA4E;AAC5E,sDAAsD;AACtD,2FAA2F;AAC3F,+FAA+F;AAC/F,sFAAsF;AACtF,6FAA6F;AAC7F,4BAA4B;AAC5B,8FAA8F;AAC9F,oFAAoF;AACpF,0FAA0F;AAC1F,8FAA8F;AAC9F,mBAAmB;AACnB,oFAAoF;AACpF,4FAA4F;AAC5F,0FAA0F;AAC1F,sFAAsF;AACtF,2FAA2F;AAC3F,EAAE;AACF,uFAAuF;AACvF,6FAA6F;AAC7F,2FAA2F;AAC3F,4FAA4F;AAC5F,6FAA6F;AAC7F,4FAA4F;AAC5F,gFAAgF;AAChF,EAAE;AACF,uFAAuF;AACvF,wFAAwF;AACxF,+FAA+F;AAC/F,+FAA+F;AAC/F,wEAAwE;AAExE,OAAO,QAAQ,MAAM,QAAQ,CAAC;AAE9B,0FAA0F;AAC1F,MAAM,cAAc,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAE1F,MAAM,QAAQ,GAAG,OAAO,CAAC;AAEzB;;;;;;;;GAQG;AACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAiB;IACxC,CAAC,OAAO,EAAE,SAAS,CAAC;IACpB,CAAC,MAAM,EAAE,QAAQ,CAAC;IAClB,CAAC,WAAW,EAAE,aAAa,CAAC;IAC5B,CAAC,QAAQ,EAAE,UAAU,CAAC;IACtB,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAClC,CAAC,WAAW,EAAE,aAAa,CAAC;IAC5B,CAAC,QAAQ,EAAE,UAAU,CAAC;IACtB,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;IACxC,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;IACxC,CAAC,UAAU,EAAE,YAAY,CAAC;IAC1B,CAAC,eAAe,EAAE,iBAAiB,CAAC;IACpC,CAAC,eAAe,EAAE,iBAAiB,CAAC;IACpC,CAAC,YAAY,EAAE,cAAc,CAAC;IAC9B,CAAC,MAAM,EAAE,QAAQ,CAAC;IAClB,CAAC,SAAS,EAAE,WAAW,CAAC;IACxB,CAAC,UAAU,EAAE,YAAY,CAAC;IAC1B,CAAC,YAAY,EAAE,cAAc,CAAC;CAC/B,CAAC,CAAC;AAEH,sFAAsF;AACtF,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI;AACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,IAAI;AAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI;AACzB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI;AACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,EAAE,GAAG,IAAI,CAAC;AAChB,MAAM,EAAE,GAAG,IAAI,CAAC;AAChB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,IAAI,GAAG,IAAI,CAAC;AAClB,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,IAAI;AAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,IAAI;AAE1B,SAAS,OAAO,CAAC,CAAS;IACxB,OAAO,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;AAC1D,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,KAAa,EAAE,EAAU;IAC1C,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE;QAAE,CAAC,EAAE,CAAC;IAC7E,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnE,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,qBAAqB,CAAC,EAAU;IACvC,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,EAAE,IAAI,IAAI;QAAE,OAAO,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC5D,OAAO,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;AACtC,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC7B,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YACrC,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,4FAA4F;YAC5F,2FAA2F;YAC3F,uEAAuE;YACvE,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACjC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBACnC,MAAM,GAAG,GAAG,CAAC,KAAK,OAAO,IAAI,CAAC,KAAK,OAAO,CAAC;gBAC3C,yFAAyF;gBACzF,+DAA+D;gBAC/D,MAAM,EAAE,GAAG,GAAG;oBACZ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC;oBACxC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3C,IAAI,qBAAqB,CAAC,EAAE,CAAC;oBAAE,OAAO,IAAI,CAAC;YAC7C,CAAC;YACD,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,CAAC,wBAAwB;QACvC,CAAC;QACD,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,WAAW,CAAC,GAAW,EAAE,MAAe;IAC/C,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,GAAG,CAAC,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC/B,6FAA6F;IAC7F,mEAAmE;IACnE,IAAI,OAAO,KAAK,EAAE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC,CAAC,qCAAqC;IAC9E,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACtC,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,wFAAwF;IACxF,4FAA4F;IAC5F,0CAA0C;IAC1C,MAAM,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,KAAK,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;QAClF,OAAO,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+FAA+F;AAC/F,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,OAAO,CAAC;AACjF,CAAC;AAED;;;;GAIG;AACH,SAAS,MAAM,CAAC,MAA+B,EAAE,GAAW,EAAE,KAAc;IAC1E,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;QACjF,MAAM,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACjD,OAAO;IACT,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAClC,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAY,EAAE,IAAY,EAAE,EAAU;IACvD,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;QAClB,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACzD,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3D,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7D,KAAK,EAAE;YACL,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CACrB,IAAY,EACZ,IAAY,EACZ,KAAa,EACb,IAAc,EACd,IAAc;IAEd,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;QACjB,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAChC,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,CAAC,CAAC;QACpB,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;YACjB,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM;gBAAE,MAAM;YACtC,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,CAAC,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAAE,CAAC,EAAE,CAAC;QACrD,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7C,CAAC,EAAE,CAAC;QACJ,OAAO,CAAC,GAAG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAAE,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACvE,IAAI,QAAQ,KAAK,CAAC,CAAC,IAAI,QAAQ,GAAG,KAAK;YAAE,OAAO,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC7C,2EAA2E;QAC3E,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,OAAO,KAAK,IAAI;gBAAE,OAAO,CAAC,CAAC,CAAC;YAChC,KAAK,GAAG,OAAO,CAAC;QAClB,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACpB,KAAK,EAAE,CAAC;QACR,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,MAAM,IAAI,GAA4B,EAAE,CAAC;IAEzC,2FAA2F;IAC3F,0FAA0F;IAC1F,gFAAgF;IAChF,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAmC,EAAE,CAAC;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAiD,EAAE,CAAC;IACjE,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,kFAAkF;IAClF,MAAM,SAAS,GAAG,CAAC,CAAS,EAAW,EAAE;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,CAAC;QACxE,IAAI,OAAO;YAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC;QACnC,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YACX,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE;gBAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YAChD,OAAO,GAAG,CAAC,CAAC,6CAA6C;QAC3D,CAAC;QACD,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,mFAAmF;IACnF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,GAAW,EAAE,KAAc,EAAW,EAAE;QAC/D,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC,CAAC,0BAA0B;QACzC,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;QAChC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACb,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAEhC,qFAAqF;QACrF,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,oFAAoF;YACpF,qFAAqF;YACrF,qFAAqF;YACrF,2DAA2D;YAC3D,EAAE;YACF,uFAAuF;YACvF,uFAAuF;YACvF,4EAA4E;YAC5E,0DAA0D;YAC1D,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAAE,CAAC,EAAE,CAAC;YACnD,MAAM,QAAQ,GAAG,CAAC,KAAK,GAAG,CAAC;YAE3B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,oFAAoF;gBACpF,qEAAqE;gBACrE,IAAI,CAAC,QAAQ;oBAAE,OAAO,IAAI,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,sFAAsF;gBACtF,qFAAqF;gBACrF,yDAAyD;gBACzD,MAAM,OAAO,GAAG,EAAE,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC;gBAC/D,MAAM,IAAI,GAAG,OAAO,IAAI,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;gBAClD,mFAAmF;gBACnF,qFAAqF;gBACrF,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACtB,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;oBACxD,IAAI,SAAS,KAAK,IAAI;wBAAE,OAAO,IAAI,CAAC;oBACpC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;wBAC5B,MAAM,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;wBACpB,OAAO,CAAC,CAAC,CAAC;4BACR,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC/E,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,MAAM;YACrB,CAAC,GAAG,EAAE,CAAC;QACT,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAErC,sFAAsF;QACtF,wFAAwF;QACxF,sFAAsF;QACtF,oEAAoE;QACpE,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAEhC,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvB,gFAAgF;YAChF,uEAAuE;YACvE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACrC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC9C,IAAI,OAAO,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YACxC,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1D,IAAI,KAAK,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC3B,MAAM,IAAI,GAA4B,EAAE,CAAC;YACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACxD,+EAA+E;YAC/E,kFAAkF;YAClF,IAAI,CAAC,MAAM,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,qFAAqF;QACrF,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACpB,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACpC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC1C,IAAI,GAAG,GAAG,EAAE,CAAC;YACb,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;gBAAE,GAAG,EAAE,CAAC;YAC/D,KAAK,EAAE,CAAC;YACR,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YACxB,wFAAwF;YACxF,6CAA6C;YAC7C,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAC;YAChD,IAAI,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC;YACvD,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,qFAAqF;QACrF,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACpC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,qDAAqD;QACjF,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;QACpB,OAAO,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QACzE,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;QAC5C,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC9B,IAAI,KAAK,KAAK,CAAC,IAAI,OAAO;YAAE,OAAO,IAAI,CAAC,CAAC,mCAAmC;QAE5E,MAAM,CAAC,GAAG,KAAK,CAAC;QAChB,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACrB,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YACrB,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAClB,CAAC;QACD,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAE3B,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QACd,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChB,OAAO,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;QACvB,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACjB,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;QAErB,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,KAAK,EAAE,CAAC;QACV,CAAC;QACD,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACb,CAAC;IAED,wFAAwF;IACxF,qFAAqF;IACrF,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,IAAI,CAAC;AACd,CAAC"} \ No newline at end of file diff --git a/dist/datamodel/parser/XmlReader.d.ts b/dist/datamodel/parser/XmlReader.d.ts index e861bf9f..4a7799c8 100644 --- a/dist/datamodel/parser/XmlReader.d.ts +++ b/dist/datamodel/parser/XmlReader.d.ts @@ -8,6 +8,18 @@ * caller already does. */ export declare function readDictionaryXml(text: string): unknown; +/** + * The same document through the general engine, with no fast path consulted. + * + * NOT for callers — `readDictionaryXml` is the reader, and reaching past it would give up the + * speed for nothing. This exists because the fast reader's entire contract is "identical to the + * general engine or `null`", and a test of that cannot go through `readDictionaryXml`: that + * would compare the fast reader against itself. The alternative was for the test to build its + * own `XMLParser` with a copy of the options above, which passes just as happily when the copy + * has gone stale — the one failure that would let the two readers diverge in production while + * the test stays green. + */ +export declare function readDictionaryXmlGeneric(text: string): unknown; /** * One part of a `.slx` or `.mdl` package. Engine defaults: text is trimmed, and a lone * child stays an object, so a walker over one of these has to coerce to an array itself. diff --git a/dist/datamodel/parser/XmlReader.d.ts.map b/dist/datamodel/parser/XmlReader.d.ts.map index 57ed8c0e..f15664ce 100644 --- a/dist/datamodel/parser/XmlReader.d.ts.map +++ b/dist/datamodel/parser/XmlReader.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"XmlReader.d.ts","sourceRoot":"","sources":["../../../src/datamodel/parser/XmlReader.ts"],"names":[],"mappings":"AA2JA;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEvD;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAElD;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEpD"} \ No newline at end of file +{"version":3,"file":"XmlReader.d.ts","sourceRoot":"","sources":["../../../src/datamodel/parser/XmlReader.ts"],"names":[],"mappings":"AA6JA;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAcvD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE9D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAElD;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEpD"} \ No newline at end of file diff --git a/dist/datamodel/parser/XmlReader.js b/dist/datamodel/parser/XmlReader.js index 1a131d01..4e42fd1d 100644 --- a/dist/datamodel/parser/XmlReader.js +++ b/dist/datamodel/parser/XmlReader.js @@ -85,6 +85,7 @@ // stays three characters where the model reader would make it the number 7. Interior // whitespace is untouched either way — trimming is not normalization. import { XMLParser } from 'fast-xml-parser'; +import { readDictionaryXmlFast } from './DictionaryXmlFast.js'; const ATTRIBUTE_PREFIX = '@_'; const TEXT_KEY = '#text'; /** The shape every reader produces: attributes under `@_`, text under `#text`. */ @@ -149,6 +150,32 @@ const defaultParser = new XMLParser(SHAPE); * caller already does. */ export function readDictionaryXml(text) { + // Two engines, ONE contract. `DictionaryXmlFast` reads the subset of XML a dictionary is + // actually written in — five element names, quoted attributes, XML's five entities — and + // returns `null` for anything else, which is every question about damage, DOCTYPE, CDATA + // or a numeric character reference. Those go to the general engine, which is what the + // contract above was measured against and still is: the fast reader is held to the + // engine's output byte for byte, key order included, over the whole corpus + // (`.scratch/probe-differential.mjs`) and case by case in `test/dictionaryXmlFast.test.ts`. + // + // So this line is a speed change and not a behaviour change. It is written as a fallback + // rather than a feature flag because the fallback IS the specification — there is no + // configuration in which the fast reader is the only reader, and a dictionary it declines + // is read exactly as it was before. + return readDictionaryXmlFast(text) ?? readDictionaryXmlGeneric(text); +} +/** + * The same document through the general engine, with no fast path consulted. + * + * NOT for callers — `readDictionaryXml` is the reader, and reaching past it would give up the + * speed for nothing. This exists because the fast reader's entire contract is "identical to the + * general engine or `null`", and a test of that cannot go through `readDictionaryXml`: that + * would compare the fast reader against itself. The alternative was for the test to build its + * own `XMLParser` with a copy of the options above, which passes just as happily when the copy + * has gone stale — the one failure that would let the two readers diverge in production while + * the test stays green. + */ +export function readDictionaryXmlGeneric(text) { return dictionaryParser.parse(text); } /** diff --git a/dist/datamodel/parser/XmlReader.js.map b/dist/datamodel/parser/XmlReader.js.map index 7b9b1fe8..ceb9dcdc 100644 --- a/dist/datamodel/parser/XmlReader.js.map +++ b/dist/datamodel/parser/XmlReader.js.map @@ -1 +1 @@ -{"version":3,"file":"XmlReader.js","sourceRoot":"","sources":["../../../src/datamodel/parser/XmlReader.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,EAAE;AACF,qDAAqD;AACrD,EAAE;AACF,sFAAsF;AACtF,2FAA2F;AAC3F,0FAA0F;AAC1F,2FAA2F;AAC3F,qDAAqD;AACrD,EAAE;AACF,4FAA4F;AAC5F,wFAAwF;AACxF,0FAA0F;AAC1F,2BAA2B;AAC3B,EAAE;AACF,0FAA0F;AAC1F,sEAAsE;AACtE,EAAE;AACF,4BAA4B;AAC5B,EAAE;AACF,iFAAiF;AACjF,wFAAwF;AACxF,sFAAsF;AACtF,uFAAuF;AACvF,qFAAqF;AACrF,uEAAuE;AACvE,mFAAmF;AACnF,wFAAwF;AACxF,yFAAyF;AACzF,yFAAyF;AACzF,uEAAuE;AACvE,wEAAwE;AACxE,uFAAuF;AACvF,uDAAuD;AACvD,yFAAyF;AACzF,qFAAqF;AACrF,yEAAyE;AACzE,EAAE;AACF,mFAAmF;AACnF,EAAE;AACF,uFAAuF;AACvF,0FAA0F;AAC1F,sFAAsF;AACtF,sEAAsE;AACtE,iCAAiC;AACjC,uFAAuF;AACvF,wFAAwF;AACxF,qFAAqF;AACrF,uFAAuF;AACvF,mEAAmE;AACnE,+EAA+E;AAC/E,EAAE;AACF,0FAA0F;AAC1F,sCAAsC;AACtC,EAAE;AACF,6DAA6D;AAC7D,EAAE;AACF,2FAA2F;AAC3F,kFAAkF;AAClF,6EAA6E;AAC7E,sFAAsF;AACtF,oFAAoF;AACpF,4FAA4F;AAC5F,yFAAyF;AACzF,2CAA2C;AAC3C,EAAE;AACF,wFAAwF;AACxF,qFAAqF;AACrF,sFAAsF;AACtF,sFAAsF;AACtF,yFAAyF;AACzF,2CAA2C;AAC3C,EAAE;AACF,oFAAoF;AACpF,+EAA+E;AAC/E,oFAAoF;AACpF,yFAAyF;AACzF,uFAAuF;AACvF,oFAAoF;AACpF,kDAAkD;AAClD,EAAE;AACF,yFAAyF;AACzF,qFAAqF;AACrF,wFAAwF;AACxF,uFAAuF;AACvF,wEAAwE;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC;AAEzB,kFAAkF;AAClF,MAAM,KAAK,GAAG;IACZ,gBAAgB,EAAE,KAAK;IACvB,mBAAmB,EAAE,gBAAgB;IACrC,YAAY,EAAE,QAAQ;CACvB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,SAAS,oBAAoB,CAC3B,QAAgB,EAChB,KAAa,EACb,MAAe,EACf,cAAuB,EACvB,UAAmB;IAEnB,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AACzD,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,SAAS,CAAC;IACrC,GAAG,KAAK;IACR,OAAO,EAAE,CAAC,IAAY,EAAE,EAAE,CACxB,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,OAAO;IAC7E,UAAU,EAAE,KAAK;IACjB,iBAAiB,EAAE,oBAAoB;CACxC,CAAC,CAAC;AAEH,wFAAwF;AACxF,oFAAoF;AACpF,sFAAsF;AACtF,kFAAkF;AAClF,wFAAwF;AACxF,iFAAiF;AACjF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;AAE3C;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,OAAO,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACnC,CAAC"} \ No newline at end of file +{"version":3,"file":"XmlReader.js","sourceRoot":"","sources":["../../../src/datamodel/parser/XmlReader.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,EAAE;AACF,qDAAqD;AACrD,EAAE;AACF,sFAAsF;AACtF,2FAA2F;AAC3F,0FAA0F;AAC1F,2FAA2F;AAC3F,qDAAqD;AACrD,EAAE;AACF,4FAA4F;AAC5F,wFAAwF;AACxF,0FAA0F;AAC1F,2BAA2B;AAC3B,EAAE;AACF,0FAA0F;AAC1F,sEAAsE;AACtE,EAAE;AACF,4BAA4B;AAC5B,EAAE;AACF,iFAAiF;AACjF,wFAAwF;AACxF,sFAAsF;AACtF,uFAAuF;AACvF,qFAAqF;AACrF,uEAAuE;AACvE,mFAAmF;AACnF,wFAAwF;AACxF,yFAAyF;AACzF,yFAAyF;AACzF,uEAAuE;AACvE,wEAAwE;AACxE,uFAAuF;AACvF,uDAAuD;AACvD,yFAAyF;AACzF,qFAAqF;AACrF,yEAAyE;AACzE,EAAE;AACF,mFAAmF;AACnF,EAAE;AACF,uFAAuF;AACvF,0FAA0F;AAC1F,sFAAsF;AACtF,sEAAsE;AACtE,iCAAiC;AACjC,uFAAuF;AACvF,wFAAwF;AACxF,qFAAqF;AACrF,uFAAuF;AACvF,mEAAmE;AACnE,+EAA+E;AAC/E,EAAE;AACF,0FAA0F;AAC1F,sCAAsC;AACtC,EAAE;AACF,6DAA6D;AAC7D,EAAE;AACF,2FAA2F;AAC3F,kFAAkF;AAClF,6EAA6E;AAC7E,sFAAsF;AACtF,oFAAoF;AACpF,4FAA4F;AAC5F,yFAAyF;AACzF,2CAA2C;AAC3C,EAAE;AACF,wFAAwF;AACxF,qFAAqF;AACrF,sFAAsF;AACtF,sFAAsF;AACtF,yFAAyF;AACzF,2CAA2C;AAC3C,EAAE;AACF,oFAAoF;AACpF,+EAA+E;AAC/E,oFAAoF;AACpF,yFAAyF;AACzF,uFAAuF;AACvF,oFAAoF;AACpF,kDAAkD;AAClD,EAAE;AACF,yFAAyF;AACzF,qFAAqF;AACrF,wFAAwF;AACxF,uFAAuF;AACvF,wEAAwE;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAE/D,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC;AAEzB,kFAAkF;AAClF,MAAM,KAAK,GAAG;IACZ,gBAAgB,EAAE,KAAK;IACvB,mBAAmB,EAAE,gBAAgB;IACrC,YAAY,EAAE,QAAQ;CACvB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,SAAS,oBAAoB,CAC3B,QAAgB,EAChB,KAAa,EACb,MAAe,EACf,cAAuB,EACvB,UAAmB;IAEnB,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AACzD,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,SAAS,CAAC;IACrC,GAAG,KAAK;IACR,OAAO,EAAE,CAAC,IAAY,EAAE,EAAE,CACxB,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,OAAO;IAC7E,UAAU,EAAE,KAAK;IACjB,iBAAiB,EAAE,oBAAoB;CACxC,CAAC,CAAC;AAEH,wFAAwF;AACxF,oFAAoF;AACpF,sFAAsF;AACtF,kFAAkF;AAClF,wFAAwF;AACxF,iFAAiF;AACjF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;AAE3C;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,yFAAyF;IACzF,yFAAyF;IACzF,yFAAyF;IACzF,sFAAsF;IACtF,mFAAmF;IACnF,2EAA2E;IAC3E,4FAA4F;IAC5F,EAAE;IACF,yFAAyF;IACzF,qFAAqF;IACrF,0FAA0F;IAC1F,oCAAoC;IACpC,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,wBAAwB,CAAC,IAAI,CAAC,CAAC;AACvE,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAY;IACnD,OAAO,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACnC,CAAC"} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 74a3d04e..e044205f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "license": "BSD-3-Clause", "dependencies": { "fast-xml-parser": "^5.8.0", - "fflate": "^0.8.3" + "fflate": "^0.8.3", + "strnum": "^2.4.1" }, "devDependencies": { "@types/node": "^26.2.0", diff --git a/package.json b/package.json index d9002a79..7877341c 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,8 @@ }, "dependencies": { "fast-xml-parser": "^5.8.0", - "fflate": "^0.8.3" + "fflate": "^0.8.3", + "strnum": "^2.4.1" }, "devDependencies": { "@types/node": "^26.2.0", diff --git a/src/datamodel/parser/DictionaryXmlFast.ts b/src/datamodel/parser/DictionaryXmlFast.ts new file mode 100644 index 00000000..0f6aafa2 --- /dev/null +++ b/src/datamodel/parser/DictionaryXmlFast.ts @@ -0,0 +1,598 @@ +// src/datamodel/parser/DictionaryXmlFast.ts +// Copyright 2026 The MathWorks, Inc. +// +// A reader for ONE XML vocabulary: the `data/chunk0.xml` part of a binary `.sldd`. +// +// WHY THIS EXISTS +// +// A zipped dictionary is small on disk and enormous once inflated — the corpus's +// `sldd-zip-entries` is 2.75 MB of zip and 74.7 MB of XML — and reading that XML was +// 94% of the whole ingest: +// +// unzip (native zlib under the Inflate seam) ~140 ms +// utf8 decode 74.7 MB -> JS string 26 ms +// readDictionaryXml (fast-xml-parser) 3358 ms <- 94% +// everything else, data model included ~80 ms +// +// 22 MB/s, where a bare `charCodeAt` pass over the same string is 149 ms (478 MB/s). The cost +// was the general-purpose engine, not the XML: it must cope with namespaces, DOCTYPE, CDATA, +// stop nodes, path matching and entity-expansion limits, and it pays for all of that on every +// one of the 6.9 M elements a dictionary holds. +// +// This file pays for none of it, because a dictionary's XML is a closed vocabulary. Over all +// 32 zipped dictionaries available (469.4 MB of XML, `.scratch/probe-schema.mjs`): +// +// elements

6,022,235 624,550 241,644 22,237 +// 32 — and NOTHING else +// attributes Class, Name, Dimension, Source, PropertyType, EnumerationName, +// EnumerationType, IsEnum, IsComplex, Encoding, EncodedLength, +// FormatVersion, MinRelease, Arch +// +// and `.scratch/probe-unusual.mjs` finds, across those same 32 parts: no DOCTYPE, no CDATA, +// no comments, no processing instruction other than the leading declaration, no namespaced +// tag, and no named entity outside XML's own five. That is what makes a special-purpose reader +// honest rather than a gamble. +// +// One line of that census has already proved blind, and it is the reason to distrust the rest. +// It also said "no numeric character reference" — true of those 32 files, false of dictionaries: +// MATLAB writes ` ` for a CR inside a char property, and the corpus simply held no +// multi-line value. A MATLAB-written fixture found it in minutes +// (`test/fixtures/custom_object_binary.sldd`). A census measures the files you have; what makes +// this reader safe is not the census but the bail, and the fix was to read ` ` rather than +// decline a 74.7 MB file over one carriage return. +// +// The attribute line is blind in the same way and it does not matter, which is the useful half of +// the comparison. That same fixture carries a fifteenth name, `Format="decimal"`, found in none of +// the 32 — and nothing happened: an attribute outside the interned table pays a string +// concatenation and is copied faithfully (see `ATTR_KEYS`). So the ELEMENT vocabulary is what this +// reader bets on being closed. The attribute vocabulary is open, by construction. +// +// WHY IT CAN BAIL, AND WHY THAT IS THE WHOLE SAFETY ARGUMENT +// +// `readDictionaryXmlFast` returns `null` the moment it sees anything it was not built for, and +// `XmlReader.readDictionaryXml` then hands the same text to fast-xml-parser. So this file does +// not have to be a correct XML parser — it has to be a correct reader of the subset it +// recognises, and *provably silent* on everything else. Every lenient and defensive behaviour +// the contract in `XmlReader.ts` pins — an unclosed tag that keeps its attributes, a +// mismatched close tag that is accepted, a non-`DataSource` root, input with no markup reading +// as the empty object, the entity-expansion limits — is inherited by falling back, not +// reimplemented here. A bail is never wrong, only slower. +// +// This is also why the bail conditions are deliberately coarse. A single `` is an object with no `#text` +// while `

` is `''`. +// * Text is entity-decoded, then offered to `dropLayoutWhitespace`, and only then coerced — +// and coerced ONLY IF IT EQUALS ITS OWN TRIM, because `trimValues: false` makes +// fast-xml-parser hand back any padded value raw. `' 7 '` stays three characters; `7` +// becomes the number. That asymmetry is load-bearing: it is how a MATLAB char array keeps +// its padding. +// * Coercion itself is NOT reimplemented. `strnum` does it, with the same options +// fast-xml-parser passes, so `007`, `0x1F`, `1.0` and an integer too large for a double +// all land where they landed before by construction rather than by my having read its +// source carefully enough. See `strnum.d.ts` for why that dependency is declared. +// * Attribute values are never coerced and never trimmed (`parseAttributeValue: false`). +// +// `.scratch/probe-differential.mjs` compares the two readers' output over every binary +// dictionary available — 32 files, 469.4 MB, 18,505,864 keys, identical including key order. +// That corpus run is the claim that actually matters, and it is blind in one direction: it +// proves what the writer DOES emit and says nothing about the branches it never reaches. So +// `test/dictionaryXmlFast.test.ts` pins each rule above against the engine case by case, and +// `test/dictionaryXmlFastDifferential.test.ts` asserts the contract — identical or `null` — +// over every binary fixture in the repo plus 6,000 generated documents per run. +// +// Those three are all blind to one thing: whether a passing assertion would ever fail. +// `.scratch/probe-mutants.mjs` answers that by breaking this file 30 different ways and +// checking the suite goes red; it is the check that found four of the five real holes in those +// tests, and `docs/deep-work/2026-09-30-sldd-xml-fast-reader/testing.md` says which. The fifth +// was found by a new fixture, and is the numeric-reference story above. + +import toNumber from 'strnum'; + +/** The options fast-xml-parser's OptionsBuilder defaults to; see `numberParseOptions`. */ +const NUMBER_OPTIONS = { hex: true, leadingZeros: true, eNotation: true, unicode: false }; + +const TEXT_KEY = '#text'; + +/** + * Attribute keys, pre-prefixed and interned. + * + * Not a micro-optimization: `Class` and `Name` occur 6.3 M and 6.0 M times across the corpus, + * so building `'@_' + name` per occurrence is twelve million string allocations that this + * lookup replaces with twelve million pointer copies. An attribute NOT in this table is still + * read — it just pays the concatenation — because an unknown attribute is harmless to copy + * faithfully and bailing on one would strand a whole file for nothing. + */ +const ATTR_KEYS = new Map([ + ['Class', '@_Class'], + ['Name', '@_Name'], + ['Dimension', '@_Dimension'], + ['Source', '@_Source'], + ['PropertyType', '@_PropertyType'], + ['IsComplex', '@_IsComplex'], + ['IsEnum', '@_IsEnum'], + ['EnumerationName', '@_EnumerationName'], + ['EnumerationType', '@_EnumerationType'], + ['Encoding', '@_Encoding'], + ['EncodedLength', '@_EncodedLength'], + ['FormatVersion', '@_FormatVersion'], + ['MinRelease', '@_MinRelease'], + ['Arch', '@_Arch'], + ['version', '@_version'], + ['encoding', '@_encoding'], + ['standalone', '@_standalone'], +]); + +/** Character codes, named so the scanner reads as prose rather than as arithmetic. */ +const SLASH = 0x2f; // / +const BANG = 0x21; // ! +const QUESTION = 0x3f; // ? +const EQUALS = 0x3d; // = +const QUOTE = 0x22; // " +const APOS = 0x27; // ' +const SPACE = 0x20; +const TAB = 0x09; +const LF = 0x0a; +const CR = 0x0d; +const DIGIT_0 = 0x30; +const DIGIT_9 = 0x39; +const PLUS = 0x2b; +const MINUS = 0x2d; +const DOT = 0x2e; +const UPPER_P = 0x50; +const HASH = 0x23; // # +const SEMI = 0x3b; // ; +const LOWER_X = 0x78; // x +const UPPER_X = 0x58; // X + +function isSpace(c: number): boolean { + return c === SPACE || c === LF || c === TAB || c === CR; +} + +/** + * The end of the `&...;` token starting at `at`, or `-1` when there is none. The entity decoder + * looks at most 32 characters past the `&` and gives up if it has not found a `;` by then, so a + * `;` further away does not close a reference — the `&` is just text. + */ +function entityEnd(value: string, at: number): number { + let j = at + 1; + while (j < value.length && value.charCodeAt(j) !== SEMI && j - at <= 32) j++; + return j < value.length && value.charCodeAt(j) === SEMI ? j : -1; +} + +/** + * Whether the entity decoder DELETES this codepoint instead of leaving its reference standing: a + * C0 control other than tab/LF/CR — NUL included — or a surrogate half. XML 1.0 cannot carry any + * of them at all, so the decoder drops them even though it otherwise decodes no numeric reference. + * + * A codepoint it cannot parse, or one out of Unicode's range, is NOT deleted — it is left alone, + * which is the same thing that happens to an ordinary one. + * + * NUL has no case of its own even though the decoder gives it a separate, separately configurable + * rule, because every setting of that rule still deletes it and `0 <= 0x1f` already says so. It + * had one until mutation testing showed that removing it changed nothing: an untestable branch. + * + * This is deliberately a superset of what the decoder deletes, not an equal: a document declaring + * `version="1.1"` exempts the C0 block, and this does not follow it there. The consequence of the + * difference is only ever an extra bail, which costs speed and not fidelity — the direction every + * disagreement in this file has to fall. + */ +function isProhibitedCodePoint(cp: number): boolean { + if (Number.isNaN(cp) || cp < 0 || cp > 0x10ffff) return false; + if (cp <= 0x1f) return cp !== TAB && cp !== LF && cp !== CR; + return cp >= 0xd800 && cp <= 0xdfff; +} + +/** + * The five entities XML defines, and the only five a dictionary uses. `htmlEntities` is off and + * `numericAllowed` follows it, so fast-xml-parser decodes exactly these five and nothing else; + * ` ` and a bare `&` are a bail, so this never has to be the authority on what the other + * engine would have done with them. + * + * A numeric character reference is the one case worth more than a bail. The engine does not + * decode ` ` — it leaves it standing as literal text — and MATLAB writes exactly that for a + * CR inside a char property, which `test/fixtures/custom_object_binary.sldd` now contains. The + * 32-dictionary corpus had none and said these never occur; it was simply blind, since none of + * those files held a multi-line value. Declining would have been correct and would have cost the + * whole file: one CR anywhere in 74 MB sends every byte of it back to the slow engine. + * + * Callers check for `&` before calling, so the no-entity case costs one `indexOf` and no call. + */ +function decodeEntities(value: string): string | null { + let out = ''; + let from = 0; + let at = value.indexOf('&'); + while (at !== -1) { + out += value.slice(from, at); + if (value.startsWith('amp;', at + 1)) { + out += '&'; + from = at + 5; + } else if (value.startsWith('lt;', at + 1)) { + out += '<'; + from = at + 4; + } else if (value.startsWith('gt;', at + 1)) { + out += '>'; + from = at + 4; + } else if (value.startsWith('quot;', at + 1)) { + out += '"'; + from = at + 6; + } else if (value.startsWith('apos;', at + 1)) { + out += "'"; + from = at + 6; + } else if (value.charCodeAt(at + 1) === HASH) { + // Left standing, reference and all — EXCEPT a codepoint XML 1.0 prohibits, which the engine + // deletes from the value. That table is not reimplemented here: nothing a dictionary holds + // reaches it, so this bails and lets the engine be the one that knows. + const end = entityEnd(value, at); + if (end !== -1) { + const c = value.charCodeAt(at + 2); + const hex = c === LOWER_X || c === UPPER_X; + // The same two `parseInt` calls the decoder makes, lenient tail and all, because a token + // like `zz;` has to be classified the way it classifies it. + const cp = hex + ? parseInt(value.slice(at + 3, end), 16) + : parseInt(value.slice(at + 2, end), 10); + if (isProhibitedCodePoint(cp)) return null; + } + out += '&'; + from = at + 1; + } else { + return null; // not ours to interpret + } + at = value.indexOf('&', from); + } + return out + value.slice(from); +} + +/** + * The text pipeline, in fast-xml-parser's order: decode, then offer to the layout-whitespace + * hook, then coerce only an unpadded value. + * + * Returns `undefined` for "add no `#text` key", which is how both the empty chunk and the + * dropped layout whitespace arrive — `saveTextToParentTag` adds the key only for a non-empty + * processed value. Returns `null` for "bail", which text can only do by carrying an entity + * this reader will not interpret. + * + * `trim` is called ONCE. It used to be called twice, which cost a second pass over every + * value in the file for a question the first pass had already answered. + */ +function processText(raw: string, isLeaf: boolean): string | number | boolean | undefined | null { + let decoded = raw; + if (raw.indexOf('&') !== -1) { + const d = decodeEntities(raw); + if (d === null) return null; + decoded = d; + } + const trimmed = decoded.trim(); + // `dropLayoutWhitespace`: whitespace between two tags is layout, and removing the KEY rather + // than emptying it is what the real hook achieves by returning ''. + if (trimmed === '' && !isLeaf) return undefined; + if (trimmed !== decoded) return decoded; // padded: handed back raw, uncoerced + if (decoded === 'true') return true; + if (decoded === 'false') return false; + // `strnum` can only ever return a number for a value whose first character is a digit, a + // sign or a dot — EXCEPT when it is leading whitespace, and the line above has just proved + // there is none. Verified exhaustively over all 65,535 BMP first characters: the only ones + // that coerce despite not being digit-like are the 25 that `trim` strips. So this guard + // skips the call for every name, class and enum literal in the file, which is most of them, + // without being able to change an answer. + const c0 = decoded.charCodeAt(0); + if ((c0 >= DIGIT_0 && c0 <= DIGIT_9) || c0 === PLUS || c0 === MINUS || c0 === DOT) { + return toNumber(decoded, NUMBER_OPTIONS); + } + return decoded; +} + +/** `Object`, `P`, `Element` and `Field` are forced to arrays by `isArray`; nothing else is. */ +function alwaysArray(tag: string): boolean { + return tag === 'P' || tag === 'Element' || tag === 'Object' || tag === 'Field'; +} + +/** + * Attach a finished child value to its parent, matching `compress`'s accumulation: the + * forced-array names start as a one-element array and push; anything else is set once and only + * becomes an array if the name repeats. + */ +function attach(parent: Record, tag: string, value: unknown): void { + const existing = parent[tag]; + if (existing === undefined && !Object.prototype.hasOwnProperty.call(parent, tag)) { + parent[tag] = alwaysArray(tag) ? [value] : value; + return; + } + if (Array.isArray(existing)) { + existing.push(value); + return; + } + parent[tag] = [existing, value]; +} + +/** + * The five element names a dictionary uses, interned — so 6.9 M elements share five strings + * instead of allocating 6.9 M. Any other name is a bail: it would mean this is not the + * document this reader was measured against. + */ +function internTag(text: string, from: number, to: number): string | null { + switch (to - from) { + case 1: + return text.charCodeAt(from) === UPPER_P ? 'P' : null; + case 5: + return text.startsWith('Field', from) ? 'Field' : null; + case 6: + return text.startsWith('Object', from) ? 'Object' : null; + case 7: + return text.startsWith('Element', from) ? 'Element' : null; + case 10: + return text.startsWith('DataSource', from) ? 'DataSource' : null; + default: + return null; + } +} + +/** + * Read `name="value"` pairs out of `text[from, limit)` into the two parallel buffers, and + * answer how many there were — or `-1` to bail. + * + * Quoted values only. fast-xml-parser's own attribute regex tolerates a name with no value; a + * dictionary has never written one, and guessing what the engine would make of it is exactly + * the kind of judgement this reader defers instead of imitating. + */ +function readAttributes( + text: string, + from: number, + limit: number, + keys: string[], + vals: string[], +): number { + let k = from; + let count = 0; + while (k < limit) { + if (isSpace(text.charCodeAt(k))) { + k++; + continue; + } + const nameStart = k; + while (k < limit) { + const c = text.charCodeAt(k); + if (isSpace(c) || c === EQUALS) break; + k++; + } + if (k === nameStart) return -1; + const name = text.slice(nameStart, k); + while (k < limit && isSpace(text.charCodeAt(k))) k++; + if (text.charCodeAt(k) !== EQUALS) return -1; + k++; + while (k < limit && isSpace(text.charCodeAt(k))) k++; + const quote = text.charCodeAt(k); + if (quote !== QUOTE && quote !== APOS) return -1; + const valueStart = k + 1; + const valueEnd = text.indexOf(quote === QUOTE ? '"' : "'", valueStart); + if (valueEnd === -1 || valueEnd > limit) return -1; + let value = text.slice(valueStart, valueEnd); + // Attribute values are entity-decoded but never coerced and never trimmed. + if (value.indexOf('&') !== -1) { + const decoded = decodeEntities(value); + if (decoded === null) return -1; + value = decoded; + } + keys[count] = ATTR_KEYS.get(name) ?? `@_${name}`; + vals[count] = value; + count++; + k = valueEnd + 1; + } + return count; +} + +/** + * Read a dictionary XML part, or decline. + * + * `null` means "I did not read this" — never "this is broken". The caller must fall back to the + * general engine, which owns every judgement about damaged input. + * + * HOW THE STATE IS HELD, and why it is seven arrays instead of a stack of objects. + * + * The obvious shape is a `Frame` object per open element, pushed and popped. It works and it + * was the first version, and it cost one object plus one attribute array for every element in + * the file — 13.8 M allocations on the corpus's largest dictionary, all of them dying + * immediately. Dictionaries nest shallowly (single digits), so the arrays below are indexed by + * DEPTH and reused: a `

` at depth 3 writes over the state of the previous `

` at depth 3. + * The element's own object (`kids[d]`) is of course fresh each time, because it is the output. + */ +export function readDictionaryXmlFast(text: string): Record | null { + const n = text.length; + const root: Record = {}; + + // Per-depth element state, grown on demand and then reused. `kidCount` is how the collapse + // rules are decided without calling `Object.keys` 6.9 M times: "no child keys" is exactly + // "no child was attached", and that is the only thing those rules need to know. + const tags: string[] = []; + const kids: Array> = []; + const kidCount: number[] = []; + const textVal: Array = []; + const textCount: number[] = []; + const attrKey: string[][] = []; + const attrVal: string[][] = []; + const attrCount: number[] = []; + + /** Finish the element in slot `d` and answer the value its parent should hold. */ + const closeSlot = (d: number): unknown => { + const val = kids[d]; + const text0 = textVal[d]; + const hasText = textCount[d] > 0 && text0 !== undefined && text0 !== ''; + if (hasText) val[TEXT_KEY] = text0; + const ac = attrCount[d]; + if (ac > 0) { + const ks = attrKey[d]; + const vs = attrVal[d]; + for (let a = 0; a < ac; a++) val[ks[a]] = vs[a]; + return val; // an element with attributes never collapses + } + if (kidCount[d] === 0) return hasText ? text0 : ''; + return val; + }; + + /** Hand a finished value to its parent, or to the root when there is no parent. */ + const emit = (d: number, tag: string, value: unknown): boolean => { + if (d === 0) { + attach(root, tag, value); + return true; // a root element was seen + } + attach(kids[d - 1], tag, value); + kidCount[d - 1]++; + return false; + }; + + let depth = 0; + let i = 0; + let sawRoot = false; + + while (i < n) { + const lt = text.indexOf('<', i); + + // --- text between tags ------------------------------------------------------------ + if (lt !== i) { + const end = lt === -1 ? n : lt; + // Find the first non-whitespace WITHOUT slicing. The gap between two pretty-printed + // tags is layout and is dropped; there are 204,373 such gaps in the corpus's largest + // dictionary, and slicing each one only to `trim` it and throw it away is two string + // allocations per gap for an answer this loop already has. + // + // Only XML's four whitespace characters are tested here, while `trim` strips the whole + // Unicode set — which is safe in this direction: "all ASCII whitespace" implies "trims + // to empty", and anything this loop calls non-whitespace still goes through + // `processText`, where the real `trim` has the final say. + let w = i; + while (w < end && isSpace(text.charCodeAt(w))) w++; + const allSpace = w === end; + + if (depth === 0) { + // Outside any element. Only whitespace may live here; anything else is content this + // reader has no place to put, so it is the general engine's problem. + if (!allSpace) return null; + } else { + // A close tag makes this the element's last chunk, and "leaf" means no child was ever + // attached — which is what `OrderedObjParser` asks when it flushes there. An opening + // tag follows, so it hands `isLeafNode: false` outright. + const isClose = lt !== -1 && text.charCodeAt(lt + 1) === SLASH; + const leaf = isClose && kidCount[depth - 1] === 0; + // `allSpace && !leaf` is precisely the dropped-layout case, so skipping it here is + // not a shortcut past `processText` — it is the same answer without the two strings. + if (!allSpace || leaf) { + const processed = processText(text.slice(i, end), leaf); + if (processed === null) return null; + if (processed !== undefined) { + const d = depth - 1; + textVal[d] = + textCount[d] === 0 ? processed : `${String(textVal[d])}${String(processed)}`; + textCount[d]++; + } + } + } + if (lt === -1) break; + i = lt; + } + + const after = text.charCodeAt(i + 1); + + // --- anything this reader refuses to interpret ------------------------------------- + // One `', i + 2); + if (pi === -1) return null; + if (!text.startsWith(' = {}; + for (let a = 0; a < count; a++) decl[keys[a]] = vals[a]; + // A declaration has no children and no text, so `compress` gives `{}` and then + // `assignAttributes` fills it; with no attributes at all it would collapse to ''. + root['?xml'] = count > 0 ? decl : ''; + i = pi + 2; + continue; + } + + // --- close tag -------------------------------------------------------------------- + if (after === SLASH) { + const gt = text.indexOf('>', i + 2); + if (gt === -1 || depth === 0) return null; + let end = gt; + while (end > i + 2 && isSpace(text.charCodeAt(end - 1))) end--; + depth--; + const tag = tags[depth]; + // A mismatched close is not ours to forgive — the engine accepts it, and what it builds + // from there is its own documented leniency. + if (text.slice(i + 2, end) !== tag) return null; + if (emit(depth, tag, closeSlot(depth))) sawRoot = true; + i = gt + 1; + continue; + } + + // --- open tag --------------------------------------------------------------------- + const gt = text.indexOf('>', i + 1); + if (gt === -1) return null; // unclosed: the general engine decides what survives + const selfClosing = text.charCodeAt(gt - 1) === SLASH; + const tagEnd = selfClosing ? gt - 1 : gt; + let nameEnd = i + 1; + while (nameEnd < tagEnd && !isSpace(text.charCodeAt(nameEnd))) nameEnd++; + const tag = internTag(text, i + 1, nameEnd); + if (tag === null) return null; + if (depth === 0 && sawRoot) return null; // a second root is a shape we lack + + const d = depth; + let ks = attrKey[d]; + if (ks === undefined) { + ks = attrKey[d] = []; + attrVal[d] = []; + } + const count = readAttributes(text, nameEnd, tagEnd, ks, attrVal[d]); + if (count < 0) return null; + + tags[d] = tag; + kids[d] = {}; + kidCount[d] = 0; + textVal[d] = undefined; + textCount[d] = 0; + attrCount[d] = count; + + if (selfClosing) { + if (emit(d, tag, closeSlot(d))) sawRoot = true; + } else { + depth++; + } + i = gt + 1; + } + + // An unclosed element, or nothing recognised at all. Both are the fallback's to report: + // "no markup" reads as the empty object there, and this reader must not invent that. + if (depth > 0 || !sawRoot) return null; + return root; +} diff --git a/src/datamodel/parser/XmlReader.ts b/src/datamodel/parser/XmlReader.ts index 99bf29e7..3a8e20f1 100644 --- a/src/datamodel/parser/XmlReader.ts +++ b/src/datamodel/parser/XmlReader.ts @@ -87,6 +87,8 @@ import { XMLParser } from 'fast-xml-parser'; +import { readDictionaryXmlFast } from './DictionaryXmlFast.js'; + const ATTRIBUTE_PREFIX = '@_'; const TEXT_KEY = '#text'; @@ -163,6 +165,33 @@ const defaultParser = new XMLParser(SHAPE); * caller already does. */ export function readDictionaryXml(text: string): unknown { + // Two engines, ONE contract. `DictionaryXmlFast` reads the subset of XML a dictionary is + // actually written in — five element names, quoted attributes, XML's five entities — and + // returns `null` for anything else, which is every question about damage, DOCTYPE, CDATA + // or a numeric character reference. Those go to the general engine, which is what the + // contract above was measured against and still is: the fast reader is held to the + // engine's output byte for byte, key order included, over the whole corpus + // (`.scratch/probe-differential.mjs`) and case by case in `test/dictionaryXmlFast.test.ts`. + // + // So this line is a speed change and not a behaviour change. It is written as a fallback + // rather than a feature flag because the fallback IS the specification — there is no + // configuration in which the fast reader is the only reader, and a dictionary it declines + // is read exactly as it was before. + return readDictionaryXmlFast(text) ?? readDictionaryXmlGeneric(text); +} + +/** + * The same document through the general engine, with no fast path consulted. + * + * NOT for callers — `readDictionaryXml` is the reader, and reaching past it would give up the + * speed for nothing. This exists because the fast reader's entire contract is "identical to the + * general engine or `null`", and a test of that cannot go through `readDictionaryXml`: that + * would compare the fast reader against itself. The alternative was for the test to build its + * own `XMLParser` with a copy of the options above, which passes just as happily when the copy + * has gone stale — the one failure that would let the two readers diverge in production while + * the test stays green. + */ +export function readDictionaryXmlGeneric(text: string): unknown { return dictionaryParser.parse(text); } diff --git a/src/datamodel/parser/strnum.d.ts b/src/datamodel/parser/strnum.d.ts new file mode 100644 index 00000000..db9efc9a --- /dev/null +++ b/src/datamodel/parser/strnum.d.ts @@ -0,0 +1,52 @@ +// src/datamodel/parser/strnum.d.ts +// Copyright 2026 The MathWorks, Inc. +// +// `strnum` ships no types — the package is three files and none of them is a `.d.ts`, and +// `@types/strnum` does not exist. So the declaration lives here, next to its only consumer. +// +// This is deliberately a PRECISE signature rather than `declare module 'strnum';`, which +// would make the import `any` and silently delete type checking at the one call site that +// decides what every dictionary text value becomes. The shape below is read off +// `node_modules/strnum/strnum.js` and off how `fast-xml-parser` calls it in +// `xmlparser/OrderedObjParser.js`: +// +// * The return is `string | number | boolean` — NOT just `number`. A value it declines +// comes back as the string it was given, and that decline is load-bearing: an integer +// too large for a double keeps its spelling instead of losing precision. (Booleans are +// in the union because the option `numericStrings`/`skipLike` paths can return the input +// unchanged; `fast-xml-parser` handles `true`/`false` before calling, and so do we.) +// * Every option is optional, and the defaults are NOT the ones we want — which is the +// whole reason `DictionaryXmlFast` passes an explicit object instead of relying on them. +// +// WHY THIS PACKAGE AT ALL, rather than ten lines of our own coercion. +// +// `readDictionaryXml`'s contract says `007` reads as 7, `1.0` as 1, `0x1F` as 31, and an +// integer too large for a double stays a string. That is not a rule anyone would +// re-derive correctly from a sentence; it is the behaviour of this function, which +// `fast-xml-parser` already calls with the options in `DictionaryXmlFast.NUMBER_OPTIONS`. +// Calling the same function with the same options makes the two readers agree BY +// CONSTRUCTION. Reimplementing it would make them agree only as long as a test happened +// to cover the case that diverged — and `strnum` is already in the tree either way, as +// `fast-xml-parser`'s own dependency, so this costs no install surface. It is declared +// directly in `package.json` because importing a transitive dependency without declaring +// it is how a working build breaks on somebody else's minor release; +// `test/moduleBoundaries.test.ts` enforces that, and caught exactly this. + +declare module 'strnum' { + interface StrnumOptions { + /** Read `0x1F` as 31. `fast-xml-parser` defaults it ON. */ + hex?: boolean; + /** Read `007` as 7 rather than keeping the spelling. Defaults ON. */ + leadingZeros?: boolean; + /** Read `1e3` as 1000. Defaults ON. */ + eNotation?: boolean; + /** Accept non-ASCII digits. Defaults OFF, and we keep it off. */ + unicode?: boolean; + /** Patterns to leave alone entirely. Unused here. */ + skipLike?: RegExp; + } + export default function toNumber( + value: string, + options?: StrnumOptions, + ): string | number | boolean; +} diff --git a/test/dictionaryXmlFast.test.ts b/test/dictionaryXmlFast.test.ts new file mode 100644 index 00000000..5719d434 --- /dev/null +++ b/test/dictionaryXmlFast.test.ts @@ -0,0 +1,680 @@ +// Copyright 2026 The MathWorks, Inc. +// The hand-rolled dictionary reader (src/datamodel/parser/DictionaryXmlFast.ts), rule by rule. +// +// WHY THIS FILE EXISTS WHEN A 469 MB CORPUS DIFFERENTIAL ALREADY PASSES +// +// `.scratch/probe-differential.mjs` reads all 32 binary dictionaries available through both +// readers and compares 18,505,864 keys including insertion order. That is the claim that +// matters — and it is blind in exactly one direction. A corpus proves what the reader DOES +// emit for the shapes the writer HAPPENS to produce, and says nothing about the branches it +// never reaches. No dictionary in that corpus has mixed content, a `>` inside an attribute +// value, a comment, a numeric character reference, or two siblings whose attribute counts +// differ in a way that would expose a stale slot. Those are this file's job. +// +// THE TWO CLAIMS, AND WHY EACH CASE ASSERTS BOTH +// +// The reader's whole contract is "indistinguishable from the general engine, or `null`". So +// every accepted case asserts three things at once (`expectRead`): the reader produced the +// written-down value, the general engine produces the same written-down value, and the key +// INSERTION ORDER matches the literal — which `toEqual` does not check and which +// `BinarySlddParser` depends on, because it walks keys to decide what a property is. +// +// A DELIBERATE DEPARTURE from `xmlReader.test.ts`, whose rule is that expected values are +// literal and never computed from the engine. That rule is kept — every value below is a +// literal, harvested once by `.scratch/probe-fast-expected.mjs` and written down — but this +// file ALSO compares against the live engine, which that file never does. The difference is +// what is under test. There, the engine is the subject and comparing it to itself would prove +// nothing. Here the subject is a SECOND implementation, and "it still matches the engine" is +// not circularity, it is the specification. The literal catches the case both readers change +// together; the engine comparison catches the case only one does. Neither alone is enough. +// +// `readDictionaryXmlGeneric` exists for the engine half. Going through `readDictionaryXml` +// instead would compare the fast reader against itself, since the seam consults it first. +// +// WHAT THE DECLINE CASES ASSERT, which is the part worth reading twice +// +// `null` from this reader is not an error — it means the seam falls back, and the fallback is +// the specification. So `expectDeclined` pins the decline AND pins what the seam still answers, +// taken from the engine. That second assertion is the entire safety argument of the design: a +// decline cannot be wrong, only slower. If someone later teaches the reader to accept one of +// those inputs, the decline assertion goes red and the engine value sitting right beside it +// says what the new code has to produce. + +import { describe, it, expect } from 'vitest'; +import { + readDictionaryXml, + readDictionaryXmlGeneric, +} from '../src/datamodel/parser/XmlReader.js'; +import { readDictionaryXmlFast } from '../src/datamodel/parser/DictionaryXmlFast.js'; + +/** + * Every key in insertion order, as `$.path.key` strings. + * + * `toEqual` compares objects by content and ignores the order their keys were inserted in, so + * without this a reader that emitted attributes before children would pass every assertion + * below. The order is not cosmetic: fast-xml-parser's `compress` fills children, then text, + * then attributes, and this reader has to land in the same sequence. + */ +function keyOrder(value: unknown, path = '$', out: string[] = []): string[] { + if (Array.isArray(value)) { + value.forEach((v, i) => keyOrder(v, `${path}[${i}]`, out)); + } else if (value !== null && typeof value === 'object') { + for (const k of Object.keys(value as Record)) { + out.push(`${path}.${k}`); + keyOrder((value as Record)[k], `${path}.${k}`, out); + } + } + return out; +} + +/** An input the fast reader reads, and the one value both readers must produce for it. */ +function expectRead(xml: string, expected: unknown): void { + const fast = readDictionaryXmlFast(xml); + // Checked first and on its own: a case that silently declines would still pass the two + // comparisons below (through the fallback) while proving nothing about the reader. + expect(fast, `the fast reader DECLINED ${JSON.stringify(xml)}`).not.toBeNull(); + expect(fast, 'fast reader').toEqual(expected); + expect(readDictionaryXmlGeneric(xml), 'general engine').toEqual(expected); + expect(keyOrder(fast), 'fast reader key order').toEqual(keyOrder(expected)); + expect(keyOrder(readDictionaryXmlGeneric(xml)), 'engine key order').toEqual(keyOrder(expected)); +} + +/** + * An input the fast reader refuses, and what the seam answers anyway. + * + * The second assertion is the point. A decline is only harmless because `readDictionaryXml` + * hands the same text to the engine, so each of these pins a behaviour the seam KEEPS — not + * a gap. + */ +function expectDeclined(xml: string, stillReadsAs: unknown): void { + expect(readDictionaryXmlFast(xml), `should have declined ${JSON.stringify(xml)}`).toBeNull(); + expect(readDictionaryXmlGeneric(xml), 'general engine').toEqual(stillReadsAs); + expect(readDictionaryXml(xml), 'the seam, which is what callers get').toEqual(stillReadsAs); +} + +describe('the object shape', () => { + it('puts an attribute under `@_` and text under `#text`', () => { + expectRead('

x

', { P: [{ '#text': 'x', '@_Name': 'v' }] }); + }); + + it('collapses an element with only text to that text', () => { + expectRead('

x

', { P: ['x'] }); + }); + + it('collapses an empty element with no attributes to the empty STRING', () => { + // Not `{}` and not `null`. Half of `BinarySlddParser`'s branches test for this. + expectRead('

', { P: [''] }); + expectRead('

', { P: [''] }); + }); + + it('never collapses an element that has attributes', () => { + // The asymmetry worth stating twice: `

` is `''` but `

` is an object with + // no `#text` key at all. A reader that collapsed both, or neither, would be wrong twice. + expectRead('

', { P: [{ '@_Name': 'x' }] }); + expectRead('

', { P: [{ '@_Name': 'x' }] }); + }); + + it('keeps `#text` beside attributes when both are present', () => { + expectRead('

', { P: [{ '#text': ' ', '@_Name': 'x' }] }); + }); + + it('nests', () => { + expectRead('

7

', { + DataSource: { Object: [{ P: [{ '#text': 7, '@_Name': 'n' }], '@_Class': 'c' }] }, + }); + }); +}); + +describe('array forcing', () => { + it('makes a SINGLE `P`, `Element`, `Object` or `Field` an array', () => { + expectRead('

1

', { DataSource: { P: [1] } }); + expectRead('1', { DataSource: { Element: [1] } }); + expectRead('1', { DataSource: { Object: [1] } }); + expectRead('1', { DataSource: { Field: [1] } }); + }); + + it('does NOT force a name outside that list', () => { + // `DataSource` is the only other name in the vocabulary, so it is the only name available + // to prove the un-forced path is still there. One stays an object... + expectRead('1', { + DataSource: { DataSource: 1 }, + }); + // ...and a repeat becomes an array, which is the behaviour `isArray` exists to remove. + expectRead( + '12', + { DataSource: { DataSource: [1, 2] } }, + ); + }); + + it('collects interleaved repeats under one key, in document order', () => { + expectRead('

1

2

3

', { + DataSource: { P: [1, 3], Element: [2] }, + }); + }); +}); + +describe('key insertion order', () => { + it('is children in document order, then `#text`, then attributes', () => { + // The order `compress` produces, and the reason `closeSlot` writes `#text` before it + // copies the attribute buffers rather than after. + const xml = '

t'; + expectRead(xml, { + DataSource: { P: [''], Element: [''], '#text': 't', '@_A': '1', '@_B': '2' }, + }); + expect(keyOrder(readDictionaryXmlFast(xml))).toEqual([ + '$.DataSource', + '$.DataSource.P', + '$.DataSource.Element', + '$.DataSource.#text', + '$.DataSource.@_A', + '$.DataSource.@_B', + ]); + }); + + it('keeps attributes in DOCUMENT order, not alphabetical and not table order', () => { + // `B` before `A` proves it is not sorted; `Name` and `Class` are in `ATTR_KEYS` and `B`/`A` + // are not, so this also proves the interned-key lookup does not reorder around a miss. + expectRead('', { + DataSource: { '@_B': '1', '@_A': '2', '@_Name': '3', '@_Class': '4' }, + }); + expect(keyOrder(readDictionaryXmlFast(''))).toEqual( + ['$.DataSource', '$.DataSource.@_B', '$.DataSource.@_A', '$.DataSource.@_Name', '$.DataSource.@_Class'], + ); + }); +}); + +describe('the XML declaration', () => { + it('is a `?xml` key at the root, beside the document element', () => { + expectRead('', { + '?xml': { '@_version': '1.0', '@_encoding': 'utf-8' }, + DataSource: '', + }); + }); + + it('collapses to the empty string when it carries no attributes', () => { + // Same collapse rule as an element, which is why it is not special-cased. + expectRead('', { '?xml': '', DataSource: '' }); + }); +}); + +describe('text coercion', () => { + it('coerces a numeric-looking value, in every spelling the engine accepts', () => { + expectRead('

7

', { P: [7] }); + expectRead('

007

', { P: [7] }); // leadingZeros + expectRead('

1.0

', { P: [1] }); // the spelling is gone + expectRead('

0x1F

', { P: [31] }); // hex + expectRead('

1e3

', { P: [1000] }); // eNotation + expectRead('

+7

', { P: [7] }); + expectRead('

-7

', { P: [-7] }); + expectRead('

.5

', { P: [0.5] }); + }); + + it('leaves a value whose first character is not digit-like alone', () => { + // The `charCodeAt` guard in `processText` skips `strnum` entirely for these, which is most + // of a real dictionary — every name, class and enum literal. `e3` is the interesting one: + // it LOOKS like e-notation and is not, and the guard and `strnum` agree that it is not. + expectRead('

e3

', { P: ['e3'] }); + expectRead('

True

', { P: ['True'] }); + }); + + it('coerces `true` and `false` to booleans, lowercase only', () => { + expectRead('

true

', { P: [true] }); + expectRead('

false

', { P: [false] }); + }); + + it('declines an integer that cannot round-trip through a double', () => { + // `strnum` protecting itself, not us. Length is not the rule — the 23-digit value below + // DOES become a number, because it was never going to be exact either way. + expectRead('

9007199254740993

', { P: ['9007199254740993'] }); + expectRead('

12345678901234567890123

', { P: [1.2345678901234568e22] }); + }); + + it('hands back a PADDED value raw and uncoerced', () => { + // The load-bearing half of `trimValues: false`: this is how a MATLAB char array keeps its + // padding. `' 7 '` is three characters and `' true '` is six, not a number and not a + // boolean, and the in-place editor is seeded with exactly what the file said. + expectRead('

7

', { P: [' 7 '] }); + expectRead('

true

', { P: [' true '] }); + }); + + it('leaves interior whitespace alone — trimming is not normalization', () => { + expectRead('

a b

', { P: ['a b'] }); + }); + + it('reads `>` in text, which is legal XML and not a tag', () => { + expectRead('

a>b

', { P: ['a>b'] }); + }); +}); + +describe('attribute values', () => { + it('are never coerced', () => { + // One rule for text, another for attributes (`parseAttributeValue: false`). A walker that + // forgot which side it was on is the bug this pins. + expectRead('

', { P: [{ '@_Name': '7' }] }); + expectRead('

', { P: [{ '@_Name': 'true' }] }); + }); + + it('are never trimmed', () => { + expectRead('

', { P: [{ '@_Name': ' 7 ' }] }); + }); + + it('can be empty', () => { + expectRead('

', { P: [{ '@_Name': '' }] }); + }); + + it('may be single-quoted, and then may contain a double quote', () => { + expectRead("

", { P: [{ '@_Name': 'v' }] }); + expectRead('

', { P: [{ '@_Name': 'a"b' }] }); + expectRead('

', { P: [{ '@_Name': "a'b" }] }); + }); + + it('survives whitespace around the `=` and between the pairs', () => { + expectRead('

', { P: [{ '@_Name': 'v' }] }); + expectRead('

', { P: [{ '@_Name': 'v' }] }); + expectRead('', { P: [{ '@_Name': 'v' }] }); + expectRead('', { P: [{ '@_Name': 'v' }] }); + expectRead('

', { P: [{ '@_Name': 'v' }] }); + }); + + it('lets a repeated name win last, as a plain object assignment does', () => { + expectRead('

', { P: [{ '@_Name': '2' }] }); + }); + + it('reads an attribute that is NOT in the interned key table', () => { + // `ATTR_KEYS` holds the fourteen names the corpus uses, and a miss falls back to `'@_' + + // name`. Not a hypothetical: `test/fixtures/custom_object_binary.sldd`, written by MATLAB, + // carries a `Format` attribute that is outside that table. Bailing on an unknown attribute + // would have stranded that whole file for nothing. + expectRead('

', { P: [{ '@_Format': '3' }] }); + expectRead('

', { P: [{ '@_Unlisted': 'x', '@_Name': 'y' }] }); + }); +}); + +describe('entity decoding', () => { + it('decodes XML’s five, in text and in attribute values alike', () => { + expectRead('

&<>"'

', { P: ['&<>"\''] }); + expectRead('

', { P: [{ '@_Name': '&<>"\'' }] }); + }); + + it('decodes once, so a double-encoded value keeps its inner markup', () => { + expectRead('

&amp;

', { P: ['&'] }); + }); + + it('coerces the DECODED value, not the raw one', () => { + // Also found by mutation testing. These are the only inputs that can tell the two apart: a + // value that enters the coercion branch (first character digit-like, equal to its own trim) + // and ALSO carries an entity. `strnum` hands back its own argument when it cannot parse, so + // coercing the raw text would return the undecoded string and leave `&` in a value — + // visible in the table, and wrong, with every other entity test still green. + expectRead('

7&8

', { P: ['7&8'] }); + expectRead('

1<2

', { P: ['1<2'] }); + expectRead('

-1>0

', { P: ['-1>0'] }); + expectRead('

.5&

', { P: ['.5&'] }); + expectRead('

0"

', { P: ['0"'] }); + expectRead('

7&8

', { P: [' 7&8 '] }); // the padded path decodes too + }); +}); + +describe('numeric character references, which the engine does NOT decode', () => { + // This reader used to decline every `&#`. A MATLAB-written fixture then turned up carrying + // ` ` — `test/fixtures/custom_object_binary.sldd`, where a char property holds a CR — and + // declining would have sent a whole 74 MB file back to the slow engine over one carriage + // return. The corpus census that said numeric references never occur was not wrong about the + // corpus; it was blind, because no dictionary in it held a multi-line value. + // + // `htmlEntities` is off, so `numericAllowed` is off with it, so the engine leaves the reference + // standing as literal text. Reading these as text is therefore not a shortcut — it IS the + // engine's answer, and the pinned values below are what it returns. + + it('leaves a reference standing, exactly as the engine does', () => { + expectRead('

a b

', { P: ['a b'] }); // the real fixture's value + expectRead('

A

', { P: ['A'] }); + expectRead('

A

', { P: ['A'] }); + expectRead('

A

', { P: ['A'] }); // capital X is a reference too + expectRead('

', { P: [' '] }); + expectRead('

x

', { P: [' x'] }); + expectRead('

😀

', { P: ['😀'] }); + expectRead('

', { P: [{ '@_Name': 'a b' }] }); + }); + + it('does not let a left-standing reference reach the number coercion', () => { + // `&` is not digit-like, so the guard skips `strnum` entirely for a leading reference; a + // trailing one reaches it and must come back unchanged rather than as its leading digits. + expectRead('

70

', { P: ['70'] }); + expectRead('

10

', { P: ['10'] }); + expectRead('

7

', { P: [' 7'] }); + expectRead('

00

', { P: ['00'] }); + }); + + it('leaves a MALFORMED reference standing too, which is also what the engine does', () => { + expectRead('

&#;

', { P: ['&#;'] }); + expectRead('

&#x;

', { P: ['&#x;'] }); + expectRead('

&#xZZ;

', { P: ['&#xZZ;'] }); + expectRead('

&#-1;

', { P: ['&#-1;'] }); + expectRead('

�

', { P: ['�'] }); // one past U+10FFFF + expectRead('

&amp;

', { P: ['&amp;'] }); // NOT decoded, so NOT `&` + }); + + it('leaves one standing with no `;` in the engine’s 32-character window', () => { + // The decoder gives up looking for `;` 32 characters past the `&`, so a `;` further off does + // not close a reference at all and the `&` is just text. Written with `repeat` rather than as + // a literal, against this file's habit, because the exact length IS the subject and nobody + // can count 29 zeros in a review. + // + // The two sides of the boundary disagree, which is what makes it worth a test: the codepoint + // is 1 either way, and 1 is prohibited, so the shorter form is a reference and gets deleted + // while the longer one is ordinary text. A reader that scanned for `;` without the limit + // would decline both and hand the engine a document it could have read itself. + const ref = (zeros: number): string => `

&#x${'0'.repeat(zeros)}1;

`; + expectDeclined(ref(29), { P: [''] }); // token `#x0…01` is 32 characters: a reference + expectRead(ref(30), { P: [`&#x${'0'.repeat(30)}1;`] }); // 33 characters: not one + expectRead('

A

', { P: ['A'] }); // and no `;` at all is not one either + }); + + it('declines a codepoint XML 1.0 prohibits, which the engine DELETES', () => { + // The engine drops these from the value no matter what `numericAllowed` says, because XML + // cannot carry them at all. That table is not reimplemented, so the seam answers instead — + // and the engine's answers are pinned here, since "deleted" is a surprising thing for a + // reference to do and a future reader of this file should not have to guess. + expectDeclined('

�

', { P: [''] }); // NUL + expectDeclined('



', { P: [''] }); + expectDeclined('



', { P: [''] }); // BEL + expectDeclined('



', { P: [''] }); // last C0 + expectDeclined('

ab

', { P: ['ab'] }); // deleted from the MIDDLE of a value + expectDeclined('

7

', { P: [7] }); // and then what is left COERCES + expectDeclined('

7

', { P: [7] }); + expectDeclined('

', { P: [{ '@_Name': 'ab' }] }); // attributes too + expectDeclined('

�

', { P: [''] }); // lone surrogate + expectDeclined('

�x

', { P: ['x'] }); + expectDeclined('

zz;

', { P: [''] }); // `parseInt` stops at `z`, so this is U+0001 + expectDeclined('

zz;

', { P: [''] }); + expectDeclined('



', { P: [''] }); // leading zeros do not hide it + // Capital `X`, which until mutation testing asked was the one spelling no test reached with a + // prohibited codepoint: `A` above reads the same whether or not `X` is understood as hex, + // so a reader that only knew lowercase `x` passed every test while mis-reading these. + expectDeclined('



', { P: [''] }); + expectDeclined('

ab

', { P: ['ab'] }); + expectDeclined('



', { P: [''] }); + expectDeclined('

�

', { P: [''] }); + }); + + it('accepts the codepoints next to the prohibited ones', () => { + // The boundaries themselves, because an off-by-one in either direction is the likely bug and + // every one of these would be WRONG to decline. + expectRead('

', { P: [' '] }); // tab, LF and CR are the three C0 exceptions + expectRead('

', { P: [' '] }); + expectRead('

', { P: [' '] }); + expectRead('

', { P: [' '] }); // one past the C0 block + expectRead('

퟿

', { P: ['퟿'] }); // one before the surrogates + expectRead('

', { P: [''] }); // one after them + expectRead('

􏿿

', { P: ['􏿿'] }); // the last codepoint there is + }); + + it('mixes a reference with the entities it DOES decode', () => { + expectRead('

&

', { P: [' &'] }); + expectRead('

&

', { P: ['& '] }); + expectRead('

A<B

', { P: ['A<B'] }); + }); + + it('declines a value where ONE of two references is prohibited', () => { + // The whole value goes to the engine, not the prohibited reference alone — a decline is a + // property of the document, which is the only granularity this reader has. + expectDeclined('

a

', { P: [' a'] }); + expectDeclined('



', { P: [''] }); + }); + + it('still declines the entities it never owned, reference or not', () => { + expectDeclined('

 

', { P: ['  '] }); + expectDeclined('

a&b

', { P: ['a&b '] }); + }); +}); + +describe('whitespace', () => { + it('drops the layout whitespace of a pretty-printed file', () => { + // 204,373 such gaps in the corpus's largest dictionary. The KEY is removed, not emptied. + expectRead('\n

1

\n

2

\n', { + DataSource: { P: [1, 2] }, + }); + }); + + it('keeps whitespace that is the whole value of a LEAF', () => { + // The difference between layout and data is whether anything else is in the element. + expectRead('

', { P: [' '] }); + expectRead('

\n

', { P: ['\n'] }); + }); + + it('drops a whitespace chunk beside a child, on either side of it', () => { + expectRead('

', { P: [{ Element: [''] }] }); + expectRead('

', { P: [{ Element: [''] }] }); + }); + + it('concatenates the real text chunks a child splits', () => { + expectRead('

ab

', { P: [{ Element: [''], '#text': 'ab' }] }); + }); + + it('keeps mixed content’s text and drops only the layout chunk', () => { + // No dictionary in the corpus holds this shape, which is exactly why it is pinned here + // rather than left to be discovered: the trailing `\n` chunk is dropped and the + // `'\n abc\n '` chunk is not. + expectRead('

\n abc\n \n

', { + P: [{ Element: [''], '#text': '\n abc\n ' }], + }); + }); + + it('drops UNICODE whitespace beside a child, which is the only way into that branch', () => { + // Found by mutation testing, not by reading the code. The main loop pre-scans for the four + // ASCII whitespace characters and skips the slice entirely when a non-leaf chunk is all of + // them — so `processText`'s own drop branch is UNREACHABLE for ordinary indentation, and + // deleting it changed nothing that the suite was asserting. These are the inputs that + // reach it: characters `trim()` strips and that pre-scan does not. + expectRead('

\f

', { P: [{ Element: [''] }] }); // form feed + expectRead('

\v

', { P: [{ Element: [''] }] }); // vertical tab + expectRead('

　

', { P: [{ Element: [''] }] }); // ideographic space + expectRead('

 

', { P: [{ Element: [''] }] }); // no-break space + expectRead('

\f\n

', { P: [{ Element: [''] }] }); // mixed with ASCII + expectRead('\f

1

', { DataSource: { P: [1] } }); + }); + + it('keeps unicode whitespace that is the whole value of a leaf', () => { + // The same characters on the other side of the leaf test, so the two branches are pinned + // against each other rather than one at a time. + expectRead('

\f

', { P: ['\f'] }); + expectRead('

　

', { P: ['　'] }); + expectRead('

 

', { P: [' '] }); + }); + + it('allows whitespace around the root and inside a close tag', () => { + expectRead('\n \n ', { DataSource: '' }); + expectRead('

1', { DataSource: { P: [1] } }); + expectRead('

', { P: [''] }); + }); +}); + +describe('the per-depth state is reused, so a stale slot must not leak', () => { + // `readDictionaryXmlFast` indexes seven parallel arrays by DEPTH and writes over them rather + // than allocating a frame per element — 13.8 M allocations saved on the largest dictionary, + // and one new bug class in exchange. Every slot gets a case where the second sibling needs + // LESS than the first, because "forgot to reset" always shows up in that direction. + + it('keeps two sibling subtrees apart (`kids`, `tags`)', () => { + expectRead( + '

1

2

', + { + DataSource: { + Object: [ + { P: [{ '#text': 1, '@_Name': 'p1' }], '@_Class': 'a' }, + { P: [{ '#text': 2, '@_Name': 'p2' }], '@_Class': 'b' }, + ], + }, + }, + ); + }); + + it('lets a different tag name occupy a depth the previous subtree used (`tags`)', () => { + expectRead( + '

1

2

', + { DataSource: { Object: [{ Object: [{ P: [1] }] }, { P: [2] }] } }, + ); + }); + + it('forgets the extra attributes of the previous sibling (`attrCount`)', () => { + expectRead('

', { + DataSource: { P: [{ '@_A': '1', '@_B': '2', '@_C': '3' }, { '@_A': '4' }] }, + }); + }); + + it('collapses a sibling with no attributes after one with three (`attrCount`)', () => { + expectRead('

', { + DataSource: { P: [{ '@_A': '1', '@_B': '2', '@_C': '3' }, ''] }, + }); + }); + + it('forgets the previous sibling’s text (`textVal`, `textCount`)', () => { + expectRead('

1

', { DataSource: { P: [1, ''] } }); + }); + + it('forgets the previous sibling’s children (`kidCount`)', () => { + expectRead('

', { + DataSource: { Object: [{ P: [''] }, ''] }, + }); + }); + + it('lets text follow a subtree at the same depth (`kidCount` decides the collapse)', () => { + // The second `Object` is a leaf whose text must coerce; if `kidCount` had carried over from + // the first it would have been treated as a parent and the `2` would be a `#text` key. + expectRead('

1

2', { + DataSource: { Object: [{ P: [1] }, 2] }, + }); + }); +}); + +describe('what it declines, and what the seam still answers', () => { + // Each of these pins TWO facts: the fast reader returns `null`, and `readDictionaryXml` + // produces the value it produced before the fast reader existed. The second is the whole + // safety argument — a decline is never wrong, only slower. + + describe('markup this reader deliberately does not own', () => { + it('declines a comment', () => { + expectDeclined('

1

', { DataSource: { P: [1] } }); + }); + + it('declines CDATA, which the engine reads as ordinary text', () => { + expectDeclined('

]]>

', { P: [''] }); + }); + + it('declines a DOCTYPE, and with it every entity-expansion question', () => { + expectDeclined('', { DataSource: '' }); + }); + + it('declines a processing instruction that is not the declaration', () => { + expectDeclined('', { '?pi': '', DataSource: '' }); + }); + + it('declines a declaration anywhere but first', () => { + expectDeclined('', { + DataSource: '', + '?xml': { '@_version': '1.0' }, + }); + expectDeclined('', { + DataSource: { '?xml': { '@_version': '1.0' } }, + }); + }); + }); + + describe('entities it will not interpret', () => { + // Numeric character references used to be here, and are not any more: the engine leaves them + // standing, so reading them as text is its answer rather than a liberty. Their own describe + // block above covers them, including the prohibited codepoints that still decline. + + it('declines an unknown named entity', () => { + expectDeclined('

 

', { P: [' '] }); + }); + + it('declines a bare ampersand', () => { + expectDeclined('

a & b

', { P: ['a & b'] }); + }); + }); + + describe('names outside the vocabulary', () => { + it('declines an unknown element name', () => { + expectDeclined('', { Q: '' }); + }); + + it('declines a namespaced name', () => { + expectDeclined('', { 'x:P': '' }); + }); + + it('declines a name that differs only in case', () => { + // `internTag` compares bytes, so `p` is not `P`. XML agrees; this only pins that the + // interning is not accidentally case-insensitive. + expectDeclined('

', { p: '' }); + }); + + it('declines an empty or whitespace-led name', () => { + expectDeclined('', {}); + expectDeclined('< P/>', { '': '' }); + }); + }); + + describe('damage, every judgement about which belongs to the engine', () => { + it('declines a mismatched close tag, which the engine ACCEPTS', () => { + expectDeclined('

1', { DataSource: { P: [1] } }); + }); + + it('declines a close tag carrying trailing junk', () => { + expectDeclined('

1

', { DataSource: { P: [1] } }); + }); + + it('declines an unclosed element', () => { + expectDeclined('

1', { DataSource: { P: [1] } }); + }); + + it('declines a close tag with nothing open', () => { + expectDeclined('

', {}); + }); + + it('declines a second root element', () => { + expectDeclined('

', { P: ['', ''] }); + }); + + it('declines non-whitespace text outside the root', () => { + expectDeclined('x', { DataSource: '' }); + }); + + it('declines input with no markup — the shape a truncated write takes', () => { + // The engine's empty object here is load-bearing: every caller reports it as a loss. + // This reader must not invent that answer, because `null` from it means "not read". + expectDeclined('hello', {}); + expectDeclined('', {}); + }); + }); + + describe('attribute syntax it will not guess at', () => { + it('declines an attribute with no value', () => { + // fast-xml-parser's own regex tolerates it and drops it. Imitating that would mean + // owning a judgement about damaged input, which is the thing this reader defers. + expectDeclined('

', { P: [''] }); + }); + + it('declines an unquoted value', () => { + expectDeclined('

', { P: [''] }); + }); + + it('declines a `>` inside an attribute value', () => { + // A known, accepted limitation rather than a bug: finding the tag's end needs the + // attributes parsed and parsing them needs the tag's end. No dictionary attribute + // (`Class`, `Name`, `Dimension`, `Source`, ...) has ever held one, and the cost of + // being wrong is a fallback. + expectDeclined('

', { P: [{ '@_Name': 'a>b' }] }); + }); + }); + + it('declines an unterminated tag, where the engine THROWS', () => { + // The one decline whose fallback does not return a value. Pinned because the seam's + // behaviour here is "throws", and a caller that treated the fast reader's `null` as + // "empty document" would turn a thrown error into silent data loss. + expect(readDictionaryXmlFast('

readDictionaryXmlGeneric('

readDictionaryXml('

); + const kb = Object.keys(b as Record); + if (ka.length !== kb.length) return `${path}: keys [${ka}] vs [${kb}]`; + for (let i = 0; i < ka.length; i++) { + // Positional, not by lookup: this is where a reader that emitted the right keys in the + // wrong sequence gets caught. + if (ka[i] !== kb[i]) return `${path}: key ${i} is '${ka[i]}' vs '${kb[i]}'`; + } + for (const k of ka) { + const d = diff( + (a as Record)[k], + (b as Record)[k], + `${path}.${k}`, + ); + if (d !== null) return d; + } + return null; +} + +/** + * The property, for one document. Returns whether the reader took it, so a caller can tell a + * generator that produces nothing but declines from one that is actually exercising the reader. + */ +function assertEquivalent(xml: string, label: string): boolean { + const fast = readDictionaryXmlFast(xml); + if (fast === null) return false; + let generic: unknown; + try { + generic = readDictionaryXmlGeneric(xml); + } catch (e) { + // Accepted where the engine throws: the seam would stop throwing, which is a behaviour + // change and not a speedup. + expect.fail( + `${label}: the fast reader ACCEPTED a document the engine throws on ` + + `(${(e as Error).message})\n xml: ${JSON.stringify(xml)}`, + ); + } + const d = diff(fast, generic); + if (d !== null) { + expect.fail(`${label}: diverged at ${d}\n xml: ${JSON.stringify(xml)}`); + } + return true; +} + +// --------------------------------------------------------------------------------------- +// Part 1 — every binary `.sldd` fixture in the repo +// --------------------------------------------------------------------------------------- + +const TEST_DIR = fileURLToPath(new URL('.', import.meta.url)); + +/** Every `.sldd` under `test/`, recursively. */ +function findSldd(dir: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) findSldd(full, out); + else if (entry.endsWith('.sldd')) out.push(full); + } + return out; +} + +/** The XML part of a compressed-binary dictionary, or `null` if the file is the JSON format. */ +function chunkXml(path: string): string | null { + const bytes = readFileSync(path); + // A text `.sldd` is JSON and holds no zip; the local-file-header magic is the cheap test. + if (bytes.length < 4 || bytes[0] !== 0x50 || bytes[1] !== 0x4b) return null; + const part = unzipSync(new Uint8Array(bytes))['data/chunk0.xml']; + return part === undefined ? null : new TextDecoder().decode(part); +} + +describe('every binary dictionary fixture reads identically through both readers', () => { + const binary = findSldd(TEST_DIR) + .map((p) => [p, chunkXml(p)] as const) + .filter((e): e is readonly [string, string] => e[1] !== null); + + // A guard on the guard: if the fixtures move or the zip detection breaks, this suite would + // pass by testing nothing. The number only has to be a floor. + it('finds the fixtures at all', () => { + expect(binary.length).toBeGreaterThanOrEqual(15); + }); + + for (const [path, xml] of binary) { + const name = path.slice(TEST_DIR.length); + it(`${name} (${(xml.length / 1024).toFixed(0)} KB of XML)`, () => { + // Every fixture in the repo is a document MATLAB wrote, so the reader is expected to + // take all of them — a decline here is not a correctness failure but it is a silent + // loss of the whole point, so it fails. + expect(assertEquivalent(xml, name), `${name} was DECLINED by the fast reader`).toBe(true); + }); + } +}); + +// --------------------------------------------------------------------------------------- +// Part 2 — generated documents +// --------------------------------------------------------------------------------------- + +/** mulberry32: four lines, deterministic, and nobody has to install anything. */ +function rng(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const TAGS = ['P', 'Element', 'Object', 'Field', 'DataSource']; + +/** The real vocabulary, plus names outside it so the bail path is generated too. */ +const ATTR_NAMES = [ + 'Class', + 'Name', + 'Dimension', + 'Source', + 'PropertyType', + 'EnumerationName', + 'EnumerationType', + 'IsEnum', + 'IsComplex', + 'Encoding', + 'EncodedLength', + 'FormatVersion', + 'MinRelease', + 'Arch', + 'Unlisted', + 'x:ns', +]; + +/** + * Text chunks chosen so each decides something: the coercion spellings, the padding rule, the + * entity table and its edges, the layout-whitespace hook, and the values that only LOOK + * coercible. + */ +const TEXTS = [ + '', + ' ', + '\n ', + // Whitespace `trim()` strips but the reader's four-character ASCII pre-scan does not — the + // only route into `processText`'s own drop branch. Mutation testing found that branch + // unreachable without these. + '\f', + '\v', + '　', + ' ', + '7', + '007', + '-0', + '+0', + '0.0', + '1.0', + '0x1F', + '1e3', + '.5', + '9007199254740993', + '12345678901234567890123', + 'true', + 'false', + 'True', + 'e3', + 'abc', + 'a b', + ' 7 ', + ' true ', + 'a>b', + '&', + '<x>', + '"'', + '&amp;', + // Digit-leading AND entity-bearing: the only shape that reaches the coercion branch with + // something to decode, so the only shape that can tell `toNumber(decoded)` from + // `toNumber(raw)`. Mutation testing found that pair indistinguishable without these. + '7&8', + '1<2', + '-1>0', + '.5&', + // Numeric character references, which the engine leaves standing — so the reader reads them as + // text and these are not bails. The prohibited codepoints next to them ARE bails, and both + // kinds belong here: the generator is the only check that the two are told apart in + // combinations nobody wrote down, including a value carrying one of each. + 'A', + ' ', + 'a b', + '70', + 'A', + '&#;', + '&#xZZ;', + '�', + ' ', + ' ', + '퟿', + '', + '', // deleted by the engine, so a bail + '', + '', // capital X, the spelling mutation testing caught no test reaching + '�', + ' a', + ' ', + 'a & b', + 'Simulink.Parameter', + '[1 2]', +]; + +const ATTR_VALUES = [ + '', + 'v', + '7', + ' 7 ', + 'true', + 'Simulink.Signal', + '&', + 'A', + 'a b', + '', // deleted inside an attribute value too + "a'b", + 'a"b', + 'a>b', + 'a\nb', +]; + +/** One generated document. */ +function generate(rand: () => number): string { + const pick = (xs: readonly T[]): T => xs[Math.floor(rand() * xs.length)]; + + const attrs = (): string => { + let s = ''; + for (let n = Math.floor(rand() * 4); n > 0; n--) { + const value = pick(ATTR_VALUES); + // Quote style has to follow the content, or the generator produces nothing but + // malformations past the first `"` and stops exercising anything else. + const q = value.includes('"') ? "'" : rand() < 0.15 ? "'" : '"'; + const sep = rand() < 0.1 ? ' = ' : '='; + s += `${pick([' ', ' ', '\t', '\n '])}${pick(ATTR_NAMES)}${sep}${q}${value}${q}`; + } + return s; + }; + + const element = (depth: number): string => { + const tag = pick(TAGS); + const a = attrs(); + if (depth >= 4 || rand() < 0.25) { + return rand() < 0.35 ? `<${tag}${a}${rand() < 0.5 ? ' ' : ''}/>` : `<${tag}${a}>`; + } + let body = ''; + const kids = Math.floor(rand() * 4); + if (kids === 0) { + body = pick(TEXTS); + } else { + // Mixed content on purpose: no real dictionary has it, so only a generator will. + const indent = rand() < 0.5 ? `\n${' '.repeat(depth + 1)}` : ''; + if (rand() < 0.2) body += pick(TEXTS); + for (let k = 0; k < kids; k++) body += indent + element(depth + 1); + if (rand() < 0.2) body += pick(TEXTS); + if (indent !== '') body += `\n${' '.repeat(depth)}`; + } + const close = rand() < 0.1 ? `` : ``; + return `<${tag}${a}>${body}${close}`; + }; + + let doc = ''; + if (rand() < 0.3) doc += ``; + // Hostile injections, each one a documented bail. They are generated rather than listed so + // they land in positions a list would not think to put them in. + if (rand() < 0.08) doc += pick(['', '', '']); + doc += element(0); + if (rand() < 0.05) doc += pick(['

', 'trailing', '

', '']); + return doc; +} + +describe('generated documents are read identically or declined', () => { + // Three seeds rather than one: a single stream can get unlucky about which branches it + // reaches, and three is still instant. + for (const seed of [1, 20260930, 0x5eed]) { + it(`seed ${seed}: 2000 documents`, () => { + const rand = rng(seed); + let accepted = 0; + for (let i = 0; i < 2000; i++) { + if (assertEquivalent(generate(rand), `seed ${seed} case ${i}`)) accepted++; + } + // The generator has to actually reach the reader. Without this the suite would pass + // gloriously on a generator that emitted nothing but DOCTYPEs. + expect(accepted, 'documents the fast reader accepted').toBeGreaterThan(400); + }); + } + + it('also reads a document shaped like a real dictionary entry', () => { + // The generator above is adversarial and produces shapes MATLAB never writes. This one is + // the shape it always writes, so the common path is covered by something other than luck — + // and it must be ACCEPTED, which is why its values are drawn from a list with none of + // `TEXTS`'s deliberate bails in it. + const rand = rng(99); + const pick = (xs: readonly T[]): T => xs[Math.floor(rand() * xs.length)]; + const values = ['', '0', '7', '1.5', '-1', 'true', 'false', 'double', 'auto', 'inherit', ' pad ']; + let xml = '\n'; + for (let e = 0; e < 300; e++) { + xml += `\n `; + xml += `\n `; + for (const prop of ['Value', 'Min', 'Max', 'DataType', 'Description', 'Unit']) { + xml += `\n

${pick(values)}

`; + } + xml += '\n \n '; + } + xml += '\n'; + expect(assertEquivalent(xml, 'dictionary-shaped'), 'was declined').toBe(true); + }); +}); diff --git a/test/fixtures/bus_types_binary.sldd b/test/fixtures/bus_types_binary.sldd new file mode 100644 index 0000000000000000000000000000000000000000..6a3b95e1afa36799123af7e65aacfb4febd3916b GIT binary patch literal 3710 zcmaJ^2{e>#8y-Ukt621pxp!XF5G)&S9w7 z$rd$Z-J~_}`a``mHkNWKFvNnm8E%lAyGzOboJm?-Qa!h$VWb20<*6)@ zjg5iXO9=U-P0j$+`xfUoiJE$^P^sOiMd!iy^!7rah4~&5trP|b>$>c_=CTJm=i4%; zebN*QUMQ!{ak6|@%a}Gcy6i2GEV(3maC>Fd-v40G9hP@13fjw8*ITs-OPNlwf^QFB z6T37FZ!XIW4GNl;@3-WCm+81i9&J-r4w>u}ac5r1H433TPM>Lh+c&TB0EC%hi0Jwn z-Cd32V#`i{rQmXzqlNj=S)oEV*{ZRxX_8Bgu{>Xv%J6PwWgO@(KvBJW(R*KFy?~L( z5w+W1pR*&2TnnhhgVGaeaTGaHcG`6e^w{ZW2~K_8JyBG)=6T?J`fxWLGe_ZyzUgE$eHvAkD~}7xFw;oa~gpN4AF{U6w);*0AoP*kJ&c&||+B zU8cRKx-qqZ>B;uZ??N}0NMIY2C+jcH>RKm!T?P8bZsviLEr_!}Uf}OKyw}_j^fL?| zRCsLX>CWHZ^Kh;?lMH>{KZYhDB3jzF#?++oldx&MmyrxL7(z=MONPG^x|8{5zmTB6sG0#=)(1(1a1+I8j^2);Z};!uBN^jVeGz5S3=v| zjJ&T_%l2w*Op9p9BIOJwAAI(%5eam7zwL|{Z^Qb1xl82<*J8#l@}wXY`O!)X;*A1( zfioi5rr3R@1P3e1YNLa4Hl1ma2 zR?m8j9XTNo?aeTLK`PS3xPJ1<_c87y?1a**4+By;57d?0j1fHPJ~+)>9T0C`GDK25 zfB(2Xps?`fBLYQG5xC3}K*-TCH9`6-sgt@xbriq#b(scQwzrRGi*RjL=Aj^_R=X^w zmI=^p5y1T|@}Qhv$*seYKo?j$5-Re8Nyb+5Ey~Oo+SIyuOHLmT=UnfoUXs53N)Tg1 z?aPY87+UX{s*kQ}&sF_E@epHjypdSUS)a=)EIxmAp$|JJGCWtD?#V3$5;Hp>fr&Mi z6-6j&TdM|1csO#j;o~OM&(YU8Urxn7_XM|XIs&Rh#R~iuQoyDEOGT8ef^Yl~z z{1GU$lJYuE)~VWR(fPAz@XH?zCcn8S<%uCHgaH8XVgvw=(cIHU+XL^7!+YDB2i(MY zUH$2x^_O28bt!;acTUOKnyz**h^N-Y0J)DpGT9AFtG!V{c6;q>8+vaf-R!xP`OPZ> zs~Q#)t~K@1m5MrRk0|d7=h^j|kRvseb!zEsQf+3KfI`w=6!nB0R8Ns>JC)CR`k@t7 zSt?W>XMP$J+Zj-tAa~`j^iNCn8rA%R_+I6s=!#d@?2)3U&3wn*v(%ZyiCv){he1+2uMww#w(G= zD;tehTTh(3mpt_6lXxG6k5B+1cXB#6HTj$|^*qw?WocwIIxsBdLD->&lr`AQw$fEt zfmW;oTwvf^F2c7RUX~P`?=$RdG@@F-RIiXWs#Ll@nei4OMT&Tme^vl4JY-rUQ@GEe zzG2s^RO2UUQa4xkxjodO|1LAnQ}UNb+D=XtKv8|0S`i&Kr5ZWCPnig(AqfeGa4B5( z@? zT>9hBxX1M_6vW7L`G9(6%Q+V7t`*u;csEDEzS+T~NG$K=K!KWpqjRoRiZ+8;Aw^E$ zcWmG;q3|xV^?oG+XDz{LgMVOB3wB(7yHJHBy^q!!_aaG$mc?q)DKtN3%K9>m$yy)) zAV_2K2F}~=-x=@Wi8Jx^xQX-hcENf5R4S2h-Yr4_{Br#OEbfP@C}KBdvsrS9Ixj5P z67|(p8^x}tw$f-@H@uh7&>dyCRac1&~xuzr>XO-IB+# zj{6<5VqW^kb;7YyUI|}1&lw{%WDPWBw%&|JH!G_NryHwqIH)?}0h9h!$J&uvOg(m|@P0;11CB)SQ&sO|Z4> z7FhIBvt{Dqa5f)Oz3ak_2q~R6KKQ61FL=LE4D+xoH+vgd%=9{Oh@|CuZ8hrRcdYZN z`D)(fFpKXsZ}c}bogNCZH%hg2Kl2!A||MJW$JF9^bb) zF8{l#vDYdx-_d5^NSlZ7KWA~n5Bp!!@EJPqO7o3FLv6<5byW-gBW4b`V>NQ(C{Bg6 z7%M9Q)0puREz0Ew`Ym#zI^K z#Oo5xllxrsoW-Qp1lcVDOV2vlhWO$I)omS^9tI*l)aq-04yJ2AzDkrGr zNNi{%dhW?v29frAAG!Djs@L}{hg^xuO8$EZF1P8_iCTUUT1I1Hnzp3>#K=W|nyq@d zc;J5u^=gtjcc&N_@GU32020f95(-Zn;|&BvW(=N8W|&>vzOAjqnA> zTQ_yEw6gks35hp;iFmB2Zyi)#^;|nXFIi!t>_T%3E7ws)LlBjPwM|>I-oH0s!X&F7 zU{pH1q%o!4T0G383%LPpKHPjo!6@|L{fUSlSFWSQBo_nf;;J$X$run+)@U!s+21J%0A#o!VCOrVi2(!f^s)bc=%8`&kBjaILSIau zxxXqkY3G>#k-+o-^x^UgAf5TQsG%>U_tIa59}fS~S?Pi3jo=rMJL^Ay{+BJ#L(%V` xUr@J?`~m8>{X~yRuhUQU!w2k&YjCW^F78K(cQ@`q(A17eC!R}Q{%~ty`7vn93Ge}-COczm6qEs<}5{86)1Nv zx#}!*?#mHuTl$cLUXk@PN|f1UizO%LQq0d3eUGghbX)bZry(&7`3ZIEtU5S_*i?-)rQFvsVI+4` zfZZ^^J9k%NwWGn+uuEzTzJAZ9GT1U@_nd$rn3w&b zpqAuga3)(0psiF!t}WxcZSvz0?$I;_S?{1{Nc%JL(^Kip;pWFuN7@NU#8&TAR1OlUP#k`=k!uKx@4E0EVhJ8-MPr1#Y z*s|YsBrr`!F5Hd^V!>NMXs#gMhOv1C*uEHWoF5D*TU~m+jJeQte;%S3O_tA*P^y*T zy1_@vsmGqA3VrrWG2$iWTTsV?bZ<&U+R+`a_iv}UFGyq<#?TnDAx6ci)Y^=vKdBv~ zJH%}?C1+yh-ZUUOvJb5itF)dfMWF-_Vosy8L(>MPDPM!=<=Wd)+sTnXNK zs)(oqntBqEi`LFKK(C5ce{s(m&6}zv5Q~Y=u;PSOsH)PK};UWf%75RN(Xf- z7NBL%&(P`nTp&Wytvhx%@FG_8n_Z5g7MvH&_N(q2-bqOAbnC}fL^ys+FPiZT$=q>Q zZD`Io)QtVAuZ4YH!AWMHjF$!<);*!|H>iTAY(AfOK$tKb%$+5yj<(TVrIONIgl;vW zAj9GiXc>Wxe0HyNl>U<1qR#z2`KJ5{!oN#4W)q7sjOBUTA~btCnn${|cdP9y_*J;K z!~SM@Wt|fK@IyOI5URTDkyk`wKa5~#rdiMeFOj%*&&R5>4%@2XYLL3eX|A*N3{Q6G zG^Qk$zceQjr1h8SxtB7#%8b$;Q`+1yVcXBYbXCCkBYLs3#2SGd14+%}uk~Iz^6h7c zxw9M^XEM%Hncr?yA+jS8it;kwtZ$n^&3aGkF3*j1`WmzE@u1b3joC#QUr!OO)_Gm- z1TvQ8OuDiGgI+`=PgWU)n~ph_A!-p zvY6@>HLUGBXRZC3hR@j?ddVD{0?=tP82PN1ruEq zesfI~)Ak}w-H@P9D?$ElxKWS8rj_%Go~iW_JCTjRu1b7UQ?QqNYq<@%W*{t|dguM% zgPh%s1z>Tr-OyrR)9{TA0*6n|U9S*mBIEISm~Uv8LY7SLF&`qGB&~~yN3@8|F7D-x zwd~K7!+m#XWL?{0U3l2tmzRKOuieedlllhN7T0CT?#S;@6-B)tnHu{uWlMbpsS`bv zJ$>(a+NkabAUl0J0)v}R5vlkMvUwM*D5K#{l)(HFJJ8)?{2+wB_)@XMPM*PN@xXN1 z#I^ri=4tzO`WEN$%bXj#c!UDq4I6lz<|4x|b@j}bYF5XV+J6}L(mQn$E<6ChjQ{|k z#u>M%f(z0eigdTs^L`3-Gyi4YwNLsqKM8}HkNNm5bvD}Z*b-~P@TqRT)jsi0s&+ zidFHeY^BP=2HDx|0y&_aB?4uM@dA?!vc)uBR(aTU@dyG*NGP-4Vrb2hmD9S7HP}se zQ@Ug-HjQ&c-9SZz)+(%+QWe`qg;ja@CQOLZ1Jw1sjBL5aBF_}<-WUQovxaaC5Kr@i zeB^va=MnEkOuwYR%7=j`NWq_O1%A@q?iY)vsE&v2&|{|LAvcAN@+1a0&-LZ8ZUdZt zrQvdTe=?aH_SN_aj>&3#0Duw4q%+jr`hR{eTwI~rt}aiZuI_NC+pkK^ihtl3Bngj7=dw5DFSV6{+=l%bM}z+Twteg`Y3v_v2FpwW^fltmEas4NSCE@nx?YyjRsy816+H$x>9dz8+Qj+0SljAuY8V$-_AVkFG zW^(Az#qsfltV3kbAA%)n2ySmGcs=Iav+I0R>&#fU+kUdIyx_QB$7DhlytW-lFaRSi z%$NN}lo~|kt?Vi{#c>j4Y1qlI>?UPNNJ-|PH+U}sP9^)QctPu8MMj7*p`O*Gs5Cp{ zNU4CZFM4o59$~Q&qIhZRz_(CIyXJ3jSv91(BL^#D1lDu3bhf!*&{IEjCP}z08`pns zKecf7wEaJ7cnm&3;C$oSV2c)8 z&HF{K5nVedbrnCG28D1^n2`|!Np$>Z{d;`#&jCpJeU_Y|shqh^n!8PpgTB^efgg*b zPy<2dQk#}D1#|e2z?k45v)IDPbfvJ2XnG1s&i+oe*Ec9BTzNTK_&T3=0KO7&j|eQK znKA~cTEXt+uQh4BAAbdBV5^DNi|vK0IIwbT^Cvc{o5Nq`PKLbpgw)>#PfEU!4`_Bc zH1hj?gk_8NJ@NNn+NS_D-3a#!)}MbjhR56*vqDLSsoXv_97JS^i+PnYjt>IZ4-gI>W<6hvP`n2_=M%_3AL>IE;!KxgnrGaG-D3>$UxX55h{BlRs^ zGgg}G^lb`ko_2yztZ?!saAu)5Lv+?wYPnbG3$5)Jd;G3-r3y>GV{VY#%>re*_4P5b z7OKn`D`j3V9j}3-N)m(QVy!tnk14ja*4`2fKk^&Rzxk*9>?8{PA8|=a;^gQ4w`6sp zPEcz%s0z~FeVNme)d1TTHI4N&@qR+m)%}Zttf-=xT9Bs7xtihnyZrvGX zqd#)7y7ePLEzcC^p?CYsc239rs)nLV>o#^$KOR{THbPVoY46hTn#m!ujvB3RnQ3B9 zKN-p&MdP#^+RF;E5o%lWc9Lv*v5FT?*&|Hu^bDj(l2MtK%d#co(-W^ylbbUQmrPer z$$Ie!2|>l@ww-T2k?m!A-Yz+r+Td~TdjYx~EqZLCc>-DDx8Hv>D0D!0aePUtt%ip$ zK>h!ZJ2)=>aaAN=rmUYuUhHvKvUcp_b quot \" apos ' end", + "Lf": "a\nb", + "Matrix": { + "_type": "double", + "_value": "Matrix(2,3)\n[[1.0, 2.0, 3.0]; [4.0, 5.0, 6.0]]" + }, + "Mode": { + "_enum_value": { + "_type": "int32", + "_value": "-1" + }, + "_id": "2", + "_object_class": "GearMode" + }, + "Nested": { + "_array_type": "Struct", + "_dimensions": [ + 1, + 1 + ], + "_elements": [ + { + "inner": { + "_array_type": "Struct", + "_dimensions": [ + 1, + 1 + ], + "_elements": [ + { + "leaf": 7 + } + ] + } + } + ] + }, + "Nothing": [], + "Padded": " pad me ", + "Part": { + "_id": "3", + "_object_class": "WidgetPart", + "_properties": { + "Label": "lead", + "PartId": 1 + } + }, + "Parts": { + "_array_class": "WidgetPart", + "_dimensions": [ + 1, + 3 + ], + "_elements": [ + { + "_id": "4", + "_properties": { + "Label": "part1", + "PartId": 10 + } + }, + { + "_id": "5", + "_properties": { + "Label": "part2", + "PartId": 20 + } + }, + { + "_id": "6", + "_properties": { + "Label": "part3", + "PartId": 30 + } + } + ] + }, + "Scalar": 1.5, + "Str": [ + "a string" + ], + "Tab": "a\tb", + "Tuning": { + "_id": "7", + "_object_class": "Simulink.Parameter", + "_properties": { + "CoderInfo": { + "_id": "8", + "_object_class": "Simulink.CoderInfo", + "_properties": { + "CSCPackageName": "Simulink", + "CustomAttributes": { + "_id": "9", + "_object_class": "SimulinkCSC.AttribClass_Simulink_Default", + "_properties": {} + }, + "CustomStorageClass": "Default", + "ParameterOrSignal": "Parameter", + "StorageClass": "Auto" + } + }, + "Complexity": "real", + "DataType": "single", + "Description": "nested MathWorks object", + "Dimensions": [ + 1, + 1 + ], + "Value": 2.5 + } + }, + "ZeroStruct": { + "_type": "struct", + "_value": "[]" + } + } + } + ], + "_mw_element_type": "MATLABArray" + } + } + ], + "Dictionary References": [], + "AllowAccessBWS": false + } + } + } +} \ No newline at end of file diff --git a/test/fixtures/derived_class_binary.sldd b/test/fixtures/derived_class_binary.sldd new file mode 100644 index 0000000000000000000000000000000000000000..aef2b9a06b3dd8e39b66e03fefa8591798f5e887 GIT binary patch literal 3617 zcmaJ^2{e@N9v(A;#*(onBxET|MvWMZAzNdop|Xu_>{|v2Wf}XvFG+TlC5$a8YobD9 zr->v~_I(L;r*qG_{d4c_`_B8F^L^+1p7XuW?|Gl+ef70KU^V~%aEvne0sI~F@ikxo zV2mFCV4{4nweqr(v~loubB3Mwb#+-XG1dDl%XT*x%RGCts1}_9kHc_Eg=STNRWdk? z_u+CkKa9krV0_wVKUNO8w-z!p&&Lt z!OYy(JAmSLT(YgqxKSEWI=-$L>`vlc-i z&HTB6NiRoNM-FKVuRJSi-OE~(%_+kUHnqrgoIDr zY}?-7Pf8ho5*#Hl;}|WiuPpWbyHYiB)X45q<~_zz=j|JcGPo!0Ih+NgEB9ls<{c^!6eTFf=89+IJ5Bd@{xU z*j#5G^DWpJJz4)Mljnz24zWzu5>eYn+mIwf+&Qo&j(n(jcJ zL1)b=Iz1e*!d~%mpT}3()3OODznK2sXlb8Xs32&k?~Hb5F+$bDmF}?2f;D;*No9UM zul>GfQB0vvD`NL9J5df;(5l*-uxkX}M`N9R*7X=%;GxwwA=RWOYs&6UK>Is&{raa%!Q>0yV|NMMYR=hYMRz zd$_>^D5^xzcX5d7>ca!7fR+}ki`_}u1&M7um87hv<}VG#Vg(zwQ8xV~J8%tLtAQ%O z=zfLe1|P)cl0j;CbHeWIj3>ia1Mrlw;Kef!A_xIzxwnbP8*M=!EpA>c9sb(JY}eo- za36fnid-R6D?VrN^_p27!5%vJm9swPto6&dGyRyiQetLkVvL9Ph+tXRM{@4ZNzSWT zAa_MRo*w2iZ;PUE)}Z8u5zqkf!o0s|K7Vp__iA}^O^HTIo@IKe_tG4Vg(ock#k{3% zcHUbL?Tj>`#hvODK+0F1%U5}@Tz#c1lRgbiwyL!wYe-6*5h5^rl|l z;`FVK?d4O{Ze2eO4{$5TO3?Ry=cs4*52A_A&${etO)r*?1PtYlPd9BfZkFYjh8wTf zAE37uO1@x+Fn$)i^YYZ zTc$*q?qyr#F7(sOlo1Y>E5~WyT~dDYLoj(qV2@p*+L|ZYdb+}unIP0rCEbb9%GEGt zaQ(837R@bMZ_=FIV$zdCG_A&0Ha}OG@gB&EvCv-YpWZW*A}j2TglO^xoSxil9OXX? z5vdv#cpVT7ojZ{CIJnl$I|!`LTilLdY}yYTko|c={||qgd!h{yp#lK#AOL`s;%}xZ z?rvT$U)uD;dnfI*+{h>2qj7F((KQ)?rDEGHi6?**mRxR#GOzxBZe zCJd$%uFfRF@vjjTX!>pc~Z(iE}F(52xd6GK08a@l|v3vG;Xe z6*@I9jXZA2#pF18 zb2bBVTK`ZK4NIhY*#nT3jO(?b!_{wJ2))kD_9R9NjCx~G7}H5c^ylh6hRc3b*+xa3 z;5)qIto9-gb}J6A_2R#b18kc1>=yhKKI(`c8LYRJMz`Q=}iq$y% zC$9z~P&s}IuQ?Q6=_$NoJ#j90$@9OSIVn1B1Q=9(=YIRTBBz5%9lLN$Y1){PkzY_! zRuFZ4(h~gBwzN4s9~Ehf;OgCffb?lbl*R|-c@H?~4$9_(>tJa^(hpb28N*1SKEji{ zi(H6PBm?5Pf&)hRRjY1k;w?e_+RwFX&4IQ(Q8esN$2K3S*xQ!_Pivai@T$^F6UEWX znMiw*fPig?5U%sFL|JOG#26RyA-k_-!T7?ZC=dc3&Ko!zUOQ*$vS4k+hBsW2e@sry z5+2kxMoV#6Mm%KJ9B*M6M_{Kab`tIJHLVLDRdbSRpkPgF@9_eJJnq>Rclf7gV{DkCoflQz9fe~sP-zOYc8mUP2 zCzE**A8uTwFj)fx0Jteky5hX7{++_^o;ZC^cMqJWmm?1UOQ|`@%FYBBTi41V8~-hx z@#~-y&9Y3t({ix#%B8z6w`kpx7bYNxzwr~7vn2PRd;r(&eOitNkk6RdXHDI zCSN3#=~i4)&tCE4NV|hpQ9iup3DpoAA$;s+yON%|f~c0_xvl;oLf#D6DKYQ9kO60s zm{`~XT%r7Fs7x*B%#w=l72$Q;PSrYB?)tUXy$y{S=Z*SPCXAu;E771n`(wq0$WJtx z1eV(xo{D6ly;$tEPOdL_IV_l&(E&q}jdEl`-g!8qclcF7k~^h=&!nXELH4$K5%_H! zsSo93u^6WI!^S~urkZ0u$oL1bUvpK_zJ!~hLFh$ii+gW5dDmbJau)k+;pcK8*=_de z;fjwW5M?{#_GW!t^3Pt_Sd?duQhH!V>Bp&m_QLg+&3|>no22YS@eOLy3qAhY%30q* zLt7jxQJi0g36>Ud{W=#UE_tY-5@`N1*bTLDI=7#kJJrd4vFQroLv0S*l}qvQeS`zK zCG2F;6c8Sg5K1shEFQb39+4e)k_jQ))5#x0$IRp@D)d6E)2AKqf#&R>#9WpsH$l_V zUbb+)NvCq;jw2U;Z5$@?jU(EDPiRFv^_jN0V?f?mSj;VW!x^?Qgg+|yxx?1=!0GLA z{}_R?fX5}@$M0Ij%2xYl~I_O zqzvJ|F{0*6u~odIyW20Jn#W7AguvLS{dJSjOw2%_Zjh!kJyPjPbE^`IFeIem#8NGJ zdMO*%TOzy(UZ5!y(3XA8&dQXelYIi6_}R_#$|N3%fd z;eZ-=(*9y+BMK_wr~g4MdjGDpk5>{by2mW`IVhQDI`!cDmtMjT%#@r`q{Lgkh3j)t z$N~6FDyki0C!Z$4h*QRA38Bxaz3slm1zX2rXx;x5fB;0Lr;RdA1VsR6|2ADioD0qh zk3+lJx&NB5cf1x;2n-s!auA*7??^A2oa(n>XC1(+11f{gByq%{J{psJ*Xs9s7ki>- zYnt0Yb?3;!Ten*ST7{iAh>aybtXc%<1J5C)_`&f6`Zsd}^Cnwv^{nMa*!w2tC&$9Y zW7%ES!r$;gttDILZ@VCx{J&8}GIzs{yWE7TL9(l}@=h-+_;n|%)aK<*I3n(8XXN;+ zy20a<^I8s;t|H%@qHF@kG8?^Rr;Ey4%P;Jh=y1ZjzS$>LcgK-=>7ur$AUVFc~qyrF_|c7{GSLp`g=!ItFJ{(p%Vb0rQBN- K#d9$HdixizGY2sM literal 0 HcmV?d00001 diff --git a/test/fixtures/derived_class_text.sldd b/test/fixtures/derived_class_text.sldd new file mode 100644 index 00000000..8df2e057 --- /dev/null +++ b/test/fixtures/derived_class_text.sldd @@ -0,0 +1,209 @@ +{ + "__MW_TEXT_COREPROPERTIES__": { + "release": "R2027a" + }, + "__MW_TEXT_PARTS__": { + "__MW_TEXT_PART__/data/chunk0": { + "__MW_TEXT_content": { + "entries": [ + { + "name": "dpGain", + "metadata": { + "uuid": "7a8f51eb-e176-481d-8c8f-b578b979872e", + "namespace": "dacaf35e-55a5-454d-a7c1-93db038a210e", + "lastmod": "20261001T033014.428555", + "modifiedby": "weiwang", + "isderived": "0" + }, + "value": { + "_array_class": "dexdata.DerivedParam", + "_dimensions": [ + 1, + 1 + ], + "_elements": [ + { + "_id": "1", + "_properties": { + "Aliases": { + "_array_type": "Cell", + "_dimensions": [ + 1, + 3 + ], + "_elements": [ + "alpha", + "beta", + "gamma" + ] + }, + "CalibLevel": 42, + "CalibTag": "high", + "CoderInfo": { + "_id": "3", + "_object_class": "Simulink.CoderInfo", + "_properties": { + "CSCPackageName": "Simulink", + "CustomAttributes": { + "_id": "4", + "_object_class": "SimulinkCSC.AttribClass_Simulink_Default", + "_properties": {} + }, + "CustomStorageClass": "Default", + "ParameterOrSignal": "Parameter", + "StorageClass": "Auto" + } + }, + "Complexity": "real", + "DataType": "int32", + "Description": "a derived parameter", + "Dimensions": [ + 1, + 1 + ], + "DocUnits": "m/s", + "IsLocked": true, + "Limits": { + "_array_type": "Struct", + "_dimensions": [ + 1, + 1 + ], + "_elements": [ + { + "hi": 9, + "lo": -1 + } + ], + "_fields": [ + "lo", + "hi" + ] + }, + "Max": 100, + "Min": -10, + "Mode": { + "_enum_value": { + "_type": "int32", + "_value": "-1" + }, + "_id": "2", + "_object_class": "GearMode" + }, + "Value": { + "_type": "int32", + "_value": "7" + } + } + } + ], + "_mw_element_type": "MATLABArray" + } + }, + { + "name": "dpPlain", + "metadata": { + "uuid": "6422e816-d4ad-4678-8a63-73c5cd7e396e", + "namespace": "dacaf35e-55a5-454d-a7c1-93db038a210e", + "lastmod": "20261001T033014.431172", + "modifiedby": "weiwang", + "isderived": "0" + }, + "value": { + "_array_class": "dexdata.DerivedParam", + "_dimensions": [ + 1, + 1 + ], + "_elements": [ + { + "_id": "1", + "_properties": { + "CoderInfo": { + "_id": "2", + "_object_class": "Simulink.CoderInfo", + "_properties": { + "CSCPackageName": "Simulink", + "CustomAttributes": { + "_id": "3", + "_object_class": "SimulinkCSC.AttribClass_Simulink_Default", + "_properties": {} + }, + "CustomStorageClass": "Default", + "ParameterOrSignal": "Parameter", + "StorageClass": "Auto" + } + }, + "Complexity": "real", + "Dimensions": [ + 0, + 0 + ] + } + } + ], + "_mw_element_type": "MATLABArray" + } + }, + { + "name": "dsFlow", + "metadata": { + "uuid": "a156ac1e-6f1c-41c4-a1f1-852c4447e667", + "namespace": "dacaf35e-55a5-454d-a7c1-93db038a210e", + "lastmod": "20261001T033014.432643", + "modifiedby": "weiwang", + "isderived": "0" + }, + "value": { + "_array_class": "dexdata.DerivedSignal", + "_dimensions": [ + 1, + 1 + ], + "_elements": [ + { + "_id": "1", + "_properties": { + "CoderInfo": { + "_id": "3", + "_object_class": "Simulink.CoderInfo", + "_properties": { + "CSCPackageName": "Simulink", + "CustomAttributes": { + "_id": "4", + "_object_class": "SimulinkCSC.AttribClass_Simulink_Default", + "_properties": {} + }, + "CustomStorageClass": "Default", + "ParameterOrSignal": "Signal", + "StorageClass": "Auto" + } + }, + "DataType": "single", + "Description": "a derived signal", + "DocUnits": "V", + "LoggingInfo": { + "_id": "2", + "_object_class": "Simulink.LoggingInfo", + "_properties": {} + }, + "Max": 5, + "Min": -5, + "Priority": { + "_type": "int32", + "_value": "9" + }, + "Routing": "external" + } + } + ], + "_mw_element_type": "MATLABArray" + } + } + ], + "Dictionary References": [], + "AllowAccessBWS": false + } + } + } +} \ No newline at end of file diff --git a/test/fixtures/mcos/+dexdata/@DerivedParam/DerivedParam.m b/test/fixtures/mcos/+dexdata/@DerivedParam/DerivedParam.m new file mode 100644 index 00000000..994a56ec --- /dev/null +++ b/test/fixtures/mcos/+dexdata/@DerivedParam/DerivedParam.m @@ -0,0 +1,36 @@ +classdef DerivedParam < Simulink.Parameter + % A SUBCLASS of a MathWorks data class — the shape derived_class_{binary,text}.sldd + % exists for, and the one the corpus had no instance of at all. Everything here is a + % Simulink.Parameter plus six extra properties, which is how customers extend the + % class (the documented "define data classes" workflow). + % + % WHY IT LIVES IN +dexdata AND NOT BESIDE Widget.m. MATLAB R2027a refuses outright: + % Class 'DerivedParam' is a subclass of 'Simulink.DataObject' and must be defined + % inside of a package, but not in a nested package + % So the on-disk class name a customer's dictionary carries for an extended data class + % is ALWAYS dotted — `dexdata.DerivedParam` here — and never bare. That is a fact about + % every such entry, not about this fixture: any rule that splits a Class attribute on + % '.' to find a MathWorks namespace sees a user package in exactly the same shape as + % `Simulink.Parameter`. A plain custom class (Widget) has the opposite constraint — it + % cannot be in a nested package either, but it needs no package at all. + % + % Why it is worth a fixture: an entry's `Class` attribute becomes DerivedParam, not + % Simulink.Parameter, so every class-gated rule in the data model sees a name it has + % never heard of while the VALUE still carries Simulink.Parameter's own properties + % (Value, DataType, Min, Max, Unit, CoderInfo...). A reader that keys off the exact + % class name degrades this to a generic object and silently loses the parameter + % columns; a reader that keys off "is-a" keeps them. The fixture is what makes that + % choice visible either way. + % + % The six extras are one per value kind (double, char, logical, struct, cell, custom + % enum) so that a derived class's own properties are serialized through the same + % value paths as a plain custom class's, and any divergence lands on a named property. + properties + CalibLevel = 3 + CalibTag = 'cal' + IsLocked = false + Limits = struct('lo', 0, 'hi', 1) + Aliases = {'a', 'b'} + Mode = GearMode.Drive + end +end diff --git a/test/fixtures/mcos/+dexdata/@DerivedSignal/DerivedSignal.m b/test/fixtures/mcos/+dexdata/@DerivedSignal/DerivedSignal.m new file mode 100644 index 00000000..4f693116 --- /dev/null +++ b/test/fixtures/mcos/+dexdata/@DerivedSignal/DerivedSignal.m @@ -0,0 +1,15 @@ +classdef DerivedSignal < Simulink.Signal + % The Signal half of the derived-class pair (see DerivedParam for the argument, and + % for why both are in the +dexdata package rather than beside Widget.m). + % Separate from DerivedParam because Simulink.Parameter and Simulink.Signal are + % different classes with different property sets and different data-model nodes, and + % a fixture that only subclassed one would leave the other's path unmeasured. + % + % Priority is int32 on purpose: a typed integer property is written with + % Class="int32", so the derived class also pins that an extra property keeps its + % MATLAB type rather than widening to double on the way through XML. + properties + Routing = 'bus' + Priority = int32(5) + end +end diff --git a/test/fixtures/mcos/GearMode.m b/test/fixtures/mcos/GearMode.m new file mode 100644 index 00000000..896925d6 --- /dev/null +++ b/test/fixtures/mcos/GearMode.m @@ -0,0 +1,19 @@ +classdef GearMode < Simulink.IntEnumType + % A CUSTOM enumeration, used three ways by make_class_fixtures.m: as a property + % value on a custom class, as a property value on a Simulink.Parameter subclass, + % and as a dictionary entry value in its own right. + % + % The corpus had no custom enum before this. It matters because an enum is the one + % value whose serialization is not inferable from its class name — MATLAB writes it + % with the IsEnum / EnumerationName / EnumerationType attributes, three of the + % fourteen attributes DictionaryXmlFast interns that no fixture here exercised. + % + % Reverse is deliberately NEGATIVE: the underlying storage is int32, and a minus + % sign is the one character in an enum's serialized text that a scanner reading + % digits could get wrong. + enumeration + Park(0) + Drive(1) + Reverse(-1) + end +end diff --git a/test/fixtures/mcos/Widget.m b/test/fixtures/mcos/Widget.m new file mode 100644 index 00000000..40c428ce --- /dev/null +++ b/test/fixtures/mcos/Widget.m @@ -0,0 +1,101 @@ +classdef Widget + % The authoring class for custom_object_{binary,text}.sldd: ONE plain MCOS value class + % (no superclass) whose properties cover every value shape a dictionary can hold, so a + % reader change that breaks one of them breaks a NAMED property rather than "the + % fixture". + % + % WHY EVERY DEFAULT IS THE SENTINEL 0, AND WHY THE VALUES LIVE IN THE GENERATOR. + % The two dictionary formats disagree about defaults, measured on the first cut of + % these fixtures (R2027a): + % + % compressed-binary writes EVERY property of an object, touched or not + % JSON text writes only properties away from the CLASSDEF DEFAULT, and + % reconstructs the rest from the class at load time + % + % The first cut put the interesting values in the defaults here and set only the four + % object-valued properties in the generator. MATLAB read both twins back perfectly — and + % custom_object_text.sldd contained four properties out of twenty. The literal string + % `pad me` was not in the file at all. A twin pair like that is worse than no twin: it + % looks like a control and is not one, because the text member's values live in a .m file + % the reader under test has never seen. + % + % So nothing here is a real value. `0` is a sentinel no property in the fixture uses, and + % make_class_fixtures.m assigns all twenty — which makes all twenty non-default and so + % present in BOTH formats. The fixture's content is in the generator, in one place, where + % the next person looking for it will be. + % + % WHAT EACH PROPERTY IS FOR (values in make_class_fixtures.m/custom_items): + % + % Padded ' pad me ' with LEADING AND TRAILING SPACES. The single most load-bearing + % value here. DictionaryXmlFast coerces a text body ONLY IF IT EQUALS ITS OWN + % TRIM, precisely so a padded char array survives; a reader that trims turns + % this into 'pad me' and nothing else in the corpus notices. + % Hostile all five characters XML must escape (& < > " ') in one char array. MATLAB + % escapes only the first three — `"` and `'` are written RAW, because they + % need escaping in an attribute value and not in element text — so this also + % pins which of the five entities a dictionary ever actually contains. + % Ctrl / the CONTROL-CHARACTER cases, and the reason this class is worth more than + % Cr / its property count. Measured one character at a time in + % Tab / Lf .scratch/probe_char_format.m, MATLAB has THREE encodings for a char array + % and picks by content: + % + % printable, incl. tab(9) and LF(10) written literally + % CR(13) written as + % NUL(0), BEL(7), DEL(127), or any the WHOLE array becomes + % non-BMP char (surrogate pair) Format="decimal" and the body is + % the char CODES, space separated + % + % Both of the last two are outside what the readers were built for: `Format` + % is a FIFTEENTH attribute (DictionaryXmlFast interns fourteen, and the + % 32-dictionary corpus census found no other), and ` ` is a numeric + % character reference, which that census found zero of. One property each: + % Ctrl -> Format="decimal">9 10 13 7 1 (one bad character decimalizes all) + % Cr -> a b + % Tab -> ab literal tab in element text + % Lf -> ab literal newline in element text + % Tab and Lf are not filler: a reader that strips layout whitespace from + % element text — which DictionaryXmlFast deliberately does — has to not strip + % these, and nothing in the corpus made it prove that. + % + % AND CR IS WHERE THE TWO FORMATS STOP AGREEING. With these values finally in + % the files rather than in this classdef, MATLAB round-trips char 13 through + % compressed-binary and LOSES it through JSON text: Ctrl comes back + % [9 10 7 1] and Cr comes back 'ab'. custom_object_text.sldd holds zero raw + % CR bytes and zero \r escapes, while \t, \n, \u0007 and \u0001 all survive in + % the same strings — so the character is dropped on write, not mis-escaped. + % verify_class_fixtures.m asserts both behaviours by format on purpose. + % Cplx complex scalar; MATLAB writes IsComplex="1" and a `3.0+4.0i` body. + % Matrix 2x3, so a column-major flattening is visible as a transpose rather than as + % the same six numbers in a different order. + % Nothing / ZeroStruct / EmptyStruct — the three distinct EMPTIES, which are not + % interchangeable: [] is 0x0 double, struct([]) is a 0x0 struct, struct() is a + % 1x1 struct with no fields. They serialize three different ways. + % Tuning a Simulink.Parameter INSIDE a custom class. The nesting direction the corpus + % lacked: object_props and rt_bin both have custom-in-custom, nothing had + % MathWorks-in-custom. + % Part / a scalar custom object and a 1x3 object ARRAY. An object array cannot be an + % Parts entry value at all (see make_class_fixtures.m), so a property is the only + % place real MATLAB output puts one. + properties + Scalar = 0 + Matrix = 0 + Cplx = 0 + Padded = 0 + Str = 0 + Flag = 0 + Nothing = 0 + EmptyStruct = 0 + ZeroStruct = 0 + Nested = 0 + Cells = 0 + Hostile = 0 + Ctrl = 0 + Cr = 0 + Tab = 0 + Lf = 0 + Mode = 0 + Part = 0 + Parts = 0 + Tuning = 0 + end +end diff --git a/test/fixtures/mcos/WidgetPart.m b/test/fixtures/mcos/WidgetPart.m new file mode 100644 index 00000000..6f61bd56 --- /dev/null +++ b/test/fixtures/mcos/WidgetPart.m @@ -0,0 +1,14 @@ +classdef WidgetPart + % The leaf custom class behind Widget's two object-valued properties: one scalar + % (Widget.Part) and one ARRAY (Widget.Parts). Kept deliberately tiny — its job is to + % be countable, not to hold interesting values, because what is under test is the + % `

` + one classed `` per element shape, + % and a fat leaf only makes that shape harder to read in the XML. + % + % Label carries the index so a mis-ordered or truncated element list is visible by + % reading, not by counting. + properties + PartId = 0 + Label = 'part' + end +end diff --git a/test/fixtures/mcos/make_class_fixtures.m b/test/fixtures/mcos/make_class_fixtures.m new file mode 100644 index 00000000..eb188fe0 --- /dev/null +++ b/test/fixtures/mcos/make_class_fixtures.m @@ -0,0 +1,344 @@ +function make_class_fixtures() + % Copyright 2026 The MathWorks, Inc. + % Authors the derived-class / custom-class dictionary fixtures, each as a TWIN PAIR: + % one compressed-binary (zip whose data/chunk0.xml is the XML we read) and one JSON + % text, from the SAME in-memory values. Run: + % + % mw -using Bmain matlab -batch "cd('/test/fixtures/mcos'); make_class_fixtures" + % + % (A MULTI-LINE -batch argument silently runs nothing and exits 0, so the whole call has + % to be one line. The `cd` is what puts this folder — and so the classdefs and the + % +dexdata package — on the path before the function resolves. Verify by the WROTE + % lines and the trailing MAKE_CLASS_FIXTURES_DONE, not by the exit code.) + % + % Writes six files into test/fixtures/: + % derived_class_{binary,text}.sldd dexdata.DerivedParam / dexdata.DerivedSignal — + % subclasses of Simulink.Parameter and .Signal + % custom_object_{binary,text}.sldd Widget — a plain MCOS value class covering every + % value shape, plus bare enum entries (scalar and + % 1x3) and a 2x2 CELL of custom objects, which is + % as close to an object-array entry as a dictionary + % allows (see custom_items) + % bus_types_{binary,text}.sldd the type-defining MathWorks classes as MATLAB + % actually writes them in BINARY (arch_binary.sldd + % is hand-authored and typeLink.sldd is text-only, + % so neither is evidence of MATLAB's output) + % + % WHY TWINS. The same dictionary written both ways is the control that separates FORMAT + % from CONTENT: any difference the two readers produce is a reader bug, because the + % values went in from one variable. Both members of a pair therefore come from one + % `items` list, built once below — a second hand-written list is a second thing to get + % wrong, which is exactly how rt_text/rt_bin were already done here. + % + % BUT A TWIN PAIR IS NOT TWO COPIES OF THE SAME PROPERTY SET, and that is the most + % surprising thing these fixtures measured (R2027a): + % + % compressed-binary writes EVERY property of an object + % JSON text writes only properties away from the CLASSDEF DEFAULT, and + % reconstructs the rest from the class on load + % + % Measured on the entries below: dpGain (all fifteen touched) is 15 properties in both + % formats, but dpPlain (nothing touched) is 15 in the binary and THREE in the text, and + % btValueType is 9 against 3. So "same entries, same properties" is the wrong assertion + % across a twin pair; "same entries, and every property PRESENT IN BOTH agrees" is the + % right one. It also means a text dictionary is not self-contained for a custom class — + % its defaults live in a .m file — which is why Widget's defaults are a sentinel and + % custom_items() assigns all twenty properties explicitly. + % + % WHY THE SECTION IS PROBED, NOT DECLARED. Design Data accepts only values Simulink + % recognises as design data and rejects the rest; which side of that line a + % Simulink.Parameter SUBCLASS or a bare enum falls on is not documented anywhere we + % trust, so resolve_sections() finds out once, against a throwaway dictionary, and both + % twins then use the answer. Guessing would mean a silent twin divergence if the guess + % were wrong for one format only. + here = fileparts(mfilename('fullpath')); + fixtures = fileparts(here); + addpath(here); + Simulink.data.dictionary.closeAll('-discard'); + + derived = resolve_sections('derived', derived_items()); + custom = resolve_sections('custom', custom_items()); + types = resolve_sections('types', type_items()); + + write_twins(fullfile(fixtures, 'derived_class'), derived); + write_twins(fullfile(fixtures, 'custom_object'), custom); + write_twins(fullfile(fixtures, 'bus_types'), types); + fprintf('MAKE_CLASS_FIXTURES_DONE\n'); +end + +% --------------------------------------------------------------------------- +% The three item lists. An item is {name, preferredSection, value}; the section +% is a PREFERENCE, narrowed by resolve_sections. +% --------------------------------------------------------------------------- + +function items = derived_items() + % A fully populated subclass instance: every INHERITED property set away from its + % default too, so each one can be checked by value rather than taken on trust. + % dexdata.* and not DerivedParam: MATLAB refuses to DEFINE a Simulink.DataObject + % subclass outside a package ("must be defined inside of a package, but not in a + % nested package"), so the dotted name is forced, not stylistic. See + % +dexdata/DerivedParam.m. + p = dexdata.DerivedParam; + p.Value = int32(7); + p.DataType = 'int32'; + p.Min = -10; + p.Max = 100; + p.Unit = 'm/s'; + p.Description = 'a derived parameter'; + p.CalibLevel = 42; + p.CalibTag = 'high'; + p.IsLocked = true; + p.Limits = struct('lo', -1, 'hi', 9); + p.Aliases = {'alpha', 'beta', 'gamma'}; + p.Mode = GearMode.Reverse; + + % The same class at ITS DEFAULTS, as the control. The measured answer (R2027a) is that + % MATLAB omits NOTHING: dpPlain writes all six extra properties and all of + % Simulink.Parameter's, CoderInfo tree included, exactly as dpGain does. So a derived + % class costs the same bytes whether it was touched or not, and no reader may infer + % "unset" from "absent" here. + pd = dexdata.DerivedParam; + + s = dexdata.DerivedSignal; + s.DataType = 'single'; + s.Min = -5; + s.Max = 5; + s.Unit = 'V'; + s.Description = 'a derived signal'; + s.Routing = 'external'; + s.Priority = int32(9); + + items = { ... + {'dpGain', 'Design Data', p}, ... + {'dpPlain', 'Design Data', pd}, ... + {'dsFlow', 'Design Data', s}, ... + }; +end + +function items = custom_items() + % EVERY Widget property is assigned, none left at its classdef default — see Widget.m + % for the measurement, but the short version is that the JSON text format writes only + % non-default properties, so a value left in the classdef lands in the binary twin and + % NOT in the text one. Widget's defaults are therefore the sentinel 0 and this function + % is the fixture's content. Adding a property to Widget means adding a line here. + w = Widget; + w.Scalar = 1.5; + w.Matrix = [1 2 3; 4 5 6]; + w.Cplx = 3 + 4i; + w.Padded = ' pad me '; + w.Str = "a string"; + w.Flag = true; + w.Nothing = []; + w.EmptyStruct = struct(); + w.ZeroStruct = struct([]); + w.Nested = struct('inner', struct('leaf', 7)); + w.Cells = {1, 'two', true}; + w.Hostile = 'amp & lt < gt > quot " apos '' end'; + % The four control-character cases, one encoding each. char() and not an escape, because + % MATLAB single quotes have no escapes and '\t' would be a literal backslash-t. + w.Ctrl = char([9 10 13 7 1]); + w.Cr = ['a' char(13) 'b']; + w.Tab = ['a' char(9) 'b']; + w.Lf = ['a' char(10) 'b']; + w.Mode = GearMode.Reverse; + + part = WidgetPart; + part.PartId = 1; + part.Label = 'lead'; + w.Part = part; + + % 1x3 object ARRAY as a PROPERTY. Built by growing from .empty rather than by + % repmat, so each element carries a distinct PartId/Label and a dropped or + % reordered element is legible in the XML instead of being invisible. + parts = WidgetPart.empty; + for k = 1:3 + q = WidgetPart; + q.PartId = k * 10; + q.Label = sprintf('part%d', k); + parts(k) = q; + end + w.Parts = parts; + + % A Simulink.Parameter nested inside a custom class: MathWorks-in-custom, the + % nesting direction the corpus had no instance of. + t = Simulink.Parameter; + t.Value = 2.5; + t.DataType = 'single'; + t.Description = 'nested MathWorks object'; + w.Tuning = t; + + % A 2x2 CELL of objects, which is as close as a dictionary gets to an object array at + % entry level. An MCOS object ARRAY cannot be an entry value AT ALL — measured in + % .scratch/probe_class_refusals.m, R2027a, both sections refuse every one: + % + % WidgetPart 1x3 / 2x2 "Values of class 'WidgetPart' are not supported in the + % 'Other_Data' section of the dictionary." (scalar: accepted) + % Widget 1x2 same, and the scalar is accepted + % Simulink.Parameter "Arrays of class 'Simulink.Parameter' are not supported in + % 1x3 / 2x2 the dictionary." — a DIFFERENT message, i.e. a deliberate + % array check, not a class-support check + % Simulink.BusElement refused in both sections, scalar included + % + % That is worth writing down twice over, because object_array_binary.sldd hand-authors + % exactly this shape (3x1 Simulink.Parameter and 2x1 Simulink.VariableUsage as entry + % VALUES) and MATLAB will not write it. The reader should keep coping with it, but the + % fixture is synthetic and no round-trip can ever confirm it. Object arrays reach real + % XML only as a PROPERTY — Widget.Parts above, a Bus's Elements_internal — which is + % where this corpus now covers them. + % + % The cell, by contrast, is accepted in DESIGN DATA as well as Other Data, so custom + % MCOS objects do reach the design-data section after all, just wrapped. 2x2 and not + % 1x4 so the column-major order of the list is observable. + grid = cell(2, 2); + for k = 1:4 + q = WidgetPart; + q.PartId = 100 + k; + q.Label = sprintf('cell%d', k); + grid{k} = q; + end + + % gearRow: an enum ARRAY is accepted where an object array is not, so this is the one + % non-scalar object-ish entry value a dictionary will hold. Mixed signs on purpose. + items = { ... + {'widget', 'Other Data', w}, ... + {'partCell', 'Design Data', grid}, ... + {'gearValue', 'Design Data', GearMode.Reverse}, ... + {'gearRow', 'Design Data', [GearMode.Park GearMode.Drive GearMode.Reverse]}, ... + }; +end + +function items = type_items() + a = Simulink.AliasType; + a.BaseType = 'uint8'; + a.Description = 'alias of uint8'; + + n = Simulink.NumericType; + n.DataTypeMode = 'Fixed-point: binary point scaling'; + n.Signedness = 'Signed'; + n.WordLength = 16; + n.FractionLength = 3; + n.Description = 'fixdt(1,16,3)'; + + % A bus whose four elements each differ in ONE property from the others, so the + % element list is a table of cases rather than three copies: a plain double, one typed + % by the alias entry above (the Data Type link's binary-format case), one typed by the + % CUSTOM enum class, and one that is both 2x3 and complex. + e1 = Simulink.BusElement; e1.Name = 'plain'; e1.DataType = 'double'; + e2 = Simulink.BusElement; e2.Name = 'aliased'; e2.DataType = 'btAlias'; + e3 = Simulink.BusElement; e3.Name = 'geared'; e3.DataType = 'Enum: GearMode'; + e4 = Simulink.BusElement; e4.Name = 'matrix'; e4.DataType = 'double'; + e4.Dimensions = [2 3]; + e4.Complexity = 'complex'; + b = Simulink.Bus; + b.Elements = [e1 e2 e3 e4]; + b.Description = 'four elements, four cases'; + + items = { ... + {'btAlias', 'Design Data', a}, ... + {'btNumeric', 'Design Data', n}, ... + {'btBus', 'Design Data', b}, ... + }; + + % Simulink.ValueType and EnumTypeDefinition are both newer and both optional to the + % point of this fixture; a release that lacks either must still produce the bus. + try + v = Simulink.ValueType; + v.DataType = 'btAlias'; + v.Dimensions = [1 4]; + v.Description = 'a value type over an alias'; + items{end + 1} = {'btValueType', 'Design Data', v}; + catch err + fprintf('SKIP btValueType: %s\n', err.message); + end + try + % appendEnumeral, not addEnumeral: R2027a's EnumTypeDefinition has no addEnumeral at + % all (`methods` lists appendEnumeral / removeEnumeral), so the obvious spelling fails + % as an "Unrecognized method" rather than as a bad argument. + ed = Simulink.data.dictionary.EnumTypeDefinition; + ed.appendEnumeral('OFF', 0, 'off state'); + ed.appendEnumeral('ON', 1, 'on state'); + % A fresh EnumTypeDefinition is NOT empty — it ships one enumeral named `enum1` + % with value 0, which the two appends land behind. Removed so the fixture holds the + % two enumerals it says it does, and so OFF is not the second zero in the list. + % By INDEX, not by name: removeEnumeral('enum1') fails with "Invalid argument at + % position 1. Value must be a scalar." + ed.removeEnumeral(1); + ed.DefaultValue = 'OFF'; + ed.StorageType = 'int32'; + ed.Description = 'a dictionary-defined enum'; + items{end + 1} = {'btEnumDef', 'Design Data', ed}; + catch err + fprintf('SKIP btEnumDef: %s\n', err.message); + end +end + +% --------------------------------------------------------------------------- +% Writing +% --------------------------------------------------------------------------- + +function items = resolve_sections(tag, items) + % Narrow each item's section to one Simulink will actually accept, and DROP any item + % no section takes — reported loudly, because a silently missing entry is the failure + % mode that makes a fixture lie. Runs against a throwaway dictionary in tempdir so the + % real files are only ever written from a list already known to work. + probe = [tempname '_' tag '.sldd']; + cleanup = onCleanup(@() cleanup_file(probe)); + dd = Simulink.data.dictionary.create(probe); + keep = {}; + for k = 1:numel(items) + it = items{k}; + sections = unique({it{2}, 'Other Data'}, 'stable'); + placed = ''; + for s = sections + try + addEntry(getSection(dd, s{1}), it{1}, it{3}); + placed = s{1}; + break + catch err + fprintf(' %s/%s refused by %s: %s\n', tag, it{1}, s{1}, err.message); + end + end + if isempty(placed) + fprintf('SKIP %s/%s: no section accepted it\n', tag, it{1}); + else + if ~strcmp(placed, it{2}) + fprintf(' %s/%s moved to %s\n', tag, it{1}, placed); + end + keep{end + 1} = {it{1}, placed, it{3}}; %#ok + end + end + Simulink.data.dictionary.closeAll('-discard'); + items = keep; +end + +function write_twins(stem, items) + write_dict([stem '_binary.sldd'], items, 'compressed-binary'); + write_dict([stem '_text.sldd'], items, 'text'); +end + +function write_dict(path, items, fmt) + cleanup_file(path); + dd = Simulink.data.dictionary.create(path); + for k = 1:numel(items) + it = items{k}; + addEntry(getSection(dd, it{2}), it{1}, it{3}); + end + % FileFormat must be set BEFORE saveChanges; without it the write is JSON text, + % which is R2027a's default and is NOT what the XML reader is being measured on. + if strcmp(fmt, 'compressed-binary') + dd.FileFormat = 'compressed-binary'; + end + saveChanges(dd); + Simulink.data.dictionary.closeAll('-discard'); + fprintf('WROTE %s bytes=%d entries=%d\n', path, dir(path).bytes, numel(items)); +end + +function cleanup_file(path) + % A dictionary left open holds a lock, and `create` on an existing path errors, so a + % re-run has to clear both. closeAll first: deleting a file still open leaves the + % in-memory dictionary pointing at nothing and the NEXT create fails instead. + Simulink.data.dictionary.closeAll('-discard'); + if isfile(path) + delete(path); + end +end diff --git a/test/fixtures/mcos/verify_class_fixtures.m b/test/fixtures/mcos/verify_class_fixtures.m new file mode 100644 index 00000000..7af23dab --- /dev/null +++ b/test/fixtures/mcos/verify_class_fixtures.m @@ -0,0 +1,219 @@ +function verify_class_fixtures() + % Copyright 2026 The MathWorks, Inc. + % Reopens the six fixtures make_class_fixtures.m wrote and asserts the values came back + % — so a fixture that MATLAB *wrote* but cannot *read* is caught here rather than + % becoming a test that pins a MATLAB accident. Run: + % + % mw -using Bmain matlab -batch "cd('/test/fixtures/mcos'); verify_class_fixtures" + % + % Every twin is checked SEPARATELY and against the same expectations, which is the whole + % point of a twin: a value that only survives one of the two formats is a finding. + % + % The checks are deliberately uneven. Most properties get one equality; the ones that + % have a plausible way to go quietly wrong get a stricter check: + % Padded compared with strcmp against a value WITH its spaces, because a trimming + % bug produces a string that still looks right in a printout. + % Cr/Tab/Lf/Ctrl compared by DOUBLE(char), because these characters are invisible and + % an equality on the text would read as passing either way. + % Matrix compared whole, so a column-major/row-major transpose fails rather than + % comparing equal element-by-element in the wrong order. + here = fileparts(mfilename('fullpath')); + fixtures = fileparts(here); + addpath(here); + Simulink.data.dictionary.closeAll('-discard'); + + state.pass = 0; + state.fail = 0; + for fmt = {'binary', 'text'} + state = verify_derived(state, fullfile(fixtures, ['derived_class_' fmt{1} '.sldd'])); + state = verify_custom(state, fullfile(fixtures, ['custom_object_' fmt{1} '.sldd']), fmt{1}); + state = verify_types(state, fullfile(fixtures, ['bus_types_' fmt{1} '.sldd'])); + end + fprintf('\nTOTAL pass=%d fail=%d\n', state.pass, state.fail); + if state.fail > 0 + fprintf('VERIFY_CLASS_FIXTURES_FAILED\n'); + else + fprintf('VERIFY_CLASS_FIXTURES_OK\n'); + end +end + +function state = verify_derived(state, path) + fprintf('\n=== %s ===\n', path); + [vals, state] = load_entries(state, path, {'dpGain', 'dpPlain', 'dsFlow'}); + if isempty(vals), return, end + p = vals.dpGain; + state = chk(state, 'dpGain class', class(p), 'dexdata.DerivedParam'); + state = chk(state, 'dpGain Value', p.Value, int32(7)); + state = chk(state, 'dpGain DataType', p.DataType, 'int32'); + state = chk(state, 'dpGain Min/Max', [p.Min p.Max], [-10 100]); + state = chk(state, 'dpGain Unit', p.Unit, 'm/s'); + state = chk(state, 'dpGain CalibLevel', p.CalibLevel, 42); + state = chk(state, 'dpGain CalibTag', p.CalibTag, 'high'); + state = chk(state, 'dpGain IsLocked', p.IsLocked, true); + state = chk(state, 'dpGain Limits', [p.Limits.lo p.Limits.hi], [-1 9]); + state = chk(state, 'dpGain Aliases', p.Aliases, {'alpha', 'beta', 'gamma'}); + state = chk(state, 'dpGain Mode', p.Mode, GearMode.Reverse); + + pd = vals.dpPlain; + state = chk(state, 'dpPlain class', class(pd), 'dexdata.DerivedParam'); + state = chk(state, 'dpPlain CalibLevel (default)', pd.CalibLevel, 3); + state = chk(state, 'dpPlain Mode (default)', pd.Mode, GearMode.Drive); + + s = vals.dsFlow; + state = chk(state, 'dsFlow class', class(s), 'dexdata.DerivedSignal'); + state = chk(state, 'dsFlow DataType', s.DataType, 'single'); + state = chk(state, 'dsFlow Routing', s.Routing, 'external'); + state = chk(state, 'dsFlow Priority', s.Priority, int32(9)); + state = chk(state, 'dsFlow Priority stays int32', class(s.Priority), 'int32'); +end + +function state = verify_custom(state, path, fmt) + fprintf('\n=== %s ===\n', path); + [vals, state] = load_entries(state, path, {'widget', 'partCell', 'gearValue', 'gearRow'}); + if isempty(vals), return, end + w = vals.widget; + state = chk(state, 'widget class', class(w), 'Widget'); + state = chk(state, 'widget Scalar', w.Scalar, 1.5); + state = chk(state, 'widget Matrix', w.Matrix, [1 2 3; 4 5 6]); + state = chk(state, 'widget Cplx', w.Cplx, 3 + 4i); + state = chk(state, 'widget Padded KEEPS SPACES', w.Padded, ' pad me '); + state = chk(state, 'widget Str', w.Str, "a string"); + state = chk(state, 'widget Flag', w.Flag, true); + state = chk(state, 'widget Nothing is 0x0', size(w.Nothing), [0 0]); + state = chk(state, 'widget EmptyStruct is 1x1 no fields', ... + [numel(w.EmptyStruct) numel(fieldnames(w.EmptyStruct))], [1 0]); + state = chk(state, 'widget ZeroStruct is 0x0', size(w.ZeroStruct), [0 0]); + state = chk(state, 'widget Nested.inner.leaf', w.Nested.inner.leaf, 7); + state = chk(state, 'widget Cells', w.Cells, {1, 'two', true}); + state = chk(state, 'widget Hostile all five', w.Hostile, 'amp & lt < gt > quot " apos '' end'); + % By code point: these four are invisible, so an equality on the text would pass on a + % value that silently lost a character — which is exactly what one format does. + % + % THE ONE PLACE THE TWINS DISAGREE ON A VALUE, and the reason these checks are + % format-aware rather than shared. Measured R2027a, after moving the values out of + % Widget.m's defaults so that both files actually carry them: + % + % char code 13 (CR) SURVIVES a compressed-binary round-trip and is LOST by JSON text. + % + % Ctrl = char([9 10 13 7 1]) binary -> [9 10 13 7 1] text -> [9 10 7 1] + % Cr = ['a' char(13) 'b'] binary -> [97 13 98] text -> [97 98] + % + % It is not a JSON escaping problem: custom_object_text.sldd contains zero raw CR bytes + % AND zero \r escapes, while tab, LF, BEL(7) and SOH(1) all come through the same string + % as \t, \n, \u0007 and \u0001. The CR never reaches the file at all. The binary + % format writes it as the reference and reads it back intact. + % + % So a reader may NOT assume a text dictionary and its binary twin hold equal char data, + % and must not "fix" a CR-less text value by inferring one. Asserting the measured + % behaviour per format, rather than skipping these two, is deliberate: if a later release + % makes text preserve CR, this fails and someone re-reads this comment. + if strcmp(fmt, 'text') + state = chk(state, 'widget Ctrl codes (text LOSES CR 13)', double(w.Ctrl), [9 10 7 1]); + state = chk(state, 'widget Cr codes (text LOSES CR 13)', double(w.Cr), [97 98]); + else + state = chk(state, 'widget Ctrl codes', double(w.Ctrl), [9 10 13 7 1]); + state = chk(state, 'widget Cr codes ( survives)', double(w.Cr), [97 13 98]); + end + state = chk(state, 'widget Tab codes', double(w.Tab), [97 9 98]); + state = chk(state, 'widget Lf codes', double(w.Lf), [97 10 98]); + state = chk(state, 'widget Mode', w.Mode, GearMode.Reverse); + state = chk(state, 'widget Part', [w.Part.PartId double(string(w.Part.Label) == "lead")], [1 1]); + state = chk(state, 'widget Parts size', size(w.Parts), [1 3]); + state = chk(state, 'widget Parts ids in order', [w.Parts.PartId], [10 20 30]); + state = chk(state, 'widget Tuning class', class(w.Tuning), 'Simulink.Parameter'); + state = chk(state, 'widget Tuning Value', w.Tuning.Value, 2.5); + + g = vals.partCell; + state = chk(state, 'partCell size', size(g), [2 2]); + state = chk(state, 'partCell column-major ids', cellfun(@(c) c.PartId, g(:))', [101 102 103 104]); + state = chk(state, 'partCell element class', class(g{1}), 'WidgetPart'); + + state = chk(state, 'gearValue', vals.gearValue, GearMode.Reverse); + state = chk(state, 'gearRow', vals.gearRow, [GearMode.Park GearMode.Drive GearMode.Reverse]); + state = chk(state, 'gearRow underlying int32', int32(vals.gearRow), int32([0 1 -1])); +end + +function state = verify_types(state, path) + fprintf('\n=== %s ===\n', path); + [vals, state] = load_entries(state, path, {'btAlias', 'btNumeric', 'btBus', 'btValueType', 'btEnumDef'}); + if isempty(vals), return, end + state = chk(state, 'btAlias BaseType', vals.btAlias.BaseType, 'uint8'); + state = chk(state, 'btNumeric WordLength', vals.btNumeric.WordLength, 16); + state = chk(state, 'btNumeric FractionLength', vals.btNumeric.FractionLength, 3); + b = vals.btBus; + state = chk(state, 'btBus element count', numel(b.Elements), 4); + state = chk(state, 'btBus element names', {b.Elements.Name}, {'plain', 'aliased', 'geared', 'matrix'}); + state = chk(state, 'btBus aliased type', b.Elements(2).DataType, 'btAlias'); + state = chk(state, 'btBus enum type', b.Elements(3).DataType, 'Enum: GearMode'); + state = chk(state, 'btBus matrix dims', b.Elements(4).Dimensions, [2 3]); + state = chk(state, 'btBus matrix complexity', b.Elements(4).Complexity, 'complex'); + if isfield(vals, 'btValueType') + state = chk(state, 'btValueType DataType', vals.btValueType.DataType, 'btAlias'); + state = chk(state, 'btValueType Dimensions', vals.btValueType.Dimensions, [1 4]); + end + if isfield(vals, 'btEnumDef') + e = vals.btEnumDef; + state = chk(state, 'btEnumDef enumerals', {e.Enumerals.Name}, {'OFF', 'ON'}); + state = chk(state, 'btEnumDef DefaultValue', e.DefaultValue, 'OFF'); + end +end + +% --------------------------------------------------------------------------- + +function [vals, state] = load_entries(state, path, names) + % Read every named entry out in one open/close, searching BOTH sections: which section + % an entry landed in is make_class_fixtures's probe's business, not this file's, and + % hard-coding it here would make a correct relocation look like a missing entry. + vals = struct(); + if ~isfile(path) + state = fail(state, sprintf('%s does not exist', path)); + vals = []; + return + end + d = Simulink.data.dictionary.open(path); + for k = 1:numel(names) + found = false; + for sec = {'Design Data', 'Other Data'} + try + vals.(names{k}) = getValue(getEntry(getSection(d, sec{1}), names{k})); + found = true; + break + catch + % next section + end + end + if ~found + state = fail(state, sprintf('entry %s missing from %s', names{k}, path)); + end + end + Simulink.data.dictionary.closeAll('-discard'); +end + +function state = chk(state, name, actual, expected) + if isequal(actual, expected) + state.pass = state.pass + 1; + fprintf('PASS %s\n', name); + else + state = fail(state, sprintf('%s: got %s expected %s', name, brief(actual), brief(expected))); + end +end + +function state = fail(state, msg) + state.fail = state.fail + 1; + fprintf('FAIL %s\n', msg); +end + +function s = brief(v) + % mat2str throws on a cell/object/enum, and a verifier that errors while REPORTING a + % failure hides the failure it was reporting. + try + s = mat2str(v); + catch + try + s = formattedDisplayText(v); + s = strtrim(regexprep(char(s), '\s+', ' ')); + catch + s = sprintf('<%s %s>', class(v), mat2str(size(v))); + end + end +end From ff49161e2c89658d173d9b304ad8f7e36616d9dc Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Thu, 1 Oct 2026 00:32:42 -0400 Subject: [PATCH 2/2] Normalize line endings, and decline where the engine knows more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five checks all asked whether this reader agrees with fast-xml-parser on input I thought of. None asked the only question the fallback cannot absorb: is there input where it answers non-null and disagrees? There were four, and every one produced a wrong answer rather than a decline. A literal CR was handed back as-is. XML 1.0 §2.11 requires `\r\n` and a lone `\r` to reach the application as `\n`, and the engine does it in one `replace` before parsing — a step a reader that slices the input string skips without any branch to suspect. It is now done with the engine's own expression behind an `indexOf('\r')` guard, which costs 4-7 ms on 74.7 MB and skips the copy on all 32 corpus dictionaries, because MATLAB spells a CR inside a value ` ` precisely to survive this normalization. The other three are declines, and they sharpen the rule the file rests on. Knowing less than the engine is safe only while it makes this reader stop; it is a bug when it makes the reader continue on a narrower reading. `isSpace` is XML's four whitespace characters and the engine splits names on JavaScript's `\s`, so `

` read as two attributes where the engine folds one, and `` read as a declaration where the engine sees a PI named `?xmlfoo`. Both now decline, `` at depth 3 writes over the state of the previous `

` at depth 3. * The element's own object (`kids[d]`) is of course fresh each time, because it is the output. */ -export declare function readDictionaryXmlFast(text: string): Record | null; +export declare function readDictionaryXmlFast(input: string): Record | null; //# sourceMappingURL=DictionaryXmlFast.d.ts.map \ No newline at end of file diff --git a/dist/datamodel/parser/DictionaryXmlFast.d.ts.map b/dist/datamodel/parser/DictionaryXmlFast.d.ts.map index 0fe13cc1..bda96aa1 100644 --- a/dist/datamodel/parser/DictionaryXmlFast.d.ts.map +++ b/dist/datamodel/parser/DictionaryXmlFast.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"DictionaryXmlFast.d.ts","sourceRoot":"","sources":["../../../src/datamodel/parser/DictionaryXmlFast.ts"],"names":[],"mappings":"AAqZA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAiLlF"} \ No newline at end of file +{"version":3,"file":"DictionaryXmlFast.d.ts","sourceRoot":"","sources":["../../../src/datamodel/parser/DictionaryXmlFast.ts"],"names":[],"mappings":"AA0bA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAgNnF"} \ No newline at end of file diff --git a/dist/datamodel/parser/DictionaryXmlFast.js b/dist/datamodel/parser/DictionaryXmlFast.js index 25d00d41..c6aeeb4d 100644 --- a/dist/datamodel/parser/DictionaryXmlFast.js +++ b/dist/datamodel/parser/DictionaryXmlFast.js @@ -111,6 +111,15 @@ import toNumber from 'strnum'; /** The options fast-xml-parser's OptionsBuilder defaults to; see `numberParseOptions`. */ const NUMBER_OPTIONS = { hex: true, leadingZeros: true, eNotation: true, unicode: false }; const TEXT_KEY = '#text'; +/** + * fast-xml-parser's `maxNestedTags` default, which `XmlReader` does not override. + * + * Past it the engine throws `Maximum nested tags exceeded` rather than returning a tree, so this + * is one of the two numbers in this file copied from the engine's options rather than from the + * corpus (`NUMBER_OPTIONS` is the other). Dictionaries nest in single digits; this exists only so + * that a document the engine refuses is refused here too. + */ +const MAX_NESTED_TAGS = 100; /** * Attribute keys, pre-prefixed and interned. * @@ -163,6 +172,20 @@ const UPPER_X = 0x58; // X function isSpace(c) { return c === SPACE || c === LF || c === TAB || c === CR; } +/** + * The whitespace XML does not have and JavaScript does: `\f`, `\v`, NBSP, the Unicode spaces, + * LS/PS and the BOM. + * + * `isSpace` above is XML's definition and it is the correct one for a reader. It is NOT the one + * fast-xml-parser uses: its tag and attribute regexes split on `\s`, so every character in this + * set is a separator to the engine and part of a name to this scanner. That asymmetry is the one + * way a stricter reader can be WRONG rather than merely narrower, so wherever it can bite, this + * reader declines — see `readAttributes` and the declaration. Written as a complement — a + * whitespace that is none of XML's four — so it is this runtime's `\s` rather than my + * transcription of it, the engine's regexes and this one then being the same table by + * construction. + */ +const ALIEN_SPACE = /[^\S \t\n\r]/; /** * The end of the `&...;` token starting at `at`, or `-1` when there is none. The entity decoder * looks at most 32 characters past the `&` and gives up if it has not found a `;` by then, so a @@ -398,7 +421,21 @@ function readAttributes(text, from, limit, keys, vals) { return -1; value = decoded; } - keys[count] = ATTR_KEYS.get(name) ?? `@_${name}`; + const interned = ATTR_KEYS.get(name); + if (interned === undefined) { + // A name the table does not hold is read faithfully, but first: the engine would have + // SPLIT it at any `ALIEN_SPACE`, so `

` is two attributes to + // this scanner and one to the engine, which folds the duplicate and keeps the last. That + // is a non-null disagreement, the one failure the fallback cannot absorb, so decline. + // Nothing interned can contain one — that table is written out literally above — so this + // test is paid only by a name that is already paying a concatenation. + if (ALIEN_SPACE.test(name)) + return -1; + keys[count] = `@_${name}`; + } + else { + keys[count] = interned; + } vals[count] = value; count++; k = valueEnd + 1; @@ -420,7 +457,20 @@ function readAttributes(text, from, limit, keys, vals) { * DEPTH and reused: a `

` at depth 3 writes over the state of the previous `

` at depth 3. * The element's own object (`kids[d]`) is of course fresh each time, because it is the output. */ -export function readDictionaryXmlFast(text) { +export function readDictionaryXmlFast(input) { + // XML line-ending normalization, with the engine's own expression: `OrderedObjParser.parseXml` + // opens with `xmlData = xmlData.replace(/\r\n?/g, "\n")` (marked TODO there, and load-bearing + // here). It is not optional leniency — XML 1.0 §2.11 requires a parser to present `\r\n` and a + // lone `\r` as `\n` — and a reader that slices raw substrings skips it silently: `

a b` + // written as a literal CR came back `'a\r\nb'` here and `'a\nb'` from the engine, a non-null + // disagreement in a plain char property. MATLAB writes ` ` for a CR *inside* a value, which + // is a reference and survives both (neither decodes it), so this is reachable mainly by a + // CRLF-lineended or hand-edited part — and it is a wrong answer either way. + // + // The guard is the point: a scan for one byte costs a few ms on 74.7 MB and nothing is copied + // unless a CR is really there. Declining instead would have been correct and would have cost + // the whole file, which is the cliff the numeric-reference branch exists to avoid. + const text = input.indexOf('\r') === -1 ? input : input.replace(/\r\n?/g, '\n'); const n = text.length; const root = {}; // Per-depth element state, grown on demand and then reused. `kidCount` is how the collapse @@ -529,8 +579,22 @@ export function readDictionaryXmlFast(text) { return null; if (!text.startsWith('` is a PI named + // `?xmlversion="1.0"` to it — as is ``, and as is a name split by an + // `ALIEN_SPACE` this reader would have swallowed. Requiring one of XML's four spaces, or + // `?>` immediately, is the narrow reading: `` lands here too. + if (i + 5 !== pi && !isSpace(text.charCodeAt(i + 5))) + return null; if (sawRoot || depth !== 0) return null; + // A second declaration is not an overwrite. A repeated key ARRAYS in `compress`, so the + // engine answers `'?xml': [{...}, {...}]` where overwriting answers the last one and loses + // the first. Measured, not assumed — and declined rather than imitated, because a document + // with two declarations is malformed in a way no dictionary is, and this reader's special + // case for `?xml` does not go through `attach`. + if (root['?xml'] !== undefined) + return null; const keys = []; const vals = []; const count = readAttributes(text, i + 5, pi, keys, vals); @@ -578,6 +642,13 @@ export function readDictionaryXmlFast(text) { return null; if (depth === 0 && sawRoot) return null; // a second root is a shape we lack + // The engine THROWS past its own nesting limit, and a reader that answers where the engine + // raises turns a loud failure into quiet data — the mirror of the `null`-as-empty-document + // trap. Only a non-self-closing open counts, because that is where the engine checks (it + // pushes no frame for `

`), and the boundary is exact: 101 nested opens parse, the 102nd + // throws. See `MAX_NESTED_TAGS`. + if (!selfClosing && depth > MAX_NESTED_TAGS) + return null; const d = depth; let ks = attrKey[d]; if (ks === undefined) { diff --git a/dist/datamodel/parser/DictionaryXmlFast.js.map b/dist/datamodel/parser/DictionaryXmlFast.js.map index d7209174..ade86808 100644 --- a/dist/datamodel/parser/DictionaryXmlFast.js.map +++ b/dist/datamodel/parser/DictionaryXmlFast.js.map @@ -1 +1 @@ -{"version":3,"file":"DictionaryXmlFast.js","sourceRoot":"","sources":["../../../src/datamodel/parser/DictionaryXmlFast.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,qCAAqC;AACrC,EAAE;AACF,mFAAmF;AACnF,EAAE;AACF,kBAAkB;AAClB,EAAE;AACF,iFAAiF;AACjF,qFAAqF;AACrF,2BAA2B;AAC3B,EAAE;AACF,4DAA4D;AAC5D,4DAA4D;AAC5D,qEAAqE;AACrE,6DAA6D;AAC7D,EAAE;AACF,8FAA8F;AAC9F,6FAA6F;AAC7F,8FAA8F;AAC9F,gDAAgD;AAChD,EAAE;AACF,6FAA6F;AAC7F,mFAAmF;AACnF,EAAE;AACF,qFAAqF;AACrF,+DAA+D;AAC/D,8EAA8E;AAC9E,4EAA4E;AAC5E,+CAA+C;AAC/C,EAAE;AACF,4FAA4F;AAC5F,2FAA2F;AAC3F,+FAA+F;AAC/F,+BAA+B;AAC/B,EAAE;AACF,+FAA+F;AAC/F,iGAAiG;AACjG,uFAAuF;AACvF,iEAAiE;AACjE,gGAAgG;AAChG,+FAA+F;AAC/F,mDAAmD;AACnD,EAAE;AACF,kGAAkG;AAClG,mGAAmG;AACnG,uFAAuF;AACvF,mGAAmG;AACnG,kFAAkF;AAClF,EAAE;AACF,6DAA6D;AAC7D,EAAE;AACF,+FAA+F;AAC/F,+FAA+F;AAC/F,uFAAuF;AACvF,8FAA8F;AAC9F,qFAAqF;AACrF,+FAA+F;AAC/F,uFAAuF;AACvF,0DAA0D;AAC1D,EAAE;AACF,6FAA6F;AAC7F,6FAA6F;AAC7F,qDAAqD;AACrD,EAAE;AACF,gGAAgG;AAChG,iGAAiG;AACjG,gGAAgG;AAChG,oFAAoF;AACpF,EAAE;AACF,2CAA2C;AAC3C,EAAE;AACF,wFAAwF;AACxF,6FAA6F;AAC7F,kFAAkF;AAClF,4FAA4F;AAC5F,EAAE;AACF,6FAA6F;AAC7F,yFAAyF;AACzF,4EAA4E;AAC5E,sDAAsD;AACtD,2FAA2F;AAC3F,+FAA+F;AAC/F,sFAAsF;AACtF,6FAA6F;AAC7F,4BAA4B;AAC5B,8FAA8F;AAC9F,oFAAoF;AACpF,0FAA0F;AAC1F,8FAA8F;AAC9F,mBAAmB;AACnB,oFAAoF;AACpF,4FAA4F;AAC5F,0FAA0F;AAC1F,sFAAsF;AACtF,2FAA2F;AAC3F,EAAE;AACF,uFAAuF;AACvF,6FAA6F;AAC7F,2FAA2F;AAC3F,4FAA4F;AAC5F,6FAA6F;AAC7F,4FAA4F;AAC5F,gFAAgF;AAChF,EAAE;AACF,uFAAuF;AACvF,wFAAwF;AACxF,+FAA+F;AAC/F,+FAA+F;AAC/F,wEAAwE;AAExE,OAAO,QAAQ,MAAM,QAAQ,CAAC;AAE9B,0FAA0F;AAC1F,MAAM,cAAc,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAE1F,MAAM,QAAQ,GAAG,OAAO,CAAC;AAEzB;;;;;;;;GAQG;AACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAiB;IACxC,CAAC,OAAO,EAAE,SAAS,CAAC;IACpB,CAAC,MAAM,EAAE,QAAQ,CAAC;IAClB,CAAC,WAAW,EAAE,aAAa,CAAC;IAC5B,CAAC,QAAQ,EAAE,UAAU,CAAC;IACtB,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAClC,CAAC,WAAW,EAAE,aAAa,CAAC;IAC5B,CAAC,QAAQ,EAAE,UAAU,CAAC;IACtB,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;IACxC,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;IACxC,CAAC,UAAU,EAAE,YAAY,CAAC;IAC1B,CAAC,eAAe,EAAE,iBAAiB,CAAC;IACpC,CAAC,eAAe,EAAE,iBAAiB,CAAC;IACpC,CAAC,YAAY,EAAE,cAAc,CAAC;IAC9B,CAAC,MAAM,EAAE,QAAQ,CAAC;IAClB,CAAC,SAAS,EAAE,WAAW,CAAC;IACxB,CAAC,UAAU,EAAE,YAAY,CAAC;IAC1B,CAAC,YAAY,EAAE,cAAc,CAAC;CAC/B,CAAC,CAAC;AAEH,sFAAsF;AACtF,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI;AACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,IAAI;AAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI;AACzB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI;AACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,EAAE,GAAG,IAAI,CAAC;AAChB,MAAM,EAAE,GAAG,IAAI,CAAC;AAChB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,IAAI,GAAG,IAAI,CAAC;AAClB,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,IAAI;AAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,IAAI;AAE1B,SAAS,OAAO,CAAC,CAAS;IACxB,OAAO,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;AAC1D,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,KAAa,EAAE,EAAU;IAC1C,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE;QAAE,CAAC,EAAE,CAAC;IAC7E,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnE,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,qBAAqB,CAAC,EAAU;IACvC,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,EAAE,IAAI,IAAI;QAAE,OAAO,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC5D,OAAO,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;AACtC,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC7B,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YACrC,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,4FAA4F;YAC5F,2FAA2F;YAC3F,uEAAuE;YACvE,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACjC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBACnC,MAAM,GAAG,GAAG,CAAC,KAAK,OAAO,IAAI,CAAC,KAAK,OAAO,CAAC;gBAC3C,yFAAyF;gBACzF,+DAA+D;gBAC/D,MAAM,EAAE,GAAG,GAAG;oBACZ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC;oBACxC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3C,IAAI,qBAAqB,CAAC,EAAE,CAAC;oBAAE,OAAO,IAAI,CAAC;YAC7C,CAAC;YACD,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,CAAC,wBAAwB;QACvC,CAAC;QACD,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,WAAW,CAAC,GAAW,EAAE,MAAe;IAC/C,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,GAAG,CAAC,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC/B,6FAA6F;IAC7F,mEAAmE;IACnE,IAAI,OAAO,KAAK,EAAE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC,CAAC,qCAAqC;IAC9E,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACtC,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,wFAAwF;IACxF,4FAA4F;IAC5F,0CAA0C;IAC1C,MAAM,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,KAAK,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;QAClF,OAAO,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+FAA+F;AAC/F,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,OAAO,CAAC;AACjF,CAAC;AAED;;;;GAIG;AACH,SAAS,MAAM,CAAC,MAA+B,EAAE,GAAW,EAAE,KAAc;IAC1E,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;QACjF,MAAM,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACjD,OAAO;IACT,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAClC,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAY,EAAE,IAAY,EAAE,EAAU;IACvD,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;QAClB,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACzD,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3D,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7D,KAAK,EAAE;YACL,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CACrB,IAAY,EACZ,IAAY,EACZ,KAAa,EACb,IAAc,EACd,IAAc;IAEd,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;QACjB,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAChC,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,CAAC,CAAC;QACpB,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;YACjB,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM;gBAAE,MAAM;YACtC,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,CAAC,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAAE,CAAC,EAAE,CAAC;QACrD,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7C,CAAC,EAAE,CAAC;QACJ,OAAO,CAAC,GAAG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAAE,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACvE,IAAI,QAAQ,KAAK,CAAC,CAAC,IAAI,QAAQ,GAAG,KAAK;YAAE,OAAO,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC7C,2EAA2E;QAC3E,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,OAAO,KAAK,IAAI;gBAAE,OAAO,CAAC,CAAC,CAAC;YAChC,KAAK,GAAG,OAAO,CAAC;QAClB,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACpB,KAAK,EAAE,CAAC;QACR,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,MAAM,IAAI,GAA4B,EAAE,CAAC;IAEzC,2FAA2F;IAC3F,0FAA0F;IAC1F,gFAAgF;IAChF,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAmC,EAAE,CAAC;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAiD,EAAE,CAAC;IACjE,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,kFAAkF;IAClF,MAAM,SAAS,GAAG,CAAC,CAAS,EAAW,EAAE;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,CAAC;QACxE,IAAI,OAAO;YAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC;QACnC,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YACX,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE;gBAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YAChD,OAAO,GAAG,CAAC,CAAC,6CAA6C;QAC3D,CAAC;QACD,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,mFAAmF;IACnF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,GAAW,EAAE,KAAc,EAAW,EAAE;QAC/D,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC,CAAC,0BAA0B;QACzC,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;QAChC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACb,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAEhC,qFAAqF;QACrF,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,oFAAoF;YACpF,qFAAqF;YACrF,qFAAqF;YACrF,2DAA2D;YAC3D,EAAE;YACF,uFAAuF;YACvF,uFAAuF;YACvF,4EAA4E;YAC5E,0DAA0D;YAC1D,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAAE,CAAC,EAAE,CAAC;YACnD,MAAM,QAAQ,GAAG,CAAC,KAAK,GAAG,CAAC;YAE3B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,oFAAoF;gBACpF,qEAAqE;gBACrE,IAAI,CAAC,QAAQ;oBAAE,OAAO,IAAI,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,sFAAsF;gBACtF,qFAAqF;gBACrF,yDAAyD;gBACzD,MAAM,OAAO,GAAG,EAAE,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC;gBAC/D,MAAM,IAAI,GAAG,OAAO,IAAI,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;gBAClD,mFAAmF;gBACnF,qFAAqF;gBACrF,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACtB,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;oBACxD,IAAI,SAAS,KAAK,IAAI;wBAAE,OAAO,IAAI,CAAC;oBACpC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;wBAC5B,MAAM,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;wBACpB,OAAO,CAAC,CAAC,CAAC;4BACR,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC/E,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,MAAM;YACrB,CAAC,GAAG,EAAE,CAAC;QACT,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAErC,sFAAsF;QACtF,wFAAwF;QACxF,sFAAsF;QACtF,oEAAoE;QACpE,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAEhC,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvB,gFAAgF;YAChF,uEAAuE;YACvE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACrC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC9C,IAAI,OAAO,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YACxC,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1D,IAAI,KAAK,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC3B,MAAM,IAAI,GAA4B,EAAE,CAAC;YACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACxD,+EAA+E;YAC/E,kFAAkF;YAClF,IAAI,CAAC,MAAM,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,qFAAqF;QACrF,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACpB,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACpC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC1C,IAAI,GAAG,GAAG,EAAE,CAAC;YACb,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;gBAAE,GAAG,EAAE,CAAC;YAC/D,KAAK,EAAE,CAAC;YACR,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YACxB,wFAAwF;YACxF,6CAA6C;YAC7C,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAC;YAChD,IAAI,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC;YACvD,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,qFAAqF;QACrF,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACpC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,qDAAqD;QACjF,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;QACpB,OAAO,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QACzE,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;QAC5C,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC9B,IAAI,KAAK,KAAK,CAAC,IAAI,OAAO;YAAE,OAAO,IAAI,CAAC,CAAC,mCAAmC;QAE5E,MAAM,CAAC,GAAG,KAAK,CAAC;QAChB,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACrB,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YACrB,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAClB,CAAC;QACD,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAE3B,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QACd,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChB,OAAO,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;QACvB,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACjB,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;QAErB,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,KAAK,EAAE,CAAC;QACV,CAAC;QACD,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACb,CAAC;IAED,wFAAwF;IACxF,qFAAqF;IACrF,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,IAAI,CAAC;AACd,CAAC"} \ No newline at end of file +{"version":3,"file":"DictionaryXmlFast.js","sourceRoot":"","sources":["../../../src/datamodel/parser/DictionaryXmlFast.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,qCAAqC;AACrC,EAAE;AACF,mFAAmF;AACnF,EAAE;AACF,kBAAkB;AAClB,EAAE;AACF,iFAAiF;AACjF,qFAAqF;AACrF,2BAA2B;AAC3B,EAAE;AACF,4DAA4D;AAC5D,4DAA4D;AAC5D,qEAAqE;AACrE,6DAA6D;AAC7D,EAAE;AACF,8FAA8F;AAC9F,6FAA6F;AAC7F,8FAA8F;AAC9F,gDAAgD;AAChD,EAAE;AACF,6FAA6F;AAC7F,mFAAmF;AACnF,EAAE;AACF,qFAAqF;AACrF,+DAA+D;AAC/D,8EAA8E;AAC9E,4EAA4E;AAC5E,+CAA+C;AAC/C,EAAE;AACF,4FAA4F;AAC5F,2FAA2F;AAC3F,+FAA+F;AAC/F,+BAA+B;AAC/B,EAAE;AACF,+FAA+F;AAC/F,iGAAiG;AACjG,uFAAuF;AACvF,iEAAiE;AACjE,gGAAgG;AAChG,+FAA+F;AAC/F,mDAAmD;AACnD,EAAE;AACF,kGAAkG;AAClG,mGAAmG;AACnG,uFAAuF;AACvF,mGAAmG;AACnG,kFAAkF;AAClF,EAAE;AACF,6DAA6D;AAC7D,EAAE;AACF,+FAA+F;AAC/F,+FAA+F;AAC/F,uFAAuF;AACvF,8FAA8F;AAC9F,qFAAqF;AACrF,+FAA+F;AAC/F,uFAAuF;AACvF,0DAA0D;AAC1D,EAAE;AACF,6FAA6F;AAC7F,6FAA6F;AAC7F,qDAAqD;AACrD,EAAE;AACF,gGAAgG;AAChG,iGAAiG;AACjG,gGAAgG;AAChG,oFAAoF;AACpF,EAAE;AACF,2CAA2C;AAC3C,EAAE;AACF,wFAAwF;AACxF,6FAA6F;AAC7F,kFAAkF;AAClF,4FAA4F;AAC5F,EAAE;AACF,6FAA6F;AAC7F,yFAAyF;AACzF,4EAA4E;AAC5E,sDAAsD;AACtD,2FAA2F;AAC3F,+FAA+F;AAC/F,sFAAsF;AACtF,6FAA6F;AAC7F,4BAA4B;AAC5B,8FAA8F;AAC9F,oFAAoF;AACpF,0FAA0F;AAC1F,8FAA8F;AAC9F,mBAAmB;AACnB,oFAAoF;AACpF,4FAA4F;AAC5F,0FAA0F;AAC1F,sFAAsF;AACtF,2FAA2F;AAC3F,EAAE;AACF,uFAAuF;AACvF,6FAA6F;AAC7F,2FAA2F;AAC3F,4FAA4F;AAC5F,6FAA6F;AAC7F,4FAA4F;AAC5F,gFAAgF;AAChF,EAAE;AACF,uFAAuF;AACvF,wFAAwF;AACxF,+FAA+F;AAC/F,+FAA+F;AAC/F,wEAAwE;AAExE,OAAO,QAAQ,MAAM,QAAQ,CAAC;AAE9B,0FAA0F;AAC1F,MAAM,cAAc,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAE1F,MAAM,QAAQ,GAAG,OAAO,CAAC;AAEzB;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;;;;;;GAQG;AACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAiB;IACxC,CAAC,OAAO,EAAE,SAAS,CAAC;IACpB,CAAC,MAAM,EAAE,QAAQ,CAAC;IAClB,CAAC,WAAW,EAAE,aAAa,CAAC;IAC5B,CAAC,QAAQ,EAAE,UAAU,CAAC;IACtB,CAAC,cAAc,EAAE,gBAAgB,CAAC;IAClC,CAAC,WAAW,EAAE,aAAa,CAAC;IAC5B,CAAC,QAAQ,EAAE,UAAU,CAAC;IACtB,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;IACxC,CAAC,iBAAiB,EAAE,mBAAmB,CAAC;IACxC,CAAC,UAAU,EAAE,YAAY,CAAC;IAC1B,CAAC,eAAe,EAAE,iBAAiB,CAAC;IACpC,CAAC,eAAe,EAAE,iBAAiB,CAAC;IACpC,CAAC,YAAY,EAAE,cAAc,CAAC;IAC9B,CAAC,MAAM,EAAE,QAAQ,CAAC;IAClB,CAAC,SAAS,EAAE,WAAW,CAAC;IACxB,CAAC,UAAU,EAAE,YAAY,CAAC;IAC1B,CAAC,YAAY,EAAE,cAAc,CAAC;CAC/B,CAAC,CAAC;AAEH,sFAAsF;AACtF,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI;AACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,IAAI;AAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI;AACzB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI;AACxB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,EAAE,GAAG,IAAI,CAAC;AAChB,MAAM,EAAE,GAAG,IAAI,CAAC;AAChB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,IAAI,GAAG,IAAI,CAAC;AAClB,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI;AACvB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,IAAI;AAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,IAAI;AAE1B,SAAS,OAAO,CAAC,CAAS;IACxB,OAAO,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;AAC1D,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,GAAG,cAAc,CAAC;AAEnC;;;;GAIG;AACH,SAAS,SAAS,CAAC,KAAa,EAAE,EAAU;IAC1C,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE;QAAE,CAAC,EAAE,CAAC;IAC7E,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnE,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,qBAAqB,CAAC,EAAU;IACvC,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,EAAE,IAAI,IAAI;QAAE,OAAO,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC5D,OAAO,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;AACtC,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC7B,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YACrC,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,4FAA4F;YAC5F,2FAA2F;YAC3F,uEAAuE;YACvE,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACjC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBACnC,MAAM,GAAG,GAAG,CAAC,KAAK,OAAO,IAAI,CAAC,KAAK,OAAO,CAAC;gBAC3C,yFAAyF;gBACzF,+DAA+D;gBAC/D,MAAM,EAAE,GAAG,GAAG;oBACZ,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC;oBACxC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3C,IAAI,qBAAqB,CAAC,EAAE,CAAC;oBAAE,OAAO,IAAI,CAAC;YAC7C,CAAC;YACD,GAAG,IAAI,GAAG,CAAC;YACX,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,CAAC,wBAAwB;QACvC,CAAC;QACD,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,WAAW,CAAC,GAAW,EAAE,MAAe;IAC/C,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,GAAG,CAAC,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC/B,6FAA6F;IAC7F,mEAAmE;IACnE,IAAI,OAAO,KAAK,EAAE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAChD,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC,CAAC,qCAAqC;IAC9E,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACtC,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,wFAAwF;IACxF,4FAA4F;IAC5F,0CAA0C;IAC1C,MAAM,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,KAAK,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;QAClF,OAAO,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+FAA+F;AAC/F,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,OAAO,CAAC;AACjF,CAAC;AAED;;;;GAIG;AACH,SAAS,MAAM,CAAC,MAA+B,EAAE,GAAW,EAAE,KAAc;IAC1E,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;QACjF,MAAM,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACjD,OAAO;IACT,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAClC,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAY,EAAE,IAAY,EAAE,EAAU;IACvD,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;QAClB,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACzD,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3D,KAAK,CAAC;YACJ,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7D,KAAK,EAAE;YACL,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CACrB,IAAY,EACZ,IAAY,EACZ,KAAa,EACb,IAAc,EACd,IAAc;IAEd,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;QACjB,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAChC,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,CAAC,CAAC;QACpB,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;YACjB,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM;gBAAE,MAAM;YACtC,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,CAAC,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAAE,CAAC,EAAE,CAAC;QACrD,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7C,CAAC,EAAE,CAAC;QACJ,OAAO,CAAC,GAAG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAAE,CAAC,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACvE,IAAI,QAAQ,KAAK,CAAC,CAAC,IAAI,QAAQ,GAAG,KAAK;YAAE,OAAO,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC7C,2EAA2E;QAC3E,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,OAAO,KAAK,IAAI;gBAAE,OAAO,CAAC,CAAC,CAAC;YAChC,KAAK,GAAG,OAAO,CAAC;QAClB,CAAC;QACD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,sFAAsF;YACtF,wFAAwF;YACxF,yFAAyF;YACzF,sFAAsF;YACtF,yFAAyF;YACzF,sEAAsE;YACtE,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,CAAC,CAAC,CAAC;YACtC,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC;QACzB,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;QACpB,KAAK,EAAE,CAAC;QACR,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAa;IACjD,+FAA+F;IAC/F,8FAA8F;IAC9F,+FAA+F;IAC/F,8FAA8F;IAC9F,6FAA6F;IAC7F,gGAAgG;IAChG,0FAA0F;IAC1F,4EAA4E;IAC5E,EAAE;IACF,8FAA8F;IAC9F,6FAA6F;IAC7F,mFAAmF;IACnF,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAChF,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,MAAM,IAAI,GAA4B,EAAE,CAAC;IAEzC,2FAA2F;IAC3F,0FAA0F;IAC1F,gFAAgF;IAChF,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAmC,EAAE,CAAC;IAChD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAiD,EAAE,CAAC;IACjE,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,kFAAkF;IAClF,MAAM,SAAS,GAAG,CAAC,CAAS,EAAW,EAAE;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,CAAC;QACxE,IAAI,OAAO;YAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC;QACnC,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YACX,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE;gBAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YAChD,OAAO,GAAG,CAAC,CAAC,6CAA6C;QAC3D,CAAC;QACD,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,mFAAmF;IACnF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,GAAW,EAAE,KAAc,EAAW,EAAE;QAC/D,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC,CAAC,0BAA0B;QACzC,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;QAChC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACb,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAEhC,qFAAqF;QACrF,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,oFAAoF;YACpF,qFAAqF;YACrF,qFAAqF;YACrF,2DAA2D;YAC3D,EAAE;YACF,uFAAuF;YACvF,uFAAuF;YACvF,4EAA4E;YAC5E,0DAA0D;YAC1D,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAAE,CAAC,EAAE,CAAC;YACnD,MAAM,QAAQ,GAAG,CAAC,KAAK,GAAG,CAAC;YAE3B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,oFAAoF;gBACpF,qEAAqE;gBACrE,IAAI,CAAC,QAAQ;oBAAE,OAAO,IAAI,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,sFAAsF;gBACtF,qFAAqF;gBACrF,yDAAyD;gBACzD,MAAM,OAAO,GAAG,EAAE,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC;gBAC/D,MAAM,IAAI,GAAG,OAAO,IAAI,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;gBAClD,mFAAmF;gBACnF,qFAAqF;gBACrF,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACtB,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;oBACxD,IAAI,SAAS,KAAK,IAAI;wBAAE,OAAO,IAAI,CAAC;oBACpC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;wBAC5B,MAAM,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;wBACpB,OAAO,CAAC,CAAC,CAAC;4BACR,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC/E,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,MAAM;YACrB,CAAC,GAAG,EAAE,CAAC;QACT,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAErC,sFAAsF;QACtF,wFAAwF;QACxF,sFAAsF;QACtF,oEAAoE;QACpE,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAEhC,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvB,gFAAgF;YAChF,uEAAuE;YACvE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACrC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC9C,2FAA2F;YAC3F,mFAAmF;YACnF,oFAAoF;YACpF,yFAAyF;YACzF,oFAAoF;YACpF,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;YAClE,IAAI,OAAO,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YACxC,wFAAwF;YACxF,2FAA2F;YAC3F,2FAA2F;YAC3F,0FAA0F;YAC1F,gDAAgD;YAChD,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YAC5C,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1D,IAAI,KAAK,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC3B,MAAM,IAAI,GAA4B,EAAE,CAAC;YACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACxD,+EAA+E;YAC/E,kFAAkF;YAClF,IAAI,CAAC,MAAM,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,qFAAqF;QACrF,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACpB,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACpC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC1C,IAAI,GAAG,GAAG,EAAE,CAAC;YACb,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;gBAAE,GAAG,EAAE,CAAC;YAC/D,KAAK,EAAE,CAAC;YACR,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YACxB,wFAAwF;YACxF,6CAA6C;YAC7C,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAC;YAChD,IAAI,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC;YACvD,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,qFAAqF;QACrF,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACpC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,qDAAqD;QACjF,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;QACpB,OAAO,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QACzE,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;QAC5C,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC9B,IAAI,KAAK,KAAK,CAAC,IAAI,OAAO;YAAE,OAAO,IAAI,CAAC,CAAC,mCAAmC;QAC5E,2FAA2F;QAC3F,2FAA2F;QAC3F,yFAAyF;QACzF,4FAA4F;QAC5F,iCAAiC;QACjC,IAAI,CAAC,WAAW,IAAI,KAAK,GAAG,eAAe;YAAE,OAAO,IAAI,CAAC;QAEzD,MAAM,CAAC,GAAG,KAAK,CAAC;QAChB,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACrB,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YACrB,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAClB,CAAC;QACD,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAE3B,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QACd,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChB,OAAO,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;QACvB,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACjB,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;QAErB,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,KAAK,EAAE,CAAC;QACV,CAAC;QACD,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACb,CAAC;IAED,wFAAwF;IACxF,qFAAqF;IACrF,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,IAAI,CAAC;AACd,CAAC"} \ No newline at end of file diff --git a/src/datamodel/parser/DictionaryXmlFast.ts b/src/datamodel/parser/DictionaryXmlFast.ts index 0f6aafa2..21795776 100644 --- a/src/datamodel/parser/DictionaryXmlFast.ts +++ b/src/datamodel/parser/DictionaryXmlFast.ts @@ -115,6 +115,16 @@ const NUMBER_OPTIONS = { hex: true, leadingZeros: true, eNotation: true, unicode const TEXT_KEY = '#text'; +/** + * fast-xml-parser's `maxNestedTags` default, which `XmlReader` does not override. + * + * Past it the engine throws `Maximum nested tags exceeded` rather than returning a tree, so this + * is one of the two numbers in this file copied from the engine's options rather than from the + * corpus (`NUMBER_OPTIONS` is the other). Dictionaries nest in single digits; this exists only so + * that a document the engine refuses is refused here too. + */ +const MAX_NESTED_TAGS = 100; + /** * Attribute keys, pre-prefixed and interned. * @@ -170,6 +180,21 @@ function isSpace(c: number): boolean { return c === SPACE || c === LF || c === TAB || c === CR; } +/** + * The whitespace XML does not have and JavaScript does: `\f`, `\v`, NBSP, the Unicode spaces, + * LS/PS and the BOM. + * + * `isSpace` above is XML's definition and it is the correct one for a reader. It is NOT the one + * fast-xml-parser uses: its tag and attribute regexes split on `\s`, so every character in this + * set is a separator to the engine and part of a name to this scanner. That asymmetry is the one + * way a stricter reader can be WRONG rather than merely narrower, so wherever it can bite, this + * reader declines — see `readAttributes` and the declaration. Written as a complement — a + * whitespace that is none of XML's four — so it is this runtime's `\s` rather than my + * transcription of it, the engine's regexes and this one then being the same table by + * construction. + */ +const ALIEN_SPACE = /[^\S \t\n\r]/; + /** * The end of the `&...;` token starting at `at`, or `-1` when there is none. The entity decoder * looks at most 32 characters past the `&` and gives up if it has not found a `;` by then, so a @@ -395,7 +420,19 @@ function readAttributes( if (decoded === null) return -1; value = decoded; } - keys[count] = ATTR_KEYS.get(name) ?? `@_${name}`; + const interned = ATTR_KEYS.get(name); + if (interned === undefined) { + // A name the table does not hold is read faithfully, but first: the engine would have + // SPLIT it at any `ALIEN_SPACE`, so `

` is two attributes to + // this scanner and one to the engine, which folds the duplicate and keeps the last. That + // is a non-null disagreement, the one failure the fallback cannot absorb, so decline. + // Nothing interned can contain one — that table is written out literally above — so this + // test is paid only by a name that is already paying a concatenation. + if (ALIEN_SPACE.test(name)) return -1; + keys[count] = `@_${name}`; + } else { + keys[count] = interned; + } vals[count] = value; count++; k = valueEnd + 1; @@ -418,7 +455,20 @@ function readAttributes( * DEPTH and reused: a `

` at depth 3 writes over the state of the previous `

` at depth 3. * The element's own object (`kids[d]`) is of course fresh each time, because it is the output. */ -export function readDictionaryXmlFast(text: string): Record | null { +export function readDictionaryXmlFast(input: string): Record | null { + // XML line-ending normalization, with the engine's own expression: `OrderedObjParser.parseXml` + // opens with `xmlData = xmlData.replace(/\r\n?/g, "\n")` (marked TODO there, and load-bearing + // here). It is not optional leniency — XML 1.0 §2.11 requires a parser to present `\r\n` and a + // lone `\r` as `\n` — and a reader that slices raw substrings skips it silently: `

a b` + // written as a literal CR came back `'a\r\nb'` here and `'a\nb'` from the engine, a non-null + // disagreement in a plain char property. MATLAB writes ` ` for a CR *inside* a value, which + // is a reference and survives both (neither decodes it), so this is reachable mainly by a + // CRLF-lineended or hand-edited part — and it is a wrong answer either way. + // + // The guard is the point: a scan for one byte costs a few ms on 74.7 MB and nothing is copied + // unless a CR is really there. Declining instead would have been correct and would have cost + // the whole file, which is the cliff the numeric-reference branch exists to avoid. + const text = input.indexOf('\r') === -1 ? input : input.replace(/\r\n?/g, '\n'); const n = text.length; const root: Record = {}; @@ -526,7 +576,19 @@ export function readDictionaryXmlFast(text: string): Record | n const pi = text.indexOf('?>', i + 2); if (pi === -1) return null; if (!text.startsWith('` is a PI named + // `?xmlversion="1.0"` to it — as is ``, and as is a name split by an + // `ALIEN_SPACE` this reader would have swallowed. Requiring one of XML's four spaces, or + // `?>` immediately, is the narrow reading: `` lands here too. + if (i + 5 !== pi && !isSpace(text.charCodeAt(i + 5))) return null; if (sawRoot || depth !== 0) return null; + // A second declaration is not an overwrite. A repeated key ARRAYS in `compress`, so the + // engine answers `'?xml': [{...}, {...}]` where overwriting answers the last one and loses + // the first. Measured, not assumed — and declined rather than imitated, because a document + // with two declarations is malformed in a way no dictionary is, and this reader's special + // case for `?xml` does not go through `attach`. + if (root['?xml'] !== undefined) return null; const keys: string[] = []; const vals: string[] = []; const count = readAttributes(text, i + 5, pi, keys, vals); @@ -566,6 +628,12 @@ export function readDictionaryXmlFast(text: string): Record | n const tag = internTag(text, i + 1, nameEnd); if (tag === null) return null; if (depth === 0 && sawRoot) return null; // a second root is a shape we lack + // The engine THROWS past its own nesting limit, and a reader that answers where the engine + // raises turns a loud failure into quiet data — the mirror of the `null`-as-empty-document + // trap. Only a non-self-closing open counts, because that is where the engine checks (it + // pushes no frame for `

`), and the boundary is exact: 101 nested opens parse, the 102nd + // throws. See `MAX_NESTED_TAGS`. + if (!selfClosing && depth > MAX_NESTED_TAGS) return null; const d = depth; let ks = attrKey[d]; diff --git a/test/dictionaryXmlFast.test.ts b/test/dictionaryXmlFast.test.ts index 5719d434..73bb4085 100644 --- a/test/dictionaryXmlFast.test.ts +++ b/test/dictionaryXmlFast.test.ts @@ -195,6 +195,29 @@ describe('the XML declaration', () => { // Same collapse rule as an element, which is why it is not special-cased. expectRead('', { '?xml': '', DataSource: '' }); }); + + it('is only a declaration when the NAME ends at `xml`', () => { + // The engine reads a processing instruction's name up to the first `\s` and keys whatever it + // got, so each of these is a PI named something else entirely — and a prefix test alone read + // all three as the declaration, inventing a `?xml` key the engine never produced. Found by + // reasoning out the family the fuzzer had stumbled into, not by the fuzzer. + expectDeclined('', { '?xmlfoo': '', DataSource: '' }); + expectDeclined('', { '?xml-foo="1"': '', DataSource: '' }); + expectDeclined('', { + '?xml-stylesheet': { '@_href': 'a.xsl' }, + DataSource: '', + }); + }); + + it('declines a SECOND declaration rather than overwriting the first', () => { + // A repeated key arrays in `compress`, so the engine keeps both and overwriting would lose + // one. Declined rather than imitated: two declarations is a shape no dictionary has. + expectDeclined('', { + '?xml': [{ '@_version': '1.0' }, { '@_version': '1.0' }], + DataSource: '', + }); + expectDeclined('', { '?xml': ['', ''], DataSource: '' }); + }); }); describe('text coercion', () => { @@ -490,6 +513,60 @@ describe('whitespace', () => { }); }); +describe('line endings, which XML normalizes and a raw slice does not', () => { + // XML 1.0 §2.11: a parser presents `\r\n` and a lone `\r` as `\n`, and the engine implements + // exactly that with one `replace(/\r\n?/g, "\n")` before it parses. A reader that hands back + // slices of the original string skips the step without noticing — which is what this one did + // until an adversarial probe compared the two on `

a\rb

`. Every case here FAILED before + // the pre-pass existed, returning the carriage return it was given. + // + // Reachable in practice by a CRLF-lineended or hand-edited part, not by MATLAB: for a CR + // *inside* a value MATLAB writes ` `, a reference neither reader decodes, which is why the + // corpus differential over 469.4 MB could not see this. + + it('turns CRLF and a lone CR into a single LF, in text', () => { + expectRead('

a\r\nb

', { P: ['a\nb'] }); + expectRead('

a\rb

', { P: ['a\nb'] }); + expectRead('

\r

', { P: ['\n'] }); + }); + + it('collapses only `\\r\\n`, so a doubled CR stays two line endings', () => { + // The engine's regex is `\r\n?`, which is not the same as "delete every CR": `\r\r\n` is a CR + // (-> LF) followed by a CRLF (-> LF). A `replace(/\r/g, '\n')` would agree here and a + // `replace(/\r\n/g, '\n')` would not, so both near-misses are pinned. + expectRead('

a\r\r\nb

', { P: ['a\n\nb'] }); + expectRead('

a\n\rb

', { P: ['a\n\nb'] }); + }); + + it('normalizes inside an attribute value too', () => { + // Attribute values are NOT whitespace-normalized to spaces — fast-xml-parser's line doing + // that is commented out in its own source — so the LF survives as an LF. + expectRead('

', { P: [{ '@_Name': 'a\nb' }] }); + expectRead('

', { P: [{ '@_Name': 'a\n' }] }); + }); + + it('normalizes BEFORE the padding test, so a trailing CR still defeats coercion', () => { + // `1\r` is padded, and a padded value is handed back raw and uncoerced. The order matters in + // both directions: normalizing after would compare `'1\r'` against `'1'` and reach the same + // answer by luck, while trimming CR as layout would coerce it to the number 1. + expectRead('

1\r

', { P: ['1\n'] }); + expectRead('

1\r\n

', { P: ['1\n'] }); + expectRead('

\r

', { P: [' \n '] }); + }); + + it('leaves ` ` standing beside a literal it normalized', () => { + // The two halves of the same character, one of which is text and one of which is not. + expectRead('

\r

', { P: [' \n'] }); + }); + + it('reads a CRLF-lineended document, layout and all', () => { + expectRead('\r\n

1

\r\n', { + DataSource: { P: [{ '#text': 1, '@_Name': 'a' }] }, + }); + expectRead('

\r

', { P: [{ Element: [''] }] }); + }); +}); + describe('the per-depth state is reused, so a stale slot must not leak', () => { // `readDictionaryXmlFast` indexes seven parallel arrays by DEPTH and writes over them rather // than allocating a frame per element — 13.8 M allocations saved on the largest dictionary, @@ -660,6 +737,44 @@ describe('what it declines, and what the seam still answers', () => { expectDeclined('

', { P: [''] }); }); + it('declines a name split by whitespace only JAVASCRIPT counts as whitespace', () => { + // The one way a stricter reader can be WRONG rather than narrower. `isSpace` is XML's four + // characters, which is correct; the engine's regexes split on `\s`, which also includes + // `\f`, `\v`, NBSP and the Unicode spaces. So the engine sees TWO attributes where this + // scanner sees one name with a form feed inside it — and when the halves repeat a name, + // the engine folds the duplicate and keeps the last, so even the key COUNT differs. + expectDeclined('

', { P: [{ '@_Value': 'b' }] }); + expectDeclined('

', { P: [{ '@_Value': 'b' }] }); + expectDeclined('

', { P: [{ '@_Value': '1' }] }); + expectDeclined('

', { P: [{ '@_Value': '1' }] }); + expectDeclined('

', { P: [{ '@_Value': '1' }] }); + expectDeclined('

', { P: [{ '@_Value': '1' }] }); + // The same hole on the declaration's attributes, which is where it was actually found: the + // element path happened to decline already, because the tag name then holds the character. + expectDeclined('', { + '?xml': { '@_version': '1.0' }, + DataSource: '', + }); + }); + + it('still reads an attribute name that is merely UNKNOWN', () => { + // The control for the test above, and the one that matters for real files: the decline is + // keyed on the alien whitespace, not on the name being absent from `ATTR_KEYS`. MATLAB + // writes `Format="decimal"` for a char value XML cannot hold plainly, and that attribute + // appears in none of the 32 corpus dictionaries — it must still be copied faithfully. + expectRead('

9 10 13 7 1

', { + P: [ + { + '#text': '9 10 13 7 1', + '@_Name': 'Ctrl', + '@_Class': 'char', + '@_Dimension': '1*5', + '@_Format': 'decimal', + }, + ], + }); + }); + it('declines a `>` inside an attribute value', () => { // A known, accepted limitation rather than a bug: finding the tag's end needs the // attributes parsed and parsing them needs the tag's end. No dictionary attribute @@ -669,6 +784,46 @@ describe('what it declines, and what the seam still answers', () => { }); }); + describe('the engine’s nesting limit, past which it throws rather than answers', () => { + // `maxNestedTags` defaults to 100 and `XmlReader` does not override it. A reader that keeps + // going past the point where the engine raises is the mirror image of a caller reading `null` + // as "empty document": a loud failure becomes quiet data. Dictionaries nest in single digits, + // so nothing here is about real files — it is about the contract holding at the edge. + const deep = (d: number, inner = 'x'): string => + '

'.repeat(d) + inner + '

'.repeat(d); + + it('reads the deepest document the engine reads', () => { + // 101 opens, not 100: the engine checks its stack BEFORE pushing, so the limit admits one + // more level than the number suggests. Compared against the live engine rather than a + // literal, which at this depth would be 101 lines of nesting and would pin the shape + // instead of the boundary — the shape is pinned by every other case in this file. + for (const d of [99, 100, 101]) { + const fast = readDictionaryXmlFast(deep(d)); + expect(fast, `depth ${d} should still be read`).not.toBeNull(); + expect(fast, `depth ${d}`).toEqual(readDictionaryXmlGeneric(deep(d))); + } + }); + + it('declines one level past it, where the engine throws', () => { + for (const d of [102, 103, 150]) { + expect(readDictionaryXmlFast(deep(d)), `depth ${d}`).toBeNull(); + expect(() => readDictionaryXmlGeneric(deep(d))).toThrow(/Maximum nested tags/); + expect(() => readDictionaryXml(deep(d))).toThrow(/Maximum nested tags/); + } + }); + + it('does not count a self-closing tag, because the engine pushes no frame for one', () => { + // The off-by-one that a `depth > limit` check placed one line earlier would introduce: at + // the limit exactly, `

` is still fine and `

` is not. Declining here would cost + // nothing in practice and would still be a divergence from "identical or null" in the only + // direction that is allowed — but it would mean the boundary was copied, not understood. + const edge = '

'.repeat(101) + '

'.repeat(5) + '

'.repeat(101); + const fast = readDictionaryXmlFast(edge); + expect(fast).not.toBeNull(); + expect(fast).toEqual(readDictionaryXmlGeneric(edge)); + }); + }); + it('declines an unterminated tag, where the engine THROWS', () => { // The one decline whose fallback does not return a value. Pinned because the seam's // behaviour here is "throws", and a caller that treated the fast reader's `null` as diff --git a/test/dictionaryXmlFastDifferential.test.ts b/test/dictionaryXmlFastDifferential.test.ts index da6336e5..b19c747c 100644 --- a/test/dictionaryXmlFastDifferential.test.ts +++ b/test/dictionaryXmlFastDifferential.test.ts @@ -254,10 +254,31 @@ const TEXTS = [ ' a', ' ', 'a & b', + // Carriage returns, which XML normalizes (`\r\n` and a lone `\r` both become `\n`) and a reader + // slicing the raw string does not. These are not bails — they must come back normalized — and + // they belong in a generator because the interesting part is the COMBINATION: a CR beside a + // reference, a CR that turns a coercible value into a padded one, a CR in layout position. + '\r', + '\r\n ', + 'a\r\nb', + 'a\rb', + 'a\r\r\nb', + '1\r', + ' \r', 'Simulink.Parameter', '[1 2]', ]; +/** + * Whitespace JavaScript's `\s` counts and XML's four do not. + * + * Every one of these is a separator to the engine's regexes and part of a name to this reader's + * scanner, so wherever one can end a name the reader must decline. Kept out of `ATTR_NAMES` and + * injected at low probability instead: at list frequency almost every document would decline and + * the stream would stop reaching anything else. + */ +const ALIEN_WS = ['\f', '\v', ' ', ' ', '']; + const ATTR_VALUES = [ '', 'v', @@ -273,6 +294,8 @@ const ATTR_VALUES = [ 'a"b', 'a>b', 'a\nb', + 'a\r\nb', + 'a\r', ]; /** One generated document. */ @@ -286,8 +309,16 @@ function generate(rand: () => number): string { // Quote style has to follow the content, or the generator produces nothing but // malformations past the first `"` and stops exercising anything else. const q = value.includes('"') ? "'" : rand() < 0.15 ? "'" : '"'; - const sep = rand() < 0.1 ? ' = ' : '='; - s += `${pick([' ', ' ', '\t', '\n '])}${pick(ATTR_NAMES)}${sep}${q}${value}${q}`; + const name = pick(ATTR_NAMES); + // An `ALIEN_WS` in either place a name can end. The engine splits there and this reader + // does not, so the engine sees one more attribute than the scanner does. + const lead = rand() < 0.04 ? pick(ALIEN_WS) : pick([' ', ' ', '\t', '\n ']); + const sep = rand() < 0.04 ? `${pick(ALIEN_WS)}=` : rand() < 0.1 ? ' = ' : '='; + s += `${lead}${name}${sep}${q}${value}${q}`; + // The same name twice. Where the split above lands inside a repeated name, the engine folds + // the duplicate and keeps the last — so the divergence is a key COUNT rather than a key + // value, which is the form it was first caught in. + if (rand() < 0.03) s += `${rand() < 0.3 ? pick(ALIEN_WS) : ' '}${name}="dup"`; } return s; }; @@ -304,7 +335,10 @@ function generate(rand: () => number): string { body = pick(TEXTS); } else { // Mixed content on purpose: no real dictionary has it, so only a generator will. - const indent = rand() < 0.5 ? `\n${' '.repeat(depth + 1)}` : ''; + // A CRLF indent now and then, so whole documents arrive line-ended the way a Windows editor + // leaves them rather than only carrying a CR inside one value. + const indent = + rand() < 0.5 ? `${rand() < 0.15 ? '\r\n' : '\n'}${' '.repeat(depth + 1)}` : ''; if (rand() < 0.2) body += pick(TEXTS); for (let k = 0; k < kids; k++) body += indent + element(depth + 1); if (rand() < 0.2) body += pick(TEXTS); @@ -319,6 +353,18 @@ function generate(rand: () => number): string { // Hostile injections, each one a documented bail. They are generated rather than listed so // they land in positions a list would not think to put them in. if (rand() < 0.08) doc += pick(['', '', '']); + // Processing instructions whose name merely STARTS with `xml`, and a repeat of the declaration. + // A prefix test read all of these as the declaration; the engine keys the name it actually read, + // and arrays a repeated key rather than overwriting it. The last one is only a repeat when the + // line above fired, which is the point of generating it instead of listing it. + if (rand() < 0.06) { + doc += pick([ + '', + '', + '', + '', + ]); + } doc += element(0); if (rand() < 0.05) doc += pick(['

', 'trailing', '

', '']); return doc;