From fd04b44728b8c383a3072000c50e5c9f0a120db9 Mon Sep 17 00:00:00 2001 From: Nicola Isotta Date: Tue, 4 Aug 2026 08:54:52 +0200 Subject: [PATCH] XMLSyntaxParser: tolerate a UTF-8 BOM at the start of XML documents XMLSyntaxParser rejected any leading text before the document element with "Invalid token ... found in document", so a BOM character (U+FEFF) caused every BOM-prefixed XML file to fail parsing. Treat a trimmed BOM the same as blank/whitespace when validating text under the Document node. This also adds a test with a BOM xml file and removes some dead code (tokenSequence.token() is always null before the first moveNext()) --- .../xml/xdm/nodes/XMLSyntaxParser.java | 26 ++++++------------- .../xml/xdm/nodes/XMLSyntaxParserTest.java | 18 ++++++++++++- .../modules/xml/xdm/nodes/testBOM.xml | 9 +++++++ 3 files changed, 34 insertions(+), 19 deletions(-) create mode 100644 ide/xml.xdm/test/unit/src/org/netbeans/modules/xml/xdm/nodes/testBOM.xml diff --git a/ide/xml.xdm/src/org/netbeans/modules/xml/xdm/nodes/XMLSyntaxParser.java b/ide/xml.xdm/src/org/netbeans/modules/xml/xdm/nodes/XMLSyntaxParser.java index 7627ccc8debb..ad40dd91852d 100644 --- a/ide/xml.xdm/src/org/netbeans/modules/xml/xdm/nodes/XMLSyntaxParser.java +++ b/ide/xml.xdm/src/org/netbeans/modules/xml/xdm/nodes/XMLSyntaxParser.java @@ -31,7 +31,9 @@ import org.netbeans.api.xml.lexer.XMLTokenId; public class XMLSyntaxParser { - + + private static final String BYTE_ORDER_MARK = "\uFEFF"; + public Document parse(BaseDocument basedoc) throws IOException, BadLocationException { try { @@ -44,22 +46,8 @@ public Document parse(BaseDocument basedoc) List currentTokens = new ArrayList(); TokenHierarchy th = TokenHierarchy.get(basedoc); TokenSequence tokenSequence = th.tokenSequence(); - org.netbeans.api.lexer.Token token = tokenSequence.token(); - // Add the text token, if any, before xml decalration to document node - if(token != null && token.id() == XMLTokenId.TEXT) { - currentTokens.add(Token.create(token.text().toString(),TokenType.TOKEN_CHARACTER_DATA)); - if(tokenSequence.moveNext()) { - token = tokenSequence.token(); - } - // if the xml decalration is not there assign this token to document - if(token.id() != XMLTokenId.PI_START) { - currentNode.setTokens(new ArrayList(currentTokens)); - currentTokens.clear(); - } - } - while (tokenSequence.moveNext()) { - token = tokenSequence.token(); + org.netbeans.api.lexer.Token token = tokenSequence.token(); XMLTokenId tokenId = token.id(); String image = token.text().toString(); TokenType tokenType = TokenType.TOKEN_WHITESPACE; @@ -244,9 +232,11 @@ public Document parse(BaseDocument basedoc) if (parent instanceof Element) { ((Element)parent).appendChild(currentNode, false); } else {//parent is Document + // isBlank cannot be used because it strips unicode whitespaces, + // which are not allowed before the prologue if(token.id() != XMLTokenId.BLOCK_COMMENT && - token.text().toString().trim().length() > 0) { - throw new IOException("Invalid token '" + token.text() + + !image.trim().isEmpty() && !BYTE_ORDER_MARK.equals(image.trim())) { + throw new IOException("Invalid token '" + image + "' found in document: " + "Please use the text editor to resolve the issues..."); } diff --git a/ide/xml.xdm/test/unit/src/org/netbeans/modules/xml/xdm/nodes/XMLSyntaxParserTest.java b/ide/xml.xdm/test/unit/src/org/netbeans/modules/xml/xdm/nodes/XMLSyntaxParserTest.java index fff1190ec0b3..c6401fb139c3 100644 --- a/ide/xml.xdm/test/unit/src/org/netbeans/modules/xml/xdm/nodes/XMLSyntaxParserTest.java +++ b/ide/xml.xdm/test/unit/src/org/netbeans/modules/xml/xdm/nodes/XMLSyntaxParserTest.java @@ -64,6 +64,7 @@ public static Test suite() { suite.addTest(new XMLSyntaxParserTest("testParseWSDL")); // Disabled as referenced files were partly not donated by oracle to apache // suite.addTest(new XMLSyntaxParserTest("testParsePerformace")); + suite.addTest(new XMLSyntaxParserTest("testParseBOM")); return suite; } @@ -254,5 +255,20 @@ public void testParsePerformace() throws Exception { //FlushVisitor fv = new FlushVisitor(); //String docBuf = fv.flushModel(doc); //assertEquals("The document should be unaltered",basedoc.getText(0,basedoc.getLength()),docBuf); - } + } + + /** + * Test of parse method, of class org.netbeans.modules.xmltools.xmlmodel.nodes.XMLSyntaxParser. + * XMLSyntaxParser should handle xml files with BOM + */ + public void testParseBOM() throws Exception { + BaseDocument basedoc = getDocument("nodes/testBOM.xml"); + XMLSyntaxParser parser = new XMLSyntaxParser(); + Document doc = parser.parse(basedoc); + assertNotNull("Document can not be null", doc); + FlushVisitor fv = new FlushVisitor(); + String docBuf = fv.flushModel(doc); + assertEquals("The document should be unaltered",basedoc.getText(0,basedoc.getLength()),docBuf); + } + } diff --git a/ide/xml.xdm/test/unit/src/org/netbeans/modules/xml/xdm/nodes/testBOM.xml b/ide/xml.xdm/test/unit/src/org/netbeans/modules/xml/xdm/nodes/testBOM.xml new file mode 100644 index 000000000000..83943ceda3ff --- /dev/null +++ b/ide/xml.xdm/test/unit/src/org/netbeans/modules/xml/xdm/nodes/testBOM.xml @@ -0,0 +1,9 @@ + + + Vidhya Narayanan + + + +