From 7d939261aac70646088de9960ec00d948c77d054 Mon Sep 17 00:00:00 2001 From: Nico Lube Date: Thu, 21 May 2026 10:06:42 +0200 Subject: [PATCH 1/8] fix: tokenize Gerber commands on end-of-block (*), not newlines Commands are delimited by `*`, not newlines (spec 4.1). The old line-based tokenizer merged single-line files into one block, causing a spurious CoordinateDataWithoutOperationCode error. Now splits on `*`, treats `%...%` spans (incl. macros) as one block, and ends a block at a newline outside such spans. --- src/parser.rs | 113 +++++++++++++++++++++++++++++++++------ tests/component_tests.rs | 53 +++++++++++++++++- 2 files changed, 149 insertions(+), 17 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 433ff61..57fce8b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -22,7 +22,7 @@ use gerber_types::{ use lazy_regex::*; use regex::Regex; use std::collections::HashMap; -use std::io::{BufRead, BufReader, Lines, Read}; +use std::io::{BufReader, Bytes, Read}; use std::iter::FromIterator; use std::str::Chars; use std::sync::LazyLock; @@ -79,36 +79,116 @@ enum ModalOperationMode { } struct ParserContext { + // Physical source line on which the most recently returned block started (1-based). + // Used only for error context; Gerber itself treats newlines as insignificant. line_number: usize, - lines: Lines>, + bytes: Bytes>, + // Running count of newlines consumed so far. + current_line: usize, aperture_attributes: HashMap, object_attributes: HashMap, modal_operation: ModalOperationMode, } impl ParserContext { - pub fn new(lines: Lines>) -> ParserContext { + pub fn new(reader: BufReader) -> ParserContext { ParserContext { line_number: 0, - lines, + bytes: reader.bytes(), + current_line: 1, aperture_attributes: HashMap::new(), object_attributes: HashMap::new(), modal_operation: ModalOperationMode::Undefined, } } + // Yield the next Gerber command block. Commands are delimited by the end-of-block + // char `*` (gerber spec 4.1), so a single physical line may hold many commands. + // Extended commands `%...%` may contain several `*`-terminated words and span many + // lines (e.g. an aperture macro), so the whole `%...%` span is one block. Outside + // such a span a newline also ends the block, isolating each physical line so stray + // junk errors on its own rather than merging into the next command. Whitespace + // inside a block is kept (attribute/comment text may contain spaces) but trimmed. pub fn next(&mut self) -> Option> { - let line = self.lines.next(); - if line.is_some() { - self.line_number += 1; + let mut buf: Vec = Vec::new(); + let mut in_extended = false; + let mut block_line: Option = None; + // Skip whitespace that leads a block (or follows a newline within one). + let mut skip_ws = true; + + loop { + let byte = match self.bytes.next() { + None => break, // EOF + Some(Ok(byte)) => byte, + Some(Err(e)) => { + return Some(Err(ContentError::IoError(format!( + "IO error on line: {}, error: {}", + self.current_line, e + )))); + } + }; + + if byte == b'\r' { + continue; // carriage returns are never significant + } + + if byte == b'\n' { + self.current_line += 1; + // Trim trailing whitespace and skip whatever leads the next line. + while buf.last().is_some_and(u8::is_ascii_whitespace) { + buf.pop(); + } + skip_ws = true; + // Inside a `%...%` span newlines are insignificant; elsewhere a newline + // ends the block, isolating each physical line (junk included). + if !in_extended && !buf.is_empty() { + break; + } + continue; + } + + if skip_ws && byte.is_ascii_whitespace() { + continue; + } + skip_ws = false; + + if block_line.is_none() { + block_line = Some(self.current_line); + } + + match byte { + b'%' if in_extended => { + buf.push(byte); + break; // end of extended `%...%` block + } + b'%' if buf.is_empty() => { + in_extended = true; + buf.push(byte); // start of extended `%...%` block + } + b'*' if !in_extended => { + buf.push(byte); + break; // end-of-block for a normal command + } + _ => buf.push(byte), + } } - line.map(|result| { - result.map_err(|e| { - ContentError::IoError( - format!("IO error on line: {}, error: {}", self.line_number, e).to_string(), - ) - }) - }) + + // Leading whitespace was already skipped; only a block ended by EOF (rather than a + // delimiter) can still carry trailing whitespace, so trim that here. + while buf.last().is_some_and(u8::is_ascii_whitespace) { + buf.pop(); + } + if buf.is_empty() { + return None; // nothing left but trailing whitespace / EOF + } + + self.line_number = block_line.unwrap_or(self.current_line); + Some(String::from_utf8(buf).map_err(|e| { + ContentError::IoError(format!( + "Invalid UTF-8 on line: {}, error: {}", + self.line_number, e + )) + })) } // Update the modal operation mode after a command was parsed (gerber spec 8.3). @@ -141,7 +221,7 @@ impl ParserContext { pub fn parse(reader: BufReader) -> Result { let mut gerber_doc = GerberDoc::default(); - let mut parser_context = ParserContext::new(reader.lines()); + let mut parser_context = ParserContext::new(reader); let mut parse_error: Option = None; @@ -912,6 +992,9 @@ fn parse_aperture_macro_definition( log::trace!("macro chunks: {:?}", chunks); for chunk in chunks { + // A block may span several physical lines; the tokenizer drops newlines but + // keeps the surrounding indentation, so trim each primitive before matching. + let chunk = chunk.trim(); if let Some(stripped) = chunk.strip_prefix("0 ") { // Handle the special-case comment primitive diff --git a/tests/component_tests.rs b/tests/component_tests.rs index e35cc4a..8770fd5 100644 --- a/tests/component_tests.rs +++ b/tests/component_tests.rs @@ -568,6 +568,55 @@ fn deprecated_modal_d01_invalid_without_preceding_d01() { assert_eq!(filtered_commands.len(), 3) } +/// Commands are delimited by the end-of-block char `*`, not by newlines (gerber spec 4.1): +/// a file may pack the whole stream onto a single physical line. Each `*` must still be +/// parsed as its own command, including modal D01 coordinate-only blocks. +#[test] +fn commands_separated_by_block_terminator_on_one_line() { + // given + logging_init(); + + // Header, aperture, and a modal-D01 run all on one line, separated only by `*`. + let reader = gerber_to_reader( + "%FSLAX23Y23*%%MOMM*%%ADD10C, 0.01*%D10*X700Y1000D01*X1200Y1000*X1200Y1300*M02*", + ); + + let fs = CoordinateFormat::new(ZeroOmission::Leading, CoordinateMode::Absolute, 2, 3); + + // when + parse_and_filter!(reader, commands, filtered_commands, |cmd| matches!( + cmd, + Ok(Command::FunctionCode(FunctionCode::DCode( + DCode::Operation(Operation::Interpolate(_, _)) + ))) + )); + + // then + assert_eq_commands!( + filtered_commands, + vec![ + Ok(Command::FunctionCode(FunctionCode::DCode( + DCode::Operation(Operation::Interpolate( + coordinates_from_gerber(700, 1000, fs).unwrap(), + None, + )) + ))), + Ok(Command::FunctionCode(FunctionCode::DCode( + DCode::Operation(Operation::Interpolate( + coordinates_from_gerber(1200, 1000, fs).unwrap(), + None, + )) + ))), + Ok(Command::FunctionCode(FunctionCode::DCode( + DCode::Operation(Operation::Interpolate( + coordinates_from_gerber(1200, 1300, fs).unwrap(), + None, + )) + ))), + ] + ) +} + /// Test the D01* statements (circular) #[test] #[allow(non_snake_case)] @@ -2280,7 +2329,7 @@ fn missing_eof() { let reader = gerber_to_reader( " %FSLAX23Y23*% - %MOMM*%- + %MOMM*% G04 We should have a MO2 at the end, but what if we forget it?* ", @@ -3874,7 +3923,7 @@ fn malformed_aperture_definition() { let reader = gerber_to_reader( " %FSLAX23Y23*% - %MOMM*%- + %MOMM*% G04 Too many parameters * From 0f57c8dc657109b107768049be333f72fb660456 Mon Sep 17 00:00:00 2001 From: Nico Lube Date: Thu, 21 May 2026 11:20:54 +0200 Subject: [PATCH 2/8] feat: add line, offset, and failing token to parse error context Parse errors carried only a (line, content) tuple. Replace it with an ErrorContext { line, offset, token } so callers can pinpoint which command failed, which matters when several are packed onto one line. Also funnel outer parse_line errors through the context path instead of dropping them. --- src/error.rs | 36 ++++++++++---- src/parser.rs | 100 +++++++++++++++++++++++---------------- tests/component_tests.rs | 84 ++++++++++++++++++++++++++------ 3 files changed, 156 insertions(+), 64 deletions(-) diff --git a/src/error.rs b/src/error.rs index 4706d7b..7caed91 100644 --- a/src/error.rs +++ b/src/error.rs @@ -137,9 +137,11 @@ pub enum ContentError { } impl ContentError { - /// line number is 1-based, for humans. - pub fn to_with_context(self, line: Option<(usize, String)>) -> GerberParserErrorWithContext { - GerberParserErrorWithContext { error: self, line } + pub fn to_with_context(self, context: Option) -> GerberParserErrorWithContext { + GerberParserErrorWithContext { + error: self, + context, + } } } @@ -150,18 +152,36 @@ impl PartialEq for ContentError { } } +/// Where in the source a parse error occurred. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct ErrorContext { + /// 1-based line number on which the failing token starts. + pub line: usize, + /// 1-based column offset of the failing token within that line. + pub offset: usize, + /// The token (command block) that failed to parse. + pub token: String, +} + #[derive(Error, Debug, PartialEq)] pub struct GerberParserErrorWithContext { pub error: ContentError, - /// line number is 1-based, for humans. - pub line: Option<(usize, String)>, + pub context: Option, } impl std::fmt::Display for GerberParserErrorWithContext { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match &self.line { - Some((number, content)) => { - write!(f, "Error: {}\nLine {}: '{}'", self.error, number, content) + match &self.context { + Some(ErrorContext { + line, + offset, + token, + }) => { + write!( + f, + "Error: {}\nLine {}:{}: '{}'", + self.error, line, offset, token + ) } _ => { write!(f, "Error at unspecified line: {}", self.error) diff --git a/src/parser.rs b/src/parser.rs index 57fce8b..11547df 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,5 +1,5 @@ use crate::document::GerberDoc; -use crate::error::ContentError; +use crate::error::{ContentError, ErrorContext}; use crate::gerber_types::{ Aperture, ApertureAttribute, ApertureFunction, ApertureMacro, CenterLinePrimitive, Circle, CirclePrimitive, Command, CoordinateFormat, DCode, ExtendedCode, FiducialScope, FileAttribute, @@ -78,13 +78,18 @@ enum ModalOperationMode { Interpolate, } +// 1-based source line and column. Gerber treats newlines as insignificant, so this is +// only carried for error reporting. +#[derive(Copy, Clone)] +struct SourcePos { + line: usize, + offset: usize, +} + struct ParserContext { - // Physical source line on which the most recently returned block started (1-based). - // Used only for error context; Gerber itself treats newlines as insignificant. - line_number: usize, bytes: Bytes>, - // Running count of newlines consumed so far. - current_line: usize, + // Position of the byte stream consumed so far. + current: SourcePos, aperture_attributes: HashMap, object_attributes: HashMap, modal_operation: ModalOperationMode, @@ -93,26 +98,26 @@ struct ParserContext { impl ParserContext { pub fn new(reader: BufReader) -> ParserContext { ParserContext { - line_number: 0, bytes: reader.bytes(), - current_line: 1, + current: SourcePos { line: 1, offset: 0 }, aperture_attributes: HashMap::new(), object_attributes: HashMap::new(), modal_operation: ModalOperationMode::Undefined, } } - // Yield the next Gerber command block. Commands are delimited by the end-of-block - // char `*` (gerber spec 4.1), so a single physical line may hold many commands. - // Extended commands `%...%` may contain several `*`-terminated words and span many - // lines (e.g. an aperture macro), so the whole `%...%` span is one block. Outside - // such a span a newline also ends the block, isolating each physical line so stray - // junk errors on its own rather than merging into the next command. Whitespace - // inside a block is kept (attribute/comment text may contain spaces) but trimmed. - pub fn next(&mut self) -> Option> { + // Yield the next Gerber command block together with the position where it starts. + // Commands are delimited by the end-of-block char `*` (gerber spec 4.1), so a single + // physical line may hold many commands. Extended commands `%...%` may contain several + // `*`-terminated words and span many lines (e.g. an aperture macro), so the whole + // `%...%` span is one block. Outside such a span a newline also ends the block, + // isolating each physical line so stray junk errors on its own rather than merging + // into the next command. Whitespace inside a block is kept (attribute/comment text may + // contain spaces) but trimmed. + pub fn next(&mut self) -> Option> { let mut buf: Vec = Vec::new(); let mut in_extended = false; - let mut block_line: Option = None; + let mut block_start: Option = None; // Skip whitespace that leads a block (or follows a newline within one). let mut skip_ws = true; @@ -123,18 +128,20 @@ impl ParserContext { Some(Err(e)) => { return Some(Err(ContentError::IoError(format!( "IO error on line: {}, error: {}", - self.current_line, e + self.current.line, e )))); } }; + self.current.offset += 1; if byte == b'\r' { continue; // carriage returns are never significant } if byte == b'\n' { - self.current_line += 1; - // Trim trailing whitespace and skip whatever leads the next line. + self.current.line += 1; + self.current.offset = 0; // next byte is column 1 + // Trim trailing whitespace and skip whatever leads the next line. while buf.last().is_some_and(u8::is_ascii_whitespace) { buf.pop(); } @@ -152,8 +159,8 @@ impl ParserContext { } skip_ws = false; - if block_line.is_none() { - block_line = Some(self.current_line); + if block_start.is_none() { + block_start = Some(self.current); } match byte { @@ -182,13 +189,17 @@ impl ParserContext { return None; // nothing left but trailing whitespace / EOF } - self.line_number = block_line.unwrap_or(self.current_line); - Some(String::from_utf8(buf).map_err(|e| { - ContentError::IoError(format!( - "Invalid UTF-8 on line: {}, error: {}", - self.line_number, e - )) - })) + let position = block_start.unwrap_or(self.current); + Some( + String::from_utf8(buf) + .map(|text| (text, position)) + .map_err(|e| { + ContentError::IoError(format!( + "Invalid UTF-8 on line: {}, error: {}", + position.line, e + )) + }), + ) } // Update the modal operation mode after a command was parsed (gerber spec 8.3). @@ -230,10 +241,8 @@ pub fn parse(reader: BufReader) -> Result line, + let (raw_line, position) = match line_result { + Ok(block) => block, Err(ContentError::IoError(error)) => { parse_error = Some(ParseError::IoError(error)); break; @@ -242,12 +251,17 @@ pub fn parse(reader: BufReader) -> Result results, + Err(error) => vec![Err(error)], + }; + for result in line_results { let final_result = match result { Ok(command) => { log::trace!("Parsed command: {:?}", command); @@ -260,8 +274,12 @@ pub fn parse(reader: BufReader) -> Result { - let contexted_error = error_without_context - .to_with_context(Some((line_number, line.to_string()))); + let contexted_error = + error_without_context.to_with_context(Some(ErrorContext { + line: position.line, + offset: position.offset, + token: line.to_string(), + })); log::error!("Content error: {}", contexted_error); Err(contexted_error) } @@ -964,8 +982,8 @@ fn parse_aperture_macro_definition( let Some(line_result) = parser_context.next() else { break; }; - let line = line_result?.trim().to_string(); - macro_content.push_str(&line); + let (block, _) = line_result?; + macro_content.push_str(block.trim()); } // Extract the macro name from the AM command diff --git a/tests/component_tests.rs b/tests/component_tests.rs index 8770fd5..dea5e09 100644 --- a/tests/component_tests.rs +++ b/tests/component_tests.rs @@ -1,4 +1,4 @@ -use gerber_parser::{parse, ContentError, GerberParserErrorWithContext}; +use gerber_parser::{parse, ContentError, ErrorContext, GerberParserErrorWithContext}; use gerber_types::{ Aperture, ApertureAttribute, ApertureBlock, ApertureDefinition, ApertureFunction, ApertureMacro, AxisSelect, Circle, CirclePrimitive, Command, CommentContent, @@ -617,6 +617,52 @@ fn commands_separated_by_block_terminator_on_one_line() { ) } +/// Error context reports the line, column offset, and failing token. With several commands +/// packed onto one line, the offset is what pinpoints which one failed. +#[test] +fn error_context_reports_line_offset_and_token() { + // given + logging_init(); + + // `X100Y100*` at column 21 is invalid: no D01 in modal effect (gerber spec 8.3). + let reader = gerber_to_reader("%FSLAX23Y23*%%MOMM*%X100Y100*M02*"); + + // when + let doc = parse(reader).unwrap(); + + // then + assert!(matches!( + doc.errors().first().unwrap(), + GerberParserErrorWithContext { + error: ContentError::CoordinateDataWithoutOperationCode, + context: Some(ErrorContext { line, offset, token }), + } if *line == 1 && *offset == 21 && token.eq("X100Y100*") + )); +} + +/// A truncated command makes `parse_line` bail before producing any command. That outer +/// error must still be recorded with context, not silently dropped. +#[test] +fn outer_parse_error_is_recorded_with_context() { + // given + logging_init(); + + // A lone `G` is truncated: `parse_line` returns an outer Err before any command. + let reader = gerber_to_reader("%FSLAX23Y23*%\n%MOMM*%\nG\nM02*\n"); + + // when + let doc = parse(reader).unwrap(); + + // then + assert!(doc.errors().iter().any(|error| matches!( + error, + GerberParserErrorWithContext { + error: ContentError::UnknownCommand {}, + context: Some(ErrorContext { line, token, .. }), + } if *line == 3 && token.eq("G") + ))); +} + /// Test the D01* statements (circular) #[test] #[allow(non_snake_case)] @@ -955,7 +1001,11 @@ fn test_load_scaling_zero() { error: ContentError::InvalidParameter { parameter, }, - line: Some((number, content)), + context: Some(ErrorContext { + line: number, + token: content, + .. + }), } if parameter.eq("0") && *number == 2 && content.eq("%LS0*%") )); } @@ -2433,7 +2483,11 @@ fn coordinates_not_within_format() { format, cause: GerberError::CoordinateFormatError(_) }, - line: Some((number, content)), + context: Some(ErrorContext { + line: number, + token: content, + .. + }), } if format.integer == 2 && format.decimal == 3 && *number == 8 && content.eq("X100000Y0D01*") )); } @@ -3969,7 +4023,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("C") && content.eq("%ADD10C,1X2X3*%") )); @@ -3981,7 +4035,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("C") && content.eq("%ADD10C*%") )); @@ -3993,7 +4047,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("R") && content.eq("%ADD10R,1X2X3X4*%") )); @@ -4005,7 +4059,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("R") && content.eq("%ADD10R,1*%") )); @@ -4017,7 +4071,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("R") && content.eq("%ADD10R*%") )); @@ -4029,7 +4083,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("O") && content.eq("%ADD10O,1X2X3X4*%") )); @@ -4041,7 +4095,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("O") && content.eq("%ADD10O,1*%") )); @@ -4053,7 +4107,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("O") && content.eq("%ADD10O*%") )); @@ -4065,7 +4119,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("P") && content.eq("%ADD10P,1X2X3X4X5*%") )); @@ -4077,7 +4131,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("P") && content.eq("%ADD10P,1*%") )); @@ -4089,7 +4143,7 @@ fn malformed_aperture_definition() { aperture_code, aperture_name, }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if *aperture_code == 10 && aperture_name.eq("P") && content.eq("%ADD10P*%") )); @@ -4100,7 +4154,7 @@ fn malformed_aperture_definition() { error: ContentError::UnknownApertureType { type_str }, - line: Some((_line_number, content)), + context: Some(ErrorContext { token: content, .. }), } if type_str.eq("T") && content.eq("%ADD10T*%") )); } From 0e2a68d3dcaadf933d7b502e85450ff4e8f991d7 Mon Sep 17 00:00:00 2001 From: Nico Lube Date: Thu, 21 May 2026 11:26:35 +0200 Subject: [PATCH 3/8] fix: support deprecated single-digit G-codes (G1/G2/G3) ViewMate and other tools emit G1/G2/G3 instead of G01/G02/G03 (spec 8.3 style variation). These fell through to UnknownCommand, so the interpolate never registered and modal D01 never armed, cascading into spurious CoordinateDataWithoutOperationCode errors across every region. Accept the single-digit forms, disambiguating G3 from the G36/G37 region commands. --- src/parser.rs | 141 ++++++++++++++++++++++----------------- tests/component_tests.rs | 51 ++++++++++++++ 2 files changed, 130 insertions(+), 62 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 11547df..2f7bee3 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -305,6 +305,32 @@ pub fn parse(reader: BufReader) -> Result Vec> { + // Sized for two: the combined form adds a second command, and these G-codes are the + // hot path in large flashed/region files. + let mut commands = Vec::with_capacity(2); + commands.push(Ok( + FunctionCode::GCode(GCode::InterpolationMode(mode)).into() + )); + // More than the trailing `*` means operation data follows on the same block. + if remaining_line.len() > 1 { + commands.push(parse_interpolate_move_or_flash( + remaining_line, + gerber_doc, + modal, + )); + } + commands +} + fn parse_line( line: &str, gerber_doc: &mut GerberDoc, @@ -316,68 +342,59 @@ fn parse_line( // Safety: already explicitly checked that the line is not empty 'G' => { match linechars.next().ok_or(ContentError::UnknownCommand {})? { - '0' => { - let remaining_line = &line[3..]; - let using_deprecated_syntax = remaining_line.len() > 1; - let mut commands = Vec::with_capacity(1); - match linechars.next().ok_or(ContentError::UnknownCommand {})? { - '1' => { - // G01 - commands.push(Ok(FunctionCode::GCode(GCode::InterpolationMode( - InterpolationMode::Linear, - )) - .into())); - if using_deprecated_syntax { - commands.push(parse_interpolate_move_or_flash( - remaining_line, - gerber_doc, - parser_context.modal_operation, - )); - } - } - '2' => { - // G02 - commands.push(Ok(FunctionCode::GCode(GCode::InterpolationMode( - InterpolationMode::ClockwiseCircular, - )) - .into())); - if using_deprecated_syntax { - commands.push(parse_interpolate_move_or_flash( - remaining_line, - gerber_doc, - parser_context.modal_operation, - )); - } - } - '3' => { - // G03 - commands.push(Ok(FunctionCode::GCode(GCode::InterpolationMode( - InterpolationMode::CounterclockwiseCircular, - )) - .into())); - if using_deprecated_syntax { - commands.push(parse_interpolate_move_or_flash( - remaining_line, - gerber_doc, - parser_context.modal_operation, - )); - } - } - '4' => { - // G04 - commands.push(parse_comment(line, parser_context)) - } - _ => commands.push(Err(ContentError::UnknownCommand {})), - } - Ok(commands) - } - '3' => Ok(vec![ - match linechars.next().ok_or(ContentError::UnknownCommand {})? { - '6' => Ok(FunctionCode::GCode(GCode::RegionMode(true)).into()), // G36 - '7' => Ok(FunctionCode::GCode(GCode::RegionMode(false)).into()), // G37 - _ => Err(ContentError::UnknownCommand {}), - }, - ]), + '0' => match linechars.next().ok_or(ContentError::UnknownCommand {})? { + '1' => Ok(interpolation_mode_commands( + InterpolationMode::Linear, + &line[3..], + gerber_doc, + parser_context.modal_operation, + )), + '2' => Ok(interpolation_mode_commands( + InterpolationMode::ClockwiseCircular, + &line[3..], + gerber_doc, + parser_context.modal_operation, + )), + '3' => Ok(interpolation_mode_commands( + InterpolationMode::CounterclockwiseCircular, + &line[3..], + gerber_doc, + parser_context.modal_operation, + )), + '4' => Ok(vec![parse_comment(line, parser_context)]), + _ => Ok(vec![Err(ContentError::UnknownCommand {})]), + }, + // Deprecated single-digit interpolation modes `G1`/`G2`/`G3` (gerber spec + // 8.3 style variations); ViewMate and others emit these instead of `G0n`. + '1' => Ok(interpolation_mode_commands( + InterpolationMode::Linear, + &line[2..], + gerber_doc, + parser_context.modal_operation, + )), + '2' => Ok(interpolation_mode_commands( + InterpolationMode::ClockwiseCircular, + &line[2..], + gerber_doc, + parser_context.modal_operation, + )), + // `G3` is ambiguous: the G36/G37 region commands, or deprecated single-digit + // `G3` (= G03). Peek the next char; `6`/`7` select a region, anything else is + // taken as G03 (matching how `G1`/`G01` tolerate trailing operation data). + '3' => match linechars.next() { + Some('6') => Ok(vec![ + Ok(FunctionCode::GCode(GCode::RegionMode(true)).into()), + ]), + Some('7') => Ok(vec![Ok( + FunctionCode::GCode(GCode::RegionMode(false)).into() + )]), + _ => Ok(interpolation_mode_commands( + InterpolationMode::CounterclockwiseCircular, + &line[2..], + gerber_doc, + parser_context.modal_operation, + )), + }, '7' => Ok(vec![ match linechars.next().ok_or(ContentError::UnknownCommand {})? { // the G74 command is technically part of the Deprecated commands diff --git a/tests/component_tests.rs b/tests/component_tests.rs index dea5e09..70ec04b 100644 --- a/tests/component_tests.rs +++ b/tests/component_tests.rs @@ -663,6 +663,57 @@ fn outer_parse_error_is_recorded_with_context() { ))); } +/// Deprecated single-digit G-codes `G1`/`G2`/`G3` (gerber spec 8.3 style variations) must +/// parse like `G01`/`G02`/`G03`, including the combined `G1X..Y..D1*` form that arms modal +/// D01. `G3` must still not shadow the `G36`/`G37` region commands. As emitted by ViewMate. +#[test] +fn deprecated_single_digit_g_codes() { + // given + logging_init(); + + // A region drawn with single-digit `G1` + combined D01, then modal-D01 coordinate lines. + let reader = gerber_to_reader( + "%FSLAX25Y25*%%MOIN*%%ADD111C,0.03937*%D111*X0Y0D2*G36*G1X200Y100D1*X200Y200*X100Y200*G37*M02*", + ); + + // when + let doc = parse(reader).unwrap(); + + // then + assert!( + doc.errors().is_empty(), + "unexpected errors: {:?}", + doc.errors() + ); + let ok_commands: Vec<_> = doc + .commands + .iter() + .filter_map(|c| c.as_ref().ok()) + .collect(); + // G36 open and G37 close survived the `G3` disambiguation. + assert!(ok_commands.iter().any(|c| matches!( + c, + Command::FunctionCode(FunctionCode::GCode(GCode::RegionMode(true))) + ))); + assert!(ok_commands.iter().any(|c| matches!( + c, + Command::FunctionCode(FunctionCode::GCode(GCode::RegionMode(false))) + ))); + // The explicit `G1...D1` plus two modal-D01 lines = three interpolations. + let interpolations = ok_commands + .iter() + .filter(|c| { + matches!( + c, + Command::FunctionCode(FunctionCode::DCode(DCode::Operation( + Operation::Interpolate(..) + ))) + ) + }) + .count(); + assert_eq!(interpolations, 3); +} + /// Test the D01* statements (circular) #[test] #[allow(non_snake_case)] From 27b5d5a18f16349e487725e8478934dc6bb70161 Mon Sep 17 00:00:00 2001 From: Nico Lube Date: Fri, 4 Sep 2026 00:59:20 +0200 Subject: [PATCH 4/8] fix: don't split blocks on backslash-escaped * or % in strings Gerber spec 3.4.3 requires \, *, and % to be backslash-escaped inside strings. The tokenizer treated every raw * and % as a delimiter, so an escaped one inside a comment or TF/TA/TO value truncated the block early. --- src/parser.rs | 45 ++++++++++++++++++++++++++-------------- tests/component_tests.rs | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 15 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 2f7bee3..b0f6fdb 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -120,6 +120,10 @@ impl ParserContext { let mut block_start: Option = None; // Skip whitespace that leads a block (or follows a newline within one). let mut skip_ws = true; + // Whether the previous byte was an unescaped `\`, the string escape character + // (gerber spec 3.4.3). An escaped `*` or `%` is string content, not a delimiter; + // this only keeps such a byte from splitting the block, it does not decode escapes. + let mut escaped = false; loop { let byte = match self.bytes.next() { @@ -146,8 +150,9 @@ impl ParserContext { buf.pop(); } skip_ws = true; - // Inside a `%...%` span newlines are insignificant; elsewhere a newline - // ends the block, isolating each physical line (junk included). + escaped = false; // a lone `\` right before a newline isn't a valid escape + // Inside a `%...%` span newlines are insignificant; elsewhere a newline + // ends the block, isolating each physical line (junk included). if !in_extended && !buf.is_empty() { break; } @@ -163,20 +168,30 @@ impl ParserContext { block_start = Some(self.current); } - match byte { - b'%' if in_extended => { - buf.push(byte); - break; // end of extended `%...%` block - } - b'%' if buf.is_empty() => { - in_extended = true; - buf.push(byte); // start of extended `%...%` block - } - b'*' if !in_extended => { - buf.push(byte); - break; // end-of-block for a normal command + if escaped { + // The byte right after an unescaped `\` is always literal content. + escaped = false; + buf.push(byte); + } else { + match byte { + b'\\' => { + escaped = true; + buf.push(byte); + } + b'%' if in_extended => { + buf.push(byte); + break; // end of extended `%...%` block + } + b'%' if buf.is_empty() => { + in_extended = true; + buf.push(byte); // start of extended `%...%` block + } + b'*' if !in_extended => { + buf.push(byte); + break; // end-of-block for a normal command + } + _ => buf.push(byte), } - _ => buf.push(byte), } } diff --git a/tests/component_tests.rs b/tests/component_tests.rs index 70ec04b..1768019 100644 --- a/tests/component_tests.rs +++ b/tests/component_tests.rs @@ -254,6 +254,48 @@ fn G04_comments() { ); } +#[test] +fn escaped_reserved_characters_in_strings() { + // given + logging_init(); + + // Gerber spec 3.4.3: `\` escapes reserved characters in strings, notably `*` and `%` + // (the block/extended-block delimiters). An escaped `*` inside a comment, or an + // escaped `%` inside a `%...%` extended block, must not end the block early. + let reader = gerber_to_reader( + " + G04 escaped \\* asterisk and \\% percent survive as comment text* + %TF.MD5,abc\\%def*% + M02* + ", + ); + + // when + parse_and_filter!(reader, commands, filtered_commands, |cmd| matches!( + cmd, + Ok(Command::FunctionCode(FunctionCode::GCode(GCode::Comment( + _ + )))) | Ok(Command::ExtendedCode(ExtendedCode::FileAttribute( + FileAttribute::Md5(_) + ))) + )); + + // then + assert_eq!( + filtered_commands, + vec![ + Ok(Command::FunctionCode(FunctionCode::GCode(GCode::Comment( + CommentContent::String( + "escaped \\* asterisk and \\% percent survive as comment text".to_string() + ) + )))), + Ok(Command::ExtendedCode(ExtendedCode::FileAttribute( + FileAttribute::Md5("abc\\%def".to_string()) + ))), + ] + ); +} + #[test] fn aperture_selection() { // given From 84b53d2cefc64b8130327f814c0e2f9349f6721d Mon Sep 17 00:00:00 2001 From: Nico Lube Date: Fri, 4 Sep 2026 14:34:35 +0200 Subject: [PATCH 5/8] Revert "fix: don't split blocks on backslash-escaped * or %" This reverts commit 7d1822ee309ebd0bf861c4252928d7dfdee5100c. Gerber spec 3.4.3 defines only unicode escapes (\uXXXX, \UXXXXXXXX); there is no C-style `\*` form, and 3.4.4 excludes `*`/`%` from the field grammar outright. A conforming string therefore never contains a raw delimiter, so the tokenizer needs no escape awareness. --- src/parser.rs | 45 ++++++++++++++-------------------------- tests/component_tests.rs | 42 ------------------------------------- 2 files changed, 15 insertions(+), 72 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index b0f6fdb..2f7bee3 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -120,10 +120,6 @@ impl ParserContext { let mut block_start: Option = None; // Skip whitespace that leads a block (or follows a newline within one). let mut skip_ws = true; - // Whether the previous byte was an unescaped `\`, the string escape character - // (gerber spec 3.4.3). An escaped `*` or `%` is string content, not a delimiter; - // this only keeps such a byte from splitting the block, it does not decode escapes. - let mut escaped = false; loop { let byte = match self.bytes.next() { @@ -150,9 +146,8 @@ impl ParserContext { buf.pop(); } skip_ws = true; - escaped = false; // a lone `\` right before a newline isn't a valid escape - // Inside a `%...%` span newlines are insignificant; elsewhere a newline - // ends the block, isolating each physical line (junk included). + // Inside a `%...%` span newlines are insignificant; elsewhere a newline + // ends the block, isolating each physical line (junk included). if !in_extended && !buf.is_empty() { break; } @@ -168,30 +163,20 @@ impl ParserContext { block_start = Some(self.current); } - if escaped { - // The byte right after an unescaped `\` is always literal content. - escaped = false; - buf.push(byte); - } else { - match byte { - b'\\' => { - escaped = true; - buf.push(byte); - } - b'%' if in_extended => { - buf.push(byte); - break; // end of extended `%...%` block - } - b'%' if buf.is_empty() => { - in_extended = true; - buf.push(byte); // start of extended `%...%` block - } - b'*' if !in_extended => { - buf.push(byte); - break; // end-of-block for a normal command - } - _ => buf.push(byte), + match byte { + b'%' if in_extended => { + buf.push(byte); + break; // end of extended `%...%` block + } + b'%' if buf.is_empty() => { + in_extended = true; + buf.push(byte); // start of extended `%...%` block + } + b'*' if !in_extended => { + buf.push(byte); + break; // end-of-block for a normal command } + _ => buf.push(byte), } } diff --git a/tests/component_tests.rs b/tests/component_tests.rs index 1768019..70ec04b 100644 --- a/tests/component_tests.rs +++ b/tests/component_tests.rs @@ -254,48 +254,6 @@ fn G04_comments() { ); } -#[test] -fn escaped_reserved_characters_in_strings() { - // given - logging_init(); - - // Gerber spec 3.4.3: `\` escapes reserved characters in strings, notably `*` and `%` - // (the block/extended-block delimiters). An escaped `*` inside a comment, or an - // escaped `%` inside a `%...%` extended block, must not end the block early. - let reader = gerber_to_reader( - " - G04 escaped \\* asterisk and \\% percent survive as comment text* - %TF.MD5,abc\\%def*% - M02* - ", - ); - - // when - parse_and_filter!(reader, commands, filtered_commands, |cmd| matches!( - cmd, - Ok(Command::FunctionCode(FunctionCode::GCode(GCode::Comment( - _ - )))) | Ok(Command::ExtendedCode(ExtendedCode::FileAttribute( - FileAttribute::Md5(_) - ))) - )); - - // then - assert_eq!( - filtered_commands, - vec![ - Ok(Command::FunctionCode(FunctionCode::GCode(GCode::Comment( - CommentContent::String( - "escaped \\* asterisk and \\% percent survive as comment text".to_string() - ) - )))), - Ok(Command::ExtendedCode(ExtendedCode::FileAttribute( - FileAttribute::Md5("abc\\%def".to_string()) - ))), - ] - ); -} - #[test] fn aperture_selection() { // given From 655f9cf1e8fc343f8e5c24ecbac88d4821f8a4f5 Mon Sep 17 00:00:00 2001 From: Nico Lube Date: Fri, 4 Sep 2026 14:34:55 +0200 Subject: [PATCH 6/8] test: round-trip unicode-escaped strings and fields Gerber spec 3.4.3/3.4.4: reserved characters (`*`, `%`, `\`, and `,` in fields) are written as unicode escapes, so a conforming string cannot contain a block delimiter. Escapes are kept verbatim, making the gbr -> rust -> gbr round-trip byte-exact. --- assets/reference_files/escaped_strings.gbr | 10 +++++ tests/component_tests.rs | 47 ++++++++++++++++++++++ tests/reference_file_tests.rs | 25 ++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 assets/reference_files/escaped_strings.gbr diff --git a/assets/reference_files/escaped_strings.gbr b/assets/reference_files/escaped_strings.gbr new file mode 100644 index 0000000..363740b --- /dev/null +++ b/assets/reference_files/escaped_strings.gbr @@ -0,0 +1,10 @@ +G04 escaped asterisk \u002A, percent \u0025 and backslash \u005C in a comment* +%MOMM*% +%FSLAX26Y26*% +%TFMyEscapes,escaped comma \u002C in a field,escaped asterisk \u002A*% +%ADD10C,0.01*% +D10* +X0Y0D02* +G01* +X5000000Y0D01* +M02* diff --git a/tests/component_tests.rs b/tests/component_tests.rs index 70ec04b..451bd4f 100644 --- a/tests/component_tests.rs +++ b/tests/component_tests.rs @@ -254,6 +254,53 @@ fn G04_comments() { ); } +#[test] +fn unicode_escaped_reserved_characters_in_strings() { + // given + logging_init(); + + // Gerber spec 3.4.3: reserved characters in a string are written as a unicode escape, + // `\u002A` for `*` and `\u0025` for `%`, never as a raw byte (3.4.4 excludes them from the + // field grammar outright). A conforming string therefore cannot contain the block + // delimiters, and the parser keeps the escape sequence verbatim. + let reader = gerber_to_reader( + " + G04 escaped \\u002A asterisk and \\u0025 percent survive as comment text* + %TFMyEscapes,escaped comma \\u002C in a field*% + M02* + ", + ); + + // when + parse_and_filter!(reader, commands, filtered_commands, |cmd| matches!( + cmd, + Ok(Command::FunctionCode(FunctionCode::GCode(GCode::Comment( + _ + )))) | Ok(Command::ExtendedCode(ExtendedCode::FileAttribute( + FileAttribute::UserDefined { .. } + ))) + )); + + // then + assert_eq!( + filtered_commands, + vec![ + Ok(Command::FunctionCode(FunctionCode::GCode(GCode::Comment( + CommentContent::String( + "escaped \\u002A asterisk and \\u0025 percent survive as comment text" + .to_string() + ) + )))), + Ok(Command::ExtendedCode(ExtendedCode::FileAttribute( + FileAttribute::UserDefined { + name: "MyEscapes".to_string(), + values: vec!["escaped comma \\u002C in a field".to_string()], + } + ))), + ] + ); +} + #[test] fn aperture_selection() { // given diff --git a/tests/reference_file_tests.rs b/tests/reference_file_tests.rs index 509fdad..16401dc 100644 --- a/tests/reference_file_tests.rs +++ b/tests/reference_file_tests.rs @@ -74,3 +74,28 @@ fn a_drill_file_to_rust_and_back() { doc.commands ) } + +/// Gerber spec 3.4.3 and 3.4.4: the reserved characters `*` (`*`), `%` (`%`), +/// `\` (`\`) and, in fields, `,` (`,`) are expressed as unicode escapes and never +/// as raw bytes, so they cannot be confused with the block delimiters. Escape sequences are +/// kept verbatim by the parser, which is what makes the round-trip byte-exact. +#[test] +fn escaped_strings_to_rust() { + let gbr_string = include_str!("../assets/reference_files/escaped_strings.gbr"); + let reader = gerber_to_reader(gbr_string); + parse(reader).unwrap(); +} + +#[test] +fn escaped_strings_to_rust_and_back() { + let gbr_string = include_str!("../assets/reference_files/escaped_strings.gbr"); + let reader = gerber_to_reader(gbr_string); + let doc = parse(reader).unwrap(); + + assert_eq!( + gerber_doc_as_str(&doc), + gbr_string, + "unexpected differences, commands: {:?}", + doc.commands + ) +} From 7b80a15888e641b7388f2a3de9fa678acf048704 Mon Sep 17 00:00:00 2001 From: Nico Lube Date: Wed, 7 Oct 2026 21:59:07 +0200 Subject: [PATCH 7/8] feat: decode Gerber Unicode escapes in strings and fields --- README.md | 6 +++ src/error.rs | 2 + src/parser.rs | 88 +++++++++++++++++++++++++++++---- tests/component_tests.rs | 91 +++++++++++++++++++++++++++++++++-- tests/reference_file_tests.rs | 25 ++++++---- 5 files changed, 189 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 9dd1c8a..dbd7371 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,12 @@ The gerber specification 2024.05 is the latest version of the Gerber file format Contributions to improve support welcomed! +### Unicode escapes + +String and attribute field values are decoded from Gerber `\uXXXX` and `\UXXXXXXXX` +escapes into Unicode characters. Invalid escapes produce content errors with source context. +Escaped delimiters are decoded after splitting commands and fields. + ### Image Transformations Image transformations have been deprecated since December 2012 (I1 revision). diff --git a/src/error.rs b/src/error.rs index 7caed91..82bf7fb 100644 --- a/src/error.rs +++ b/src/error.rs @@ -14,6 +14,8 @@ pub enum ParseError { #[non_exhaustive] #[derive(Error, Debug)] pub enum ContentError { + #[error("Invalid Unicode escape in string: {value:?}")] + InvalidUnicodeEscape { value: String }, #[error("Document included a line that isn't valid.")] UnknownCommand {}, #[error("Document included a line that isn't supported.")] diff --git a/src/parser.rs b/src/parser.rs index 2f7bee3..42b6421 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -904,7 +904,8 @@ fn parse_comment( Some(regmatch) => { let string_content = regmatch .name("string") - .map(|string| string.as_str().to_string()); + .map(|string| decode_unicode_escapes(string.as_str())) + .transpose()?; let standard_content = regmatch .name("standard") @@ -977,7 +978,7 @@ fn parse_image_name(line: &str, gerber_doc: &GerberDoc) -> Result Err(ContentError::NoRegexMatch { regex: RE_IMAGE_NAME.clone(), @@ -1035,7 +1036,9 @@ fn parse_aperture_macro_definition( // Gerber spec: 4.5.1.2 "The comment primitive starts with the ‘0’ code followed by a space and then a // single-line text string" - content.push(MacroContent::Comment(stripped.trim().to_string())); + content.push(MacroContent::Comment(decode_unicode_escapes( + stripped.trim(), + )?)); continue; } @@ -1933,7 +1936,8 @@ fn parse_file_attribute(line: Chars) -> Result { }; } - let attr_args = attr_args(line); + let decoded_args = attr_args(line)?; + let attr_args: Vec<&str> = decoded_args.iter().map(String::as_str).collect(); log::trace!("TF args: {:?}, len: {}", attr_args, attr_args.len()); @@ -2236,7 +2240,8 @@ fn parse_aperture_attribute( } let raw_line = partial_line.as_str().to_string(); - let attr_args = attr_args(partial_line); + let decoded_args = attr_args(partial_line)?; + let attr_args: Vec<&str> = decoded_args.iter().map(String::as_str).collect(); log::trace!("TA ARGS: {:?}", attr_args); @@ -2372,7 +2377,8 @@ fn parse_object_attribute(partial_line: Chars) -> Result<(String, ObjectAttribut }}; } - let attr_args = attr_args(partial_line); + let decoded_args = attr_args(partial_line)?; + let attr_args: Vec<&str> = decoded_args.iter().map(String::as_str).collect(); log::trace!("TO ARGS: {:?}", attr_args); @@ -2589,11 +2595,50 @@ fn parse_macro_integer(value: &str) -> Result { } } -fn attr_args(partial_line: Chars<'_>) -> Vec<&str> { +// Decode only after tokenizing commands and splitting fields. Decoded delimiters are +// text, and a decoded backslash must never start a second round of decoding. +fn decode_unicode_escapes(value: &str) -> Result { + let mut decoded = String::with_capacity(value.len()); + let mut chars = value.chars(); + let invalid = || ContentError::InvalidUnicodeEscape { + value: value.to_string(), + }; + while let Some(ch) = chars.next() { + if ch != '\\' { + decoded.push(ch); + continue; + } + let digits = match chars.next() { + Some('u') => 4, + Some('U') => 8, + _ => return Err(invalid()), + }; + let mut codepoint = 0u32; + for _ in 0..digits { + let digit = chars + .next() + .and_then(|ch| ch.to_digit(16)) + .ok_or_else(invalid)?; + codepoint = (codepoint << 4) | digit; + } + decoded.push(char::from_u32(codepoint).ok_or_else(invalid)?); + } + Ok(decoded) +} + +fn attr_args(partial_line: Chars<'_>) -> Result, ContentError> { partial_line .as_str() .split(',') - .map(|el| el.trim()) + .enumerate() + .map(|(index, el)| { + // Attribute names use the ASCII name grammar, not the string grammar. + if index == 0 { + Ok(el.trim().to_string()) + } else { + decode_unicode_escapes(el.trim()) + } + }) .collect() } @@ -2612,10 +2657,35 @@ fn trim_attr_line(mut partial_line: Chars) -> Result { mod attr_args_tests { use super::*; + #[test] + fn unicode_escapes_follow_spec_3_4_3() { + for (encoded, expected) in [ + (r"\u00A9", "©"), + (r"\U000000a9", "©"), + (r"\u0025\u002A\u005C\u002C", "%*\\,"), + (r"\u0041B", "AB"), + (r"\U00000041B", "AB"), + (r"\U0010FFFF", "\u{10ffff}"), + (r"\u0000", "\0"), + (r"\u005Cu00A9", r"\u00A9"), + ("日本語", "日本語"), + ] { + assert_eq!(decode_unicode_escapes(encoded).unwrap(), expected); + } + } + + #[test] + fn attribute_names_are_not_string_fields() { + let args = attr_args(r"Custom,\u0041,\u002C".chars()).unwrap(); + assert_eq!(args, ["Custom", "A", ","]); + let args = attr_args(r"\u0043ustom,\u0041".chars()).unwrap(); + assert_eq!(args[0], r"\u0043ustom"); + } + #[test] pub fn test_attr_args() { let attribute_chars = " .DrillTolerance , 0.02 , 0.01 ".chars(); - let arguments = attr_args(attribute_chars); + let arguments = attr_args(attribute_chars).unwrap(); println!("arguments: {:?}", arguments); assert_eq!(arguments, vec![".DrillTolerance", "0.02", "0.01"]) } diff --git a/tests/component_tests.rs b/tests/component_tests.rs index 451bd4f..edda3c8 100644 --- a/tests/component_tests.rs +++ b/tests/component_tests.rs @@ -262,7 +262,7 @@ fn unicode_escaped_reserved_characters_in_strings() { // Gerber spec 3.4.3: reserved characters in a string are written as a unicode escape, // `\u002A` for `*` and `\u0025` for `%`, never as a raw byte (3.4.4 excludes them from the // field grammar outright). A conforming string therefore cannot contain the block - // delimiters, and the parser keeps the escape sequence verbatim. + // delimiters. The parser decodes escapes only after splitting commands and fields. let reader = gerber_to_reader( " G04 escaped \\u002A asterisk and \\u0025 percent survive as comment text* @@ -287,14 +287,13 @@ fn unicode_escaped_reserved_characters_in_strings() { vec![ Ok(Command::FunctionCode(FunctionCode::GCode(GCode::Comment( CommentContent::String( - "escaped \\u002A asterisk and \\u0025 percent survive as comment text" - .to_string() + "escaped * asterisk and % percent survive as comment text".to_string() ) )))), Ok(Command::ExtendedCode(ExtendedCode::FileAttribute( FileAttribute::UserDefined { name: "MyEscapes".to_string(), - values: vec!["escaped comma \\u002C in a field".to_string()], + values: vec!["escaped comma , in a field".to_string()], } ))), ] @@ -991,7 +990,7 @@ fn omitted_coordinate() { // Compare the two results nanos for (i, cmd) in filtered_commands.iter().enumerate() { let cmd2 = &filtered_commands2[i]; - match { (cmd, cmd2) } { + match (cmd, cmd2) { ( Ok(Command::FunctionCode(FunctionCode::DCode(DCode::Operation(Operation::Flash( Some(c1), @@ -4296,3 +4295,85 @@ fn oversized_coordinate_formats_do_not_panic() { assert!(doc.commands.iter().any(Result::is_err), "{gerber}"); } } + +#[test] +fn unicode_escape_decoding_across_string_fields() { + let input = r"G04 café \u00a9 \U0001F680 \u005Cu002A* +%TFCustom,\u0020hello\u0020,\u002C\u002A\u0025*% +%TALabel,\u03A9*% +%TO.C,R\u0031*% +G04 #@! TFNote,\u65E5\u672C* +%AMExample*0 \u00A9 \U0001F680*1,1,1,0,0*% +M02*"; + let doc = parse(gerber_to_reader(input)).unwrap(); + let expected: Vec = vec![ + GCode::Comment(CommentContent::String("café © 🚀 \\u002A".into())).into(), + ExtendedCode::FileAttribute(FileAttribute::UserDefined { + name: "Custom".into(), + values: vec![" hello ".into(), ",*%".into()], + }) + .into(), + ExtendedCode::ApertureAttribute(ApertureAttribute::UserDefined { + name: "Label".into(), + values: vec!["Ω".into()], + }) + .into(), + ExtendedCode::ObjectAttribute(ObjectAttribute::Component("R1".into())).into(), + GCode::Comment(CommentContent::Standard(StandardComment::FileAttribute( + FileAttribute::UserDefined { + name: "Note".into(), + values: vec!["日本".into()], + }, + ))) + .into(), + ExtendedCode::ApertureMacro(ApertureMacro { + name: "Example".into(), + content: vec![ + MacroContent::Comment("© 🚀".into()), + MacroContent::Circle(CirclePrimitive { + exposure: MacroBoolean::Value(true), + diameter: MacroDecimal::Value(1.0), + center: (MacroDecimal::Value(0.0), MacroDecimal::Value(0.0)), + angle: None, + }), + ], + }) + .into(), + MCode::EndOfFile.into(), + ]; + let actual: Vec<_> = doc.commands.into_iter().map(Result::unwrap).collect(); + assert_eq!(actual, expected); +} + +#[test] +fn invalid_unicode_escapes_report_context_and_continue() { + for escape in [ + r"\u123", + r"\u12g4", + r"\U00110000", + r"\uD800", + r"\U0000DFFF", + r"\UFFFFFFFF", + r"\q", + "\\", + "\\u日000", + ] { + for token in [ + format!("G04 {}*", escape), + format!("%TFCustom,{}*%", escape), + ] { + let input = format!("{}M02*", token); + let doc = parse(gerber_to_reader(&input)).unwrap(); + let error = doc.commands[0].as_ref().unwrap_err(); + assert!(matches!( + error.error, + ContentError::InvalidUnicodeEscape { .. } + )); + assert_eq!(error.context.as_ref().unwrap().token, token); + assert!(matches!( + doc.commands[1], + Ok(Command::FunctionCode(FunctionCode::MCode(MCode::EndOfFile))) + )); + } + } +} diff --git a/tests/reference_file_tests.rs b/tests/reference_file_tests.rs index 16401dc..d7df4a1 100644 --- a/tests/reference_file_tests.rs +++ b/tests/reference_file_tests.rs @@ -77,8 +77,8 @@ fn a_drill_file_to_rust_and_back() { /// Gerber spec 3.4.3 and 3.4.4: the reserved characters `*` (`*`), `%` (`%`), /// `\` (`\`) and, in fields, `,` (`,`) are expressed as unicode escapes and never -/// as raw bytes, so they cannot be confused with the block delimiters. Escape sequences are -/// kept verbatim by the parser, which is what makes the round-trip byte-exact. +/// as raw bytes, so they cannot be confused with the block delimiters. +/// The parser decodes these escapes into their Unicode characters. #[test] fn escaped_strings_to_rust() { let gbr_string = include_str!("../assets/reference_files/escaped_strings.gbr"); @@ -87,15 +87,22 @@ fn escaped_strings_to_rust() { } #[test] -fn escaped_strings_to_rust_and_back() { +fn escaped_strings_are_decoded() { let gbr_string = include_str!("../assets/reference_files/escaped_strings.gbr"); let reader = gerber_to_reader(gbr_string); let doc = parse(reader).unwrap(); - assert_eq!( - gerber_doc_as_str(&doc), - gbr_string, - "unexpected differences, commands: {:?}", - doc.commands - ) + assert!(doc.commands.iter().all(Result::is_ok)); + assert!( + matches!(&doc.commands[0], Ok(gerber_types::Command::FunctionCode( + gerber_types::FunctionCode::GCode(gerber_types::GCode::Comment( + gerber_types::CommentContent::String(text) + )) + )) if text == "escaped asterisk *, percent % and backslash \\ in a comment") + ); + assert!( + matches!(&doc.commands[3], Ok(gerber_types::Command::ExtendedCode( + gerber_types::ExtendedCode::FileAttribute(gerber_types::FileAttribute::UserDefined { values, .. }) + )) if values == &["escaped comma , in a field", "escaped asterisk *"]) + ); } From 5b8d01ce5c22deb3e3870f2ff420163d58201213 Mon Sep 17 00:00:00 2001 From: Nico Lube Date: Wed, 7 Oct 2026 22:06:50 +0200 Subject: [PATCH 8/8] refactor: simplify Unicode field parsing and expand coverage --- src/parser.rs | 20 +++++++------------- tests/component_tests.rs | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 42b6421..4f60494 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -2627,19 +2627,13 @@ fn decode_unicode_escapes(value: &str) -> Result { } fn attr_args(partial_line: Chars<'_>) -> Result, ContentError> { - partial_line - .as_str() - .split(',') - .enumerate() - .map(|(index, el)| { - // Attribute names use the ASCII name grammar, not the string grammar. - if index == 0 { - Ok(el.trim().to_string()) - } else { - decode_unicode_escapes(el.trim()) - } - }) - .collect() + let mut fields = partial_line.as_str().split(','); + // split always yields a first field; names use the ASCII name grammar. + let mut args = vec![fields.next().unwrap().trim().to_string()]; + for field in fields { + args.push(decode_unicode_escapes(field.trim())?); + } + Ok(args) } fn trim_attr_line(mut partial_line: Chars) -> Result { diff --git a/tests/component_tests.rs b/tests/component_tests.rs index edda3c8..7c295ed 100644 --- a/tests/component_tests.rs +++ b/tests/component_tests.rs @@ -4361,6 +4361,11 @@ fn invalid_unicode_escapes_report_context_and_continue() { for token in [ format!("G04 {}*", escape), format!("%TFCustom,{}*%", escape), + format!("%TACustom,{}*%", escape), + format!("%TOCustom,{}*%", escape), + format!("G04 #@! TFCustom,{}*", escape), + format!("%IN{}*%", escape), + format!("%AMExample*0 {}*1,1,1,0,0*%", escape), ] { let input = format!("{}M02*", token); let doc = parse(gerber_to_reader(&input)).unwrap(); @@ -4377,3 +4382,27 @@ fn invalid_unicode_escapes_report_context_and_continue() { } } } + +#[test] +fn image_name_and_escaped_comment_marker_are_decoded_as_text() { + let input = r"%INBoard\u002A\U00000025\u005C\u002C\u00A9*% +G04 \u0023@! TFCustom,value* +M02*"; + let doc = parse(gerber_to_reader(input)).unwrap(); + assert_eq!(doc.image_name.as_deref(), Some("Board*%\\,©")); + let expected: Vec = vec![ + ExtendedCode::ImageName(gerber_types::ImageName { + name: "Board*%\\,©".into(), + }) + .into(), + GCode::Comment(CommentContent::String("#@! TFCustom,value".into())).into(), + MCode::EndOfFile.into(), + ]; + assert_eq!( + doc.commands + .into_iter() + .map(Result::unwrap) + .collect::>(), + expected + ); +}