fix: keep a keyword reserved when directly followed by a -- line comment

src/lexer/regexFactory.ts

@@ -31,8 +31,18 @@ export const operator = (operators: string[]) =>
 // For example "SELECT$ME" should be tokenized as:
 // - ["SELECT$ME"] when $ is allowed inside identifiers
 // - ["SELECT", "$", "ME"] when $ can't be part of identifiers.
-const rejectIdentCharsPattern = ({ rest, dashes }: IdentChars): string =>
-  rest || dashes ? `(?![${rest || ''}${dashes ? '-' : ''}])` : '';
+//
+// A dash only extends an identifier when it's followed by a character that could
+// start an identifier segment. So "SELECT-a" is a single identifier, while in
+// "SELECT--a" the dashes start a line comment and SELECT stays a keyword.
+const rejectIdentCharsPattern = (identChars: IdentChars): string => {
+  const { rest, dashes } = identChars;
+  const alternatives = [
+    ...(rest ? [`[${rest}]`] : []),
+    ...(dashes ? [`-(?=${identFirstCharsPattern(identChars)})`] : []),
+  ];
+  return alternatives.length ? `(?!${alternatives.join('|')})` : '';
+};
 
 /**
  * Builds a RegExp for all Reserved Keywords in a SQL dialect
@@ -156,6 +166,22 @@ export const string = (quoteTypes: QuoteType[]): RegExp =>
 export const identifier = (specialChars: IdentChars = {}): RegExp =>
   patternToRegex(identifierPattern(specialChars));
 
+// Unicode letters, diacritical marks and underscore
+const LETTER_CHARS = '\\p{Alphabetic}\\p{Mark}_';
+// Numbers 0..9, plus various unicode numbers
+const NUMBER_CHARS = '\\p{Decimal_Number}';
+
+/**
+ * Builds a character class matching the first character of an identifier.
+ * Dashed identifiers repeat this same pattern after every dash.
+ */
+const identFirstCharsPattern = ({ first, allowFirstCharNumber }: IdentChars = {}): string => {
+  const firstChars = escapeRegExp(first ?? '');
+  return allowFirstCharNumber
+    ? `[${LETTER_CHARS}${NUMBER_CHARS}${firstChars}]`
+    : `[${LETTER_CHARS}${firstChars}]`;
+};
+
 /**
  * Builds a RegExp string for valid identifiers in a SQL dialect
  */
@@ -165,17 +191,12 @@ export const identifierPattern = ({
   dashes,
   allowFirstCharNumber,
 }: IdentChars = {}): string => {
-  // Unicode letters, diacritical marks and underscore
-  const letter = '\\p{Alphabetic}\\p{Mark}_';
-  // Numbers 0..9, plus various unicode numbers
-  const number = '\\p{Decimal_Number}';
-
-  const firstChars = escapeRegExp(first ?? '');
   const restChars = escapeRegExp(rest ?? '');
 
-  const pattern = allowFirstCharNumber
-    ? `[${letter}${number}${firstChars}][${letter}${number}${restChars}]*`
-    : `[${letter}${firstChars}][${letter}${number}${restChars}]*`;
+  const pattern = `${identFirstCharsPattern({
+    first,
+    allowFirstCharNumber,
+  })}[${LETTER_CHARS}${NUMBER_CHARS}${restChars}]*`;
 
   return dashes ? withDashes(pattern) : pattern;
 };

test/bigquery.test.ts

@@ -636,4 +636,26 @@ describe('BigQueryFormatter', () => {
       expect(format(input, { linesBetweenQueries: 0 })).toBe(input);
     });
   });
+  describe('BigQuery dashed identifiers followed by line comments', () => {
+    // BigQuery is the only dialect that allows dashes inside identifiers, which
+    // makes "--" ambiguous with the start of a line comment. A keyword that is
+    // immediately followed by a line comment must still be recognized as a keyword.
+    it('recognizes SELECT when directly followed by a -- line comment', () => {
+      expect(format('SELECT-- c\na FROM t;')).toBe(dedent`
+        SELECT -- c
+          a
+        FROM
+          t;
+      `);
+    });
+
+    it('recognizes FROM when directly followed by a -- line comment', () => {
+      expect(format('SELECT a FROM-- c\nt;')).toBe(dedent`
+        SELECT
+          a
+        FROM -- c
+          t;
+      `);
+    });
+  });
 });