fix: reject dialect quote types that would match zero-length tokens

src/dialect.ts

@@ -3,7 +3,9 @@ import {
   ProcessedDialectFormatOptions,
 } from './formatter/ExpressionFormatter.js';
 import Tokenizer from './lexer/Tokenizer.js';
-import { TokenizerOptions } from './lexer/TokenizerOptions.js';
+import { QuoteType, TokenizerOptions, VariableType } from './lexer/TokenizerOptions.js';
+import { quotePatterns } from './lexer/regexFactory.js';
+import { ConfigError } from './validateConfig.js';
 
 export interface DialectOptions {
   name: string;
@@ -18,6 +20,47 @@ export interface Dialect {
 
 const cache = new Map<DialectOptions, Dialect>();
 
+// A quote type that resolves to no pattern at all yields a regex matching the
+// empty string, which used to leave the tokenizer looping forever.
+const validateQuoteTypes = (name: string, options: TokenizerOptions): void => {
+  const unknownQuote = (field: string, quote: string): ConfigError =>
+    new ConfigError(
+      `Unknown quote type "${quote}" given in ${field} of dialect "${name}". ` +
+        `Known ones are: ${Object.keys(quotePatterns).join(', ')}.`
+    );
+
+  const check = (field: string, types: (QuoteType | VariableType)[] | undefined) => {
+    if (types === undefined) {
+      return;
+    }
+    if (types.length === 0) {
+      throw new ConfigError(
+        `Empty ${field} given for dialect "${name}". That would result in matching zero-length tokens.`
+      );
+    }
+    // A quote type is either a plain name, a name with prefixes, or a regex.
+    for (const type of types) {
+      if (typeof type === 'string') {
+        if (!Object.prototype.hasOwnProperty.call(quotePatterns, type)) {
+          throw unknownQuote(field, type);
+        }
+      } else if ('regex' in type) {
+        if (type.regex === '') {
+          throw new ConfigError(
+            `Empty regex given in ${field} of dialect "${name}". That would result in matching zero-length tokens.`
+          );
+        }
+      } else if (!Object.prototype.hasOwnProperty.call(quotePatterns, type.quote)) {
+        throw unknownQuote(field, type.quote);
+      }
+    }
+  };
+
+  check('stringTypes', options.stringTypes);
+  check('identTypes', options.identTypes);
+  check('variableTypes', options.variableTypes);
+};
+
 /**
  * Factory function for building Dialect objects.
  * When called repeatedly with same options object returns the cached Dialect,
@@ -26,6 +69,7 @@ const cache = new Map<DialectOptions, Dialect>();
 export const createDialect = (options: DialectOptions): Dialect => {
   let dialect = cache.get(options);
   if (!dialect) {
+    validateQuoteTypes(options.name, options.tokenizerOptions);
     dialect = dialectFromOptions(options);
     cache.set(options, dialect);
   }

src/lexer/TokenizerEngine.ts

@@ -1,4 +1,5 @@
 import { Token, TokenType } from './token.js';
+import { ConfigError } from '../validateConfig.js';
 import { lineColFromIndex } from './lineColFromIndex.js';
 import { WHITESPACE_REGEX } from './regexUtil.js';
 
@@ -103,6 +104,12 @@ export default class TokenizerEngine {
     if (matches) {
       const matchedText = matches[0];
 
+      if (matchedText.length === 0) {
+        throw new ConfigError(
+          `A token rule matches the empty string at ${this.position()}, so the tokenizer would loop forever.\n${this.dialectInfo()}`
+        );
+      }
+
       const token: Token = {
         type: rule.type,
         raw: matchedText,
@@ -120,4 +127,9 @@ export default class TokenizerEngine {
     }
     return undefined;
   }
+
+  private position(): string {
+    const { line, col } = lineColFromIndex(this.input, this.index);
+    return `line ${line} column ${col}`;
+  }
 }

test/sqlFormatter.test.ts

@@ -116,4 +116,60 @@ describe('sqlFormatter', () => {
       `);
     });
   });
+  describe('when a custom dialect would match zero-length tokens', () => {
+    // A quote type that resolves to no pattern at all yields a regex matching
+    // the empty string, which never advances the tokenizer. It used to loop
+    // forever instead of reporting the problem. Issue #754 was the same failure
+    // for paramTypes.
+    const dialectWith = (
+      tokenizerOptions: Partial<typeof sqlite.tokenizerOptions>
+    ): DialectOptions => ({
+      name: 'myCustomDialect',
+      tokenizerOptions: { ...sqlite.tokenizerOptions, ...tokenizerOptions },
+      formatOptions: sqlite.formatOptions,
+    });
+
+    const expectConfigError = (dialect: DialectOptions, message: RegExp) => {
+      expect(() => formatDialect('SELECT 1;', { dialect })).toThrow(message);
+    };
+
+    it('rejects an empty quote type list', () => {
+      const empty = (field: string) =>
+        new RegExp(`Empty ${field} given for dialect "myCustomDialect"\\.`);
+      expectConfigError(dialectWith({ stringTypes: [] }), empty('stringTypes'));
+      expectConfigError(dialectWith({ identTypes: [] }), empty('identTypes'));
+      expectConfigError(dialectWith({ variableTypes: [] }), empty('variableTypes'));
+    });
+
+    it('rejects a quote type name that has no pattern', () => {
+      // The types only allow known names, so this covers plain JavaScript
+      // callers. quotePatterns has no plain "''" key; the real keys are
+      // "''-qq", "''-bs", "''-raw" and so on.
+      const unknown = (quote: string) =>
+        new RegExp(`Unknown quote type ${JSON.stringify(quote)} given in stringTypes`);
+
+      expectConfigError(dialectWith({ stringTypes: ["''"] } as never), unknown("''"));
+
+      const prefixed = { stringTypes: [{ quote: '""', prefixes: ['X'] }] } as never;
+      expectConfigError(dialectWith(prefixed), unknown('""'));
+    });
+
+    it('rejects an empty regex quote type', () => {
+      expectConfigError(
+        dialectWith({ stringTypes: [{ regex: '' }] }),
+        /Empty regex given in stringTypes of dialect "myCustomDialect"\./
+      );
+    });
+
+    it('still allows a valid custom quote type list', () => {
+      expect(
+        formatDialect('SELECT 1;', {
+          dialect: dialectWith({ stringTypes: [...sqlite.tokenizerOptions.stringTypes] }),
+        })
+      ).toBe(dedent`
+        SELECT
+          1;
+      `);
+    });
+  });
 });