diff --git a/src/dialect.ts b/src/dialect.ts index 5a4b40ec77..9abed75292 100644 --- a/src/dialect.ts +++ b/src/dialect.ts @@ -3,7 +3,9 @@ import { ProcessedDialectFormatOptions, } from './formatter/ExpressionFormatter.js'; import Tokenizer from './lexer/Tokenizer.js'; -import { TokenizerOptions } from './lexer/TokenizerOptions.js'; +import { QuoteType, TokenizerOptions, VariableType } from './lexer/TokenizerOptions.js'; +import { quotePatterns } from './lexer/regexFactory.js'; +import { ConfigError } from './validateConfig.js'; export interface DialectOptions { name: string; @@ -18,6 +20,47 @@ export interface Dialect { const cache = new Map(); +// A quote type that resolves to no pattern at all yields a regex matching the +// empty string, which used to leave the tokenizer looping forever. +const validateQuoteTypes = (name: string, options: TokenizerOptions): void => { + const unknownQuote = (field: string, quote: string): ConfigError => + new ConfigError( + `Unknown quote type "${quote}" given in ${field} of dialect "${name}". ` + + `Known ones are: ${Object.keys(quotePatterns).join(', ')}.` + ); + + const check = (field: string, types: (QuoteType | VariableType)[] | undefined) => { + if (types === undefined) { + return; + } + if (types.length === 0) { + throw new ConfigError( + `Empty ${field} given for dialect "${name}". That would result in matching zero-length tokens.` + ); + } + // A quote type is either a plain name, a name with prefixes, or a regex. + for (const type of types) { + if (typeof type === 'string') { + if (!Object.prototype.hasOwnProperty.call(quotePatterns, type)) { + throw unknownQuote(field, type); + } + } else if ('regex' in type) { + if (type.regex === '') { + throw new ConfigError( + `Empty regex given in ${field} of dialect "${name}". That would result in matching zero-length tokens.` + ); + } + } else if (!Object.prototype.hasOwnProperty.call(quotePatterns, type.quote)) { + throw unknownQuote(field, type.quote); + } + } + }; + + check('stringTypes', options.stringTypes); + check('identTypes', options.identTypes); + check('variableTypes', options.variableTypes); +}; + /** * Factory function for building Dialect objects. * When called repeatedly with same options object returns the cached Dialect, @@ -26,6 +69,7 @@ const cache = new Map(); export const createDialect = (options: DialectOptions): Dialect => { let dialect = cache.get(options); if (!dialect) { + validateQuoteTypes(options.name, options.tokenizerOptions); dialect = dialectFromOptions(options); cache.set(options, dialect); } diff --git a/src/lexer/TokenizerEngine.ts b/src/lexer/TokenizerEngine.ts index 28245af70c..d0446988e1 100644 --- a/src/lexer/TokenizerEngine.ts +++ b/src/lexer/TokenizerEngine.ts @@ -1,4 +1,5 @@ import { Token, TokenType } from './token.js'; +import { ConfigError } from '../validateConfig.js'; import { lineColFromIndex } from './lineColFromIndex.js'; import { WHITESPACE_REGEX } from './regexUtil.js'; @@ -103,6 +104,12 @@ export default class TokenizerEngine { if (matches) { const matchedText = matches[0]; + if (matchedText.length === 0) { + throw new ConfigError( + `A token rule matches the empty string at ${this.position()}, so the tokenizer would loop forever.\n${this.dialectInfo()}` + ); + } + const token: Token = { type: rule.type, raw: matchedText, @@ -120,4 +127,9 @@ export default class TokenizerEngine { } return undefined; } + + private position(): string { + const { line, col } = lineColFromIndex(this.input, this.index); + return `line ${line} column ${col}`; + } } diff --git a/test/sqlFormatter.test.ts b/test/sqlFormatter.test.ts index 87e9cca082..d22b91a144 100644 --- a/test/sqlFormatter.test.ts +++ b/test/sqlFormatter.test.ts @@ -116,4 +116,60 @@ describe('sqlFormatter', () => { `); }); }); + describe('when a custom dialect would match zero-length tokens', () => { + // A quote type that resolves to no pattern at all yields a regex matching + // the empty string, which never advances the tokenizer. It used to loop + // forever instead of reporting the problem. Issue #754 was the same failure + // for paramTypes. + const dialectWith = ( + tokenizerOptions: Partial + ): DialectOptions => ({ + name: 'myCustomDialect', + tokenizerOptions: { ...sqlite.tokenizerOptions, ...tokenizerOptions }, + formatOptions: sqlite.formatOptions, + }); + + const expectConfigError = (dialect: DialectOptions, message: RegExp) => { + expect(() => formatDialect('SELECT 1;', { dialect })).toThrow(message); + }; + + it('rejects an empty quote type list', () => { + const empty = (field: string) => + new RegExp(`Empty ${field} given for dialect "myCustomDialect"\\.`); + expectConfigError(dialectWith({ stringTypes: [] }), empty('stringTypes')); + expectConfigError(dialectWith({ identTypes: [] }), empty('identTypes')); + expectConfigError(dialectWith({ variableTypes: [] }), empty('variableTypes')); + }); + + it('rejects a quote type name that has no pattern', () => { + // The types only allow known names, so this covers plain JavaScript + // callers. quotePatterns has no plain "''" key; the real keys are + // "''-qq", "''-bs", "''-raw" and so on. + const unknown = (quote: string) => + new RegExp(`Unknown quote type ${JSON.stringify(quote)} given in stringTypes`); + + expectConfigError(dialectWith({ stringTypes: ["''"] } as never), unknown("''")); + + const prefixed = { stringTypes: [{ quote: '""', prefixes: ['X'] }] } as never; + expectConfigError(dialectWith(prefixed), unknown('""')); + }); + + it('rejects an empty regex quote type', () => { + expectConfigError( + dialectWith({ stringTypes: [{ regex: '' }] }), + /Empty regex given in stringTypes of dialect "myCustomDialect"\./ + ); + }); + + it('still allows a valid custom quote type list', () => { + expect( + formatDialect('SELECT 1;', { + dialect: dialectWith({ stringTypes: [...sqlite.tokenizerOptions.stringTypes] }), + }) + ).toBe(dedent` + SELECT + 1; + `); + }); + }); });