diff --git a/e2e/questdb b/e2e/questdb index cc1e5555b..c5dc7a1eb 160000 --- a/e2e/questdb +++ b/e2e/questdb @@ -1 +1 @@ -Subproject commit cc1e5555bab53e65ff28b1c0948e44496db51d12 +Subproject commit c5dc7a1eb1533a6971322e69515278e711ec4ec7 diff --git a/e2e/tests/console/editor.spec.js b/e2e/tests/console/editor.spec.js index 3f7ee847b..980316f39 100644 --- a/e2e/tests/console/editor.spec.js +++ b/e2e/tests/console/editor.spec.js @@ -654,14 +654,14 @@ describe("autocomplete", () => { const assertFrom = () => cy.getAutocomplete().within(() => { cy.getMonacoListRow() - .should("have.length", 4) + .should("have.length", 1) .eq(0) .should("contain", "FROM") }) - cy.typeQuery("select * from") + cy.typeQuery("select * fro") assertFrom() cy.clearEditor() - cy.typeQuery("SELECT * FROM") + cy.typeQuery("SELECT * FRO") assertFrom() }) @@ -681,37 +681,127 @@ describe("autocomplete", () => { // Columns .should("contain", "secret") .should("contain", "public") - // Tables list for the `secret` column - // list the tables containing `secret` column - .should("contain", "my_secrets, my_secrets2") .clearEditor() }) it("should suggest columns on SELECT only when applicable", () => { - cy.typeQuery("select secret") - cy.getAutocomplete().should("contain", "secret").eq(0).click() - cy.typeQuery(", public") - cy.getAutocomplete().should("contain", "public").eq(0).click() - cy.typeQuery(" ") - cy.getAutocomplete().should("not.be.visible") + cy.typeQuery("select secre") + cy.getAutocomplete().should("be.visible") + cy.typeQuery("{enter}") + cy.typeQuery(", publi") + cy.getAutocomplete().should("be.visible") + cy.typeQuery("{enter}") + cy.getAutocomplete().should("contain", "FROM") + cy.clearEditor() }) it("should suggest correct columns on 'where' filter", () => { cy.typeQuery("select * from my_secrets where ") + cy.getAutocomplete().eq(0).should("contain", "secret").clearEditor() + }) + + it("should suggest correct columns on 'on' clause", () => { + cy.typeQuery("select * from my_secrets join my_publics on ") cy.getAutocomplete() + .should("contain", "public") .should("contain", "secret") - .should("not.contain", "public") .clearEditor() }) - it("should suggest correct columns on 'on' clause", () => { - cy.typeQuery("select * from my_secrets join my_publics on ") + it("should suggest columns for dot-qualified alias", () => { + cy.typeQuery("select * from my_secrets s where s.") + cy.getAutocomplete().should("contain", "secret").clearEditor() + }) + + it("should replace partial text when accepting a suggestion", () => { + cy.typeQuery("select * from my_se") + cy.getAutocomplete().should("contain", "my_secrets") + cy.typeQuery("{enter}") + cy.window().then((win) => { + const value = win.monaco.editor.getEditors()[0].getValue() + expect(value).to.match(/select \* from my_secrets/) + expect(value).to.not.contain("my_semy_secrets") + }) + cy.clearEditor() + }) + + it("should suggest the new keyword after accepting a suggestion", () => { + cy.typeQuery("CR") + cy.getAutocomplete().should("contain", "CREATE") + cy.typeQuery("{enter}") + cy.typeQuery("T") + cy.getAutocomplete().should("contain", "TABLE") + cy.typeQuery("{enter}") + cy.getAutocomplete().should("contain", "IF") + cy.typeQuery("{enter}") + cy.getAutocomplete().should("contain", "NOT") + cy.typeQuery("{enter}") + cy.getAutocomplete().should("contain", "EXISTS") + cy.typeQuery("{enter}") + cy.clearEditor() + }) + + it("should not suggest the very same keyword when it's already typed", () => { + cy.typeQuery("SELECT * FROM") + cy.getAutocomplete().should("not.be.visible") + + cy.typeQuery(`${ctrlOrCmd}i`) + cy.getAutocomplete() + .should("be.visible") + .should("contain", "No suggestions") + cy.clearEditor() + }) + + it("should suggest tables in second statement of multi-statement buffer", () => { + cy.typeQuery("select * from my_secrets;{enter}select * from ") cy.getAutocomplete() - .should("contain", "my_publics.public") - .should("contain", "my_secrets.secret") + .should("contain", "my_secrets") + .should("contain", "my_publics") .clearEditor() }) + it("should not suggest anything immediately after opening parenthesis", () => { + cy.typeQuery("select count(") + cy.getAutocomplete().should("not.be.visible") + cy.clearEditor() + }) + + it("should suggest after parenthesis followed by space", () => { + cy.typeQuery("select count( ") + cy.getAutocomplete().should("be.visible").clearEditor() + }) + + it("should not suggest inside line comments", () => { + cy.typeQuery("-- select * from ") + cy.getAutocomplete().should("not.be.visible").clearEditor() + }) + + it("should not suggest in dead space between statements", () => { + cy.typeQuery("select 1;") + cy.typeQuery("{enter}{enter}") + cy.getAutocomplete().should("not.be.visible").clearEditor() + }) + + it("should display keywords in uppercase", () => { + cy.typeQuery("select * from my_secrets whe") + cy.getAutocomplete().should("contain", "WHERE") + // Should not contain lowercase variant + cy.getAutocomplete().within(() => { + cy.getMonacoListRow().first().should("contain", "WHERE") + }) + cy.clearEditor() + }) + + it("should suggest CTE name in FROM clause", () => { + cy.typeQuery("with cte as (select 1) select * from ") + cy.getAutocomplete().should("contain", "cte").clearEditor() + }) + + it("should not suggest inside block comments", () => { + cy.typeQuery("/* select * from ") + cy.getAutocomplete().should("not.be.visible").clearEditor() + }) + after(() => { cy.loadConsoleWithAuth() ;["my_publics", "my_secrets", "my_secrets2"].forEach((table) => { @@ -730,14 +820,6 @@ describe("errors", () => { cy.clearEditor() }) - it("should mark '(200000)' as error", () => { - const query = `create table test (\ncol symbol index CAPACITY (200000)` - cy.typeQuery(query) - cy.runLine() - cy.matchErrorMarkerPosition({ left: 237, width: 67 }) - cy.getCollapsedNotifications().should("contain", "bad integer") - }) - it("should mark date position as error", () => { const query = `select * from long_sequence(1) where cast(x as timestamp) = '2012-04-12T12:00:00A'` cy.typeQuery(query) diff --git a/e2e/tests/console/tableDetails.spec.js b/e2e/tests/console/tableDetails.spec.js index a5441bbb8..b4824ec29 100644 --- a/e2e/tests/console/tableDetails.spec.js +++ b/e2e/tests/console/tableDetails.spec.js @@ -694,8 +694,6 @@ describe("TableDetailsDrawer", () => { "contain", "AI Assistant is not configured", ) - // Workaround - tooltip grace area causes problems in the test when quickly switching multiple triggers. - // Move mouse away from trigger for 200ms cy.getByDataHook("table-details-tab-monitoring").realHover() cy.wait(200) @@ -746,8 +744,6 @@ describe("TableDetailsDrawer", () => { "contain", "Schema access is not granted to this model", ) - // Workaround - tooltip grace area causes problems in the test when quickly switching multiple triggers. - // Move mouse away from trigger for 200ms cy.getByDataHook("table-details-tab-monitoring").realHover() cy.wait(200) diff --git a/package.json b/package.json index 778044fb1..56f83f9c8 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "@monaco-editor/react": "^4.7.0", "@phosphor-icons/react": "^2.1.10", "@popperjs/core": "2.4.2", - "@questdb/sql-grammar": "1.4.2", + "@questdb/sql-parser": "0.1.3", "@radix-ui/react-alert-dialog": "^1.1.15", "@radix-ui/react-context-menu": "^2.2.16", "@radix-ui/react-dialog": "^1.1.15", @@ -179,7 +179,7 @@ }, { "path": "dist/assets/index-*.js", - "maxSize": "2.5MB", + "maxSize": "3MB", "compression": "none" }, { diff --git a/src/scenes/Editor/Monaco/index.tsx b/src/scenes/Editor/Monaco/index.tsx index 91fa96b61..a3396df96 100644 --- a/src/scenes/Editor/Monaco/index.tsx +++ b/src/scenes/Editor/Monaco/index.tsx @@ -50,7 +50,9 @@ import { createSchemaCompletionProvider } from "./questdb-sql" import { Request } from "./utils" import { appendQuery, + applyValidationMarkers, clearModelMarkers, + clearValidationMarkers, findMatches, getErrorRange, getQueryFromCursor, @@ -66,6 +68,7 @@ import { parseQueryKey, createQueryKeyFromRequest, validateQueryAtOffset, + validateQueryJIT, setErrorMarkerForQuery, getQueryStartOffset, getQueriesToRun, @@ -342,6 +345,7 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { }) const scrollTimeoutRef = useRef(null) const notificationTimeoutRef = useRef(null) + const validationTimeoutRef = useRef(null) const targetPositionRef = useRef<{ lineNumber: number column: number @@ -929,8 +933,6 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { newKey: QueryKey data: ExecutionInfo }> = [] - const keysToRemove: QueryKey[] = [] - Object.keys(bufferExecutions).forEach((key) => { const queryKey = key as QueryKey const { queryText, startOffset, endOffset } = @@ -948,36 +950,23 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { } const newOffset = startOffset + effectiveOffsetDelta - if (validateQueryAtOffset(editor, queryText, newOffset)) { - const selection = bufferExecutions[queryKey].selection - const shiftedSelection = selection - ? { - startOffset: selection.startOffset + effectiveOffsetDelta, - endOffset: selection.endOffset + effectiveOffsetDelta, - } - : undefined - keysToUpdate.push({ - oldKey: queryKey, - newKey: createQueryKey(queryText, newOffset), - data: { - ...bufferExecutions[queryKey], - startOffset: newOffset, - endOffset: endOffset + effectiveOffsetDelta, - selection: shiftedSelection, - }, - }) - } else { - keysToRemove.push(queryKey) - notificationUpdates.push(() => - dispatch( - actions.query.removeNotification(queryKey, activeBufferId), - ), - ) - } - }) - - keysToRemove.forEach((key) => { - delete bufferExecutions[key] + const selection = bufferExecutions[queryKey].selection + const shiftedSelection = selection + ? { + startOffset: selection.startOffset + effectiveOffsetDelta, + endOffset: selection.endOffset + effectiveOffsetDelta, + } + : undefined + keysToUpdate.push({ + oldKey: queryKey, + newKey: createQueryKey(queryText, newOffset), + data: { + ...bufferExecutions[queryKey], + startOffset: newOffset, + endOffset: endOffset + effectiveOffsetDelta, + selection: shiftedSelection, + }, + }) }) keysToUpdate.forEach(({ oldKey, newKey, data }) => { @@ -1024,25 +1013,16 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { } const newOffset = startOffset + effectiveOffsetDelta - - if (validateQueryAtOffset(editor, queryText, newOffset)) { - const newKey = createQueryKey(queryText, newOffset) - notificationUpdates.push(() => - dispatch( - actions.query.updateNotificationKey( - queryKey, - newKey, - activeBufferId, - ), + const newKey = createQueryKey(queryText, newOffset) + notificationUpdates.push(() => + dispatch( + actions.query.updateNotificationKey( + queryKey, + newKey, + activeBufferId, ), - ) - } else { - notificationUpdates.push(() => - dispatch( - actions.query.removeNotification(queryKey, activeBufferId), - ), - ) - } + ), + ) }) if (bufferExecutions && Object.keys(bufferExecutions).length === 0) { @@ -1081,13 +1061,38 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { contentJustChangedRef.current = false notificationUpdates.forEach((update) => update()) + + // JIT validation (debounced) + if (validationTimeoutRef.current) { + window.clearTimeout(validationTimeoutRef.current) + } + validationTimeoutRef.current = window.setTimeout(() => { + if (monacoRef.current && editorRef.current) { + const currentBufferId = activeBufferRef.current.id as number + validateQueryJIT( + monacoRef.current, + editorRef.current, + currentBufferId, + () => executionRefs.current[currentBufferId.toString()] || {}, + (q, signal) => quest.validateQuery(q, signal), + ) + } + validationTimeoutRef.current = null + }, 300) }) editor.onDidChangeModel(() => { glyphWidgetsRef.current.forEach((widget) => { - editor.removeGlyphMarginWidget(widget) + editorRef.current?.removeGlyphMarginWidget(widget) }) glyphWidgetsRef.current.clear() + const lineCount = editorRef.current?.getModel()?.getLineCount() + if (lineCount) { + setLineNumbersMinChars( + getDefaultLineNumbersMinChars(canUseAIRef.current) + + (lineCount.toString().length - 1), + ) + } setTimeout(() => { if (monacoRef.current && editorRef.current) { applyGlyphsAndLineMarkings(monacoRef.current, editorRef.current) @@ -1209,6 +1214,15 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { | null = null dispatch(actions.query.setResult(undefined)) + // Clear JIT validation markers — execution result takes over. + if (monacoRef.current) { + clearValidationMarkers(monacoRef.current, editor, activeBufferId) + } + if (validationTimeoutRef.current) { + window.clearTimeout(validationTimeoutRef.current) + validationTimeoutRef.current = null + } + dispatch( actions.query.setActiveNotification({ type: NotificationType.LOADING, @@ -1627,6 +1641,15 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { ) { if (monacoRef?.current) { clearModelMarkers(monacoRef.current, editorRef.current) + clearValidationMarkers( + monacoRef.current, + editorRef.current, + activeBufferRef.current.id as number, + ) + } + if (validationTimeoutRef.current) { + window.clearTimeout(validationTimeoutRef.current) + validationTimeoutRef.current = null } if (monacoRef?.current && editorRef?.current) { @@ -1952,6 +1975,13 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { clearModelMarkers(monacoRef.current, editorRef.current) applyGlyphsAndLineMarkings(monacoRef.current, editorRef.current) + + // Restore cached validation markers for this buffer + applyValidationMarkers( + monacoRef.current, + editorRef.current, + activeBuffer.id as number, + ) } }, [activeBuffer]) @@ -2026,6 +2056,10 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { window.clearTimeout(notificationTimeoutRef.current) } + if (validationTimeoutRef.current) { + window.clearTimeout(validationTimeoutRef.current) + } + glyphWidgetsRef.current.forEach((widget) => { editorRef.current?.removeGlyphMarginWidget(widget) }) @@ -2076,6 +2110,7 @@ const MonacoEditor = ({ hidden = false }: { hidden?: boolean }) => { scrollBeyondLastLine: false, tabSize: 2, lineNumbersMinChars, + wordBasedSuggestions: "off", }} theme="dracula" /> diff --git a/src/scenes/Editor/Monaco/questdb-sql/conf.ts b/src/scenes/Editor/Monaco/questdb-sql/conf.ts index 5c9227857..17284adba 100644 --- a/src/scenes/Editor/Monaco/questdb-sql/conf.ts +++ b/src/scenes/Editor/Monaco/questdb-sql/conf.ts @@ -7,7 +7,7 @@ export const conf: languages.LanguageConfiguration = { * An additional example is a "bad integer" error, i.e. (20000) - needs brackets to be allowed as well. */ wordPattern: - /(-?\d*\.\d\w*)|(::|:=|<<=|>>=|!=|<>|<=|>=|\|\||[-+*/%~<>^|&=!]|\b(?:not|and|or|in|between|within|like|ilike)\b|[^`~!@#$%^&*\-+[{\]}\\|;:",<>/?\s]+)/g, + /(-?\d*\.\d\w*)|(::|:=|<<=|>>=|!=|<>|<=|>=|\|\||[-+*/%~<>^|&=!]|\b(?:not|and|or|in|between|within|like|ilike)\b|[^`~!@#$%^&*()\-+[{\]}\\|;:",<>/?\s]+)/g, comments: { lineComment: "--", blockComment: ["/*", "*/"], diff --git a/src/scenes/Editor/Monaco/questdb-sql/createSchemaCompletionProvider.ts b/src/scenes/Editor/Monaco/questdb-sql/createSchemaCompletionProvider.ts index 37179125e..e9cfc1df2 100644 --- a/src/scenes/Editor/Monaco/questdb-sql/createSchemaCompletionProvider.ts +++ b/src/scenes/Editor/Monaco/questdb-sql/createSchemaCompletionProvider.ts @@ -1,175 +1,257 @@ -import { Table, uniq, InformationSchemaColumn } from "../../../../utils" +import { Table, InformationSchemaColumn } from "../../../../utils" import type { editor, languages } from "monaco-editor" -import { CompletionItemPriority } from "./types" -import { findMatches, getQueryFromCursor } from "../utils" -import { getTableCompletions } from "./getTableCompletions" -import { getColumnCompletions } from "./getColumnCompletions" -import { getLanguageCompletions } from "./getLanguageCompletions" - -const trimQuotesFromTableName = (tableName: string) => { - return tableName.replace(/(^")|("$)/g, "") +import { CompletionItemKind, CompletionItemPriority } from "./types" +import { + createAutocompleteProvider, + SuggestionKind, + SuggestionPriority, + tokenize, + type SchemaInfo, + type Suggestion, +} from "@questdb/sql-parser" + +/** + * Map parser's SuggestionKind to Monaco's CompletionItemKind + */ +const KIND_MAP: Record = { + [SuggestionKind.Keyword]: CompletionItemKind.Keyword, + [SuggestionKind.Function]: CompletionItemKind.Function, + [SuggestionKind.Table]: CompletionItemKind.Class, + [SuggestionKind.Column]: CompletionItemKind.Field, + [SuggestionKind.Operator]: CompletionItemKind.Operator, + [SuggestionKind.DataType]: CompletionItemKind.TypeParameter, +} + +/** + * Map parser's SuggestionPriority to Monaco's sortText + */ +const PRIORITY_MAP: Record = { + [SuggestionPriority.High]: CompletionItemPriority.High, + [SuggestionPriority.Medium]: CompletionItemPriority.Medium, + [SuggestionPriority.MediumLow]: CompletionItemPriority.MediumLow, + [SuggestionPriority.Low]: CompletionItemPriority.Low, +} + +/** + * Convert UI schema format to parser's SchemaInfo format + */ +const convertToSchemaInfo = ( + tables: Table[], + informationSchemaColumns: Record, +): SchemaInfo => ({ + tables: tables.map((t) => ({ + name: t.table_name, + designatedTimestamp: t.designatedTimestamp, + })), + columns: Object.fromEntries( + Object.entries(informationSchemaColumns).map(([tableName, cols]) => [ + tableName.toLowerCase(), + cols.map((c) => ({ + name: c.column_name, + type: c.data_type, + })), + ]), + ), +}) + +/** + * Convert parser's Suggestion to Monaco's CompletionItem. + * For columns, uses CompletionItemLabel to show table names inline + * and data type on the right side. + */ +const UPPERCASE_KINDS = new Set([ + SuggestionKind.Keyword, + SuggestionKind.Operator, + SuggestionKind.DataType, +]) + +const toCompletionItem = ( + suggestion: Suggestion, + range: languages.CompletionItem["range"], +): languages.CompletionItem => { + const shouldUppercase = UPPERCASE_KINDS.has(suggestion.kind) + const label = shouldUppercase + ? suggestion.label.toUpperCase() + : suggestion.label + const insertText = shouldUppercase + ? suggestion.insertText.toUpperCase() + : suggestion.insertText + + return { + label: + suggestion.detail != null || suggestion.description != null + ? { + label, + detail: suggestion.detail, + description: suggestion.description, + } + : label, + kind: KIND_MAP[suggestion.kind], + insertText: insertText + " ", + filterText: suggestion.filterText, + sortText: PRIORITY_MAP[suggestion.priority], + range, + command: { + id: "editor.action.triggerSuggest", + title: "Re-trigger suggestions", + }, + } } -const isInColumnListing = (text: string) => - text.match( - /(?:,$|,\s$|\b(?:SELECT|UPDATE|COLUMN|ON|JOIN|BY|WHERE|DISTINCT)\s$)/gim, - ) +/** + * Check if cursor is inside a line comment (--) or block comment. + * The parser already handles string literals via its own guard, + * but comments are invisible to the lexer (Lexer.SKIPPED). + */ +function isCursorInComment(text: string, cursorOffset: number): boolean { + let i = 0 + const end = Math.min(cursorOffset, text.length) + while (i < end) { + const ch = text[i] + const next = text[i + 1] + // Line comment: -- until end of line + if (ch === "-" && next === "-") { + i += 2 + while (i < end && text[i] !== "\n") i++ + if (i >= cursorOffset) return true + continue + } + // Block comment: /* until */ + if (ch === "/" && next === "*") { + i += 2 + while (i < text.length && !(text[i] === "*" && text[i + 1] === "/")) i++ + if (i >= cursorOffset) return true + i += 2 // skip */ + continue + } + // Skip over string literals so quotes inside comments don't confuse us + if (ch === "'") { + i++ + while (i < text.length && text[i] !== "'") i++ + i++ // skip closing quote + continue + } + i++ + } + return false +} export const createSchemaCompletionProvider = ( editor: editor.IStandaloneCodeEditor, tables: Table[] = [], informationSchemaColumns: Record = {}, ) => { + // Convert UI schema to parser format and create provider + const schema = convertToSchemaInfo(tables, informationSchemaColumns) + const autocompleteProvider = createAutocompleteProvider(schema) + const completionProvider: languages.CompletionItemProvider = { triggerCharacters: - 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n ."'.split(""), + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz .":'.split(""), + provideCompletionItems(model, position) { const word = model.getWordUntilPosition(position) + const cursorOffset = model.getOffsetAt(position) + const fullText = model.getValue() - const queryAtCursor = getQueryFromCursor(editor) + // Suppress suggestions inside comments (the parser handles strings itself) + if (isCursorInComment(fullText, cursorOffset)) { + return null + } - // get text value in the current line - const textInLine = model.getValueInRange({ - startLineNumber: position.lineNumber, - startColumn: 1, - endLineNumber: position.lineNumber, - endColumn: position.column, - }) + const charBeforeCursor = + cursorOffset > 0 ? fullText[cursorOffset - 1] : "" + if (charBeforeCursor === "(") { + return null + } - let tableContext: string[] = [] + // Extract the current SQL statement for autocomplete by finding the + // nearest semicolon before the cursor. This is more robust than using + // the parser's statement splitting (getQueryFromCursor), which can + // break incomplete SQL into separate statements — e.g., "select * F" + // gets split into "select *" and "F", losing context for autocomplete. + const tokens = tokenize(fullText).tokens - const isWhitespaceOnly = /^\s*$/.test(textInLine) - const isLineComment = /(-- |--|\/\/ |\/\/)$/gim.test(textInLine) + let queryStartOffset = 0 + let queryEndOffset = fullText.length + for (const token of tokens) { + const tokenEnd = token.endOffset ?? token.startOffset + if (token.tokenType.name === "Semicolon") { + if (tokenEnd < cursorOffset) { + queryStartOffset = tokenEnd + 1 + } else if ( + token.startOffset >= cursorOffset && + queryEndOffset === fullText.length + ) { + queryEndOffset = token.startOffset + } + } + } - if (isWhitespaceOnly || isLineComment) { + // If there are no tokens between the query start and the cursor, + // the cursor is in dead space (whitespace/comments between statements). + // Don't suggest anything. + const hasTokensBeforeCursor = tokens.some( + (t) => + t.startOffset >= queryStartOffset && t.startOffset < cursorOffset, + ) + if (!hasTokensBeforeCursor) { return null } - if (queryAtCursor) { - const matches = findMatches(model, queryAtCursor.query) - if (matches.length > 0) { - const cursorMatch = matches.find( - (m) => m.range.startLineNumber === queryAtCursor.row + 1, - ) - - const fromMatch = queryAtCursor.query.match(/(?<=FROM\s)([^ )]+)/gim) - const joinMatch = queryAtCursor.query.match(/(JOIN)\s+([^ ]+)/i) - const alterTableMatch = queryAtCursor.query.match( - /(ALTER TABLE)\s+([^ ]+)/i, - ) - if (fromMatch) { - tableContext = uniq(fromMatch) - } else if (alterTableMatch && alterTableMatch[2]) { - tableContext.push(alterTableMatch[2]) - } - if (joinMatch && joinMatch[2]) { - tableContext.push(joinMatch[2]) - } + // Pass the full statement (including text after cursor) so the parser + // can detect when the cursor is inside a string literal or comment. + const query = fullText.substring(queryStartOffset, queryEndOffset) - tableContext = tableContext.map(trimQuotesFromTableName) + const relativeCursorOffset = cursorOffset - queryStartOffset - const textUntilPosition = model.getValueInRange({ - startLineNumber: cursorMatch?.range.startLineNumber ?? 1, - startColumn: cursorMatch?.range.startColumn ?? 1, - endLineNumber: position.lineNumber, - endColumn: word.startColumn, - }) + // Get suggestions from the parser-based provider. + // Filter out punctuation-only suggestions (e.g. "(", ")", ";") — + // the parser may suggest structural tokens as the next expected symbol, + // but autocompleting them causes issues (e.g. accepting "(" via Enter + // when the user just wants a newline). + const suggestions = autocompleteProvider + .getSuggestions(query, relativeCursorOffset) + .filter((s) => /[a-zA-Z0-9_]/.test(s.insertText)) - const range = { - startLineNumber: position.lineNumber, - endLineNumber: position.lineNumber, - startColumn: word.startColumn, - endColumn: word.endColumn, - } + // When the "word" at cursor is an operator (e.g. :: from type cast), + // don't replace it — insert after it instead. + const isOperatorWord = + word.word.length > 0 && !/[a-zA-Z0-9_]/.test(word.word[0]) - const nextChar = model.getValueInRange({ - startLineNumber: position.lineNumber, - startColumn: word.endColumn, - endLineNumber: position.lineNumber, - endColumn: word.endColumn + 1, - }) - - const openQuote = textUntilPosition.substr(-1) === '"' - const nextCharQuote = nextChar == '"' - - if ( - /(FROM|INTO|(ALTER|BACKUP|DROP|REINDEX|RENAME|TRUNCATE|VACUUM) TABLE|JOIN|UPDATE)\s$/gim.test( - textUntilPosition, - ) || - (/'$/gim.test(textUntilPosition) && - !textUntilPosition.endsWith("= '")) - ) { - return { - suggestions: getTableCompletions({ - tables, - range, - priority: CompletionItemPriority.High, - openQuote, - nextCharQuote, - }), - } - } + // When the word contains a dot (qualified reference like "t." or "t.col"), + // only replace after the last dot. The prefix before the dot is the + // table/alias qualifier and should be kept. Without this, Monaco filters + // suggestions against "t." and nothing matches. + const dotIndex = word.word.lastIndexOf(".") + const startColumn = isOperatorWord + ? position.column + : dotIndex >= 0 + ? word.startColumn + dotIndex + 1 + : word.startColumn - if ( - /(?:(SELECT|UPDATE).*?(?:(?:,(?:COLUMN )?)|(?:ALTER COLUMN ))?(?:WHERE )?(?: BY )?(?: ON )?(?: SET )?$|ALTER COLUMN )/gim.test( - textUntilPosition, - ) && - !isWhitespaceOnly - ) { - if (tableContext.length > 0) { - const withTableName = - textUntilPosition.match(/\sON\s/gim) !== null - return { - suggestions: [ - ...(isInColumnListing(textUntilPosition) - ? getColumnCompletions({ - columns: tableContext.reduce( - (acc, tableName) => [ - ...acc, - ...(informationSchemaColumns[tableName] ?? []), - ], - [] as InformationSchemaColumn[], - ), - range, - withTableName, - priority: CompletionItemPriority.High, - }) - : []), - ...getLanguageCompletions(range), - ], - } - } else if (isInColumnListing(textUntilPosition)) { - return { - suggestions: [ - ...getColumnCompletions({ - columns: Object.values(informationSchemaColumns).reduce( - (acc, columns) => [...acc, ...columns], - [] as InformationSchemaColumn[], - ), - range, - withTableName: false, - priority: CompletionItemPriority.High, - }), - ], - } - } - } + const range = { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn, + endColumn: word.endColumn, + } - if (word.word) { - return { - suggestions: [ - ...getTableCompletions({ - tables, - range, - priority: CompletionItemPriority.High, - openQuote, - nextCharQuote, - }), - ...getLanguageCompletions(range), - ], - } - } - } + // Filter out suggestions that exactly match the word already typed, + // e.g. don't suggest "FROM" when cursor is right after "FROM". + const currentWord = isOperatorWord + ? "" + : dotIndex >= 0 + ? word.word.substring(dotIndex + 1) + : word.word + + const filtered = suggestions.filter( + (s) => s.insertText.toUpperCase() !== currentWord.toUpperCase(), + ) + + return { + incomplete: true, + suggestions: filtered.map((s) => toCompletionItem(s, range)), } }, } diff --git a/src/scenes/Editor/Monaco/questdb-sql/getLanguageCompletions.ts b/src/scenes/Editor/Monaco/questdb-sql/getLanguageCompletions.ts index d18ba4c0f..5291aff12 100644 --- a/src/scenes/Editor/Monaco/questdb-sql/getLanguageCompletions.ts +++ b/src/scenes/Editor/Monaco/questdb-sql/getLanguageCompletions.ts @@ -1,5 +1,4 @@ -import { operators } from "./operators" -import { dataTypes, functions, keywords } from "@questdb/sql-grammar" +import { dataTypes, functions, keywords, operators } from "@questdb/sql-parser" import { CompletionItemKind } from "./types" import type { IRange } from "monaco-editor" diff --git a/src/scenes/Editor/Monaco/questdb-sql/language.ts b/src/scenes/Editor/Monaco/questdb-sql/language.ts index 01654a870..2fbc2608c 100644 --- a/src/scenes/Editor/Monaco/questdb-sql/language.ts +++ b/src/scenes/Editor/Monaco/questdb-sql/language.ts @@ -1,6 +1,11 @@ -import { operators } from "./operators" import type { languages } from "monaco-editor" -import { constants, dataTypes, functions, keywords } from "@questdb/sql-grammar" +import { + constants, + dataTypes, + functions, + keywords, + operators, +} from "@questdb/sql-parser" import { escapeRegExpCharacters } from "../../../../utils/textSearch" const functionPattern = new RegExp( @@ -121,7 +126,7 @@ export const language: languages.IMonarchLanguage = { ], ], numbers: [ - [/\b(\d+)([utsmhdwmy])\b/i, "number"], // sampling rate + [/[+-]?\d+[utsmhdwyn]\b/i, "number"], // sampling rate/ horizons [/([+-]?\d+\.\d+[eE]?[+-]?\d+)/, "number"], // floating point number [/0[xX][0-9a-fA-F]*/, "number"], // hex integers [/[+-]?\d+((_)?\d+)*[Ll]?/, "number"], // integers diff --git a/src/scenes/Editor/Monaco/questdb-sql/operators.ts b/src/scenes/Editor/Monaco/questdb-sql/operators.ts deleted file mode 100644 index 05e7c32ec..000000000 --- a/src/scenes/Editor/Monaco/questdb-sql/operators.ts +++ /dev/null @@ -1,37 +0,0 @@ -export const operators = [ - // Logical - "ALL", - "AND", - "ANY", - "BETWEEN", - "EXISTS", - "IN", - "LIKE", - "NOT", - "OR", - "SOME", - // Set - "EXCEPT", - "INTERSECT", - "UNION", - // Join - "APPLY", - "CROSS", - "FULL", - "INNER", - "JOIN", - "LEFT", - "OUTER", - "RIGHT", - // Predicates - "CONTAINS", - "FREETEXT", - "IS", - "NULL", - // Pivoting - "PIVOT", - "UNPIVOT", - // Merging - "MATCHED", - "EXPLAIN", -] diff --git a/src/scenes/Editor/Monaco/utils.ts b/src/scenes/Editor/Monaco/utils.ts index 45a7a4212..25e965b2d 100644 --- a/src/scenes/Editor/Monaco/utils.ts +++ b/src/scenes/Editor/Monaco/utils.ts @@ -25,6 +25,7 @@ import type { editor, IPosition, IRange } from "monaco-editor" import type { Monaco } from "@monaco-editor/react" import type { ErrorResult } from "../../../utils" import { hashString } from "../../../utils" +import type { ValidateQueryResult } from "../../../utils/questdb" type IStandaloneCodeEditor = editor.IStandaloneCodeEditor @@ -198,7 +199,8 @@ export const getQueriesFromPosition = ( let startCol = start.column let startPos = startCharIndex - 1 let nextSql = null - let inQuote = false + let inSingleQuote = false + let inDoubleQuote = false let singleLineCommentStack: number[] = [] let multiLineCommentStack: number[] = [] let inSingleLineComment = false @@ -231,7 +233,12 @@ export const getQueriesFromPosition = ( switch (char) { case ";": { - if (inQuote || inSingleLineComment || inMultiLineComment) { + if ( + inSingleQuote || + inDoubleQuote || + inSingleLineComment || + inMultiLineComment + ) { column++ break } @@ -297,15 +304,23 @@ export const getQueriesFromPosition = ( } case "'": { - if (!inMultiLineComment && !inSingleLineComment) { - inQuote = !inQuote + if (!inMultiLineComment && !inSingleLineComment && !inDoubleQuote) { + inSingleQuote = !inSingleQuote + } + column++ + break + } + + case '"': { + if (!inMultiLineComment && !inSingleLineComment && !inSingleQuote) { + inDoubleQuote = !inDoubleQuote } column++ break } case "-": { - if (!inMultiLineComment && !inQuote) { + if (!inMultiLineComment && !inSingleQuote && !inDoubleQuote) { singleLineCommentStack.push(i) if (singleLineCommentStack.length === 2) { if (singleLineCommentStack[0] + 1 === singleLineCommentStack[1]) { @@ -326,7 +341,12 @@ export const getQueriesFromPosition = ( } case "/": { - if (!inMultiLineComment && !inSingleLineComment && !inQuote) { + if ( + !inMultiLineComment && + !inSingleLineComment && + !inSingleQuote && + !inDoubleQuote + ) { if (multiLineCommentStack.length === 0) { multiLineCommentStack.push(i) } else { @@ -352,7 +372,12 @@ export const getQueriesFromPosition = ( } case "*": { - if (!inMultiLineComment && !inSingleLineComment) { + if ( + !inMultiLineComment && + !inSingleLineComment && + !inSingleQuote && + !inDoubleQuote + ) { if ( multiLineCommentStack.length === 1 && multiLineCommentStack[0] + 1 === i @@ -1249,6 +1274,147 @@ export const setErrorMarkerForQuery = ( monaco.editor.setModelMarkers(model, QuestDBLanguageName, markers) } +const ValidationOwner = "questdb-validation" + +const validationRefs: Record< + string, + { markers: editor.IMarkerData[]; queryText: string; version: number } +> = {} + +const validationControllers: Record = {} + +export const clearValidationMarkers = ( + monaco: Monaco, + editor: IStandaloneCodeEditor, + bufferId?: number, +) => { + const model = editor.getModel() + if (model) { + monaco.editor.setModelMarkers(model, ValidationOwner, []) + } + if (bufferId !== undefined) { + const bufferKey = bufferId.toString() + delete validationRefs[bufferKey] + validationControllers[bufferKey]?.abort() + delete validationControllers[bufferKey] + } +} + +export const applyValidationMarkers = ( + monaco: Monaco, + editor: IStandaloneCodeEditor, + bufferId: number, +) => { + const model = editor.getModel() + if (!model) return + + const cached = validationRefs[bufferId.toString()] + if (cached) { + monaco.editor.setModelMarkers(model, ValidationOwner, cached.markers) + } +} + +export const validateQueryJIT = ( + monaco: Monaco, + editor: IStandaloneCodeEditor, + bufferId: number, + getBufferExecutions: () => Record, + validateQuery: ( + query: string, + signal: AbortSignal, + ) => Promise, +) => { + const model = editor.getModel() + if (!model) return + + const bufferKey = bufferId.toString() + const queryAtCursor = getQueryFromCursor(editor) + + if (!queryAtCursor) { + validationControllers[bufferKey]?.abort() + delete validationControllers[bufferKey] + monaco.editor.setModelMarkers(model, ValidationOwner, []) + delete validationRefs[bufferKey] + return + } + + const queryText = normalizeQueryText(queryAtCursor.query) + const version = model.getVersionId() + + // Skip if already validated this exact query+version + const cached = validationRefs[bufferKey] + if (cached && cached.queryText === queryText && cached.version === version) { + return + } + + // Skip if execution result already exists for this query + const queryKey = createQueryKeyFromRequest(editor, queryAtCursor) + const bufferExecutions = getBufferExecutions() + if (bufferExecutions[queryKey]) { + validationControllers[bufferKey]?.abort() + delete validationControllers[bufferKey] + monaco.editor.setModelMarkers(model, ValidationOwner, []) + delete validationRefs[bufferKey] + return + } + + // Abort any previous in-flight validation for this buffer + validationControllers[bufferKey]?.abort() + const controller = new AbortController() + validationControllers[bufferKey] = controller + + validateQuery(queryText, controller.signal) + .then((result: ValidateQueryResult) => { + if (validationControllers[bufferKey] === controller) { + delete validationControllers[bufferKey] + } + + const currentModel = editor.getModel() + if (!currentModel || currentModel.getVersionId() !== version) return + + // Query was executed while validation was in flight — skip + if (getBufferExecutions()[queryKey]) return + + if ("error" in result) { + const errorRange = getErrorRange(editor, queryAtCursor, result.position) + const markers: editor.IMarkerData[] = [] + + if (errorRange) { + markers.push({ + message: result.error, + severity: monaco.MarkerSeverity.Error, + startLineNumber: errorRange.startLineNumber, + endLineNumber: errorRange.endLineNumber, + startColumn: errorRange.startColumn, + endColumn: errorRange.endColumn, + }) + } else { + const errorPos = toTextPosition(queryAtCursor, result.position) + markers.push({ + message: result.error, + severity: monaco.MarkerSeverity.Error, + startLineNumber: errorPos.lineNumber, + endLineNumber: errorPos.lineNumber, + startColumn: errorPos.column, + endColumn: errorPos.column, + }) + } + + validationRefs[bufferKey] = { markers, queryText, version } + monaco.editor.setModelMarkers(currentModel, ValidationOwner, markers) + } else { + delete validationRefs[bufferKey] + monaco.editor.setModelMarkers(currentModel, ValidationOwner, []) + } + }) + .catch(() => { + // Abort or network error — silently ignore + if (validationControllers[bufferKey] === controller) { + delete validationControllers[bufferKey] + } + }) +} + // Creates a QueryKey for schema explanation conversations // Uses DDL hash so same schema = same queryKey = cached conversation export const createSchemaQueryKey = ( diff --git a/src/utils/questdb/client.ts b/src/utils/questdb/client.ts index 017e87e62..45c034ed2 100644 --- a/src/utils/questdb/client.ts +++ b/src/utils/questdb/client.ts @@ -339,11 +339,15 @@ export class Client { } } - async validateQuery(query: string): Promise { + async validateQuery( + query: string, + signal?: AbortSignal, + ): Promise { const response = await fetch( `api/v1/sql/validate?${Client.encodeParams({ query })}`, { headers: this.commonHeaders, + signal, }, ) if (response.ok) { diff --git a/yarn.lock b/yarn.lock index 5e869060a..367fad1f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1471,6 +1471,48 @@ __metadata: languageName: node linkType: hard +"@chevrotain/cst-dts-gen@npm:11.1.1": + version: 11.1.1 + resolution: "@chevrotain/cst-dts-gen@npm:11.1.1" + dependencies: + "@chevrotain/gast": "npm:11.1.1" + "@chevrotain/types": "npm:11.1.1" + lodash-es: "npm:4.17.23" + checksum: 10/ff69fa978abea4075cbeb8356e1d297c7063091044d4766ea9f7e037c1b0745f19bade1527f08bc7304903e54a80342faab2641004a3c9e7e9b8027c979f1cbf + languageName: node + linkType: hard + +"@chevrotain/gast@npm:11.1.1": + version: 11.1.1 + resolution: "@chevrotain/gast@npm:11.1.1" + dependencies: + "@chevrotain/types": "npm:11.1.1" + lodash-es: "npm:4.17.23" + checksum: 10/7e3e951cdf1f4c90634f75a7f284b521bf01e03580ae725deefeb4178d12c70c4eeb0e44f39938cb061943940d14e5d2a79c355dae1ca7ed26a86cfc16f72e6f + languageName: node + linkType: hard + +"@chevrotain/regexp-to-ast@npm:11.1.1": + version: 11.1.1 + resolution: "@chevrotain/regexp-to-ast@npm:11.1.1" + checksum: 10/6eef0f317107e79b72bcd8af0f05837166200e3b8dec8dd2e235a1b73f190e2461beb86a3824627d392f59ca3bd9eb9ac0361928d9fa36664772bdac1ce857ca + languageName: node + linkType: hard + +"@chevrotain/types@npm:11.1.1": + version: 11.1.1 + resolution: "@chevrotain/types@npm:11.1.1" + checksum: 10/99c3cd6f1f77af9a0929b8ec0324ba08bbba0cac8c59e74c83ded1316b13ead5546881363089ecb45c0921738ba243c0ec15ca9d1e3a45b3aac0b157e7030cb2 + languageName: node + linkType: hard + +"@chevrotain/utils@npm:11.1.1": + version: 11.1.1 + resolution: "@chevrotain/utils@npm:11.1.1" + checksum: 10/c3a0dd1fbd3062b3193d200e043b612347b116383b2940239dd3d52764f14bb50a01deeb84ec944672c17799e94af9245057a9442cf6f3165fd8e0ec2026d814 + languageName: node + linkType: hard + "@colors/colors@npm:1.5.0": version: 1.5.0 resolution: "@colors/colors@npm:1.5.0" @@ -2449,10 +2491,12 @@ __metadata: languageName: node linkType: hard -"@questdb/sql-grammar@npm:1.4.2": - version: 1.4.2 - resolution: "@questdb/sql-grammar@npm:1.4.2" - checksum: 10/55c1dd2af91bebcf31953b110f6922f03145fef6bcbb905d8758d5bb6252d7b1ce25215bb63d1811150476f901da533d6a8fff3d667ded0ec44e5856064165cd +"@questdb/sql-parser@npm:0.1.3": + version: 0.1.3 + resolution: "@questdb/sql-parser@npm:0.1.3" + dependencies: + chevrotain: "npm:^11.1.1" + checksum: 10/dfa079c6789153b6b188b926efa75d04d24a45bc2055369aa3b96de2928c7f4ab75ac76c99f179f90bb1a7744c37b0a7dc70579b6eabd1c5ef5bdf3ee84b38cd languageName: node linkType: hard @@ -2474,7 +2518,7 @@ __metadata: "@monaco-editor/react": "npm:^4.7.0" "@phosphor-icons/react": "npm:^2.1.10" "@popperjs/core": "npm:2.4.2" - "@questdb/sql-grammar": "npm:1.4.2" + "@questdb/sql-parser": "npm:0.1.3" "@radix-ui/react-alert-dialog": "npm:^1.1.15" "@radix-ui/react-context-menu": "npm:^2.2.16" "@radix-ui/react-dialog": "npm:^1.1.15" @@ -5218,6 +5262,20 @@ __metadata: languageName: node linkType: hard +"chevrotain@npm:^11.1.1": + version: 11.1.1 + resolution: "chevrotain@npm:11.1.1" + dependencies: + "@chevrotain/cst-dts-gen": "npm:11.1.1" + "@chevrotain/gast": "npm:11.1.1" + "@chevrotain/regexp-to-ast": "npm:11.1.1" + "@chevrotain/types": "npm:11.1.1" + "@chevrotain/utils": "npm:11.1.1" + lodash-es: "npm:4.17.23" + checksum: 10/e90972f939b597908843e2b6ed23ed6756b0ce103ee2822c651d0a75cd93913e1e3d6c486315922ce3334a1ea9e1f6d0e62288d1ace8ff8e419bf8613be1b694 + languageName: node + linkType: hard + "chokidar@npm:^3.6.0": version: 3.6.0 resolution: "chokidar@npm:3.6.0" @@ -8445,6 +8503,13 @@ __metadata: languageName: node linkType: hard +"lodash-es@npm:4.17.23": + version: 4.17.23 + resolution: "lodash-es@npm:4.17.23" + checksum: 10/1feae200df22eb0bd93ca86d485e77784b8a9fb1d13e91b66e9baa7a7e5e04be088c12a7e20c2250fc0bd3db1bc0ef0affc7d9e3810b6af2455a3c6bf6dde59e + languageName: node + linkType: hard + "lodash.clamp@npm:^4.0.0": version: 4.0.3 resolution: "lodash.clamp@npm:4.0.3"