diff --git a/THIRD-PARTY b/THIRD-PARTY index 8469c53d5e5..4fe2762c7e8 100644 --- a/THIRD-PARTY +++ b/THIRD-PARTY @@ -467,15 +467,15 @@ DAMAGE. ------ -** ANTLR; version 4.7.1 -- https://github.com/antlr/antlr4 +** ANTLR; version 4.13.2 -- https://github.com/antlr/antlr4 /* - * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. + * Copyright (c) 2012-2024 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that * can be found in the LICENSE.txt file in the project root. */ [The "BSD 3-clause license"] -Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. +Copyright (c) 2012-2024 The ANTLR Project. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions diff --git a/docs/user/ppl/interfaces/endpoint.md b/docs/user/ppl/interfaces/endpoint.md index d5958ba3250..3f87180dca6 100644 --- a/docs/user/ppl/interfaces/endpoint.md +++ b/docs/user/ppl/interfaces/endpoint.md @@ -201,3 +201,25 @@ Expected output (trimmed): - Plan node names use Calcite physical operator names (for example, `EnumerableCalc` or `CalciteEnumerableIndexScan`). - Plan `time_ms` is inclusive of child operators and represents wall-clock time; overlapping work can make summed plan times exceed `summary.total_time_ms`. - Scan nodes reflect operator wall-clock time; background prefetch can make scan time smaller than total request latency. + +## Grammar (Experimental) + +### Description + +You can send an HTTP GET request to endpoint **/_plugins/_ppl/_grammar** to fetch serialized PPL grammar metadata used by autocomplete clients. + +### Example + +```bash +curl -sS -X GET localhost:9200/_plugins/_ppl/_grammar | python3 -m json.tool | grep -E '"bundleVersion"|"antlrVersion"|"startRuleIndex"|"ignoredTokens"|"rulesToVisit"' +``` + +Expected output (trimmed): + +```text +"bundleVersion": "1.0", +"antlrVersion": "4.13.2", +"startRuleIndex": 0, +"ignoredTokens": [ +"rulesToVisit": [ +``` diff --git a/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java b/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java index d84ca2af1b3..ed25a1df2d9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java @@ -339,6 +339,23 @@ private JSONObject executeQueryAsUser(String query, String username) throws IOEx return new JSONObject(org.opensearch.sql.legacy.TestUtils.getResponseBody(response, true)); } + /** Executes a grammar metadata request as a specific user with basic authentication. */ + private JSONObject executeGrammarAsUser(String username) throws IOException { + Request request = new Request("GET", "/_plugins/_ppl/_grammar"); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader( + "Authorization", + "Basic " + + java.util.Base64.getEncoder() + .encodeToString((username + ":" + STRONG_PASSWORD).getBytes())); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return new JSONObject(org.opensearch.sql.legacy.TestUtils.getResponseBody(response, true)); + } + @Test public void testUserWithBankPermissionCanAccessBankIndex() throws IOException { // Test that bank_user can access bank index - this should work with the fix @@ -512,6 +529,32 @@ public void testBankUserWithEvalCommand() throws IOException { verifyColumn(result, columnName("full_name")); } + @Test + public void testUserWithPPLPermissionCanAccessGrammarEndpoint() throws IOException { + JSONObject result = executeGrammarAsUser(BANK_USER); + assertTrue(result.has("bundleVersion")); + assertTrue(result.has("antlrVersion")); + assertTrue(result.has("grammarHash")); + assertTrue(result.has("tokenDictionary")); + } + + @Test + public void testUserWithoutPPLPermissionCannotAccessGrammarEndpoint() throws IOException { + try { + executeGrammarAsUser(NO_PPL_USER); + fail("Expected security exception for user without PPL permission"); + } catch (ResponseException e) { + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + String responseBody = + org.opensearch.sql.legacy.TestUtils.getResponseBody(e.getResponse(), false); + assertTrue( + "Response should contain permission error message", + responseBody.contains("no permissions") + || responseBody.contains("Forbidden") + || responseBody.contains("cluster:admin/opensearch/ppl")); + } + } + // Negative test cases for missing permissions @Test diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/api/ppl.grammar.json b/integ-test/src/yamlRestTest/resources/rest-api-spec/api/ppl.grammar.json new file mode 100644 index 00000000000..d2be3e0189d --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/api/ppl.grammar.json @@ -0,0 +1,18 @@ +{ + "ppl.grammar": { + "documentation": { + "url": "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/interfaces/endpoint.md", + "description": "PPL Grammar Endpoint for Autocomplete" + }, + "stability": "experimental", + "url": { + "paths": [ + { + "path": "/_plugins/_ppl/_grammar", + "methods": ["GET"] + } + ] + }, + "params": {} + } +} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.grammar.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.grammar.yml new file mode 100644 index 00000000000..bb1ade36bda --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.grammar.yml @@ -0,0 +1,18 @@ +--- +"PPL grammar endpoint returns expected response shape": + - do: + ppl.grammar: {} + - is_true: bundleVersion + - is_true: antlrVersion + - is_true: grammarHash + - match: {startRuleIndex: 0} + - gt: {lexerSerializedATN.0: 0} + - is_true: lexerRuleNames.0 + - is_true: channelNames.0 + - is_true: modeNames.0 + - gt: {parserSerializedATN.0: 0} + - is_true: parserRuleNames.0 + - is_true: symbolicNames.1 + - is_true: tokenDictionary.PIPE + - gt: {ignoredTokens.0: 0} + - gt: {rulesToVisit.0: 0} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index d817e13c69f..edffd65f6bf 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -94,6 +94,7 @@ import org.opensearch.sql.opensearch.storage.OpenSearchDataSourceFactory; import org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; +import org.opensearch.sql.plugin.rest.RestPPLGrammarAction; import org.opensearch.sql.plugin.rest.RestPPLQueryAction; import org.opensearch.sql.plugin.rest.RestPPLStatsAction; import org.opensearch.sql.plugin.rest.RestQuerySettingsAction; @@ -163,6 +164,7 @@ public List getRestHandlers( return Arrays.asList( new RestPPLQueryAction(), + new RestPPLGrammarAction(), new RestSqlAction(settings, injector), new RestSqlStatsAction(settings, restController), new RestPPLStatsAction(settings, restController), diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLGrammarAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLGrammarAction.java new file mode 100644 index 00000000000..f10a3e30831 --- /dev/null +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLGrammarAction.java @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin.rest; + +import static org.opensearch.rest.RestRequest.Method.GET; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.util.List; +import lombok.extern.log4j.Log4j2; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.sql.plugin.transport.PPLQueryAction; +import org.opensearch.sql.plugin.transport.TransportPPLQueryRequest; +import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse; +import org.opensearch.sql.ppl.autocomplete.GrammarBundle; +import org.opensearch.sql.ppl.autocomplete.PPLGrammarBundleBuilder; +import org.opensearch.transport.client.node.NodeClient; + +/* + * REST handler for {@code GET /_plugins/_ppl/_grammar}. + * + * @opensearch.experimental + */ +@ExperimentalApi +@Log4j2 +public class RestPPLGrammarAction extends BaseRestHandler { + + private static final String ENDPOINT_PATH = "/_plugins/_ppl/_grammar"; + + @Override + public String getName() { + return "ppl_grammar_action"; + } + + @Override + public List routes() { + return ImmutableList.of(new Route(GET, ENDPOINT_PATH)); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) + throws IOException { + + return channel -> { + try { + authorizeRequest( + client, + new ActionListener<>() { + @Override + public void onResponse(TransportPPLQueryResponse ignored) { + try { + GrammarBundle bundle = getBundle(); + XContentBuilder builder = channel.newBuilder(); + serializeBundle(builder, bundle); + channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); + } catch (Exception e) { + log.error("Error building or serializing PPL grammar", e); + sendErrorResponse(channel, e); + } + } + + @Override + public void onFailure(Exception e) { + log.error("PPL grammar authorization failed", e); + sendErrorResponse(channel, e); + } + }); + } catch (Exception e) { + log.error("Error authorizing PPL grammar request", e); + sendErrorResponse(channel, e); + } + }; + } + + @VisibleForTesting + protected void authorizeRequest( + NodeClient client, ActionListener listener) { + client.execute( + PPLQueryAction.INSTANCE, new TransportPPLQueryRequest("", null, ENDPOINT_PATH), listener); + } + + private void sendErrorResponse(RestChannel channel, Exception e) { + try { + channel.sendResponse(new BytesRestResponse(channel, e)); + } catch (IOException ioException) { + log.error("Failed to send PPL grammar error response", ioException); + } + } + + /** Gets the grammar bundle. Override in tests to inject a custom or failing bundle provider. */ + @VisibleForTesting + protected GrammarBundle getBundle() { + return PPLGrammarBundleBuilder.getBundle(); + } + + private void serializeBundle(XContentBuilder builder, GrammarBundle bundle) throws IOException { + builder.startObject(); + + // Identity & versioning + builder.field("bundleVersion", bundle.getBundleVersion()); + builder.field("antlrVersion", bundle.getAntlrVersion()); + builder.field("grammarHash", bundle.getGrammarHash()); + builder.field("startRuleIndex", bundle.getStartRuleIndex()); + + // Lexer ATN & metadata + builder.field("lexerSerializedATN", bundle.getLexerSerializedATN()); + builder.field("lexerRuleNames", bundle.getLexerRuleNames()); + builder.field("channelNames", bundle.getChannelNames()); + builder.field("modeNames", bundle.getModeNames()); + + // Parser ATN & metadata + builder.field("parserSerializedATN", bundle.getParserSerializedATN()); + builder.field("parserRuleNames", bundle.getParserRuleNames()); + + // Vocabulary + builder.field("literalNames", bundle.getLiteralNames()); + builder.field("symbolicNames", bundle.getSymbolicNames()); + + // Autocomplete configuration + builder.field("tokenDictionary", bundle.getTokenDictionary()); + builder.field("ignoredTokens", bundle.getIgnoredTokens()); + builder.field("rulesToVisit", bundle.getRulesToVisit()); + + builder.endObject(); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index 27bfe2084f7..48bc36374a8 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -100,13 +100,21 @@ protected void doExecute( + " false")); return; } + + TransportPPLQueryRequest transportRequest = TransportPPLQueryRequest.fromActionRequest(request); + if (transportRequest.isGrammarRequest()) { + // Authorization is enforced by this transport action before returning grammar metadata in + // REST. + listener.onResponse(new TransportPPLQueryResponse("{}")); + return; + } + Metrics.getInstance().getNumericalMetric(MetricName.PPL_REQ_TOTAL).increment(); Metrics.getInstance().getNumericalMetric(MetricName.PPL_REQ_COUNT_TOTAL).increment(); QueryContext.addRequestId(); PPLService pplService = injector.getInstance(PPLService.class); - TransportPPLQueryRequest transportRequest = TransportPPLQueryRequest.fromActionRequest(request); // in order to use PPL service, we need to convert TransportPPLQueryRequest to PPLQueryRequest PPLQueryRequest transformedRequest = transportRequest.toPPLQueryRequest(); QueryContext.setProfile(transformedRequest.profile()); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java index e342d9a90f0..6db2bd249ae 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java @@ -119,7 +119,16 @@ public String getRequest() { * @return true if it is an explain request */ public boolean isExplainRequest() { - return path.endsWith("/_explain"); + return path != null && path.endsWith("/_explain"); + } + + /** + * Check if request is for grammar metadata endpoint. + * + * @return true if it is a grammar metadata request + */ + public boolean isGrammarRequest() { + return path != null && path.endsWith("/_grammar"); } /** Decide on the formatter by the requested format. */ diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestPPLGrammarActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestPPLGrammarActionTest.java new file mode 100644 index 00000000000..2eee8750360 --- /dev/null +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestPPLGrammarActionTest.java @@ -0,0 +1,271 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin.rest; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse; +import org.opensearch.sql.ppl.autocomplete.GrammarBundle; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.transport.client.node.NodeClient; + +/** Unit tests for {@link RestPPLGrammarAction}. */ +public class RestPPLGrammarActionTest { + + private RestPPLGrammarAction action; + private NodeClient client; + + @Before + public void setUp() { + action = + new RestPPLGrammarAction() { + @Override + protected void authorizeRequest( + NodeClient client, ActionListener listener) { + listener.onResponse(new TransportPPLQueryResponse("{}")); + } + }; + client = mock(NodeClient.class); + } + + @Test + public void testName() { + assertEquals("ppl_grammar_action", action.getName()); + } + + @Test + public void testRoutes() { + assertEquals(1, action.routes().size()); + assertEquals(RestRequest.Method.GET, action.routes().get(0).getMethod()); + assertEquals("/_plugins/_ppl/_grammar", action.routes().get(0).getPath()); + } + + @Test + public void testGetGrammar_ReturnsBundle() throws Exception { + FakeRestRequest request = newGrammarGetRequest(); + + MockRestChannel channel = new MockRestChannel(request, true); + action.handleRequest(request, channel, client); + + RestResponse response = channel.getResponse(); + assertNotNull("Response should not be null", response); + assertEquals("Should return 200 OK", RestStatus.OK, response.status()); + + String content = response.content().utf8ToString(); + JSONObject json = new JSONObject(content); + + // Identity & versioning + assertEquals("1.0", json.getString("bundleVersion")); + assertTrue( + "antlrVersion should be a version string", + json.getString("antlrVersion").matches("\\d+\\.\\d+.*")); + assertTrue( + "grammarHash should start with sha256:", + json.getString("grammarHash").startsWith("sha256:")); + assertEquals(0, json.getInt("startRuleIndex")); + + // Lexer ATN & metadata (non-empty arrays) + assertTrue(json.getJSONArray("lexerSerializedATN").length() > 0); + assertTrue(json.getJSONArray("lexerRuleNames").length() > 0); + assertTrue(json.getJSONArray("channelNames").length() > 0); + assertTrue(json.getJSONArray("modeNames").length() > 0); + + // Parser ATN & metadata (non-empty arrays) + assertTrue(json.getJSONArray("parserSerializedATN").length() > 0); + assertTrue(json.getJSONArray("parserRuleNames").length() > 0); + + // Vocabulary (non-empty arrays) + assertTrue(json.getJSONArray("literalNames").length() > 0); + assertTrue(json.getJSONArray("symbolicNames").length() > 0); + + // Autocomplete configuration + assertTrue(json.getJSONObject("tokenDictionary").length() > 0); + assertTrue(json.getJSONArray("ignoredTokens").length() > 0); + assertTrue(json.getJSONArray("rulesToVisit").length() > 0); + } + + @Test + public void testGetGrammar_UsesBundleProvider() throws Exception { + int[] calls = {0}; + RestPPLGrammarAction providerAction = + new RestPPLGrammarAction() { + @Override + protected void authorizeRequest( + NodeClient client, ActionListener listener) { + listener.onResponse(new TransportPPLQueryResponse("{}")); + } + + @Override + protected GrammarBundle getBundle() { + calls[0]++; + return super.getBundle(); + } + }; + + FakeRestRequest request1 = newGrammarGetRequest(); + MockRestChannel channel1 = new MockRestChannel(request1, true); + providerAction.handleRequest(request1, channel1, client); + + FakeRestRequest request2 = newGrammarGetRequest(); + MockRestChannel channel2 = new MockRestChannel(request2, true); + providerAction.handleRequest(request2, channel2, client); + + assertEquals("Bundle provider should be invoked once per request", 2, calls[0]); + } + + @Test + public void testGetGrammar_ErrorPath_Returns500() throws Exception { + RestPPLGrammarAction failingAction = + new RestPPLGrammarAction() { + @Override + protected void authorizeRequest( + NodeClient client, ActionListener listener) { + listener.onResponse(new TransportPPLQueryResponse("{}")); + } + + @Override + protected GrammarBundle getBundle() { + throw new RuntimeException("simulated build failure"); + } + }; + + FakeRestRequest request = newGrammarGetRequest(); + MockRestChannel channel = new MockRestChannel(request, true); + failingAction.handleRequest(request, channel, client); + + assertEquals(RestStatus.INTERNAL_SERVER_ERROR, channel.getResponse().status()); + } + + @Test + public void testGetGrammar_NullBundle_Returns500() throws Exception { + RestPPLGrammarAction nullBundleAction = + new RestPPLGrammarAction() { + @Override + protected void authorizeRequest( + NodeClient client, ActionListener listener) { + listener.onResponse(new TransportPPLQueryResponse("{}")); + } + + @Override + protected GrammarBundle getBundle() { + return null; + } + }; + + FakeRestRequest request = newGrammarGetRequest(); + MockRestChannel channel = new MockRestChannel(request, true); + nullBundleAction.handleRequest(request, channel, client); + + assertEquals(RestStatus.INTERNAL_SERVER_ERROR, channel.getResponse().status()); + } + + @Test + public void testGetGrammar_AuthorizationFailure_Returns403() throws Exception { + RestPPLGrammarAction unauthorizedAction = + new RestPPLGrammarAction() { + @Override + protected void authorizeRequest( + NodeClient client, ActionListener listener) { + listener.onFailure(new OpenSearchStatusException("forbidden", RestStatus.FORBIDDEN)); + } + }; + + FakeRestRequest request = newGrammarGetRequest(); + MockRestChannel channel = new MockRestChannel(request, true); + unauthorizedAction.handleRequest(request, channel, client); + + assertEquals(RestStatus.FORBIDDEN, channel.getResponse().status()); + } + + private static FakeRestRequest newGrammarGetRequest() { + return new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withMethod(RestRequest.Method.GET) + .withPath("/_plugins/_ppl/_grammar") + .build(); + } + + /** Mock RestChannel to capture responses */ + private static class MockRestChannel implements RestChannel { + private final RestRequest request; + private final boolean detailedErrorsEnabled; + private RestResponse response; + + MockRestChannel(RestRequest request, boolean detailedErrorsEnabled) { + this.request = request; + this.detailedErrorsEnabled = detailedErrorsEnabled; + } + + @Override + public void sendResponse(RestResponse response) { + this.response = response; + } + + public RestResponse getResponse() { + return response; + } + + @Override + public RestRequest request() { + return request; + } + + @Override + public boolean detailedErrorsEnabled() { + return detailedErrorsEnabled; + } + + @Override + public boolean detailedErrorStackTraceEnabled() { + return false; + } + + @Override + public XContentBuilder newBuilder() throws IOException { + return XContentBuilder.builder(XContentType.JSON.xContent()); + } + + @Override + public XContentBuilder newErrorBuilder() throws IOException { + return XContentBuilder.builder(XContentType.JSON.xContent()); + } + + @Override + public XContentBuilder newBuilder(MediaType mediaType, boolean useFiltering) + throws IOException { + return XContentBuilder.builder(XContentType.JSON.xContent()); + } + + @Override + public XContentBuilder newBuilder( + MediaType requestContentType, MediaType responseContentType, boolean useFiltering) + throws IOException { + return XContentBuilder.builder(XContentType.JSON.xContent()); + } + + @Override + public BytesStreamOutput bytesOutput() { + return new BytesStreamOutput(); + } + } +} diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/autocomplete/GrammarBundle.java b/ppl/src/main/java/org/opensearch/sql/ppl/autocomplete/GrammarBundle.java new file mode 100644 index 00000000000..552c3f39167 --- /dev/null +++ b/ppl/src/main/java/org/opensearch/sql/ppl/autocomplete/GrammarBundle.java @@ -0,0 +1,159 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.autocomplete; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import lombok.Builder; +import lombok.NonNull; +import lombok.Value; + +/** Serialized ANTLR grammar data served by {@code GET /_plugins/_ppl/_grammar}. */ +@Value +@Builder +public class GrammarBundle { + + /** Bundle format version. */ + @NonNull private String bundleVersion; + + /** ANTLR runtime version used to generate the grammar. */ + @NonNull private String antlrVersion; + + /** + * SHA-256 hash of grammar metadata used by autocomplete (ATN, rule names, vocabulary, ANTLR + * version). Clients may use this to detect grammar changes. + */ + @NonNull private String grammarHash; + + /** Serialized lexer ATN as int array (ATNSerializer output). */ + @NonNull private int[] lexerSerializedATN; + + /** Lexer rule names. */ + @NonNull private String[] lexerRuleNames; + + /** Channel names (e.g. DEFAULT_TOKEN_CHANNEL, HIDDEN). */ + @NonNull private String[] channelNames; + + /** Mode names (e.g. DEFAULT_MODE). */ + @NonNull private String[] modeNames; + + /** Serialized parser ATN as int array (ATNSerializer output). */ + @NonNull private int[] parserSerializedATN; + + /** Parser rule names. */ + @NonNull private String[] parserRuleNames; + + /** Start rule index (0 = root rule). */ + private int startRuleIndex; + + /** + * Literal token names indexed by token type (e.g. "'search'", "'|'"). Elements may be null for + * tokens with no literal form; clients must handle sparse arrays. + */ + @NonNull private String[] literalNames; + + /** + * Symbolic token names indexed by token type (e.g. "SEARCH", "PIPE"). Elements may be null for + * tokens with no symbolic name; clients must handle sparse arrays. + */ + @NonNull private String[] symbolicNames; + + /** + * Autocomplete token dictionary — maps semantic names used by the autocomplete enrichment logic + * (e.g. "SPACE", "PIPE", "SOURCE") to their token type IDs in this grammar. Clients use this to + * configure token-aware enrichment without hardcoding token IDs. + */ + @NonNull private Map tokenDictionary; + + /** + * Token type IDs that should be ignored by CodeCompletionCore during candidate collection. These + * are lexical/internal tokens that should not appear as direct keyword suggestions. + */ + @NonNull private int[] ignoredTokens; + + /** + * Parser rule indices that CodeCompletionCore should treat as preferred rules. When these rules + * are candidate alternatives, CodeCompletionCore reports them as rule candidates instead of + * expanding into their child tokens. The autocomplete enrichment uses these to trigger semantic + * suggestions (e.g. suggest fields, suggest tables). + */ + @NonNull private int[] rulesToVisit; + + public int[] getLexerSerializedATN() { + return copy(lexerSerializedATN); + } + + public String[] getLexerRuleNames() { + return copy(lexerRuleNames); + } + + public String[] getChannelNames() { + return copy(channelNames); + } + + public String[] getModeNames() { + return copy(modeNames); + } + + public int[] getParserSerializedATN() { + return copy(parserSerializedATN); + } + + public String[] getParserRuleNames() { + return copy(parserRuleNames); + } + + public String[] getLiteralNames() { + return copy(literalNames); + } + + public String[] getSymbolicNames() { + return copy(symbolicNames); + } + + public Map getTokenDictionary() { + return Collections.unmodifiableMap(new LinkedHashMap<>(tokenDictionary)); + } + + public int[] getIgnoredTokens() { + return copy(ignoredTokens); + } + + public int[] getRulesToVisit() { + return copy(rulesToVisit); + } + + public static class GrammarBundleBuilder { + public GrammarBundle build() { + return new GrammarBundle( + bundleVersion, + antlrVersion, + grammarHash, + copy(lexerSerializedATN), + copy(lexerRuleNames), + copy(channelNames), + copy(modeNames), + copy(parserSerializedATN), + copy(parserRuleNames), + startRuleIndex, + copy(literalNames), + copy(symbolicNames), + Collections.unmodifiableMap(new LinkedHashMap<>(tokenDictionary)), + copy(ignoredTokens), + copy(rulesToVisit)); + } + } + + private static int[] copy(int[] values) { + return Arrays.copyOf(values, values.length); + } + + private static String[] copy(String[] values) { + return Arrays.copyOf(values, values.length); + } +} diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/autocomplete/PPLGrammarBundleBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/autocomplete/PPLGrammarBundleBuilder.java new file mode 100644 index 00000000000..cd992c740a9 --- /dev/null +++ b/ppl/src/main/java/org/opensearch/sql/ppl/autocomplete/PPLGrammarBundleBuilder.java @@ -0,0 +1,249 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.autocomplete; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.Vocabulary; +import org.antlr.v4.runtime.atn.ATNSerializer; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; + +/** Builds the {@link GrammarBundle} for the PPL language from the generated ANTLR lexer/parser. */ +public final class PPLGrammarBundleBuilder { + private static final String ANTLR_VERSION = + org.antlr.v4.runtime.RuntimeMetaData.getRuntimeVersion(); + private static final String BUNDLE_VERSION = "1.0"; + private static final Set INTERNAL_NON_LITERAL_TOKENS = + new HashSet<>( + Arrays.asList( + "ID", + "NUMERIC_ID", + "ID_DATE_SUFFIX", + "CLUSTER", + "TIME_SNAP", + "SPANLENGTH", + "DECIMAL_SPANLENGTH", + "DQUOTA_STRING", + "SQUOTA_STRING", + "BQUOTA_STRING", + "LINE_COMMENT", + "BLOCK_COMMENT", + "ERROR_RECOGNITION")); + + private PPLGrammarBundleBuilder() {} + + private static class BundleHolder { + private static final GrammarBundle INSTANCE = build(); + } + + /** Lazily builds and returns the singleton grammar bundle for this JVM. */ + public static GrammarBundle getBundle() { + return BundleHolder.INSTANCE; + } + + private static GrammarBundle build() { + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(CharStreams.fromString("")); + CommonTokenStream tokens = new CommonTokenStream(lexer); + OpenSearchPPLParser parser = new OpenSearchPPLParser(tokens); + + int[] lexerATN = new ATNSerializer(lexer.getATN()).serialize().toArray(); + int[] parserATN = new ATNSerializer(parser.getATN()).serialize().toArray(); + + Vocabulary vocabulary = parser.getVocabulary(); + int maxTokenType = vocabulary.getMaxTokenType(); + String[] literalNames = new String[maxTokenType + 1]; + String[] symbolicNames = new String[maxTokenType + 1]; + for (int i = 0; i <= maxTokenType; i++) { + literalNames[i] = vocabulary.getLiteralName(i); + symbolicNames[i] = vocabulary.getSymbolicName(i); + } + + return GrammarBundle.builder() + .bundleVersion(BUNDLE_VERSION) + .antlrVersion(ANTLR_VERSION) + .grammarHash( + computeGrammarHash( + lexerATN, + parserATN, + lexer.getRuleNames(), + parser.getRuleNames(), + literalNames, + symbolicNames)) + .lexerSerializedATN(lexerATN) + .parserSerializedATN(parserATN) + .lexerRuleNames(lexer.getRuleNames()) + .parserRuleNames(parser.getRuleNames()) + .channelNames(lexer.getChannelNames()) + .modeNames(lexer.getModeNames()) + .startRuleIndex(resolveStartRuleIndex(parser.getRuleNames())) + .literalNames(literalNames) + .symbolicNames(symbolicNames) + .tokenDictionary(buildTokenDictionary()) + .ignoredTokens(buildIgnoredTokens(vocabulary)) + .rulesToVisit(buildRulesToVisit(parser.getRuleNames())) + .build(); + } + + /** + * Build the token dictionary — semantic name → token type ID mapping. Uses lexer constants since + * token type IDs are defined by the lexer. The frontend autocomplete enrichment uses these to + * identify tokens like PIPE and SOURCE by name. + */ + private static Map buildTokenDictionary() { + Map dict = new LinkedHashMap<>(); + dict.put("WHITESPACE", OpenSearchPPLLexer.WHITESPACE); + dict.put("FROM", OpenSearchPPLLexer.FROM); + dict.put("OPENING_BRACKET", OpenSearchPPLLexer.LT_PRTHS); + dict.put("CLOSING_BRACKET", OpenSearchPPLLexer.RT_PRTHS); + dict.put("SEARCH", OpenSearchPPLLexer.SEARCH); + dict.put("SOURCE", OpenSearchPPLLexer.SOURCE); + dict.put("PIPE", OpenSearchPPLLexer.PIPE); + dict.put("ID", OpenSearchPPLLexer.ID); + dict.put("EQUAL", OpenSearchPPLLexer.EQUAL); + dict.put("IN", OpenSearchPPLLexer.IN); + dict.put("COMMA", OpenSearchPPLLexer.COMMA); + dict.put("BACKTICK_QUOTE", OpenSearchPPLLexer.BQUOTA_STRING); + dict.put("DOT", OpenSearchPPLLexer.DOT); + return dict; + } + + /** + * Build token type IDs to ignore for autocomplete. + * + *

Only lexical/internal tokens are ignored (identifiers, literals, quoted-string tokens, + * comments, and error token). User-facing commands/functions/operators are intentionally kept so + * completion dynamically reflects grammar changes. + */ + private static int[] buildIgnoredTokens(Vocabulary vocabulary) { + List ignored = new ArrayList<>(); + + for (int tokenType = 0; tokenType <= vocabulary.getMaxTokenType(); tokenType++) { + String symbolicName = vocabulary.getSymbolicName(tokenType); + if (isLexicalInternalToken(symbolicName)) { + ignored.add(tokenType); + } + } + + return ignored.stream().mapToInt(Integer::intValue).toArray(); + } + + private static boolean isLexicalInternalToken(String symbolicName) { + if (symbolicName == null) { + return false; + } + return symbolicName.endsWith("_LITERAL") || INTERNAL_NON_LITERAL_TOKENS.contains(symbolicName); + } + + /** + * Build the list of parser rule indices for CodeCompletionCore preferredRules. These rules + * trigger semantic suggestions (suggest fields, tables, functions, etc.). + * + * @throws IllegalStateException if any expected rule name is not found in the parser grammar + */ + private static int[] buildRulesToVisit(String[] ruleNames) { + List ruleNamesToVisit = + Arrays.asList( + "statsFunctionName", + "takeAggFunction", + "integerLiteral", + "decimalLiteral", + "keywordsCanBeId", + "renameClasue", + "qualifiedName", + "tableQualifiedName", + "wcQualifiedName", + "positionFunctionName", + "searchableKeyWord", + "stringLiteral", + "searchCommand", + "searchComparisonOperator", + "comparisonOperator", + "sqlLikeJoinType"); + + List ruleNamesList = Arrays.asList(ruleNames); + int[] indices = new int[ruleNamesToVisit.size()]; + for (int i = 0; i < ruleNamesToVisit.size(); i++) { + String name = ruleNamesToVisit.get(i); + int idx = ruleNamesList.indexOf(name); + if (idx < 0) { + throw new IllegalStateException( + "Parser rule '" + + name + + "' not found in grammar — " + + "was it renamed or removed from OpenSearchPPLParser.g4?"); + } + indices[i] = idx; + } + return indices; + } + + private static int resolveStartRuleIndex(String[] ruleNames) { + int idx = Arrays.asList(ruleNames).indexOf("root"); + return Math.max(idx, 0); + } + + private static String computeGrammarHash( + int[] lexerATN, + int[] parserATN, + String[] lexerRuleNames, + String[] parserRuleNames, + String[] literalNames, + String[] symbolicNames) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + updateDigest(digest, lexerATN); + updateDigest(digest, parserATN); + updateDigest(digest, lexerRuleNames); + updateDigest(digest, parserRuleNames); + updateDigest(digest, literalNames); + updateDigest(digest, symbolicNames); + digest.update(ANTLR_VERSION.getBytes(StandardCharsets.UTF_8)); + byte[] hash = digest.digest(); + // Output is always "sha256:" (7 chars) + 64 hex chars = 71 chars. + StringBuilder sb = new StringBuilder(71); + sb.append("sha256:"); + for (byte b : hash) { + sb.append(String.format("%02x", b & 0xFF)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 not available", e); + } + } + + private static void updateDigest(MessageDigest digest, int[] data) { + for (int v : data) { + digest.update((byte) (v >>> 24)); + digest.update((byte) (v >>> 16)); + digest.update((byte) (v >>> 8)); + digest.update((byte) (v)); + } + } + + private static void updateDigest(MessageDigest digest, String[] data) { + for (String value : data) { + if (value == null) { + digest.update((byte) 0); + } else { + digest.update((byte) 1); + digest.update(value.getBytes(StandardCharsets.UTF_8)); + } + // field separator to avoid concatenation ambiguities + digest.update((byte) 0xFF); + } + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/autocomplete/PPLGrammarBundleBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/autocomplete/PPLGrammarBundleBuilderTest.java new file mode 100644 index 00000000000..3792d98555c --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/autocomplete/PPLGrammarBundleBuilderTest.java @@ -0,0 +1,291 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.autocomplete; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.junit.BeforeClass; +import org.junit.Test; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; + +public class PPLGrammarBundleBuilderTest { + + private static final int EXPECTED_ATN_SERIALIZATION_VERSION = 4; + private static final Set EXPECTED_IGNORED_NON_LITERAL_SYMBOLS = + new HashSet<>( + Arrays.asList( + "ID", + "NUMERIC_ID", + "ID_DATE_SUFFIX", + "CLUSTER", + "TIME_SNAP", + "SPANLENGTH", + "DECIMAL_SPANLENGTH", + "DQUOTA_STRING", + "SQUOTA_STRING", + "BQUOTA_STRING", + "LINE_COMMENT", + "BLOCK_COMMENT", + "ERROR_RECOGNITION")); + + private static GrammarBundle bundle; + + @BeforeClass + public static void buildBundle() { + bundle = PPLGrammarBundleBuilder.getBundle(); + } + + @Test + public void testBuildBundleNotNull() { + assertNotNull(bundle); + } + + @Test + public void testBuildBundleVersionIsSet() { + assertEquals("1.0", bundle.getBundleVersion()); + } + + @Test + public void testBuildAntlrVersionIsSet() { + String version = bundle.getAntlrVersion(); + assertNotNull("antlrVersion should not be null", version); + assertTrue( + "antlrVersion should look like a version string, got: " + version, + version.matches("\\d+\\.\\d+.*")); + } + + @Test + public void testBuildGrammarHashHasSha256Format() { + String hash = bundle.getGrammarHash(); + assertNotNull(hash); + assertTrue("grammarHash should start with 'sha256:'", hash.startsWith("sha256:")); + assertEquals("grammarHash should be 71 chars (sha256: + 64 hex)", 71, hash.length()); + } + + @Test + public void testBuildStartRuleIndexIsZero() { + assertEquals(0, bundle.getStartRuleIndex()); + } + + @Test + public void testBuildLexerATNIsNonEmpty() { + assertNotNull(bundle.getLexerSerializedATN()); + assertTrue(bundle.getLexerSerializedATN().length > 0); + } + + @Test + public void testBuildLexerATNIsSerializationVersion4() { + assertEquals( + "Lexer ATN must be serialization version 4 for antlr4ng compatibility", + EXPECTED_ATN_SERIALIZATION_VERSION, + bundle.getLexerSerializedATN()[0]); + } + + @Test + public void testBuildParserATNIsNonEmpty() { + assertNotNull(bundle.getParserSerializedATN()); + assertTrue(bundle.getParserSerializedATN().length > 0); + } + + @Test + public void testBuildParserATNIsSerializationVersion4() { + assertEquals( + "Parser ATN must be serialization version 4 for antlr4ng compatibility", + EXPECTED_ATN_SERIALIZATION_VERSION, + bundle.getParserSerializedATN()[0]); + } + + @Test + public void testBuildLexerRuleNamesAreNonEmpty() { + assertNotNull(bundle.getLexerRuleNames()); + assertTrue(bundle.getLexerRuleNames().length > 0); + } + + @Test + public void testBuildParserRuleNamesAreNonEmpty() { + assertNotNull(bundle.getParserRuleNames()); + assertTrue(bundle.getParserRuleNames().length > 0); + } + + @Test + public void testBuildChannelNamesAreNonEmpty() { + assertNotNull(bundle.getChannelNames()); + assertTrue(bundle.getChannelNames().length > 0); + } + + @Test + public void testBuildModeNamesAreNonEmpty() { + assertNotNull(bundle.getModeNames()); + assertTrue(bundle.getModeNames().length > 0); + } + + @Test + public void testBuildVocabularyIsNonEmpty() { + assertNotNull(bundle.getLiteralNames()); + assertNotNull(bundle.getSymbolicNames()); + assertTrue(bundle.getLiteralNames().length > 0); + assertTrue(bundle.getSymbolicNames().length > 0); + } + + @Test + public void testBuildIsDeterministic() { + GrammarBundle second = PPLGrammarBundleBuilder.getBundle(); + assertSame("Bundle accessor should return the same singleton instance", bundle, second); + assertEquals( + "Repeated accesses should produce the same hash", + bundle.getGrammarHash(), + second.getGrammarHash()); + } + + @Test + public void testTokenDictionaryContainsExpectedEntries() { + Map dict = bundle.getTokenDictionary(); + assertNotNull(dict); + assertEquals((Integer) OpenSearchPPLParser.PIPE, dict.get("PIPE")); + assertEquals((Integer) OpenSearchPPLParser.SOURCE, dict.get("SOURCE")); + assertEquals((Integer) OpenSearchPPLParser.FROM, dict.get("FROM")); + assertEquals((Integer) OpenSearchPPLParser.EQUAL, dict.get("EQUAL")); + assertEquals((Integer) OpenSearchPPLParser.ID, dict.get("ID")); + } + + @Test + public void testIgnoredTokensAreNonEmpty() { + assertNotNull(bundle.getIgnoredTokens()); + assertTrue("ignoredTokens should not be empty", bundle.getIgnoredTokens().length > 0); + } + + @Test + public void testRulesToVisitAreNonEmpty() { + assertNotNull(bundle.getRulesToVisit()); + assertTrue("rulesToVisit should not be empty", bundle.getRulesToVisit().length > 0); + } + + @Test + public void testIgnoredTokensContainOnlyLexicalInternalTokens() { + Set ignored = ignoredTokenSet(); + for (Integer tokenType : ignored) { + String symbol = bundle.getSymbolicNames()[tokenType]; + assertTrue( + "ignoredTokens should contain only lexical/internal tokens, got: " + + symbol + + " (" + + tokenType + + ")", + symbol != null + && (symbol.endsWith("_LITERAL") + || EXPECTED_IGNORED_NON_LITERAL_SYMBOLS.contains(symbol))); + } + } + + @Test + public void testCommandAndKeywordTokensAreNotIgnored() { + Set ignored = ignoredTokenSet(); + assertFalse("LOOKUP should not be ignored", ignored.contains(OpenSearchPPLParser.LOOKUP)); + assertFalse("REPLACE should not be ignored", ignored.contains(OpenSearchPPLParser.REPLACE)); + assertFalse("REVERSE should not be ignored", ignored.contains(OpenSearchPPLParser.REVERSE)); + assertFalse("MVCOMBINE should not be ignored", ignored.contains(OpenSearchPPLParser.MVCOMBINE)); + assertFalse("MVEXPAND should not be ignored", ignored.contains(OpenSearchPPLParser.MVEXPAND)); + assertFalse("LEFT should not be ignored", ignored.contains(OpenSearchPPLParser.LEFT)); + assertFalse("RIGHT should not be ignored", ignored.contains(OpenSearchPPLParser.RIGHT)); + assertFalse("AS should not be ignored", ignored.contains(OpenSearchPPLParser.AS)); + assertFalse("IN should not be ignored", ignored.contains(OpenSearchPPLParser.IN)); + } + + @Test + public void testExpressionFunctionTokensAreNotIgnored() { + Set ignored = ignoredTokenSet(); + assertFalse("MVAPPEND should not be ignored", ignored.contains(OpenSearchPPLParser.MVAPPEND)); + assertFalse("MVJOIN should not be ignored", ignored.contains(OpenSearchPPLParser.MVJOIN)); + assertFalse("MVINDEX should not be ignored", ignored.contains(OpenSearchPPLParser.MVINDEX)); + } + + @Test + public void testNewerGrammarKeywordsAreNotIgnoredWhenPresent() { + // These tokens exist in newer grammar variants (for example graph lookup support). + // Keep this test tolerant so it works across branches with different grammar revisions. + assertTokenNotIgnoredIfPresent("GRAPHLOOKUP"); + assertTokenNotIgnoredIfPresent("START_FIELD"); + assertTokenNotIgnoredIfPresent("FROM_FIELD"); + assertTokenNotIgnoredIfPresent("TO_FIELD"); + assertTokenNotIgnoredIfPresent("MAX_DEPTH"); + assertTokenNotIgnoredIfPresent("DEPTH_FIELD"); + assertTokenNotIgnoredIfPresent("DIRECTION"); + assertTokenNotIgnoredIfPresent("UNI"); + assertTokenNotIgnoredIfPresent("BI"); + assertTokenNotIgnoredIfPresent("SUPPORT_ARRAY"); + assertTokenNotIgnoredIfPresent("BATCH_MODE"); + assertTokenNotIgnoredIfPresent("USE_PIT"); + } + + @Test + public void testArrayGettersAreDefensiveCopies() { + int[] ignoredBefore = bundle.getIgnoredTokens(); + int original = ignoredBefore[0]; + ignoredBefore[0] = -1; + assertEquals( + "ignoredTokens getter should return a defensive copy", + original, + bundle.getIgnoredTokens()[0]); + + String[] symbolsBefore = bundle.getSymbolicNames(); + String originalSymbol = symbolsBefore[1]; + symbolsBefore[1] = "MUTATED"; + assertEquals( + "symbolicNames getter should return a defensive copy", + originalSymbol, + bundle.getSymbolicNames()[1]); + } + + @Test + public void testTokenDictionaryGetterIsUnmodifiableCopy() { + Map dict = bundle.getTokenDictionary(); + Integer pipeBefore = dict.get("PIPE"); + + try { + dict.put("PIPE", -1); + fail("tokenDictionary getter should return an unmodifiable map"); + } catch (UnsupportedOperationException expected) { + // expected + } + + assertEquals(pipeBefore, bundle.getTokenDictionary().get("PIPE")); + } + + private static Set ignoredTokenSet() { + Set ignored = new HashSet<>(); + for (int tokenType : bundle.getIgnoredTokens()) { + ignored.add(tokenType); + } + return ignored; + } + + private static void assertTokenNotIgnoredIfPresent(String symbolicTokenName) { + int tokenType = tokenTypeBySymbolicName(symbolicTokenName); + if (tokenType >= 0) { + assertFalse( + symbolicTokenName + " should not be ignored", ignoredTokenSet().contains(tokenType)); + } + } + + private static int tokenTypeBySymbolicName(String symbolicTokenName) { + String[] symbols = bundle.getSymbolicNames(); + for (int i = 0; i < symbols.length; i++) { + if (symbolicTokenName.equals(symbols[i])) { + return i; + } + } + return -1; + } +}