You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2026/08/19 18:00:50 UTC
(asterixdb) 03/05: [ASTERIXDB-3593]: Keep the text of each statement
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 28e3dcf45478228ea7693510f95ca2b3eba483e2
Author: Janhavi Tripurwar <ja...@couchbase.com>
AuthorDate: Mon Aug 17 16:11:13 2026 +0530
[ASTERIXDB-3593]: Keep the text of each statement
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- The text a statement was parsed from was not available to what
reports the statements of a request.
- The SQL++ parser now records where each statement began and ended
and builds its text on demand, through the new
IParser.getStatementTexts(); StatementInfo carries it.
Ext-ref: MB-62708
Change-Id: I2a866a51f80747e545ba4d03e237359f8e78a170
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21569
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Janhavi Tripurwar <ja...@apache.org>
---
.../asterix/translator/IStatementExecutor.java | 10 +++
.../message/ExecuteStatementRequestMessage.java | 25 +++++--
.../apache/asterix/lang/common/base/IParser.java | 8 +++
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 21 ++++++
.../lang/sqlpp/parser/StatementTextTest.java | 80 ++++++++++++++++++++++
5 files changed, 138 insertions(+), 6 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index 5c3dd65b94..be4e39ff4b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -121,6 +121,8 @@ public interface IStatementExecutor {
private final String name;
private ResultSetInfo resultSet;
private ExecutionPlans plans;
+ /** The statement's own text */
+ private String text;
private Stats stats;
private Throwable error;
private final List<Warning> warnings = new ArrayList<>();
@@ -143,6 +145,14 @@ public interface IStatementExecutor {
return name;
}
+ public String getText() {
+ return text;
+ }
+
+ public void setText(String text) {
+ this.text = text;
+ }
+
public ResultSetInfo getResultSet() {
return resultSet;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index 693019e142..4811368931 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -194,7 +194,7 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
translator.compileAndExecute(ccApp.getHcc(), requestParameters);
} finally {
// the parser's warnings belong to their statement whether or not the request went on to fail
- addParserWarningsToStatements(outMetadata, parser.getWarningsPerStatement(), maxWarnings);
+ addParserResultsToStatements(outMetadata, parser, maxWarnings);
}
translator.getWarnings(warnings, maxWarnings - warnings.size());
stats.updateTotalWarningsCount(parserTotalWarningsCount);
@@ -214,14 +214,27 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
}
/**
- * Adds the warnings the parser raised for a statement to what that statement reports, matching them by position.
- * Every warning is counted, and {@code maxWarnings} of them reported, per statement.
+ * Adds what the parser knows about a statement - its text, and the warnings it raised - to what that statement
+ * reports, matching them by position. Every warning is counted, and {@code maxWarnings} of them reported, per
+ * statement.
*/
- private static void addParserWarningsToStatements(IStatementExecutor.ResultMetadata outMetadata,
- List<Set<Warning>> warningsPerStatement, long maxWarnings) {
+ private static void addParserResultsToStatements(IStatementExecutor.ResultMetadata outMetadata, IParser parser,
+ long maxWarnings) {
+ if (outMetadata.getStatements().isEmpty()) {
+ // nothing reports the statements of this request, so their text is not worth building
+ return;
+ }
+ List<Set<Warning>> warningsPerStatement = parser.getWarningsPerStatement();
+ List<String> texts = parser.getStatementTexts();
for (IStatementExecutor.StatementInfo statement : outMetadata.getStatements()) {
int parsedAt = statement.getPosition() - 1;
- if (parsedAt >= 0 && parsedAt < warningsPerStatement.size()) {
+ if (parsedAt < 0) {
+ continue;
+ }
+ if (parsedAt < texts.size()) {
+ statement.setText(texts.get(parsedAt));
+ }
+ if (parsedAt < warningsPerStatement.size()) {
Set<Warning> parserWarnings = warningsPerStatement.get(parsedAt);
statement.addWarnings(parserWarnings.stream().limit(maxWarnings).toList());
if (statement.getStats() != null) {
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParser.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParser.java
index db9273c79d..c8bcd0e786 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParser.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParser.java
@@ -70,4 +70,12 @@ public interface IParser {
default List<Set<Warning>> getWarningsPerStatement() {
return Collections.emptyList();
}
+
+ /**
+ * The text of each statement of the request, in parse order: from a statement's first token to its last, without
+ * the semicolon that ends it. A parser that does not keep the text of a statement returns none.
+ */
+ default List<String> getStatementTexts() {
+ return Collections.emptyList();
+ }
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 763f915d97..e1fdd769c5 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -326,6 +326,13 @@ class SQLPPParser extends ScopeChecker implements IParser {
/** The warnings of each statement, in the order the statements were parsed. */
private final List<Set<Warning>> allStatementWarnings = new ArrayList<Set<Warning>>();
+ /**
+ * Where each statement began and ended - beginLine, beginColumn, endLine, endColumn - in the order the statements
+ * were parsed. Kept rather than the text itself, so that a request whose statements nobody asks about pays only
+ * for the four numbers.
+ */
+ private final List<int[]> allStatementSpans = new ArrayList<int[]>();
+
/** The warnings of the statement being parsed. */
private final Set<Warning> currentStatementWarnings = new HashSet<Warning>();
@@ -606,6 +613,15 @@ class SQLPPParser extends ScopeChecker implements IParser {
return allStatementWarnings;
}
+ @Override
+ public List<String> getStatementTexts() {
+ List<String> texts = new ArrayList<String>(allStatementSpans.size());
+ for (int[] span : allStatementSpans) {
+ texts.add(extractFragment(span[0], span[1], span[2], span[3]));
+ }
+ return texts;
+ }
+
protected String getMessage(ParseException pe) {
Token currentToken = pe.currentToken;
if (currentToken == null) {
@@ -1029,11 +1045,13 @@ List<Statement> Statement() throws ParseException:
scopeStack.push(RootScopeFactory.createRootScope(this));
List<Statement> decls = new ArrayList<Statement>();
Statement stmt = null;
+ Token statementStart = null;
}
{
(
{
currentStatementWarnings.clear();
+ statementStart = getToken(1);
}
(stmt = ExplainOrAdviseStatement()
{
@@ -1041,6 +1059,9 @@ List<Statement> Statement() throws ParseException:
// unclaimed hints are warnings of this statement too
reportUnclaimedHints();
allStatementWarnings.add(new HashSet<Warning>(currentStatementWarnings));
+ // the statement spans its first token to its last; the semicolon that ends it is not part of it
+ allStatementSpans.add(new int[] { statementStart.beginLine, statementStart.beginColumn - 1, token.endLine,
+ token.endColumn + 1 });
}
)?
(<SEMICOLON>)+
diff --git a/asterixdb/asterix-lang-sqlpp/src/test/java/org/apache/asterix/lang/sqlpp/parser/StatementTextTest.java b/asterixdb/asterix-lang-sqlpp/src/test/java/org/apache/asterix/lang/sqlpp/parser/StatementTextTest.java
new file mode 100644
index 0000000000..422c37a72a
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/test/java/org/apache/asterix/lang/sqlpp/parser/StatementTextTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.parser;
+
+import java.util.List;
+
+import org.apache.asterix.common.metadata.NamespaceResolver;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.IParserFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * The text of each statement of a request, which is what lets a deployment record a statement on its own rather than
+ * recording the whole request against every one of them.
+ */
+public class StatementTextTest {
+
+ private static List<String> textsOf(String request) throws Exception {
+ IParserFactory factory = new SqlppParserFactory(new NamespaceResolver(false));
+ IParser parser = factory.createParser(request);
+ parser.parse();
+ return parser.getStatementTexts();
+ }
+
+ @Test
+ public void eachStatementKeepsItsOwnText() throws Exception {
+ Assert.assertEquals(List.of("select 1", "select 2"), textsOf("select 1; select 2;"));
+ }
+
+ /** The semicolon ends a statement rather than belonging to it, and the text of one statement stops there. */
+ @Test
+ public void theSemicolonIsNotPartOfTheText() throws Exception {
+ Assert.assertEquals(List.of("select 1"), textsOf("select 1;"));
+ Assert.assertEquals(List.of("select 1", "select 2"), textsOf("select 1;;;select 2;"));
+ }
+
+ /** A statement is taken from its first token to its last, so what surrounds it is not part of it. */
+ @Test
+ public void whatSurroundsAStatementIsNotPartOfIt() throws Exception {
+ Assert.assertEquals(List.of("select 1", "select 2"),
+ textsOf(" /* a comment */ select 1; -- trailing\n select 2; \n"));
+ }
+
+ @Test
+ public void aStatementSpanningSeveralLinesKeepsThem() throws Exception {
+ List<String> texts = textsOf("select\n 1;\nselect value 2\n from range(1, 2) r;");
+ Assert.assertEquals(2, texts.size());
+ Assert.assertEquals("select\n 1", texts.get(0));
+ Assert.assertEquals("select value 2\n from range(1, 2) r", texts.get(1));
+ }
+
+ /** A comment inside a statement is part of it, since it falls between the statement's first and last token. */
+ @Test
+ public void aCommentInsideAStatementIsPartOfIt() throws Exception {
+ Assert.assertEquals(List.of("select /* here */ 1"), textsOf("select /* here */ 1;"));
+ }
+
+ @Test
+ public void theTextsFollowTheStatementsInOrder() throws Exception {
+ Assert.assertEquals(List.of("use dv", "select 1", "upsert into ds ([{\"id\": 1}])"),
+ textsOf("use dv; select 1; upsert into ds ([{\"id\": 1}]);"));
+ }
+}