You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by et...@apache.org on 2024/11/22 14:26:30 UTC
(iceberg) branch main updated: Spark 3.5: Correct the two-stage parsing strategy of antlr parser (#11628)
This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new ce4c44792a Spark 3.5: Correct the two-stage parsing strategy of antlr parser (#11628)
ce4c44792a is described below
commit ce4c44792a31fdadb99489bca50360d9d190877d
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Fri Nov 22 22:26:23 2024 +0800
Spark 3.5: Correct the two-stage parsing strategy of antlr parser (#11628)
---
.../parser/extensions/IcebergSparkSqlExtensionsParser.scala | 8 ++++++--
.../java/org/apache/iceberg/spark/extensions/TestBranchDDL.java | 6 +++---
.../test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java | 8 ++++----
3 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
index 30940d9cff..72023a6f2f 100644
--- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
+++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala
@@ -181,19 +181,23 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) extends ParserI
parser.removeErrorListeners()
parser.addErrorListener(IcebergParseErrorListener)
+ // https://github.com/antlr/antlr4/issues/192#issuecomment-15238595
+ // Save a great deal of time on correct inputs by using a two-stage parsing strategy.
try {
try {
- // first, try parsing with potentially faster SLL mode
+ // first, try parsing with potentially faster SLL mode and BailErrorStrategy
+ parser.setErrorHandler(new BailErrorStrategy)
parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
toResult(parser)
}
catch {
case _: ParseCancellationException =>
- // if we fail, parse with LL mode
+ // if we fail, parse with LL mode with DefaultErrorStrategy
tokenStream.seek(0) // rewind input stream
parser.reset()
// Try Again.
+ parser.setErrorHandler(new DefaultErrorStrategy)
parser.getInterpreter.setPredictionMode(PredictionMode.LL)
toResult(parser)
}
diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java
index fb7f73186a..6bad39dc58 100644
--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java
+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestBranchDDL.java
@@ -218,13 +218,13 @@ public class TestBranchDDL extends ExtensionsTestBase {
assertThatThrownBy(() -> sql("ALTER TABLE %s CREATE BRANCH %s RETAIN", tableName, branchName))
.isInstanceOf(IcebergParseException.class)
- .hasMessageContaining("mismatched input");
+ .hasMessageContaining("no viable alternative at input");
assertThatThrownBy(
() ->
sql("ALTER TABLE %s CREATE BRANCH %s RETAIN %s DAYS", tableName, branchName, "abc"))
.isInstanceOf(IcebergParseException.class)
- .hasMessageContaining("mismatched input");
+ .hasMessageContaining("no viable alternative at input");
assertThatThrownBy(
() ->
@@ -274,7 +274,7 @@ public class TestBranchDDL extends ExtensionsTestBase {
public void testDropBranchNonConformingName() {
assertThatThrownBy(() -> sql("ALTER TABLE %s DROP BRANCH %s", tableName, "123"))
.isInstanceOf(IcebergParseException.class)
- .hasMessageContaining("mismatched input '123'");
+ .hasMessageContaining("no viable alternative at input '123'");
}
@TestTemplate
diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java
index 65c2c0f713..623af87774 100644
--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java
+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestTagDDL.java
@@ -99,12 +99,12 @@ public class TestTagDDL extends ExtensionsTestBase {
"ALTER TABLE %s CREATE TAG %s AS OF VERSION %d RETAIN",
tableName, tagName, firstSnapshotId, maxRefAge))
.isInstanceOf(IcebergParseException.class)
- .hasMessageContaining("mismatched input");
+ .hasMessageContaining("no viable alternative at input '<EOF>'");
assertThatThrownBy(
() -> sql("ALTER TABLE %s CREATE TAG %s RETAIN %s DAYS", tableName, tagName, "abc"))
.isInstanceOf(IcebergParseException.class)
- .hasMessageContaining("mismatched input");
+ .hasMessageContaining("no viable alternative at input 'abc'");
assertThatThrownBy(
() ->
@@ -151,7 +151,7 @@ public class TestTagDDL extends ExtensionsTestBase {
assertThatThrownBy(() -> sql("ALTER TABLE %s CREATE TAG %s", tableName, "123"))
.isInstanceOf(IcebergParseException.class)
- .hasMessageContaining("mismatched input '123'");
+ .hasMessageContaining("no viable alternative at input '123'");
table.manageSnapshots().removeTag(tagName).commit();
List<SimpleRecord> records =
@@ -303,7 +303,7 @@ public class TestTagDDL extends ExtensionsTestBase {
public void testDropTagNonConformingName() {
assertThatThrownBy(() -> sql("ALTER TABLE %s DROP TAG %s", tableName, "123"))
.isInstanceOf(IcebergParseException.class)
- .hasMessageContaining("mismatched input '123'");
+ .hasMessageContaining("no viable alternative at input '123'");
}
@TestTemplate