You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/12/27 04:55:21 UTC
(iotdb) 01/01: move path related antlr defined files to tsfile module
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/tsfileMove
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4d984616a4750ee8fdb06134a4cafb22d3b4c32c
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed Dec 27 12:54:54 2023 +0800
move path related antlr defined files to tsfile module
---
.../db/queryengine/plan/parser/ASTVisitor.java | 7 +-
iotdb-core/tsfile/pom.xml | 56 +++++++++++--
.../antlr4/org/apache/tsfile/parser/PathLexer.g4 | 98 ++++++++++++++++++++++
.../antlr4/org/apache/tsfile/parser}/PathParser.g4 | 9 +-
.../tsfile/common/constant/TsFileConstant.java | 4 -
.../read/common/parser/PathNodesGenerator.java | 8 +-
.../tsfile/read/common/parser/PathVisitor.java | 6 +-
7 files changed, 167 insertions(+), 21 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index 929a1347868..f73cbf1f3f5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -223,6 +223,7 @@ import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_RESULT_NODES;
@@ -259,6 +260,10 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
private static final String LIMIT_CONFIGURATION_ENABLED_ERROR_MSG =
"Limit configuration is not enabled, please enable it first.";
+ private static final String NODE_NAME_IN_INTO_PATH_MATCHER = "([a-zA-Z0-9_${}\\u2E80-\\u9FFF]+)";
+ private static final Pattern NODE_NAME_IN_INTO_PATH_PATTERN =
+ Pattern.compile(NODE_NAME_IN_INTO_PATH_MATCHER);
+
private static final String IGNORENULL = "IgnoreNull";
private ZoneId zoneId;
@@ -2026,7 +2031,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
private static void checkNodeNameInIntoPath(String src) {
// ${} are allowed
- if (!TsFileConstant.NODE_NAME_IN_INTO_PATH_PATTERN.matcher(src).matches()) {
+ if (!NODE_NAME_IN_INTO_PATH_PATTERN.matcher(src).matches()) {
throw new SemanticException(
String.format(
"%s is illegal, unquoted node name in select into clause can only consist of digits, characters, $, { and }",
diff --git a/iotdb-core/tsfile/pom.xml b/iotdb-core/tsfile/pom.xml
index 07db3bbc312..4849955a60b 100644
--- a/iotdb-core/tsfile/pom.xml
+++ b/iotdb-core/tsfile/pom.xml
@@ -40,11 +40,6 @@
<artifactId>common-api</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.iotdb</groupId>
- <artifactId>iotdb-antlr</artifactId>
- <version>1.3.1-SNAPSHOT</version>
- </dependency>
<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
@@ -113,6 +108,57 @@
</dependencies>
<build>
<plugins>
+ <plugin>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr4-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>antlr4</goal>
+ </goals>
+ <configuration>
+ <listener>false</listener>
+ <visitor>true</visitor>
+ <libDirectory>src/main/antlr4/org/apache/tsfile/parser</libDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources/antlr4</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-clean-plugin</artifactId>
+ <configuration>
+ <filesets>
+ <fileset>
+ <directory>${basedir}/gen</directory>
+ </fileset>
+ <fileset>
+ <directory>${basedir}/src</directory>
+ <includes>
+ <include>**/*.tokens</include>
+ </includes>
+ </fileset>
+ </filesets>
+ </configuration>
+ </plugin>
<!--
Generate an OSGI compatible MANIFEST file.
-->
diff --git a/iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/PathLexer.g4 b/iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/PathLexer.g4
new file mode 100644
index 00000000000..334d08bc029
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/PathLexer.g4
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+lexer grammar PathLexer;
+
+ROOT
+ : R O O T
+ ;
+
+DOT : '.';
+
+STAR: '*';
+DOUBLE_STAR: '**';
+
+ID
+ : NAME_CHAR+
+ ;
+
+QUOTED_ID
+ : BQUOTA_STRING
+ ;
+
+// Number Literal
+
+INTEGER_LITERAL
+ : DEC_DIGIT+
+ ;
+
+fragment NAME_CHAR
+ : 'A'..'Z'
+ | 'a'..'z'
+ | '0'..'9'
+ | '_'
+ | ':'
+ | '@'
+ | '#'
+ | '$'
+ | '{'
+ | '}'
+ | CN_CHAR
+ ;
+
+fragment CN_CHAR
+ : '\u2E80'..'\u9FFF'
+ ;
+
+fragment BQUOTA_STRING
+ : '`' ( '``' | ~('`') )* '`'
+ ;
+
+fragment DEC_DIGIT
+ : [0-9]
+ ;
+
+// Characters and write it this way for case sensitivity
+
+fragment A: [aA];
+fragment B: [bB];
+fragment C: [cC];
+fragment D: [dD];
+fragment E: [eE];
+fragment F: [fF];
+fragment G: [gG];
+fragment H: [hH];
+fragment I: [iI];
+fragment J: [jJ];
+fragment K: [kK];
+fragment L: [lL];
+fragment M: [mM];
+fragment N: [nN];
+fragment O: [oO];
+fragment P: [pP];
+fragment Q: [qQ];
+fragment R: [rR];
+fragment S: [sS];
+fragment T: [tT];
+fragment U: [uU];
+fragment V: [vV];
+fragment W: [wW];
+fragment X: [xX];
+fragment Y: [yY];
+fragment Z: [zZ];
\ No newline at end of file
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/PathParser.g4 b/iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/PathParser.g4
similarity index 93%
rename from iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/PathParser.g4
rename to iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/PathParser.g4
index eaf38e28edc..b0c7a475524 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/PathParser.g4
+++ b/iotdb-core/tsfile/src/main/antlr4/org/apache/tsfile/parser/PathParser.g4
@@ -19,9 +19,8 @@
parser grammar PathParser;
-options { tokenVocab=SqlLexer; }
+options { tokenVocab=PathLexer; }
-import IdentifierParser;
/**
* PartialPath and Path used by Session API and TsFile API should be parsed by Antlr4.
@@ -48,11 +47,13 @@ nodeName
;
nodeNameWithoutWildcard
- : identifier
+ : ID
+ | QUOTED_ID
;
nodeNameSlice
- : identifier
+ : ID
+ | QUOTED_ID
| INTEGER_LITERAL
;
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
index b7fd8ccbe15..ae3dc1bcd04 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
@@ -48,9 +48,5 @@ public class TsFileConstant {
private static final String NODE_NAME_MATCHER = "(\\*{0,2}[a-zA-Z0-9_\\u2E80-\\u9FFF]+\\*{0,2})";
public static final Pattern NODE_NAME_PATTERN = Pattern.compile(NODE_NAME_MATCHER);
- private static final String NODE_NAME_IN_INTO_PATH_MATCHER = "([a-zA-Z0-9_${}\\u2E80-\\u9FFF]+)";
- public static final Pattern NODE_NAME_IN_INTO_PATH_PATTERN =
- Pattern.compile(NODE_NAME_IN_INTO_PATH_MATCHER);
-
private TsFileConstant() {}
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/parser/PathNodesGenerator.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/parser/PathNodesGenerator.java
index 5aa49cffbd2..7d62a62ea92 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/parser/PathNodesGenerator.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/parser/PathNodesGenerator.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.tsfile.read.common.parser;
-import org.apache.iotdb.db.qp.sql.PathParser;
-import org.apache.iotdb.db.qp.sql.SqlLexer;
import org.apache.iotdb.tsfile.exception.PathParseException;
import org.antlr.v4.runtime.CharStream;
@@ -29,6 +27,8 @@ import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.atn.PredictionMode;
import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.antlr.v4.runtime.tree.ParseTree;
+import org.apache.tsfile.parser.PathLexer;
+import org.apache.tsfile.parser.PathParser;
/** convert String path to String[] nodes * */
public class PathNodesGenerator {
@@ -60,7 +60,7 @@ public class PathNodesGenerator {
CharStream charStream1 = CharStreams.fromString(path);
- SqlLexer lexer1 = new SqlLexer(charStream1);
+ PathLexer lexer1 = new PathLexer(charStream1);
lexer1.removeErrorListeners();
lexer1.addErrorListener(PathParseError.INSTANCE);
@@ -80,7 +80,7 @@ public class PathNodesGenerator {
} catch (Exception ex) {
CharStream charStream2 = CharStreams.fromString(path);
- SqlLexer lexer2 = new SqlLexer(charStream2);
+ PathLexer lexer2 = new PathLexer(charStream2);
lexer2.removeErrorListeners();
lexer2.addErrorListener(PathParseError.INSTANCE);
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/parser/PathVisitor.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/parser/PathVisitor.java
index d45dd93c1aa..0c36f7a0d72 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/parser/PathVisitor.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/parser/PathVisitor.java
@@ -19,12 +19,12 @@
package org.apache.iotdb.tsfile.read.common.parser;
-import org.apache.iotdb.db.qp.sql.PathParser;
-import org.apache.iotdb.db.qp.sql.PathParser.NodeNameContext;
-import org.apache.iotdb.db.qp.sql.PathParserBaseVisitor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.tsfile.parser.PathParser;
+import org.apache.tsfile.parser.PathParser.NodeNameContext;
+import org.apache.tsfile.parser.PathParserBaseVisitor;
import java.util.List;