You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by el...@apache.org on 2016/05/31 23:24:01 UTC
[03/14] calcite git commit: [CALCITE-1241] Add a freemarker variable
for adding non reserved keyword list to Parser.jj template (Venki Korukanti)
[CALCITE-1241] Add a freemarker variable for adding non reserved keyword list to Parser.jj template (Venki Korukanti)
Add a test to prevent unintentionally adding reserved keywords.
Close apache/calcite#233
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/20ba4346
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/20ba4346
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/20ba4346
Branch: refs/heads/branch-avatica-1.8
Commit: 20ba43461564c983557e9d9fd1b3e04d75ef7b18
Parents: 6c633cf
Author: vkorukanti <ve...@dremio.com>
Authored: Wed Mar 9 11:00:26 2016 -0800
Committer: Julian Hyde <jh...@apache.org>
Committed: Sun May 22 12:46:46 2016 -0700
----------------------------------------------------------------------
core/src/main/codegen/config.fmpp | 7 +-
core/src/main/codegen/templates/Parser.jj | 3 +
.../calcite/sql/parser/SqlParserTest.java | 343 +++++++++++++++++++
3 files changed, 352 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/20ba4346/core/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/core/src/main/codegen/config.fmpp b/core/src/main/codegen/config.fmpp
index 6f88718..87ed18f 100644
--- a/core/src/main/codegen/config.fmpp
+++ b/core/src/main/codegen/config.fmpp
@@ -43,10 +43,15 @@ data: {
imports: [
]
- # List of new keywords. Example: "DATABASES", "TABLES".
+ # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is not a reserved
+ # keyword add it to 'nonReservedKeywords' section.
keywords: [
]
+ # List of keywords from "keywords" section that are not reserved.
+ nonReservedKeywords: [
+ ]
+
# List of methods for parsing custom SQL statements.
# Return type of method implementation should be 'SqlNode'.
# Example: SqlShowDatabases(), SqlShowTables().
http://git-wip-us.apache.org/repos/asf/calcite/blob/20ba4346/core/src/main/codegen/templates/Parser.jj
----------------------------------------------------------------------
diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj
index bad7622..cfd9f22 100644
--- a/core/src/main/codegen/templates/Parser.jj
+++ b/core/src/main/codegen/templates/Parser.jj
@@ -5619,6 +5619,9 @@ String CommonNonReservedKeyWord() :
| <WRITE>
| <XML>
| <ZONE>
+ <#list parser.nonReservedKeywords as keyword>
+ | <${keyword}>
+ </#list>
)
{
return unquotedIdentifier();
http://git-wip-us.apache.org/repos/asf/calcite/blob/20ba4346/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
index 9fe216a..f8bb2b3 100644
--- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
+++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
@@ -42,12 +42,14 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
@@ -60,6 +62,319 @@ import static org.junit.Assert.assertTrue;
public class SqlParserTest {
//~ Static fields/initializers ---------------------------------------------
+ /**
+ * List of reserved keywords in parser. If a new <b>reserved</b> keyword is added to the
+ * parser, it must be included in this list. If the keyword is not intended to be a
+ * reserved keyword, add it to the non-reserved keyword list in the parser.
+ */
+ private static final List<String> RESERVED_KEYWORDS = ImmutableList.of(
+ "ABS",
+ "ALL",
+ "ALLOCATE",
+ "ALLOW",
+ "ALTER",
+ "AND",
+ "ANY",
+ "ARE",
+ "ARRAY",
+ "AS",
+ "ASENSITIVE",
+ "ASYMMETRIC",
+ "AT",
+ "ATOMIC",
+ "AUTHORIZATION",
+ "AVG",
+ "BEGIN",
+ "BETWEEN",
+ "BIGINT",
+ "BINARY",
+ "BIT",
+ "BLOB",
+ "BOOLEAN",
+ "BOTH",
+ "BY",
+ "CALL",
+ "CALLED",
+ "CARDINALITY",
+ "CASCADED",
+ "CASE",
+ "CAST",
+ "CEIL",
+ "CEILING",
+ "CHAR",
+ "CHARACTER",
+ "CHARACTER_LENGTH",
+ "CHAR_LENGTH",
+ "CHECK",
+ "CLOB",
+ "CLOSE",
+ "COALESCE",
+ "COLLATE",
+ "COLLECT",
+ "COLUMN",
+ "COMMIT",
+ "CONDITION",
+ "CONNECT",
+ "CONSTRAINT",
+ "CONVERT",
+ "CORR",
+ "CORRESPONDING",
+ "COUNT",
+ "COVAR_POP",
+ "COVAR_SAMP",
+ "CREATE",
+ "CROSS",
+ "CUBE",
+ "CUME_DIST",
+ "CURRENT",
+ "CURRENT_CATALOG",
+ "CURRENT_DATE",
+ "CURRENT_DEFAULT_TRANSFORM_GROUP",
+ "CURRENT_PATH",
+ "CURRENT_ROLE",
+ "CURRENT_SCHEMA",
+ "CURRENT_TIME",
+ "CURRENT_TIMESTAMP",
+ "CURRENT_TRANSFORM_GROUP_FOR_TYPE",
+ "CURRENT_USER",
+ "CURSOR",
+ "CYCLE",
+ "DATE",
+ "DAY",
+ "DEALLOCATE",
+ "DEC",
+ "DECIMAL",
+ "DECLARE",
+ "DEFAULT",
+ "DELETE",
+ "DENSE_RANK",
+ "DEREF",
+ "DESCRIBE",
+ "DETERMINISTIC",
+ "DISALLOW",
+ "DISCONNECT",
+ "DISTINCT",
+ "DOUBLE",
+ "DROP",
+ "DYNAMIC",
+ "EACH",
+ "ELEMENT",
+ "ELSE",
+ "END",
+ "END-EXEC",
+ "ESCAPE",
+ "EVERY",
+ "EXCEPT",
+ "EXEC",
+ "EXECUTE",
+ "EXISTS",
+ "EXP",
+ "EXPLAIN",
+ "EXTEND",
+ "EXTERNAL",
+ "EXTRACT",
+ "FALSE",
+ "FETCH",
+ "FILTER",
+ "FIRST_VALUE",
+ "FLOAT",
+ "FLOOR",
+ "FOR",
+ "FOREIGN",
+ "FREE",
+ "FROM",
+ "FULL",
+ "FUNCTION",
+ "FUSION",
+ "GET",
+ "GLOBAL",
+ "GRANT",
+ "GROUP",
+ "GROUPING",
+ "HAVING",
+ "HOLD",
+ "HOUR",
+ "IDENTITY",
+ "IMPORT",
+ "IN",
+ "INDICATOR",
+ "INNER",
+ "INOUT",
+ "INSENSITIVE",
+ "INSERT",
+ "INT",
+ "INTEGER",
+ "INTERSECT",
+ "INTERSECTION",
+ "INTERVAL",
+ "INTO",
+ "IS",
+ "JOIN",
+ "LANGUAGE",
+ "LARGE",
+ "LAST_VALUE",
+ "LATERAL",
+ "LEADING",
+ "LEFT",
+ "LIKE",
+ "LIMIT",
+ "LN",
+ "LOCAL",
+ "LOCALTIME",
+ "LOCALTIMESTAMP",
+ "LOWER",
+ "MATCH",
+ "MAX",
+ "MEMBER",
+ "MERGE",
+ "METHOD",
+ "MIN",
+ "MINUTE",
+ "MOD",
+ "MODIFIES",
+ "MODULE",
+ "MONTH",
+ "MULTISET",
+ "NATIONAL",
+ "NATURAL",
+ "NCHAR",
+ "NCLOB",
+ "NEW",
+ "NEXT",
+ "NO",
+ "NONE",
+ "NORMALIZE",
+ "NOT",
+ "NULL",
+ "NULLIF",
+ "NUMERIC",
+ "OCTET_LENGTH",
+ "OF",
+ "OFFSET",
+ "OLD",
+ "ON",
+ "ONLY",
+ "OPEN",
+ "OR",
+ "ORDER",
+ "OUT",
+ "OUTER",
+ "OVER",
+ "OVERLAPS",
+ "OVERLAY",
+ "PARAMETER",
+ "PARTITION",
+ "PERCENTILE_CONT",
+ "PERCENTILE_DISC",
+ "PERCENT_RANK",
+ "POSITION",
+ "POWER",
+ "PRECISION",
+ "PREPARE",
+ "PRIMARY",
+ "PROCEDURE",
+ "RANGE",
+ "RANK",
+ "READS",
+ "REAL",
+ "RECURSIVE",
+ "REF",
+ "REFERENCES",
+ "REFERENCING",
+ "REGR_AVGX",
+ "REGR_AVGY",
+ "REGR_COUNT",
+ "REGR_INTERCEPT",
+ "REGR_R2",
+ "REGR_SLOPE",
+ "REGR_SXX",
+ "REGR_SXY",
+ "REGR_SYY",
+ "RELEASE",
+ "RESET",
+ "RESULT",
+ "RETURN",
+ "RETURNS",
+ "REVOKE",
+ "RIGHT",
+ "ROLLBACK",
+ "ROLLUP",
+ "ROW",
+ "ROWS",
+ "ROW_NUMBER",
+ "SAVEPOINT",
+ "SCOPE",
+ "SCROLL",
+ "SEARCH",
+ "SECOND",
+ "SELECT",
+ "SENSITIVE",
+ "SESSION_USER",
+ "SET",
+ "SIMILAR",
+ "SMALLINT",
+ "SOME",
+ "SPECIFIC",
+ "SPECIFICTYPE",
+ "SQL",
+ "SQLEXCEPTION",
+ "SQLSTATE",
+ "SQLWARNING",
+ "SQRT",
+ "START",
+ "STATIC",
+ "STDDEV_POP",
+ "STDDEV_SAMP",
+ "STREAM",
+ "SUBMULTISET",
+ "SUBSTRING",
+ "SUM",
+ "SYMMETRIC",
+ "SYSTEM",
+ "SYSTEM_USER",
+ "TABLE",
+ "TABLESAMPLE",
+ "THEN",
+ "TIME",
+ "TIMESTAMP",
+ "TIMEZONE_HOUR",
+ "TIMEZONE_MINUTE",
+ "TINYINT",
+ "TO",
+ "TRAILING",
+ "TRANSLATE",
+ "TRANSLATION",
+ "TREAT",
+ "TRIGGER",
+ "TRIM",
+ "TRUE",
+ "UESCAPE",
+ "UNION",
+ "UNIQUE",
+ "UNKNOWN",
+ "UNNEST",
+ "UPDATE",
+ "UPPER",
+ "UPSERT",
+ "USER",
+ "USING",
+ "VALUE",
+ "VALUES",
+ "VARBINARY",
+ "VARCHAR",
+ "VARYING",
+ "VAR_POP",
+ "VAR_SAMP",
+ "WHEN",
+ "WHENEVER",
+ "WHERE",
+ "WIDTH_BUCKET",
+ "WINDOW",
+ "WITH",
+ "WITHIN",
+ "WITHOUT",
+ "YEAR");
+
private static final String ANY = "(?s).*";
private static final ThreadLocal<boolean[]> LINUXIFY =
@@ -143,6 +458,10 @@ public class SqlParserTest {
getTester().checkExpFails(sql, expectedMsgPattern);
}
+ protected List<String> getReservedKeywords() {
+ return RESERVED_KEYWORDS;
+ }
+
/**
* Tests that when there is an error, non-reserved keywords such as "A",
* "ABSOLUTE" (which naturally arise whenever a production uses
@@ -5851,6 +6170,30 @@ public class SqlParserTest {
assertTrue(!jdbcKeywords.contains(",SELECT,"));
}
+ /**
+ * Tests that reserved keywords are not added to the parser unintentionally.
+ * (Most keywords are non-reserved. The set of reserved words generally
+ * only changes with a new version of the SQL standard.)
+ *
+ * <p>If the new keyword added is intended to be a reserved keyword, update
+ * the {@link #RESERVED_KEYWORDS} list. If not, add the keyword to the
+ * non-reserved keyword list in the parser.
+ */
+ @Test public void testNoUnintendedNewReservedKeywords() {
+ final SqlAbstractParserImpl.Metadata metadata = getParserMetadata();
+
+ final List<String> reservedKeywords = new ArrayList<>();
+ for (String s : metadata.getTokens()) {
+ if (metadata.isKeyword(s) && metadata.isReservedWord(s)) {
+ reservedKeywords.add(s);
+ }
+ }
+
+ assertThat("At least one new reserved keyword is added in parser. "
+ + "Make sure to check the new keywords are intended to be a reserved keywords.",
+ reservedKeywords, is(getReservedKeywords()));
+ }
+
/** Generates a copy of {@code reference.md} with the current set of key
* words. Fails if the copy is different from the original. */
@Test public void testGenerateKeyWords() throws IOException {