You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ri...@apache.org on 2022/01/11 15:34:30 UTC
[phoenix] branch 4.16 updated: PHOENIX-6596 Schema extraction double quotes expressions, resulting in un-executabe create statements
This is an automated email from the ASF dual-hosted git repository.
richardantal pushed a commit to branch 4.16
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.16 by this push:
new ecc8862 PHOENIX-6596 Schema extraction double quotes expressions, resulting in un-executabe create statements
ecc8862 is described below
commit ecc8862fb70ae7d1fc84b6aa51d2a2022a82a62f
Author: Richard Antal <an...@gmail.com>
AuthorDate: Thu Dec 9 14:00:08 2021 +0100
PHOENIX-6596 Schema extraction double quotes expressions, resulting in un-executabe create statements
Co-authored-by: Istvan Toth <st...@apache.org>
---
.../schema/tool/SchemaToolExtractionIT.java | 63 ++++++++++++++++++++--
.../schema/tool/SchemaExtractionProcessor.java | 28 ++++++++--
.../org/apache/phoenix/schema/tool/SchemaTool.java | 1 -
.../java/org/apache/phoenix/util/SchemaUtil.java | 23 ++++++++
4 files changed, 106 insertions(+), 9 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java
index 6ba411c..0a733ae 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java
@@ -93,13 +93,18 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT {
String indexName = generateUniqueName();
String indexName1 = generateUniqueName();
String indexName2 = generateUniqueName();
+ String indexName3 = generateUniqueName();
String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
- String createTableStatement = "CREATE TABLE "+pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"
+ String createTableStatement = "CREATE TABLE "+pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, \"v1\" VARCHAR, v2 VARCHAR)"
+ properties;
- String createIndexStatement = "CREATE INDEX "+indexName + " ON "+pTableFullName+"(v1 DESC) INCLUDE (v2)";
- String createIndexStatement1 = "CREATE INDEX "+indexName1 + " ON "+pTableFullName+"(v2 DESC) INCLUDE (v1)";
+ //FIXME never verified
+ String createIndexStatement = "CREATE INDEX "+indexName + " ON "+pTableFullName+"(\"v1\" DESC) INCLUDE (v2)";
+ //FIXME never verified
+ String createIndexStatement1 = "CREATE INDEX "+indexName1 + " ON "+pTableFullName+"(v2 DESC) INCLUDE (\"v1\")";
String createIndexStatement2 = "CREATE INDEX "+indexName2 + " ON "+pTableFullName+"(k)";
+ String createIndexStatement3 ="CREATE INDEX " + indexName3 + " ON " + pTableFullName +
+ "('QUOTED' || \"v1\" || V2 DESC, \"v1\" DESC, K) INCLUDE (V2)";
List<String> queries = new ArrayList<String>(){};
queries.add(createTableStatement);
queries.add(createIndexStatement);
@@ -108,6 +113,12 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT {
String result = runSchemaExtractionTool(schemaName, indexName2, null, queries);
Assert.assertEquals(createIndexStatement2.toUpperCase(), result.toUpperCase());
+
+ List<String> queries3 = new ArrayList<String>(){};
+ queries3.add(createIndexStatement3);
+
+ String result3 = runSchemaExtractionTool(schemaName, indexName3, null, queries3);
+ Assert.assertEquals(createIndexStatement3, result3);
}
@Test
@@ -144,17 +155,45 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
String indexName = generateUniqueName();
+ String indexName2 = generateUniqueName();
String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
String createTableStatement = "CREATE TABLE "+pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"
+ properties;
String createIndexStatement = "CREATE LOCAL INDEX "+indexName + " ON "+pTableFullName+"(v1 DESC, k) INCLUDE (v2)";
+ String createIndexStatement2 = "CREATE LOCAL INDEX "+indexName2 + " ON "+pTableFullName+"( LPAD(v1,10) DESC, k) INCLUDE (v2)";
+
List<String> queries = new ArrayList<String>(){};
queries.add(createTableStatement);
queries.add(createIndexStatement);
String result = runSchemaExtractionTool(schemaName, indexName, null, queries);
Assert.assertEquals(createIndexStatement.toUpperCase(), result.toUpperCase());
+
+ List<String> queries2 = new ArrayList<String>(){};
+ queries2.add(createIndexStatement2);
+
+ String result2 = runSchemaExtractionTool(schemaName, indexName2, null, queries2);
+ Assert.assertEquals(createIndexStatement2.toUpperCase(), result2.toUpperCase());
+ }
+
+ @Test
+ public void testCreateLocalIndexStatementLowerCase() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ String indexName = generateUniqueName();
+ String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String createTableStatement = "CREATE TABLE "+pTableFullName + "(K VARCHAR NOT NULL PRIMARY KEY, \"v1\" VARCHAR, V2 VARCHAR)"
+ + properties;
+ String createIndexStatement = "CREATE LOCAL INDEX "+indexName + " ON "+pTableFullName+"( LPAD(\"v1\",10) DESC, K) INCLUDE (V2)";
+
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(createTableStatement);
+ queries.add(createIndexStatement);
+
+ String result = runSchemaExtractionTool(schemaName, indexName, null, queries);
+ Assert.assertEquals(createIndexStatement, result);
}
@Test
@@ -176,6 +215,24 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT {
}
@Test
+ public void testCreateIndexStatementLowerCaseCombined() throws Exception {
+ String tableName = "lowercase" + generateUniqueName();
+ String schemaName = "lowercase" + generateUniqueName();
+ String indexName = "\"lowercaseIND" + generateUniqueName() + "\"";
+ String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
+ String pTableFullName = SchemaUtil.getEscapedTableName(schemaName, tableName);
+ String createTableStatement = "CREATE TABLE " + pTableFullName + "(ID varchar primary key, \"number\" integer, \"currency\" decimal(6,2), lista varchar[])"
+ + properties;
+ String createIndexStatement = "CREATE INDEX " + indexName + " ON "+ pTableFullName + "(\"number\" * \"currency\", ID) INCLUDE (LISTA)";
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(createTableStatement);
+ queries.add(createIndexStatement);
+
+ String result = runSchemaExtractionTool("\"" + schemaName + "\"", indexName, null, queries);
+ Assert.assertEquals(createIndexStatement, result);
+ }
+
+ @Test
public void testCreateViewStatement() throws Exception {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java
index 656bd31..8383ca5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java
@@ -134,8 +134,8 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
StringBuilder indexedColumnsBuilder = new StringBuilder();
for (PColumn indexedColumn : indexPK) {
- String indexColumn = extractIndexColumn(indexedColumn.getName().getString(), defaultCF);
- if(indexColumn.equalsIgnoreCase(MetaDataUtil.VIEW_INDEX_ID_COLUMN_NAME)) {
+ String indexColumn = extractIndexColumn(indexedColumn.getExpressionStr(), defaultCF);
+ if (indexColumn == null) {
continue;
}
indexPKName.add(indexColumn);
@@ -183,21 +183,37 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
}
private String extractIndexColumn(String columnName, String defaultCF) {
+ if (columnName == null) {
+ return null;
+ }
String [] columnNameSplit = columnName.split(":");
if(columnNameSplit[0].equals("") || columnNameSplit[0].equalsIgnoreCase(defaultCF) ||
(defaultCF.startsWith("L#") && columnNameSplit[0].equalsIgnoreCase(defaultCF.substring(2)))) {
- return SchemaUtil.formatColumnName(columnNameSplit[1]);
+ return formatColumnOrExpression(columnNameSplit[1]);
} else {
if (columnNameSplit.length > 1) {
String schema = SchemaUtil.formatSchemaName(columnNameSplit[0]);
String name = SchemaUtil.formatColumnName(columnNameSplit[1]);
return String.format("%s.%s", schema, name);
} else {
- return SchemaUtil.formatColumnName(columnNameSplit[0]);
+ return formatColumnOrExpression(columnNameSplit[0]);
}
}
}
+ private String formatColumnOrExpression(String columnOrExpression) {
+ if (columnOrExpression.startsWith("(")) {
+ //Expressions like (a*b) are always parenthesised
+ return columnOrExpression.substring(1, columnOrExpression.length()-1);
+ } else if (columnOrExpression.contains("(")) {
+ //Expressions like like func(a) are always have a parenthesis
+ return columnOrExpression;
+ } else {
+ //If there are no parentheses, this is a column name
+ return SchemaUtil.formatIndexColumnName(columnOrExpression);
+ }
+ }
+
private String getCoveredColumnsString(PTable indexPTable, String defaultCF) {
StringBuilder coveredColumnsBuilder = new StringBuilder();
List<PColumn> pkColumns = indexPTable.getColumns();
@@ -207,7 +223,9 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
}
if(cc.getFamilyName()!=null) {
String indexColumn = extractIndexColumn(cc.getName().getString(), defaultCF);
- coveredColumnsBuilder.append(indexColumn);
+ if (indexColumn != null) {
+ coveredColumnsBuilder.append(indexColumn);
+ }
}
}
return coveredColumnsBuilder.toString();
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaTool.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaTool.java
index 0386c78..fdda6e3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaTool.java
@@ -17,7 +17,6 @@
*/
package org.apache.phoenix.schema.tool;
-import org.apache.hadoop.yarn.webapp.hamlet.HamletSpec;
import org.apache.phoenix.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.phoenix.thirdparty.org.apache.commons.cli.CommandLineParser;
import org.apache.phoenix.thirdparty.org.apache.commons.cli.DefaultParser;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index 93c98c8..4e35dec 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -1315,6 +1315,29 @@ public class SchemaUtil {
return name;
}
+ public static String formatIndexColumnName(String name) {
+ String[] splitName = name.split("\\.");
+ if(splitName.length < 2) {
+ if (!name.contains("\"")) {
+ if (quotesNeededForColumn(name)) {
+ name = "\"" + name + "\"";
+ }
+ return name;
+ } else if (name.startsWith("\"") && name.endsWith("\"")) {
+ if (quotesNeededForColumn(name.substring(1, name.length() - 1))) {
+ return name;
+ } else {
+ return name.substring(1, name.length() - 1);
+ }
+ } else {
+ return name;
+ }
+ } else {
+ return String.format("%s.%s", formatIndexColumnName(splitName[0]),
+ formatIndexColumnName(splitName[1]));
+ }
+ }
+
public static boolean containsLowerCase(String name) {
if (Strings.isNullOrEmpty(name)) {
return false;