You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/12/22 19:17:42 UTC
[2/2] incubator-impala git commit: IMPALA-4033: Treat
string-partition key values as case sensitive.
IMPALA-4033: Treat string-partition key values as case sensitive.
This commit makes ADD PARTITION operations treat string partition-key
values as case sensitive in consistent with other related partition DDL
operations.
Change-Id: I6fbe67d99df8a50a16a18456fde85d03d622c7a1
Reviewed-on: http://gerrit.cloudera.org:8080/5535
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/b3636c97
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/b3636c97
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/b3636c97
Branch: refs/heads/master
Commit: b3636c97d4b872e1640955974409c57459d655e0
Parents: 226a2e6
Author: Amos Bird <am...@gmail.com>
Authored: Fri Dec 16 14:23:22 2016 +0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Dec 22 10:45:39 2016 +0000
----------------------------------------------------------------------
.../apache/impala/analysis/PartitionSet.java | 8 ++------
.../org/apache/impala/catalog/HdfsTable.java | 4 ++--
.../apache/impala/analysis/AnalyzeDDLTest.java | 2 +-
.../partition-ddl-predicates-all-fs.test | 20 ++++++++++++++++++++
4 files changed, 25 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3636c97/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
index 3ba2ad2..d5f0e70 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
@@ -137,12 +137,11 @@ public class PartitionSet extends PartitionSpecBase {
}
}
- // Transform <COL> = NULL into IsNull expr; <String COL> = '' into IsNull expr and
- // <String COL> = 'String Value' into lower case.
+ // Transform <COL> = NULL into IsNull expr; <String COL> = '' into IsNull expr.
// The reason is that COL = NULL is allowed for selecting the NULL
// partition, but a COL = NULL predicate can never be true, so we
// need to transform such predicates before feeding them into the
- // partition pruner. Same logic goes to String transformation.
+ // partition pruner.
private List<Expr> transformPartitionConjuncts(Analyzer analyzer, List<Expr> conjuncts)
throws AnalysisException {
List<Expr> transformedConjuncts = Lists.newArrayList();
@@ -162,9 +161,6 @@ public class PartitionSet extends PartitionSpecBase {
} else if (leftChild != null && stringChild != null) {
if (stringChild.getStringValue().isEmpty()) {
result = new IsNullPredicate(leftChild, false);
- } else {
- stringChild = new StringLiteral(stringChild.getStringValue().toLowerCase());
- result.setChild(1, stringChild);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3636c97/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index c2c569f..904c90f 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -570,7 +570,7 @@ public class HdfsTable extends Table {
for (FieldSchema fs: getMetaStoreTable().getPartitionKeys()) {
for (TPartitionKeyValue kv: partitionSpec) {
if (fs.getName().toLowerCase().equals(kv.getName().toLowerCase())) {
- targetValues.add(kv.getValue().toLowerCase());
+ targetValues.add(kv.getValue());
// Same key was specified twice
if (!keys.add(kv.getName().toLowerCase())) {
return null;
@@ -604,7 +604,7 @@ public class HdfsTable extends Table {
// backwards compatibility with Hive, and is clearly broken.
if (value.isEmpty()) value = getNullPartitionKeyValue();
}
- if (!targetValues.get(i).equals(value.toLowerCase())) {
+ if (!targetValues.get(i).equals(value)) {
matchFound = false;
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3636c97/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index 935edc5..5a08f98 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -424,7 +424,7 @@ public class AnalyzeDDLTest extends FrontendTestBase {
AnalyzesOk("alter table functional.stringpartitionkey PARTITION " +
"(string_col='partition1') set fileformat parquet");
AnalyzesOk("alter table functional.stringpartitionkey PARTITION " +
- "(string_col='PaRtiTion1') set location '/a/b/c'");
+ "(string_col='partition1') set location '/a/b/c'");
AnalyzesOk("alter table functional.alltypes PARTITION (year=2010, month=11) " +
"set tblproperties('a'='1')");
AnalyzesOk("alter table functional.alltypes PARTITION (year<=2010, month=11) " +
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3636c97/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
index 36104c3..1aa5c51 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test
@@ -133,3 +133,23 @@ show partitions p1
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
+---- QUERY
+# Tests case-sensitivity of string-typed partition columns.
+alter table p1 add partition (j=2,k="D");
+alter table p1 add partition (j=2,k="E");
+alter table p1 add partition (j=2,k="F");
+====
+---- QUERY
+show partitions p1
+---- RESULTS
+'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g
+'2','D',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=D
+'2','E',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=E
+'2','F',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=F
+'2','d',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d
+'2','e',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e
+'2','f',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f
+'Total','',0,0,regex:.+,regex:.+,'','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====