You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2018/05/08 15:43:10 UTC
hive git commit: HIVE-18977: Listing partitions returns different
results with JDO and direct SQL (Marta Kuczora, via Peter Vary)
Repository: hive
Updated Branches:
refs/heads/master 8c7e7a602 -> 6c2e8f951
HIVE-18977: Listing partitions returns different results with JDO and direct SQL (Marta Kuczora, via Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6c2e8f95
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6c2e8f95
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6c2e8f95
Branch: refs/heads/master
Commit: 6c2e8f951537b80744e7f6b55181e0b4eb15687e
Parents: 8c7e7a6
Author: Marta Kuczora <ku...@cloudera.com>
Authored: Tue May 8 17:42:30 2018 +0200
Committer: Peter Vary <pv...@cloudera.com>
Committed: Tue May 8 17:42:30 2018 +0200
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveMetaStore.java | 3 +
.../hadoop/hive/metastore/ObjectStore.java | 2 +-
.../hive/metastore/parser/ExpressionTree.java | 3 +-
.../metastore/client/TestListPartitions.java | 118 ++++++++++++++-----
4 files changed, 93 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6c2e8f95/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 3978b88..269798c 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -5794,6 +5794,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
final String tblName, final String filter)
throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
+ if (parsedDbName[DB_NAME] == null || tblName == null) {
+ throw new MetaException("The DB and table name cannot be null.");
+ }
startTableFunction("get_num_partitions_by_filter", parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tblName);
http://git-wip-us.apache.org/repos/asf/hive/blob/6c2e8f95/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index b43334b..b0a805f 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -3160,7 +3160,7 @@ public class ObjectStore implements RawStore, Configurable {
"table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3");
query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
query.setOrdering("partitionName ascending");
- if (max > 0) {
+ if (max >= 0) {
query.setRange(0, max);
}
mparts = (List<MPartition>) query.execute(tableName, dbName, catName);
http://git-wip-us.apache.org/repos/asf/hive/blob/6c2e8f95/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
index d608e50..32267b4 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
@@ -361,7 +361,8 @@ public class ExpressionTree {
boolean isOpEquals = operator == Operator.EQUALS;
if (isOpEquals || operator == Operator.NOTEQUALS || operator == Operator.NOTEQUALS2) {
- makeFilterForEquals(keyName, valueAsString, paramName, params,
+ String partitionKey = table.getPartitionKeys().get(partitionColumnIndex).getName();
+ makeFilterForEquals(partitionKey, valueAsString, paramName, params,
partitionColumnIndex, partitionColumnCount, isOpEquals, filterBuilder);
return;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6c2e8f95/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
index a8b6e31..e5d1058 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
@@ -52,7 +52,6 @@ import com.google.common.collect.Lists;
import org.junit.After;
import org.junit.Assert;
-import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -253,12 +252,8 @@ public class TestListPartitions extends MetaStoreClientTest {
partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)1);
assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 1));
- // HIVE-18977
- if (MetastoreConf.getBoolVar(metaStore.getConf(), MetastoreConf.ConfVars.TRY_DIRECT_SQL)) {
- partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short) 0);
- assertTrue(partitions.isEmpty());
- }
-
+ partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short) 0);
+ assertTrue(partitions.isEmpty());
}
@Test(expected = MetaException.class)
@@ -270,14 +265,14 @@ public class TestListPartitions extends MetaStoreClientTest {
@Test
public void testListPartitionsAllNoParts() throws Exception {
- Table t = createTestTable(client, DB_NAME, TABLE_NAME, Lists.newArrayList("yyyy", "mm", "dd"));
+ createTestTable(client, DB_NAME, TABLE_NAME, Lists.newArrayList("yyyy", "mm", "dd"));
List<Partition> partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1);
assertTrue(partitions.isEmpty());
}
@Test(expected = NoSuchObjectException.class)
public void testListPartitionsAllNoTable() throws Exception {
- List<Partition> partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1);
+ client.listPartitions(DB_NAME, TABLE_NAME, (short)-1);
}
@Test(expected = NoSuchObjectException.class)
@@ -302,8 +297,7 @@ public class TestListPartitions extends MetaStoreClientTest {
public void testListPartitionsAllNullTblName() throws Exception {
try {
createTable3PartCols1Part(client);
- List<Partition> partitions = client.listPartitions(DB_NAME,
- (String)null, (short)-1);
+ client.listPartitions(DB_NAME, (String)null, (short)-1);
fail("Should have thrown exception");
} catch (NullPointerException | TTransportException e) {
//TODO: should not throw different exceptions for different HMS deployment types
@@ -416,11 +410,8 @@ public class TestListPartitions extends MetaStoreClientTest {
partSpecProxy = client.listPartitionSpecs(DB_NAME, TABLE_NAME, 2);
assertPartitionsSpecProxy(partSpecProxy, testValues.subList(0, 2));
- // HIVE-18977
- if (MetastoreConf.getBoolVar(metaStore.getConf(), MetastoreConf.ConfVars.TRY_DIRECT_SQL)) {
- partSpecProxy = client.listPartitionSpecs(DB_NAME, TABLE_NAME, 0);
- assertPartitionsSpecProxy(partSpecProxy, testValues.subList(0, 0));
- }
+ partSpecProxy = client.listPartitionSpecs(DB_NAME, TABLE_NAME, 0);
+ assertPartitionsSpecProxy(partSpecProxy, testValues.subList(0, 0));
}
@Test(expected = NoSuchObjectException.class)
@@ -745,18 +736,88 @@ public class TestListPartitions extends MetaStoreClientTest {
"yyyy=\"2017\" OR " + "mm=\"02\"", (short)0);
assertTrue(partitions.isEmpty());
- // HIVE-18977
- if (MetastoreConf.getBoolVar(metaStore.getConf(), MetastoreConf.ConfVars.TRY_DIRECT_SQL)) {
- partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME,
- "yYyY=\"2017\"", (short) -1);
- assertPartitionsHaveCorrectValues(partitions, partValues.subList(2, 4));
- }
-
partitions = client.listPartitionsByFilter(DB_NAME, TABLE_NAME,
"yyyy=\"2017\" AND mm=\"99\"", (short)-1);
assertTrue(partitions.isEmpty());
}
+ @Test
+ public void testListPartitionsByFilterCaseInsensitive() throws Exception {
+ String tableName = TABLE_NAME + "_caseinsensitive";
+ Table t = createTestTable(client, DB_NAME, tableName,
+ Lists.newArrayList("yyyy", "month", "day"), false);
+ List<List<String>> testValues = Lists.newArrayList(
+ Lists.newArrayList("2017", "march", "11"),
+ Lists.newArrayList("2017", "march", "15"),
+ Lists.newArrayList("2017", "may", "15"),
+ Lists.newArrayList("2018", "march", "11"),
+ Lists.newArrayList("2018", "september", "7"));
+
+ for(List<String> vals : testValues) {
+ addPartition(client, t, vals);
+ }
+
+ List<Partition> partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "yYyY=\"2017\"", (short) -1);
+ assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 3));
+
+ partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "yYyY=\"2017\" AND mOnTh=\"may\"", (short) -1);
+ assertPartitionsHaveCorrectValues(partitions, testValues.subList(2, 3));
+
+ partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "yYyY!=\"2017\"", (short) -1);
+ assertPartitionsHaveCorrectValues(partitions, testValues.subList(3, 5));
+
+ partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "mOnTh=\"september\"", (short) -1);
+ assertPartitionsHaveCorrectValues(partitions, testValues.subList(4, 5));
+
+ partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "mOnTh like \"m.*\"", (short) -1);
+ assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 4));
+
+ partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "yYyY=\"2018\" AND mOnTh like \"m.*\"", (short) -1);
+ assertPartitionsHaveCorrectValues(partitions, testValues.subList(3, 4));
+ client.dropTable(DB_NAME, tableName);
+ }
+
+ @Test
+ public void testListPartitionsByFilterCaseSensitive() throws Exception {
+ String tableName = TABLE_NAME + "_casesensitive";
+ Table t = createTestTable(client, DB_NAME, tableName,
+ Lists.newArrayList("yyyy", "month", "day"), false);
+ List<List<String>> testValues = Lists.newArrayList(
+ Lists.newArrayList("2017", "march", "11"),
+ Lists.newArrayList("2017", "march", "15"),
+ Lists.newArrayList("2017", "may", "15"),
+ Lists.newArrayList("2018", "march", "11"),
+ Lists.newArrayList("2018", "april", "7"));
+
+ for(List<String> vals : testValues) {
+ addPartition(client, t, vals);
+ }
+
+ List<Partition> partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "month=\"mArCh\"", (short) -1);
+ Assert.assertTrue(partitions.isEmpty());
+
+ partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "yyyy=\"2017\" AND month=\"May\"", (short) -1);
+ Assert.assertTrue(partitions.isEmpty());
+
+ partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "yyyy=\"2017\" AND month!=\"mArCh\"", (short) -1);
+ assertPartitionsHaveCorrectValues(partitions, testValues.subList(0, 3));
+
+ partitions = client.listPartitionsByFilter(DB_NAME, tableName,
+ "month like \"M.*\"", (short) -1);
+ Assert.assertTrue(partitions.isEmpty());
+ client.dropTable(DB_NAME, tableName);
+
+ }
+
@Test(expected = MetaException.class)
public void testListPartitionsByFilterInvalidFilter() throws Exception {
createTable4PartColsParts(client);
@@ -987,15 +1048,10 @@ public class TestListPartitions extends MetaStoreClientTest {
client.getNumPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\"");
}
- @Test
+ @Test(expected = MetaException.class)
public void testGetNumPartitionsByFilterNullTblName() throws Exception {
- try {
- createTable4PartColsParts(client);
- client.getNumPartitionsByFilter(DB_NAME, null, "yyyy=\"2017\"");
- fail("Should have thrown exception");
- } catch (MetaException | TTransportException e) {
- //TODO: should not throw different exceptions for different HMS deployment types
- }
+ createTable4PartColsParts(client);
+ client.getNumPartitionsByFilter(DB_NAME, null, "yyyy=\"2017\"");
}
@Test(expected = MetaException.class)