You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2024/05/21 13:54:10 UTC
(hive) branch master updated: HIVE-28239: Fix bug on HMSHandler#checkLimitNumberOfPartitions (Wechar Yu, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6dc569da2eb HIVE-28239: Fix bug on HMSHandler#checkLimitNumberOfPartitions (Wechar Yu, reviewed by Denys Kuzmenko)
6dc569da2eb is described below
commit 6dc569da2eb570fda76d2edb95704c6ba15a7924
Author: Wechar Yu <yu...@gmail.com>
AuthorDate: Tue May 21 21:54:03 2024 +0800
HIVE-28239: Fix bug on HMSHandler#checkLimitNumberOfPartitions (Wechar Yu, reviewed by Denys Kuzmenko)
Closes #5246
---
.../apache/hadoop/hive/metastore/HMSHandler.java | 49 +++++++++-------------
.../hive/metastore/client/TestListPartitions.java | 28 ++++++++++---
2 files changed, 42 insertions(+), 35 deletions(-)
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index c77daf96fac..486830fd0bb 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -5562,44 +5562,36 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
private void checkLimitNumberOfPartitionsByFilter(String catName, String dbName,
String tblName, String filterString,
- int maxParts) throws TException {
- if (isPartitionLimitEnabled()) {
+ int requestMax) throws TException {
+ if (exceedsPartitionFetchLimit(requestMax)) {
checkLimitNumberOfPartitions(tblName, get_num_partitions_by_filter(prependCatalogToDbName(
- catName, dbName, conf), tblName, filterString), maxParts);
+ catName, dbName, conf), tblName, filterString));
}
}
private void checkLimitNumberOfPartitionsByPs(String catName, String dbName, String tblName,
- List<String> partVals, int maxParts)
+ List<String> partVals, int requestMax)
throws TException {
- if (isPartitionLimitEnabled()) {
+ if (exceedsPartitionFetchLimit(requestMax)) {
checkLimitNumberOfPartitions(tblName, getNumPartitionsByPs(catName, dbName, tblName,
- partVals), maxParts);
+ partVals));
}
}
- private boolean isPartitionLimitEnabled() {
- int partitionLimit = MetastoreConf.getIntVar(conf, ConfVars.LIMIT_PARTITION_REQUEST);
- return partitionLimit > -1;
- }
-
- // Check request partition limit iff:
+ // Check input count exceeding partition limit iff:
// 1. partition limit is enabled.
- // 2. request size is greater than the limit.
- private boolean needCheckPartitionLimit(int requestSize) {
+ // 2. input count is greater than the limit.
+ private boolean exceedsPartitionFetchLimit(int count) {
int partitionLimit = MetastoreConf.getIntVar(conf, ConfVars.LIMIT_PARTITION_REQUEST);
- return partitionLimit > -1 && (requestSize < 0 || requestSize > partitionLimit);
+ return partitionLimit > -1 && (count < 0 || count > partitionLimit);
}
- private void checkLimitNumberOfPartitions(String tblName, int numPartitions, int maxToFetch) throws MetaException {
- if (isPartitionLimitEnabled()) {
+ private void checkLimitNumberOfPartitions(String tblName, int numPartitions) throws MetaException {
+ if (exceedsPartitionFetchLimit(numPartitions)) {
int partitionLimit = MetastoreConf.getIntVar(conf, ConfVars.LIMIT_PARTITION_REQUEST);
- int partitionRequest = (maxToFetch < 0) ? numPartitions : maxToFetch;
- if (partitionRequest > partitionLimit) {
- String configName = ConfVars.LIMIT_PARTITION_REQUEST.toString();
- throw new MetaException(String.format(PARTITION_NUMBER_EXCEED_LIMIT_MSG, partitionRequest,
- tblName, partitionLimit, configName));
- }
+ String configName = ConfVars.LIMIT_PARTITION_REQUEST.toString();
+ throw new MetaException(String.format(PARTITION_NUMBER_EXCEED_LIMIT_MSG, numPartitions,
+ tblName, partitionLimit, configName));
}
}
@@ -7225,13 +7217,12 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
RawStore rs = getMS();
try {
authorizeTableForPartitionMetadata(catName, dbName, tblName);
- if (needCheckPartitionLimit(args.getMax())) {
+ if (exceedsPartitionFetchLimit(args.getMax())) {
// Since partition limit is configured, we need fetch at most (limit + 1) partition names
- int requestMax = args.getMax();
int max = MetastoreConf.getIntVar(conf, ConfVars.LIMIT_PARTITION_REQUEST) + 1;
args = new GetPartitionsArgs.GetPartitionsArgsBuilder(args).max(max).build();
List<String> partNames = rs.listPartitionNamesByFilter(catName, dbName, tblName, args);
- checkLimitNumberOfPartitions(tblName, partNames.size(), requestMax);
+ checkLimitNumberOfPartitions(tblName, partNames.size());
ret = rs.getPartitionsByNames(catName, dbName, tblName,
new GetPartitionsArgs.GetPartitionsArgsBuilder(args).partNames(partNames).build());
} else {
@@ -7358,11 +7349,11 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
List<Partition> partitions = new LinkedList<>();
boolean hasUnknownPartitions = false;
RawStore rs = getMS();
- if (needCheckPartitionLimit(args.getMax())) {
+ if (exceedsPartitionFetchLimit(args.getMax())) {
// Since partition limit is configured, we need fetch at most (limit + 1) partition names
int max = MetastoreConf.getIntVar(conf, ConfVars.LIMIT_PARTITION_REQUEST) + 1;
List<String> partNames = rs.listPartitionNames(catName, dbName, tblName, args.getDefaultPartName(), args.getExpr(), null, max);
- checkLimitNumberOfPartitions(tblName, partNames.size(), args.getMax());
+ checkLimitNumberOfPartitions(tblName, partNames.size());
partitions = rs.getPartitionsByNames(catName, dbName, tblName,
new GetPartitionsArgs.GetPartitionsArgsBuilder(args).partNames(partNames).build());
} else {
@@ -7462,7 +7453,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
fireReadTablePreEvent(parsedCatName, parsedDbName, tblName);
- checkLimitNumberOfPartitions(tblName, args.getPartNames().size(), -1);
+ checkLimitNumberOfPartitions(tblName, args.getPartNames().size());
ret = getMS().getPartitionsByNames(parsedCatName, parsedDbName, tblName, args);
ret = FilterUtils.filterPartitionsIfEnabled(isServerFilterEnabled, filterHook, ret);
table = getTable(parsedCatName, parsedDbName, tblName);
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
index 91ba1c311f3..abaf28217ca 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java
@@ -203,6 +203,22 @@ public class TestListPartitions extends MetaStoreClientTest {
return new ReturnTable(t, testValues);
}
+ private ReturnTable createTableNParts(IMetaStoreClient client, boolean authOn, int nParts)
+ throws Exception {
+ Table t = createTestTable(client, DB_NAME, TABLE_NAME, Lists.newArrayList("yyyy", "mm", "dd"),
+ authOn);
+ List<List<String>> testValues = new ArrayList<>(nParts);
+ for (int i = 0; i < nParts; i++) {
+ testValues.add(Lists.newArrayList("2017", "01", String.valueOf(i)));
+ }
+
+ for(List<String> vals : testValues) {
+ addPartition(client, t, vals);
+ }
+
+ return new ReturnTable(t, testValues);
+ }
+
protected ReturnTable createTable4PartColsParts(IMetaStoreClient client) throws
Exception {
return createTable4PartColsPartsGeneric(client, false);
@@ -324,7 +340,7 @@ public class TestListPartitions extends MetaStoreClientTest {
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionsAllHighMaxParts() throws Exception {
- createTable3PartCols1Part(client);
+ createTableNParts(client, false, 101);
List<Partition> partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)101);
assertTrue(partitions.isEmpty());
}
@@ -496,7 +512,7 @@ public class TestListPartitions extends MetaStoreClientTest {
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionSpecsHighMaxParts() throws Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionSpecs(DB_NAME, TABLE_NAME, 101);
}
@@ -562,7 +578,7 @@ public class TestListPartitions extends MetaStoreClientTest {
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionsWithAuthHighMaxParts() throws Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionsWithAuthInfo(DB_NAME, TABLE_NAME, (short)101, "", Lists.newArrayList());
}
@@ -754,7 +770,7 @@ public class TestListPartitions extends MetaStoreClientTest {
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionsWithAuthByValuesHighMaxParts() throws Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionsWithAuthInfo(DB_NAME, TABLE_NAME, Lists
.newArrayList("2017"), (short) 101, "", Lists.newArrayList());
}
@@ -982,7 +998,7 @@ public class TestListPartitions extends MetaStoreClientTest {
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionsByFilterHighMaxParts() throws Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\"", (short)101);
}
@@ -1095,7 +1111,7 @@ public class TestListPartitions extends MetaStoreClientTest {
@Test(expected = MetaException.class)
@ConditionalIgnoreOnSessionHiveMetastoreClient
public void testListPartitionSpecsByFilterHighMaxParts() throws Exception {
- createTable4PartColsParts(client);
+ createTableNParts(client, false, 101);
client.listPartitionSpecsByFilter(DB_NAME, TABLE_NAME, "yyyy=\"2017\"", 101);
}