You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ma...@apache.org on 2019/11/28 08:11:22 UTC
[hive] branch master updated: HIVE-22379 : Reduce db lookups during
dynamic partition loading. (Mahesh Kumar Behera reviewed by Rajesh
Balamohan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 7c796af HIVE-22379 : Reduce db lookups during dynamic partition loading. (Mahesh Kumar Behera reviewed by Rajesh Balamohan)
7c796af is described below
commit 7c796af642d3bc5d1ae72d2fb0fa31191b77ec45
Author: Mahesh Kumar Behera <ma...@apache.org>
AuthorDate: Thu Nov 28 13:39:37 2019 +0530
HIVE-22379 : Reduce db lookups during dynamic partition loading. (Mahesh Kumar Behera reviewed by Rajesh Balamohan)
---
.../TestReplicationScenariosAcrossInstances.java | 25 +++++++++++----------
.../hadoop/hive/metastore/HiveAlterHandler.java | 26 +++++++++++++++++++++-
.../apache/hadoop/hive/metastore/ObjectStore.java | 13 +++++++++--
.../metastore/InjectableBehaviourObjectStore.java | 22 ++++++++++++++++++
4 files changed, 71 insertions(+), 15 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 4f5cbbb..8df5fc8 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1387,27 +1387,28 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro
// Inject a behavior where REPL LOAD failed when try to load table "t2" and partition "uk".
// So, table "t2" will exist and partition "india" will exist, rest failed as operation failed.
- BehaviourInjection<Partition, Partition> getPartitionStub
- = new BehaviourInjection<Partition, Partition>() {
- @Nullable
+ BehaviourInjection<List<Partition>, Boolean> alterPartitionStub
+ = new BehaviourInjection<List<Partition>, Boolean>() {
@Override
- public Partition apply(@Nullable Partition ptn) {
- if (ptn.getValues().get(0).equals("india")) {
- injectionPathCalled = true;
- LOG.warn("####getPartition Stub called");
- return null;
+ public Boolean apply(List<Partition> ptns) {
+ for (Partition ptn : ptns) {
+ if (ptn.getValues().get(0).equals("india")) {
+ injectionPathCalled = true;
+ LOG.warn("####getPartition Stub called");
+ return false;
+ }
}
- return ptn;
+ return true;
}
};
- InjectableBehaviourObjectStore.setGetPartitionBehaviour(getPartitionStub);
+ InjectableBehaviourObjectStore.setAlterPartitionsBehaviour(alterPartitionStub);
// Make sure that there's some order in which the objects are loaded.
List<String> withConfigs = Arrays.asList("'hive.repl.approx.max.load.tasks'='1'",
"'hive.in.repl.test.files.sorted'='true'");
replica.loadFailure(replicatedDbName, tuple.dumpLocation, withConfigs);
- InjectableBehaviourObjectStore.resetGetPartitionBehaviour(); // reset the behaviour
- getPartitionStub.assertInjectionsPerformed(true, false);
+ InjectableBehaviourObjectStore.setAlterPartitionsBehaviour(null); // reset the behaviour
+ alterPartitionStub.assertInjectionsPerformed(true, false);
replica.run("use " + replicatedDbName)
.run("repl status " + replicatedDbName)
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 213dd9e..5eaf53f 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -64,6 +64,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.stream.Collectors;
+import java.util.LinkedList;
import static org.apache.hadoop.hive.metastore.HiveMetaHook.ALTERLOCATION;
import static org.apache.hadoop.hive.metastore.HiveMetaHook.ALTER_TABLE_OPERATION_TYPE;
@@ -732,6 +734,27 @@ public class HiveAlterHandler implements AlterHandler {
environmentContext, null, -1, null);
}
+ private Map<List<String>, Partition> getExistingPartitions(final RawStore msdb,
+ final List<Partition> new_parts, final Table tbl, final String catName,
+ final String dbname, final String name)
+ throws MetaException, NoSuchObjectException, InvalidOperationException {
+
+ // Get list of partition values
+ List<String> partValues = new LinkedList<>();
+ for (Partition tmpPart : new_parts) {
+ partValues.add(Warehouse.makePartName(tbl.getPartitionKeys(), tmpPart.getValues()));
+ }
+
+ // Get existing partitions from store
+ List<Partition> oldParts = msdb.getPartitionsByNames(catName, dbname, name, partValues);
+ if (new_parts.size() != oldParts.size()) {
+ throw new InvalidOperationException("Alter partition operation failed: "
+ + "new parts size " + new_parts.size()
+ + " not matching with old parts size " + oldParts.size());
+ }
+ return oldParts.stream().collect(Collectors.toMap(Partition::getValues, Partition -> Partition));
+ }
+
@Override
public List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String catName,
final String dbname, final String name,
@@ -760,6 +783,7 @@ public class HiveAlterHandler implements AlterHandler {
blockPartitionLocationChangesOnReplSource(msdb.getDatabase(catName, dbname), tbl,
environmentContext);
+ Map<List<String>, Partition> oldPartMap = getExistingPartitions(msdb, new_parts, tbl, catName, dbname, name);
for (Partition tmpPart: new_parts) {
// Set DDL time to now if not specified
@@ -770,7 +794,7 @@ public class HiveAlterHandler implements AlterHandler {
.currentTimeMillis() / 1000));
}
- Partition oldTmpPart = msdb.getPartition(catName, dbname, name, tmpPart.getValues());
+ Partition oldTmpPart = oldPartMap.get(tmpPart.getValues());
oldParts.add(oldTmpPart);
partValsList.add(tmpPart.getValues());
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 5525109..109b5bd 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -4286,12 +4286,21 @@ public class ObjectStore implements RawStore, Configurable {
* @param newPart Partition object containing new information
*/
private Partition alterPartitionNoTxn(String catName, String dbname, String name,
- List<String> part_vals, Partition newPart, String validWriteIds, Ref<MColumnDescriptor> oldCd)
+ List<String> part_vals, Partition newPart, String validWriteIds, Ref<MColumnDescriptor> oldCd)
+ throws InvalidObjectException, MetaException {
+ MTable table = this.getMTable(newPart.getCatName(), newPart.getDbName(), newPart.getTableName());
+ return alterPartitionNoTxn(catName, dbname, name, part_vals, newPart,
+ validWriteIds, oldCd, table);
+ }
+
+ private Partition alterPartitionNoTxn(String catName, String dbname,
+ String name, List<String> part_vals, Partition newPart,
+ String validWriteIds,
+ Ref<MColumnDescriptor> oldCd, MTable table)
throws InvalidObjectException, MetaException {
catName = normalizeIdentifier(catName);
name = normalizeIdentifier(name);
dbname = normalizeIdentifier(dbname);
- MTable table = this.getMTable(newPart.getCatName(), newPart.getDbName(), newPart.getTableName());
MPartition oldp = getMPartition(catName, dbname, name, part_vals);
MPartition newp = convertToMPart(newPart, table, false);
MColumnDescriptor oldCD = null;
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
index 6c7fe11..8673186 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
@@ -91,6 +91,8 @@ public class InjectableBehaviourObjectStore extends ObjectStore {
private static com.google.common.base.Function<CurrentNotificationEventId, CurrentNotificationEventId>
getCurrNotiEventIdModifier = null;
+ private static com.google.common.base.Function<List<Partition>, Boolean> alterPartitionsModifier = null;
+
// Methods to set/reset getTable modifier
public static void setGetTableBehaviour(com.google.common.base.Function<Table, Table> modifier){
getTableModifier = (modifier == null) ? com.google.common.base.Functions.identity() : modifier;
@@ -152,6 +154,11 @@ public class InjectableBehaviourObjectStore extends ObjectStore {
setAlterTableModifier(null);
}
+ public static void setAlterPartitionsBehaviour(com.google.common.base.Function<List<Partition>, Boolean> modifier){
+ alterPartitionsModifier = modifier;
+ }
+
+
// ObjectStore methods to be overridden with injected behavior
@Override
public Table getTable(String catName, String dbName, String tableName) throws MetaException {
@@ -295,4 +302,19 @@ public class InjectableBehaviourObjectStore extends ObjectStore {
}
return id;
}
+
+ @Override
+ public List<Partition> alterPartitions(String catName, String dbname, String name,
+ List<List<String>> part_vals, List<Partition> newParts,
+ long writeId, String queryWriteIdList)
+ throws InvalidObjectException, MetaException {
+ if (alterPartitionsModifier != null) {
+ Boolean success = alterPartitionsModifier.apply(newParts);
+ if ((success != null) && !success) {
+ throw new MetaException("InjectableBehaviourObjectStore: Invalid alterPartitions operation on Catalog : "
+ + catName + " DB: " + dbname + " table: " + name);
+ }
+ }
+ return super.alterPartitions(catName, dbname, name, part_vals, newParts, writeId, queryWriteIdList);
+ }
}