You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2018/10/06 09:31:08 UTC
hive git commit: HIVE-20680: Bootstrap is missing partitions in
replicated DB when retry after kill query (Mahesh Kumar Behera,
reviewed by Sankar Hariappan)
Repository: hive
Updated Branches:
refs/heads/master 78e45be15 -> f122e258b
HIVE-20680: Bootstrap is missing partitions in replicated DB when retry after kill query (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f122e258
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f122e258
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f122e258
Branch: refs/heads/master
Commit: f122e258b869c4cc2358a4cc4a1925765b53dd91
Parents: 78e45be
Author: Sankar Hariappan <sa...@apache.org>
Authored: Sat Oct 6 15:00:44 2018 +0530
Committer: Sankar Hariappan <sa...@apache.org>
Committed: Sat Oct 6 15:00:44 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 ++
...TestReplicationScenariosAcrossInstances.java | 51 ++++++++++++++++++++
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 9 ++++
.../filesystem/DatabaseEventsIterator.java | 49 ++++++++++++++++++-
.../InjectableBehaviourObjectStore.java | 24 +++++++++
5 files changed, 135 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f122e258/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 58951ef..27b1b73 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -612,6 +612,9 @@ public class HiveConf extends Configuration {
HIVE_MAPJOIN_TESTING_NO_HASH_TABLE_LOAD("hive.mapjoin.testing.no.hash.table.load", false, "internal use only, true when in testing map join",
true),
+ HIVE_IN_REPL_TEST_FILES_SORTED("hive.in.repl.test.files.sorted", false,
+ "internal usage only, set to true if the file listing is required in sorted order during bootstrap load", true),
+
LOCALMODEAUTO("hive.exec.mode.local.auto", false,
"Let Hive determine whether to run in local mode automatically"),
LOCALMODEMAXBYTES("hive.exec.mode.local.auto.inputbytes.max", 134217728L,
http://git-wip-us.apache.org/repos/asf/hive/blob/f122e258/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index fb797ff..1ea85eb 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1570,4 +1570,55 @@ public class TestReplicationScenariosAcrossInstances {
.run("show partitions t1")
.verifyResults(new String[] { "country=india", "country=us" });
}
+
+ // This requires the tables are loaded in a fixed sorted order.
+ @Test
+ public void testBootstrapLoadRetryAfterFailureForAlterTable() throws Throwable {
+ WarehouseInstance.Tuple tuple = primary
+ .run("use " + primaryDbName)
+ .run("create table t1 (place string)")
+ .run("insert into table t1 values ('testCheck')")
+ .run("create table t2 (place string) partitioned by (country string)")
+ .run("insert into table t2 partition(country='china') values ('shenzhen')")
+ .run("insert into table t2 partition(country='india') values ('banaglore')")
+ .dump(primaryDbName, null);
+
+ // fail setting ckpt directory property for table t1.
+ BehaviourInjection<CallerArguments, Boolean> callerVerifier
+ = new BehaviourInjection<CallerArguments, Boolean>() {
+ @Nullable
+ @Override
+ public Boolean apply(@Nullable CallerArguments args) {
+ if (args.tblName.equalsIgnoreCase("t1") && args.dbName.equalsIgnoreCase(replicatedDbName)) {
+ injectionPathCalled = true;
+ LOG.warn("Verifier - DB : " + args.dbName + " TABLE : " + args.tblName);
+ return false;
+ }
+ return true;
+ }
+ };
+
+ // Fail repl load before the ckpt proeprty is set for t1 and after it is set for t2. So in the next run, for
+ // t2 it goes directly to partion load with no task for table tracker and for t1 it loads the table
+ // again from start.
+ InjectableBehaviourObjectStore.setAlterTableModifier(callerVerifier);
+ try {
+ replica.loadFailure(replicatedDbName, tuple.dumpLocation);
+ callerVerifier.assertInjectionsPerformed(true, false);
+ } finally {
+ InjectableBehaviourObjectStore.resetAlterTableModifier();
+ }
+
+ // Retry with same dump with which it was already loaded should resume the bootstrap load. Make sure that table t1,
+ // is loaded before t2. So that scope is set to table in first iteration for table t1. In the next iteration, it
+ // loads only remaining partitions of t2, so that the table tracker has no tasks.
+ List<String> withConfigs = Arrays.asList("'hive.in.repl.test.files.sorted'='true'");
+ replica.load(replicatedDbName, tuple.dumpLocation, withConfigs);
+
+ replica.run("use " + replicatedDbName)
+ .run("repl status " + replicatedDbName)
+ .verifyResult(tuple.lastReplicationId)
+ .run("select country from t2 order by country")
+ .verifyResults(Arrays.asList("china", "india"));
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f122e258/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
index aa41301..270670d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
@@ -126,6 +126,10 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
loadTaskTracker.update(dbTracker);
if (work.hasDbState()) {
loadTaskTracker.update(updateDatabaseLastReplID(maxTasks, context, scope));
+ } else {
+ // Scope might have set to database in some previous iteration of loop, so reset it to false if database
+ // tracker has no tasks.
+ scope.database = false;
}
work.updateDbEventState(dbEvent.toState());
if (dbTracker.hasTasks()) {
@@ -151,7 +155,12 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
if (!scope.database && tableTracker.hasTasks()) {
scope.rootTasks.addAll(tableTracker.tasks());
scope.table = true;
+ } else {
+ // Scope might have set to table in some previous iteration of loop, so reset it to false if table
+ // tracker has no tasks.
+ scope.table = false;
}
+
/*
for table replication if we reach the max number of tasks then for the next run we will
try to reload the same table again, this is mainly for ease of understanding the code
http://git-wip-us.apache.org/repos/asf/hive/blob/f122e258/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
index f778cb4..d909945 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/events/filesystem/DatabaseEventsIterator.java
@@ -22,10 +22,12 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.BootstrapEvent;
import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState;
import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.parse.repl.load.EventDumpDirComparator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +36,7 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
+import java.util.ArrayList;
import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.FUNCTIONS_ROOT_DIR_NAME;
@@ -55,7 +58,51 @@ class DatabaseEventsIterator implements Iterator<BootstrapEvent> {
if (!fileSystem.exists(new Path(dbLevelPath + Path.SEPARATOR + EximUtil.METADATA_NAME))) {
databaseEventProcessed = true;
}
- remoteIterator = fileSystem.listFiles(dbLevelPath, true);
+
+ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_IN_REPL_TEST_FILES_SORTED)) {
+ LOG.info(" file sorting is enabled in DatabaseEventsIterator");
+ List<LocatedFileStatus> fileStatuses = new ArrayList<>();
+
+ // Sort the directories in tha path and then add the files recursively .
+ getSortedFileList(dbLevelPath, fileStatuses, fileSystem);
+ remoteIterator = new RemoteIterator<LocatedFileStatus>() {
+ private int idx = 0;
+ private final int numEntry = fileStatuses.size();
+ private final List<LocatedFileStatus> fileStatusesLocal = fileStatuses;
+ public boolean hasNext() throws IOException {
+ return idx < numEntry;
+ }
+ public LocatedFileStatus next() throws IOException {
+ LOG.info(" file in next is " + fileStatusesLocal.get(idx));
+ return fileStatusesLocal.get(idx++);
+ }
+ };
+ } else {
+ remoteIterator = fileSystem.listFiles(dbLevelPath, true);
+ }
+ }
+
+ private void getSortedFileList(Path eventPath, List<LocatedFileStatus> fileStatuses,
+ FileSystem fileSystem) throws IOException {
+ //Add all the files in this directory. No need to sort.
+ RemoteIterator<LocatedFileStatus> iteratorNext = fileSystem.listFiles(eventPath, false);
+ while (iteratorNext.hasNext()) {
+ LocatedFileStatus status = iteratorNext.next();
+ LOG.info(" files added at getSortedFileList" + status.getPath());
+ fileStatuses.add(status);
+ }
+
+ // get all the directories in this path and sort them
+ FileStatus[] eventDirs = fileSystem.listStatus(eventPath, EximUtil.getDirectoryFilter(fileSystem));
+ if (eventDirs.length == 0) {
+ return;
+ }
+ Arrays.sort(eventDirs, new EventDumpDirComparator());
+
+ // add files recursively for each directory
+ for (FileStatus fs : eventDirs) {
+ getSortedFileList(fs.getPath(), fileStatuses, fileSystem);
+ }
}
public Path dbLevelPath() {
http://git-wip-us.apache.org/repos/asf/hive/blob/f122e258/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
index 3d5395a..9daff37 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
@@ -85,6 +85,8 @@ public class InjectableBehaviourObjectStore extends ObjectStore {
private static com.google.common.base.Function<NotificationEvent, Boolean> addNotificationEventModifier = null;
+ private static com.google.common.base.Function<CallerArguments, Boolean> alterTableModifier = null;
+
// Methods to set/reset getTable modifier
public static void setGetTableBehaviour(com.google.common.base.Function<Table, Table> modifier){
getTableModifier = (modifier == null) ? com.google.common.base.Functions.identity() : modifier;
@@ -139,6 +141,13 @@ public class InjectableBehaviourObjectStore extends ObjectStore {
setCallerVerifier(null);
}
+ public static void setAlterTableModifier(com.google.common.base.Function<CallerArguments, Boolean> modifier) {
+ alterTableModifier = modifier;
+ }
+ public static void resetAlterTableModifier() {
+ setAlterTableModifier(null);
+ }
+
// ObjectStore methods to be overridden with injected behavior
@Override
public Table getTable(String catName, String dbName, String tableName) throws MetaException {
@@ -168,6 +177,21 @@ public class InjectableBehaviourObjectStore extends ObjectStore {
}
@Override
+ public Table alterTable(String catName, String dbname, String name, Table newTable, String queryValidWriteIds)
+ throws InvalidObjectException, MetaException {
+ if (alterTableModifier != null) {
+ CallerArguments args = new CallerArguments(dbname);
+ args.tblName = name;
+ Boolean success = alterTableModifier.apply(args);
+ if ((success != null) && !success) {
+ throw new MetaException("InjectableBehaviourObjectStore: Invalid alterTable operation on Catalog : " + catName +
+ " DB: " + dbname + " table: " + name);
+ }
+ }
+ return super.alterTable(catName, dbname, name, newTable, queryValidWriteIds);
+ }
+
+ @Override
public void addNotificationEvent(NotificationEvent entry) throws MetaException {
if (addNotificationEventModifier != null) {
Boolean success = addNotificationEventModifier.apply(entry);