You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2017/05/02 04:27:41 UTC
hive git commit: HIVE-16488 : Support replicating into existing db if
the db is empty (Sankar Hariappan, reviewed by Sushanth Sowmyan)
Repository: hive
Updated Branches:
refs/heads/master 06e486304 -> 0a235d8e1
HIVE-16488 : Support replicating into existing db if the db is empty (Sankar Hariappan, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0a235d8e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0a235d8e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0a235d8e
Branch: refs/heads/master
Commit: 0a235d8e1e5fa488a797e0001a64d2c63b371d0a
Parents: 06e4863
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Mon May 1 21:26:17 2017 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Mon May 1 21:27:38 2017 -0700
----------------------------------------------------------------------
.../hive/ql/TestReplicationScenarios.java | 66 +++++++++++++++++++-
.../ql/parse/ReplicationSemanticAnalyzer.java | 55 ++++++++++++----
2 files changed, 107 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0a235d8e/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestReplicationScenarios.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestReplicationScenarios.java
index 9b8563b..36874cd 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestReplicationScenarios.java
@@ -281,6 +281,58 @@ public class TestReplicationScenarios {
}
@Test
+ public void testBootstrapLoadOnExistingDb() throws IOException {
+ String testName = "bootstrapLoadOnExistingDb";
+ LOG.info("Testing "+testName);
+ String dbName = testName + "_" + tid;
+
+ run("CREATE DATABASE " + dbName);
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE");
+
+ String[] unptn_data = new String[]{ "eleven" , "twelve" };
+ String unptn_locn = new Path(TEST_PATH , testName + "_unptn").toUri().getPath();
+ createTestDataFile(unptn_locn, unptn_data);
+
+ run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned");
+ verifySetup("SELECT * from " + dbName + ".unptned ORDER BY a", unptn_data);
+
+ // Create an empty database to load
+ run("CREATE DATABASE " + dbName + "_empty");
+
+ advanceDumpDir();
+ run("REPL DUMP " + dbName);
+ String replDumpLocn = getResult(0,0);
+ String replDumpId = getResult(0,1,true);
+ // Load to an empty database
+ run("REPL LOAD " + dbName + "_empty FROM '" + replDumpLocn + "'");
+
+ // REPL STATUS should return same repl ID as dump
+ verifyRun("REPL STATUS " + dbName + "_empty", replDumpId);
+ verifyRun("SELECT * from " + dbName + "_empty.unptned", unptn_data);
+
+ String[] nullReplId = new String[]{ "NULL" };
+
+ // Create a database with a table
+ run("CREATE DATABASE " + dbName + "_withtable");
+ run("CREATE TABLE " + dbName + "_withtable.unptned(a string) STORED AS TEXTFILE");
+ // Load using same dump to a DB with table. It should fail as DB is not empty.
+ verifyFail("REPL LOAD " + dbName + "_withtable FROM '" + replDumpLocn + "'");
+
+ // REPL STATUS should return NULL
+ verifyRun("REPL STATUS " + dbName + "_withtable", nullReplId);
+
+ // Create a database with a view
+ run("CREATE DATABASE " + dbName + "_withview");
+ run("CREATE TABLE " + dbName + "_withview.unptned(a string) STORED AS TEXTFILE");
+ run("CREATE VIEW " + dbName + "_withview.view AS SELECT * FROM " + dbName + "_withview.unptned");
+ // Load using same dump to a DB with view. It should fail as DB is not empty.
+ verifyFail("REPL LOAD " + dbName + "_withview FROM '" + replDumpLocn + "'");
+
+ // REPL STATUS should return NULL
+ verifyRun("REPL STATUS " + dbName + "_withview", nullReplId);
+ }
+
+ @Test
public void testIncrementalAdds() throws IOException {
String testName = "incrementalAdds";
LOG.info("Testing "+testName);
@@ -1209,7 +1261,7 @@ public class TestReplicationScenarios {
LOG.info("Incremental-Dump: Dumped to {} with id {} from {}", incrementalDumpLocn, incrementalDumpId, replDumpId);
replDumpId = incrementalDumpId;
run("REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'");
- verifyRun("SELECT a from " + dbName + "_dupe.ptned_src where (b=1and c=1)", empty);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned_src where (b=1 and c=1)", empty);
verifyRun("SELECT a from " + dbName + "_dupe.ptned_src where (b=2 and c=2) ORDER BY a", ptn_data_2);
verifyRun("SELECT a from " + dbName + "_dupe.ptned_src where (b=2 and c=3) ORDER BY a", ptn_data_2);
verifyRun("SELECT a from " + dbName + "_dupe.ptned_dest where (b=1 and c=1) ORDER BY a", ptn_data_1);
@@ -1734,6 +1786,18 @@ public class TestReplicationScenarios {
verifyResults(data);
}
+ private void verifyFail(String cmd) throws RuntimeException {
+ boolean success = false;
+ try {
+ success = run(cmd,false);
+ } catch (AssertionError ae){
+ LOG.warn("AssertionError:",ae);
+ throw new RuntimeException(ae);
+ }
+
+ assertFalse(success);
+ }
+
private static void run(String cmd) throws RuntimeException {
try {
run(cmd,false); // default arg-less run simply runs, and does not care about failure
http://git-wip-us.apache.org/repos/asf/hive/blob/0a235d8e/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index c4cc9bc..37aa3ba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.ReplChangeManager;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.messaging.event.filters.AndFilter;
import org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter;
@@ -53,6 +54,7 @@ import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -1087,6 +1089,27 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
return partSpecs;
}
+ private boolean existEmptyDb(String dbName) throws InvalidOperationException, HiveException {
+ Hive hiveDb = Hive.get();
+ Database db = hiveDb.getDatabase(dbName);
+ if (null != db) {
+ List<String> allTables = hiveDb.getAllTables(dbName);
+ List<String> allFunctions = hiveDb.getFunctions(dbName, "*");
+ if (!allTables.isEmpty()) {
+ throw new InvalidOperationException(
+ "Database " + db.getName() + " is not empty. One or more tables exist.");
+ }
+ if (!allFunctions.isEmpty()) {
+ throw new InvalidOperationException(
+ "Database " + db.getName() + " is not empty. One or more functions exist.");
+ }
+
+ return true;
+ }
+
+ return false;
+ }
+
private void analyzeDatabaseLoad(String dbName, FileSystem fs, FileStatus dir)
throws SemanticException {
try {
@@ -1119,24 +1142,30 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
dbName = dbObj.getName();
}
- CreateDatabaseDesc createDbDesc = new CreateDatabaseDesc();
- createDbDesc.setName(dbName);
- createDbDesc.setComment(dbObj.getDescription());
- createDbDesc.setDatabaseProperties(dbObj.getParameters());
- // note that we do not set location - for repl load, we want that auto-created.
-
- createDbDesc.setIfNotExists(false);
- // If it exists, we want this to be an error condition. Repl Load is not intended to replace a
- // db.
- // TODO: we might revisit this in create-drop-recreate cases, needs some thinking on.
- Task<? extends Serializable> createDbTask = TaskFactory.get(new DDLWork(inputs, outputs, createDbDesc), conf);
- rootTasks.add(createDbTask);
+ Task<? extends Serializable> dbRootTask = null;
+ if (existEmptyDb(dbName)) {
+ AlterDatabaseDesc alterDbDesc = new AlterDatabaseDesc(dbName, dbObj.getParameters());
+ dbRootTask = TaskFactory.get(new DDLWork(inputs, outputs, alterDbDesc), conf);
+ } else {
+ CreateDatabaseDesc createDbDesc = new CreateDatabaseDesc();
+ createDbDesc.setName(dbName);
+ createDbDesc.setComment(dbObj.getDescription());
+ createDbDesc.setDatabaseProperties(dbObj.getParameters());
+ // note that we do not set location - for repl load, we want that auto-created.
+
+ createDbDesc.setIfNotExists(false);
+ // If it exists, we want this to be an error condition. Repl Load is not intended to replace a
+ // db.
+ // TODO: we might revisit this in create-drop-recreate cases, needs some thinking on.
+ dbRootTask = TaskFactory.get(new DDLWork(inputs, outputs, createDbDesc), conf);
+ }
+ rootTasks.add(dbRootTask);
FileStatus[] dirsInDbPath = fs.listStatus(dir.getPath(), EximUtil.getDirectoryFilter(fs));
for (FileStatus tableDir : dirsInDbPath) {
analyzeTableLoad(
- dbName, null, tableDir.getPath().toUri().toString(), createDbTask, null, null);
+ dbName, null, tableDir.getPath().toUri().toString(), dbRootTask, null, null);
}
} catch (Exception e) {
throw new SemanticException(e);