You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2017/09/01 04:08:22 UTC
hive git commit: HIVE-17385: incremental repl error for non-native
tables
Repository: hive
Updated Branches:
refs/heads/master 601c76491 -> 45f3fac55
HIVE-17385: incremental repl error for non-native tables
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/45f3fac5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/45f3fac5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/45f3fac5
Branch: refs/heads/master
Commit: 45f3fac554ada3cd4b15a7bf3c9f0047c65fa616
Parents: 601c764
Author: Daniel Dai <da...@hortonworks.com>
Authored: Thu Aug 31 21:08:02 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Thu Aug 31 21:08:02 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/parse/EximUtil.java | 30 ++++++++++++++++++++
.../hive/ql/parse/ImportSemanticAnalyzer.java | 5 ++++
.../hive/ql/parse/repl/dump/TableExport.java | 7 +----
.../repl/dump/events/CreateTableHandler.java | 8 ++++--
.../ql/parse/repl/dump/io/TableSerializer.java | 7 ++---
5 files changed, 44 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
index 22094c0..40c34bf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.DBSerializer;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.ReplicationSpecSerializer;
@@ -385,4 +386,33 @@ public class EximUtil {
};
}
+ /**
+ * Verify if a table should be exported or not
+ */
+ public static Boolean shouldExportTable(ReplicationSpec replicationSpec, Table tableHandle) throws SemanticException {
+ if (replicationSpec == null)
+ {
+ replicationSpec = new ReplicationSpec();
+ }
+
+ if (replicationSpec.isNoop())
+ {
+ return false;
+ }
+
+ if (tableHandle == null)
+ {
+ return false;
+ }
+
+ if (replicationSpec.isInReplicationScope()) {
+ return !(tableHandle == null || tableHandle.isTemporary() || tableHandle.isNonNative());
+ }
+
+ if (tableHandle.isNonNative()) {
+ throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg());
+ }
+
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index f8d5c8d..7f3460f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -197,6 +197,11 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
}
+ if (rv.getTable() == null) {
+ // nothing to do here, silently return.
+ return false;
+ }
+
ReplicationSpec replicationSpec = rv.getReplicationSpec();
if (replicationSpec.isNoop()){
// nothing to do here, silently return.
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
index 7533a39..e490633 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
@@ -156,12 +156,7 @@ public class TableExport {
}
private boolean shouldExport() throws SemanticException {
- if (replicationSpec.isInReplicationScope()) {
- return !(tableSpec.tableHandle.isTemporary() || tableSpec.tableHandle.isNonNative());
- } else if (tableSpec.tableHandle.isNonNative()) {
- throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg());
- }
- return true;
+ return EximUtil.shouldExportTable(replicationSpec, tableSpec.tableHandle);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
index 8737d50..ef6f340 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
@@ -23,13 +23,12 @@ import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
-import org.apache.hadoop.hive.ql.parse.repl.DumpType;
-
class CreateTableHandler extends AbstractEventHandler {
CreateTableHandler(NotificationEvent event) {
@@ -48,6 +47,11 @@ class CreateTableHandler extends AbstractEventHandler {
}
Table qlMdTable = new Table(tobj);
+
+ if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) {
+ return;
+ }
+
if (qlMdTable.isView()) {
withinContext.replicationSpec.setIsMetadataOnly(true);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/45f3fac5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
index 92408a4..c3a70cc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.parse.EximUtil;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.thrift.TException;
@@ -44,7 +45,7 @@ public class TableSerializer implements JsonWriter.Serializer {
@Override
public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
throws SemanticException, IOException {
- if (cannotReplicateTable(additionalPropertiesProvider)) {
+ if (!EximUtil.shouldExportTable(additionalPropertiesProvider, tableHandle)) {
return;
}
@@ -61,10 +62,6 @@ public class TableSerializer implements JsonWriter.Serializer {
}
}
- private boolean cannotReplicateTable(ReplicationSpec additionalPropertiesProvider) {
- return tableHandle == null || additionalPropertiesProvider.isNoop();
- }
-
private Table addPropertiesToTable(Table table, ReplicationSpec additionalPropertiesProvider)
throws SemanticException, IOException {
if (additionalPropertiesProvider.isInReplicationScope()) {