You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by cs...@apache.org on 2019/07/10 16:48:26 UTC
[impala] branch master updated: Bump CDP_BUILD_NUMBER to 1235229
This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new bc76175 Bump CDP_BUILD_NUMBER to 1235229
bc76175 is described below
commit bc761753125485e20f896eead73defc071ebc19a
Author: Vihang Karajgaonkar <vi...@apache.org>
AuthorDate: Wed Jul 3 10:51:50 2019 -0700
Bump CDP_BUILD_NUMBER to 1235229
The newer CDP build includes fix for HIVE-21932 which is required to fix
test failures on CDP builds.
Also, this patch fixes two test failures on CDP.
1 Fixes the MetastoreEventsProcessorTest on CDP caused due to the
serialization difference of the event messages between CDH and CDP
builds.
2. Fixes the Ranger audit test failure caused due to the missing cluster
name paramter from the Audit events from Ranger. This is needed a change
in the ranger-hive-audit.xml due to RANGER-2458
Testing:
1. Build against CDP and CDH builds
2. Run the previously failing tests on CDP which were caused by
HIVE-21932
3. Run full tests with CDP and CDH builds
Change-Id: I6d0fa856fa6c7cb1f4c9ee0e4250cfa1e8a14595
Reviewed-on: http://gerrit.cloudera.org:8080/13799
Reviewed-by: Zoltan Borok-Nagy <bo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/impala-config.sh | 10 +++++-----
.../org/apache/impala/compat/MetastoreShim.java | 12 ++++++++----
.../org/apache/impala/compat/MetastoreShim.java | 21 +++++++++++++++++----
.../impala/authorization/AuthorizationTestBase.java | 2 ++
.../events/MetastoreEventsProcessorTest.java | 6 ++++--
fe/src/test/resources/ranger-hive-audit.xml | 5 +----
6 files changed, 37 insertions(+), 19 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 06613df..3f45bf0 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -168,19 +168,19 @@ fi
export IMPALA_TOOLCHAIN_HOST
export CDH_MAJOR_VERSION=6
export CDH_BUILD_NUMBER=1173663
-export CDP_BUILD_NUMBER=1213831
+export CDP_BUILD_NUMBER=1235229
export CDH_HADOOP_VERSION=3.0.0-cdh6.x-SNAPSHOT
-export CDP_HADOOP_VERSION=3.1.1.7.0.0.0-195
+export CDP_HADOOP_VERSION=3.1.1.7.0.0.0-227
export IMPALA_HBASE_VERSION=2.1.0-cdh6.x-SNAPSHOT
export IMPALA_SENTRY_VERSION=2.1.0-cdh6.x-SNAPSHOT
-export IMPALA_RANGER_VERSION=1.2.0.7.0.0.0-195
+export IMPALA_RANGER_VERSION=1.2.0.7.0.0.0-227
export IMPALA_PARQUET_VERSION=1.9.0-cdh6.x-SNAPSHOT
export IMPALA_AVRO_JAVA_VERSION=1.8.2-cdh6.x-SNAPSHOT
export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
export IMPALA_KITE_VERSION=1.0.0-cdh6.x-SNAPSHOT
export IMPALA_KUDU_JAVA_VERSION=1.10.0-cdh6.x-SNAPSHOT
export CDH_HIVE_VERSION=2.1.1-cdh6.x-SNAPSHOT
-export CDP_HIVE_VERSION=3.1.0.7.0.0.0-195
+export CDP_HIVE_VERSION=3.1.0.7.0.0.0-227
# When IMPALA_(CDH_COMPONENT)_URL are overridden, they may contain '$(platform_label)'
# which will be substituted for the CDH platform label in bootstrap_toolchain.py
@@ -207,7 +207,7 @@ if $USE_CDP_HIVE; then
# When USE_CDP_HIVE is set we use the CDP hive version to build as well as deploy in
# the minicluster
export IMPALA_HIVE_VERSION=${CDP_HIVE_VERSION}
- export IMPALA_TEZ_VERSION=0.9.1.7.0.0.0-195
+ export IMPALA_TEZ_VERSION=0.9.1.7.0.0.0-227
export IMPALA_HADOOP_VERSION=${CDP_HADOOP_VERSION}
export HADOOP_HOME="$CDP_COMPONENTS_HOME/hadoop-${CDP_HADOOP_VERSION}/"
else
diff --git a/fe/src/compat-hive-2/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-hive-2/java/org/apache/impala/compat/MetastoreShim.java
index de58116..2064a09 100644
--- a/fe/src/compat-hive-2/java/org/apache/impala/compat/MetastoreShim.java
+++ b/fe/src/compat-hive-2/java/org/apache/impala/compat/MetastoreShim.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage;
import org.apache.hadoop.hive.metastore.messaging.InsertMessage;
import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer;
import org.apache.hadoop.hive.metastore.messaging.json.ExtendedJSONMessageFactory;
@@ -215,11 +216,14 @@ public class MetastoreShim {
return ExtendedJSONMessageFactory.getInstance().buildAlterTableMessage(before, after);
}
+ /**
+ * Wrapper around HMS-2 message serializer
+ * @param message
+ * @return serialized string to use used in the NotificationEvent's message field
+ */
@VisibleForTesting
- public static InsertMessage buildInsertMessage(Table msTbl, Partition partition,
- boolean isInsertOverwrite, List<String> newFiles) {
- return ExtendedJSONMessageFactory.getInstance().buildInsertMessage(msTbl, partition,
- isInsertOverwrite, newFiles);
+ public static String serializeEventMessage(EventMessage message) {
+ return message.toString();
}
public static String getAllColumnsInformation(List<FieldSchema> tabCols,
diff --git a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
index ea194a7..abae8e5 100644
--- a/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
+++ b/fe/src/compat-hive-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -38,11 +38,13 @@ import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage;
import org.apache.hadoop.hive.metastore.messaging.InsertMessage;
import org.apache.hadoop.hive.metastore.messaging.MessageBuilder;
import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer;
import org.apache.hadoop.hive.metastore.messaging.MessageEncoder;
import org.apache.hadoop.hive.metastore.messaging.MessageFactory;
+import org.apache.hadoop.hive.metastore.messaging.MessageSerializer;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hive.service.rpc.thrift.TGetColumnsReq;
@@ -206,6 +208,14 @@ public class MetastoreShim {
}
/**
+ * Wrapper around HMS-3 message encoder to get the serializer
+ * @return
+ */
+ public static MessageSerializer getMessageSerializer() {
+ return eventMessageEncoder_.getSerializer();
+ }
+
+ /**
* Wrapper around FileUtils.makePartName to deal with package relocation in Hive 3.
* This method uses the metastore's FileUtils method instead of one from hive-exec
* @param partitionColNames
@@ -227,11 +237,14 @@ public class MetastoreShim {
isTruncateOp, writeId);
}
+ /**
+ * Wrapper around HMS-3 message serializer
+ * @param message
+ * @return serialized string to use used in the NotificationEvent's message field
+ */
@VisibleForTesting
- public static InsertMessage buildInsertMessage(Table msTbl, Partition partition,
- boolean isInsertOverwrite, List<String> newFiles) {
- return MessageBuilder.getInstance().buildInsertMessage(msTbl, partition,
- isInsertOverwrite, newFiles.iterator());
+ public static String serializeEventMessage(EventMessage message) {
+ return getMessageSerializer().serialize(message);
}
/**
diff --git a/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java
index 7b4fcc6..a4787f3 100644
--- a/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java
+++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java
@@ -60,6 +60,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -114,6 +115,7 @@ public abstract class AuthorizationTestBase extends FrontendTestBase {
rangerImpalaPlugin_ =
((RangerAuthorizationChecker) authzFrontend_.getAuthzChecker())
.getRangerImpalaPlugin();
+ assertEquals("test-cluster", rangerImpalaPlugin_.getClusterName());
sentryService_ = null;
rangerRestClient_ = new RangerRESTClient(RANGER_ADMIN_URL, null);
rangerRestClient_.setBasicAuthInfo(RANGER_USER, RANGER_PASSWORD);
diff --git a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
index 9d1ddbe..4c2d2f8 100644
--- a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
import org.apache.impala.authorization.NoopAuthorizationFactory;
import org.apache.impala.authorization.NoopAuthorizationFactory.NoopAuthorizationManager;
import org.apache.impala.catalog.CatalogException;
@@ -1408,9 +1409,10 @@ public class MetastoreEventsProcessorTest {
fakeEvent.setTableName(tblName);
fakeEvent.setDbName(dbName);
fakeEvent.setEventId(eventIdGenerator.incrementAndGet());
+ AlterTableMessage alterTableMessage =
+ MetastoreShim.buildAlterTableMessage(tableBefore, tableAfter, false, -1L);
fakeEvent.setMessage(
- MetastoreShim.buildAlterTableMessage(tableBefore, tableAfter, false, -1L)
- .toString());
+ MetastoreShim.serializeEventMessage(alterTableMessage));
fakeEvent.setEventType("ALTER_TABLE");
return fakeEvent;
}
diff --git a/fe/src/test/resources/ranger-hive-audit.xml b/fe/src/test/resources/ranger-hive-audit.xml
index f9232f3..84b0369 100644
--- a/fe/src/test/resources/ranger-hive-audit.xml
+++ b/fe/src/test/resources/ranger-hive-audit.xml
@@ -20,11 +20,8 @@
<name>xasecure.audit.is.enabled</name>
<value>false</value>
</property>
- <!-- RANGER-2458: This property may need to be updated to
- "ranger.plugin.hive.access.cluster.name" and added to
- "ranger-hive-security.xml". -->
<property>
- <name>ranger.plugin.hive.ambari.cluster.name</name>
+ <name>ranger.plugin.hive.access.cluster.name</name>
<value>test-cluster</value>
</property>
</configuration>