You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2016/09/13 23:15:50 UTC
spark git commit: [SPARK-17531][BACKPORT] Don't initialize Hive
Listeners for the Execution Client
Repository: spark
Updated Branches:
refs/heads/branch-1.6 047bc3f13 -> bf3f6d2f1
[SPARK-17531][BACKPORT] Don't initialize Hive Listeners for the Execution Client
## What changes were proposed in this pull request?
If a user provides listeners inside the Hive Conf, the configuration for these listeners are passed to the Hive Execution Client as well. This may cause issues for two reasons:
1. The Execution Client will actually generate garbage
2. The listener class needs to be both in the Spark Classpath and Hive Classpath
This PR empties the listener configurations in HiveUtils.newTemporaryConfiguration so that the execution client will not contain the listener confs, but the metadata client will.
## How was this patch tested?
Unit tests
Author: Burak Yavuz <br...@gmail.com>
Closes #15087 from brkyvz/overwrite-hive-listeners.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bf3f6d2f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bf3f6d2f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bf3f6d2f
Branch: refs/heads/branch-1.6
Commit: bf3f6d2f10872567e083467da7951a7111975065
Parents: 047bc3f
Author: Burak Yavuz <br...@gmail.com>
Authored: Tue Sep 13 16:15:44 2016 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Tue Sep 13 16:15:44 2016 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/hive/HiveContext.scala | 7 +++++
.../spark/sql/hive/HiveContextSuite.scala | 32 ++++++++++++++++++++
2 files changed, 39 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bf3f6d2f/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index e83941c..fa64440 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -753,6 +753,13 @@ private[hive] object HiveContext {
// hive.metastore.uris is not set.
propMap.put(ConfVars.METASTOREURIS.varname, "")
+ // The execution client will generate garbage events, therefore the listeners that are generated
+ // for the execution clients are useless. In order to not output garbage, we don't generate
+ // these listeners.
+ propMap.put(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, "")
+ propMap.put(ConfVars.METASTORE_EVENT_LISTENERS.varname, "")
+ propMap.put(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname, "")
+
propMap.toMap
}
http://git-wip-us.apache.org/repos/asf/spark/blob/bf3f6d2f/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextSuite.scala
new file mode 100644
index 0000000..048a0c0
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextSuite.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars
+
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.QueryTest
+
+class HiveContextSuite extends QueryTest with TestHiveSingleton {
+ test("newTemporaryConfiguration overwrites listener configurations") {
+ val conf = HiveContext.newTemporaryConfiguration()
+ assert(conf(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname) === "")
+ assert(conf(ConfVars.METASTORE_EVENT_LISTENERS.varname) === "")
+ assert(conf(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname) === "")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org