You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2016/09/13 22:12:04 UTC
spark git commit: [SPARK-17531] Don't initialize Hive Listeners for
the Execution Client
Repository: spark
Updated Branches:
refs/heads/master 4ba63b193 -> 72edc7e95
[SPARK-17531] Don't initialize Hive Listeners for the Execution Client
## What changes were proposed in this pull request?
If a user provides listeners inside the Hive Conf, the configuration for these listeners are passed to the Hive Execution Client as well. This may cause issues for two reasons:
1. The Execution Client will actually generate garbage
2. The listener class needs to be both in the Spark Classpath and Hive Classpath
This PR empties the listener configurations in `HiveUtils.newTemporaryConfiguration` so that the execution client will not contain the listener confs, but the metadata client will.
## How was this patch tested?
Unit tests
Author: Burak Yavuz <br...@gmail.com>
Closes #15086 from brkyvz/null-listeners.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/72edc7e9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/72edc7e9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/72edc7e9
Branch: refs/heads/master
Commit: 72edc7e958271cedb01932880550cfc2c0631204
Parents: 4ba63b1
Author: Burak Yavuz <br...@gmail.com>
Authored: Tue Sep 13 15:11:55 2016 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Tue Sep 13 15:11:55 2016 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/hive/HiveUtils.scala | 7 ++++
.../apache/spark/sql/hive/HiveUtilsSuite.scala | 36 ++++++++++++++++++++
2 files changed, 43 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/72edc7e9/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index bdec611..39d71e1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -394,6 +394,13 @@ private[spark] object HiveUtils extends Logging {
// hive.metastore.uris is not set.
propMap.put(ConfVars.METASTOREURIS.varname, "")
+ // The execution client will generate garbage events, therefore the listeners that are generated
+ // for the execution clients are useless. In order to not output garbage, we don't generate
+ // these listeners.
+ propMap.put(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, "")
+ propMap.put(ConfVars.METASTORE_EVENT_LISTENERS.varname, "")
+ propMap.put(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname, "")
+
propMap.toMap
}
http://git-wip-us.apache.org/repos/asf/spark/blob/72edc7e9/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala
new file mode 100644
index 0000000..667a7dd
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars
+
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.sql.QueryTest
+
+class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
+
+ test("newTemporaryConfiguration overwrites listener configurations") {
+ Seq(true, false).foreach { useInMemoryDerby =>
+ val conf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
+ assert(conf(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname) === "")
+ assert(conf(ConfVars.METASTORE_EVENT_LISTENERS.varname) === "")
+ assert(conf(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname) === "")
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org