You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/12/30 20:24:49 UTC
spark git commit: [SPARK-4908][SQL] Prevent multiple concurrent hive
native commands
Repository: spark
Updated Branches:
refs/heads/master efa80a531 -> 480bd1d2e
[SPARK-4908][SQL] Prevent multiple concurrent hive native commands
This is just a quick fix that locks when calling `runHive`. If we can find a way to avoid the error without a global lock that would be better.
Author: Michael Armbrust <mi...@databricks.com>
Closes #3834 from marmbrus/hiveConcurrency and squashes the following commits:
bf25300 [Michael Armbrust] prevent multiple concurrent hive native commands
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/480bd1d2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/480bd1d2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/480bd1d2
Branch: refs/heads/master
Commit: 480bd1d2edd1de06af607b0cf3ff3c0b16089add
Parents: efa80a53
Author: Michael Armbrust <mi...@databricks.com>
Authored: Tue Dec 30 11:24:46 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Tue Dec 30 11:24:46 2014 -0800
----------------------------------------------------------------------
.../main/scala/org/apache/spark/sql/hive/HiveContext.scala | 2 +-
.../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 7 +++++++
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/480bd1d2/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index 56fe27a..982e059 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -284,7 +284,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {
* Execute the command using Hive and return the results as a sequence. Each element
* in the sequence is one row.
*/
- protected def runHive(cmd: String, maxRows: Int = 1000): Seq[String] = {
+ protected def runHive(cmd: String, maxRows: Int = 1000): Seq[String] = synchronized {
try {
val cmd_trimmed: String = cmd.trim()
val tokens: Array[String] = cmd_trimmed.split("\\s+")
http://git-wip-us.apache.org/repos/asf/spark/blob/480bd1d2/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 4d81acc..fb6da33 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -56,6 +56,13 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
Locale.setDefault(originalLocale)
}
+ test("SPARK-4908: concurent hive native commands") {
+ (1 to 100).par.map { _ =>
+ sql("USE default")
+ sql("SHOW TABLES")
+ }
+ }
+
createQueryTest("constant object inspector for generic udf",
"""SELECT named_struct(
lower("AA"), "10",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org