You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yu...@apache.org on 2019/09/28 23:56:12 UTC
[spark] branch master updated: [SPARK-27831][FOLLOW-UP][SQL][TEST]
Should not use maven to add Hive test jars
This is an automated email from the ASF dual-hosted git repository.
yumwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8167714 [SPARK-27831][FOLLOW-UP][SQL][TEST] Should not use maven to add Hive test jars
8167714 is described below
commit 8167714cab93a5c06c23f92c9077fe8b9677ab28
Author: Yuming Wang <yu...@ebay.com>
AuthorDate: Sat Sep 28 16:55:49 2019 -0700
[SPARK-27831][FOLLOW-UP][SQL][TEST] Should not use maven to add Hive test jars
### What changes were proposed in this pull request?
This PR moves Hive test jars(`hive-contrib-*.jar` and `hive-hcatalog-core-*.jar`) from maven dependency to local file.
### Why are the changes needed?
`--jars` can't be tested since `hive-contrib-*.jar` and `hive-hcatalog-core-*.jar` are already in classpath.
### Does this PR introduce any user-facing change?
No.
### How was this patch tested?
manual test
Closes #25690 from wangyum/SPARK-27831-revert.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: Yuming Wang <wg...@gmail.com>
---
pom.xml | 69 ----------------------
sql/hive-thriftserver/pom.xml | 9 ---
.../spark/sql/hive/thriftserver/CliSuite.scala | 12 ++--
.../thriftserver/HiveThriftServer2Suites.scala | 4 +-
sql/hive/pom.xml | 8 ---
.../spark/sql/hive/HiveSparkSubmitSuite.scala | 14 ++---
.../spark/sql/hive/execution/HiveQuerySuite.scala | 8 +--
.../spark/sql/hive/execution/SQLQuerySuite.scala | 4 +-
.../apache/spark/sql/hive/test/HiveTestUtils.scala | 32 ----------
.../org/apache/spark/sql/hive/test/TestHive.scala | 21 +++++++
10 files changed, 42 insertions(+), 139 deletions(-)
diff --git a/pom.xml b/pom.xml
index 2ea69937..e24e32f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1997,75 +1997,6 @@
</dependency>
<dependency>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-contrib</artifactId>
- <version>${hive.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-exec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>${hive.group}.hcatalog</groupId>
- <artifactId>hive-hcatalog-core</artifactId>
- <version>${hive.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-exec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
<version>${orc.version}</version>
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 5b1352a..9dd9270 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -77,15 +77,6 @@
<groupId>${hive.group}</groupId>
<artifactId>hive-beeline</artifactId>
</dependency>
- <!-- Explicit listing hive-contrib and hive-hcatalog-core. Otherwise the maven test fails. -->
- <dependency>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-contrib</artifactId>
- </dependency>
- <dependency>
- <groupId>${hive.group}.hcatalog</groupId>
- <artifactId>hive-hcatalog-core</artifactId>
- </dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index 6d45041..db33d1d 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -27,12 +27,11 @@ import scala.concurrent.Promise
import scala.concurrent.duration._
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.contrib.udf.example.UDFExampleFormat
import org.scalatest.BeforeAndAfterAll
import org.apache.spark.SparkFunSuite
import org.apache.spark.internal.Logging
-import org.apache.spark.sql.hive.test.HiveTestUtils
+import org.apache.spark.sql.hive.test.HiveTestJars
import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer
import org.apache.spark.util.{ThreadUtils, Utils}
@@ -202,7 +201,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
}
test("Commands using SerDe provided in --jars") {
- val jarFile = HiveTestUtils.getHiveHcatalogCoreJar.getCanonicalPath
+ val jarFile = HiveTestJars.getHiveHcatalogCoreJar().getCanonicalPath
val dataFilePath =
Thread.currentThread().getContextClassLoader.getResource("data/files/small_kv.txt")
@@ -297,11 +296,12 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
}
test("Support hive.aux.jars.path") {
- val hiveContribJar = HiveTestUtils.getHiveContribJar.getCanonicalPath
+ val hiveContribJar = HiveTestJars.getHiveContribJar().getCanonicalPath
runCliWithin(
1.minute,
Seq("--conf", s"spark.hadoop.${ConfVars.HIVEAUXJARS}=$hiveContribJar"))(
- s"CREATE TEMPORARY FUNCTION example_format AS '${classOf[UDFExampleFormat].getName}';" -> "",
+ "CREATE TEMPORARY FUNCTION example_format AS " +
+ "'org.apache.hadoop.hive.contrib.udf.example.UDFExampleFormat';" -> "",
"SELECT example_format('%o', 93);" -> "135"
)
}
@@ -319,7 +319,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
test("SPARK-28840 test --jars and hive.aux.jars.path command") {
val jarFile = new File("../../sql/hive/src/test/resources/SPARK-21101-1.0.jar").getCanonicalPath
- val hiveContribJar = HiveTestUtils.getHiveContribJar.getCanonicalPath
+ val hiveContribJar = HiveTestJars.getHiveContribJar().getCanonicalPath
runCliWithin(
1.minute,
Seq("--jars", s"$jarFile", "--conf",
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 3fef23f..bc62cfa 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -43,7 +43,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.spark.{SparkException, SparkFunSuite}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.hive.HiveUtils
-import org.apache.spark.sql.hive.test.HiveTestUtils
+import org.apache.spark.sql.hive.test.HiveTestJars
import org.apache.spark.sql.internal.StaticSQLConf.HIVE_THRIFT_SERVER_SINGLESESSION
import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer
import org.apache.spark.util.{ThreadUtils, Utils}
@@ -492,7 +492,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
withMultipleConnectionJdbcStatement("smallKV", "addJar")(
{
statement =>
- val jarFile = HiveTestUtils.getHiveHcatalogCoreJar.getCanonicalPath
+ val jarFile = HiveTestJars.getHiveHcatalogCoreJar().getCanonicalPath
statement.executeQuery(s"ADD JAR $jarFile")
},
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index d37f0c8..f627227 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -103,14 +103,6 @@
<groupId>${hive.group}</groupId>
<artifactId>hive-metastore</artifactId>
</dependency>
- <dependency>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-contrib</artifactId>
- </dependency>
- <dependency>
- <groupId>${hive.group}.hcatalog</groupId>
- <artifactId>hive-hcatalog-core</artifactId>
- </dependency>
<!--
<dependency>
<groupId>${hive.group}</groupId>
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index d23e0f2..3e36503 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.execution.command.DDLUtils
import org.apache.spark.sql.expressions.Window
-import org.apache.spark.sql.hive.test.{HiveTestUtils, TestHiveContext}
+import org.apache.spark.sql.hive.test.{HiveTestJars, TestHiveContext}
import org.apache.spark.sql.internal.SQLConf.SHUFFLE_PARTITIONS
import org.apache.spark.sql.internal.StaticSQLConf.WAREHOUSE_PATH
import org.apache.spark.sql.types.{DecimalType, StructType}
@@ -111,8 +111,8 @@ class HiveSparkSubmitSuite
val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
val jar1 = TestUtils.createJarWithClasses(Seq("SparkSubmitClassA"))
val jar2 = TestUtils.createJarWithClasses(Seq("SparkSubmitClassB"))
- val jar3 = HiveTestUtils.getHiveContribJar.getCanonicalPath
- val jar4 = HiveTestUtils.getHiveHcatalogCoreJar.getCanonicalPath
+ val jar3 = HiveTestJars.getHiveContribJar().getCanonicalPath
+ val jar4 = HiveTestJars.getHiveHcatalogCoreJar().getCanonicalPath
val jarsString = Seq(jar1, jar2, jar3, jar4).map(j => j.toString).mkString(",")
val args = Seq(
"--class", SparkSubmitClassLoaderTest.getClass.getName.stripSuffix("$"),
@@ -321,7 +321,7 @@ class HiveSparkSubmitSuite
"--master", "local-cluster[2,1,1024]",
"--conf", "spark.ui.enabled=false",
"--conf", "spark.master.rest.enabled=false",
- "--jars", HiveTestUtils.getHiveContribJar.getCanonicalPath,
+ "--jars", HiveTestJars.getHiveContribJar().getCanonicalPath,
unusedJar.toString)
runSparkSubmit(argsForCreateTable)
@@ -463,7 +463,7 @@ object TemporaryHiveUDFTest extends Logging {
// Load a Hive UDF from the jar.
logInfo("Registering a temporary Hive UDF provided in a jar.")
- val jar = HiveTestUtils.getHiveContribJar.getCanonicalPath
+ val jar = HiveTestJars.getHiveContribJar().getCanonicalPath
hiveContext.sql(
s"""
|CREATE TEMPORARY FUNCTION example_max
@@ -501,7 +501,7 @@ object PermanentHiveUDFTest1 extends Logging {
// Load a Hive UDF from the jar.
logInfo("Registering a permanent Hive UDF provided in a jar.")
- val jar = HiveTestUtils.getHiveContribJar.getCanonicalPath
+ val jar = HiveTestJars.getHiveContribJar().getCanonicalPath
hiveContext.sql(
s"""
|CREATE FUNCTION example_max
@@ -538,7 +538,7 @@ object PermanentHiveUDFTest2 extends Logging {
val hiveContext = new TestHiveContext(sc)
// Load a Hive UDF from the jar.
logInfo("Write the metadata of a permanent Hive UDF into metastore.")
- val jar = HiveTestUtils.getHiveContribJar.getCanonicalPath
+ val jar = HiveTestJars.getHiveContribJar().getCanonicalPath
val function = CatalogFunction(
FunctionIdentifier("example_max"),
"org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax",
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 53798e0..a6ddc1e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.catalyst.plans.logical.Project
import org.apache.spark.sql.execution.joins.BroadcastNestedLoopJoinExec
import org.apache.spark.sql.hive._
-import org.apache.spark.sql.hive.test.{HiveTestUtils, TestHive}
+import org.apache.spark.sql.hive.test.{HiveTestJars, TestHive}
import org.apache.spark.sql.hive.test.TestHive._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
@@ -817,7 +817,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
test("ADD JAR command 2") {
// this is a test case from mapjoin_addjar.q
- val testJar = HiveTestUtils.getHiveHcatalogCoreJar.toURI
+ val testJar = HiveTestJars.getHiveHcatalogCoreJar().toURI
val testData = TestHive.getHiveFile("data/files/sample.json").toURI
sql(s"ADD JAR $testJar")
sql(
@@ -827,9 +827,9 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
sql("select * from src join t1 on src.key = t1.a")
sql("DROP TABLE t1")
assert(sql("list jars").
- filter(_.getString(0).contains(HiveTestUtils.getHiveHcatalogCoreJar.getName)).count() > 0)
+ filter(_.getString(0).contains(HiveTestJars.getHiveHcatalogCoreJar().getName)).count() > 0)
assert(sql("list jar").
- filter(_.getString(0).contains(HiveTestUtils.getHiveHcatalogCoreJar.getName)).count() > 0)
+ filter(_.getString(0).contains(HiveTestJars.getHiveHcatalogCoreJar().getName)).count() > 0)
val testJar2 = TestHive.getHiveFile("TestUDTF.jar").getCanonicalPath
sql(s"ADD JAR $testJar2")
assert(sql(s"list jar $testJar").count() == 1)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 1638f6c..cfa535e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -37,7 +37,7 @@ import org.apache.spark.sql.execution.command.LoadDataCommand
import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
-import org.apache.spark.sql.hive.test.{HiveTestUtils, TestHiveSingleton}
+import org.apache.spark.sql.hive.test.{HiveTestJars, TestHiveSingleton}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
import org.apache.spark.sql.test.SQLTestUtils
@@ -1106,7 +1106,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
override def run() {
// To make sure this test works, this jar should not be loaded in another place.
sql(
- s"ADD JAR ${HiveTestUtils.getHiveContribJar.getCanonicalPath}")
+ s"ADD JAR ${HiveTestJars.getHiveContribJar().getCanonicalPath}")
try {
sql(
"""
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/HiveTestUtils.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/HiveTestUtils.scala
deleted file mode 100644
index 7631efe..0000000
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/HiveTestUtils.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.test
-
-import java.io.File
-
-import org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax
-import org.apache.hive.hcatalog.data.JsonSerDe
-
-object HiveTestUtils {
-
- val getHiveContribJar: File =
- new File(classOf[UDAFExampleMax].getProtectionDomain.getCodeSource.getLocation.getPath)
-
- val getHiveHcatalogCoreJar: File =
- new File(classOf[JsonSerDe].getProtectionDomain.getCodeSource.getLocation.getPath)
-}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
index d68a470..b48347c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -647,3 +647,24 @@ private[sql] class TestHiveSessionStateBuilder(
override protected def newBuilder: NewBuilder = new TestHiveSessionStateBuilder(_, _)
}
+
+private[hive] object HiveTestJars {
+ private val repository = SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.defaultValueString
+ private val hiveTestJarsDir = Utils.createTempDir()
+
+ def getHiveContribJar(version: String = HiveUtils.builtinHiveVersion): File =
+ getJarFromUrl(s"${repository}org/apache/hive/hive-contrib/" +
+ s"$version/hive-contrib-$version.jar")
+ def getHiveHcatalogCoreJar(version: String = HiveUtils.builtinHiveVersion): File =
+ getJarFromUrl(s"${repository}org/apache/hive/hcatalog/hive-hcatalog-core/" +
+ s"$version/hive-hcatalog-core-$version.jar")
+
+ private def getJarFromUrl(urlString: String): File = {
+ val fileName = urlString.split("/").last
+ val targetFile = new File(hiveTestJarsDir, fileName)
+ if (!targetFile.exists()) {
+ Utils.doFetchFile(urlString, hiveTestJarsDir, fileName, new SparkConf, null, null)
+ }
+ targetFile
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org