You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/04/21 07:50:30 UTC
spark git commit: [SPARK-14786] Remove hive-cli dependency from hive
subproject
Repository: spark
Updated Branches:
refs/heads/master 804581411 -> cfe472a34
[SPARK-14786] Remove hive-cli dependency from hive subproject
The `hive` subproject currently depends on `hive-cli` in order to perform a check to see whether a `SessionState` is an instance of `org.apache.hadoop.hive.cli.CliSessionState` (see #9589). The introduction of this `hive-cli` dependency has caused problems for users whose Hive metastore JAR classpaths don't include the `hive-cli` classes (such as in #11495).
This patch removes this dependency on `hive-cli` and replaces the `isInstanceOf` check by reflection. I added a Maven Enforcer rule to ban `hive-cli` from the `hive` subproject in order to make sure that this dependency is not accidentally reintroduced.
/cc rxin yhuai adrian-wang preecet
Author: Josh Rosen <jo...@databricks.com>
Closes #12551 from JoshRosen/remove-hive-cli-dep-from-hive-subproject.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cfe472a3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cfe472a3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cfe472a3
Branch: refs/heads/master
Commit: cfe472a34ea8bbf2f7a04acbf0c6ab6c48d732ff
Parents: 8045814
Author: Josh Rosen <jo...@databricks.com>
Authored: Wed Apr 20 22:50:27 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Wed Apr 20 22:50:27 2016 -0700
----------------------------------------------------------------------
sql/hive/pom.xml | 25 ++++++++++++++++----
.../spark/sql/hive/client/HiveClientImpl.scala | 13 ++++++++--
.../sql/hive/client/IsolatedClientLoader.scala | 2 +-
3 files changed, 33 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/cfe472a3/sql/hive/pom.xml
----------------------------------------------------------------------
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 61504be..177b688 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -73,10 +73,6 @@
<version>${protobuf.version}</version>
</dependency>
-->
- <dependency>
- <groupId>${hive.group}</groupId>
- <artifactId>hive-cli</artifactId>
- </dependency>
<!--
<dependency>
<groupId>${hive.group}</groupId>
@@ -225,6 +221,27 @@
<argLine>-da -Xmx3g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=512m</argLine>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-versions</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <bannedDependencies>
+ <excludes combine.children="append">
+ <exclude>*:hive-cli</exclude>
+ </excludes>
+ </bannedDependencies>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/spark/blob/cfe472a3/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 69f7dbf..703d991 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -24,7 +24,6 @@ import scala.language.reflectiveCalls
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.hive.cli.CliSessionState
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.metastore.{PartitionDropOptions, TableType => HiveTableType}
import org.apache.hadoop.hive.metastore.api.{Database => HiveDatabase, FieldSchema, Function => HiveFunction, FunctionType, PrincipalType, ResourceType, ResourceUri}
@@ -110,10 +109,20 @@ private[hive] class HiveClientImpl(
}
}
+ def isCliSessionState(state: SessionState): Boolean = {
+ var temp: Class[_] = if (state != null) state.getClass else null
+ var found = false
+ while (temp != null && !found) {
+ found = temp.getName == "org.apache.hadoop.hive.cli.CliSessionState"
+ temp = temp.getSuperclass
+ }
+ found
+ }
+
val ret = try {
// originState will be created if not exists, will never be null
val originalState = SessionState.get()
- if (originalState.isInstanceOf[CliSessionState]) {
+ if (isCliSessionState(originalState)) {
// In `SparkSQLCLIDriver`, we have already started a `CliSessionState`,
// which contains information like configurations from command line. Later
// we call `SparkSQLEnv.init()` there, which would run into this part again.
http://git-wip-us.apache.org/repos/asf/spark/blob/cfe472a3/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index f45264a..1d502e0 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -103,7 +103,7 @@ private[hive] object IsolatedClientLoader extends Logging {
hadoopVersion: String,
ivyPath: Option[String]): Seq[URL] = {
val hiveArtifacts = version.extraDeps ++
- Seq("hive-metastore", "hive-exec", "hive-common", "hive-serde", "hive-cli")
+ Seq("hive-metastore", "hive-exec", "hive-common", "hive-serde")
.map(a => s"org.apache.hive:$a:${version.fullVersion}") ++
Seq("com.google.guava:guava:14.0.1",
s"org.apache.hadoop:hadoop-client:$hadoopVersion")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org