You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2015/12/11 02:18:32 UTC
hive git commit: HIVE-11878: ClassNotFoundException can possibly
occur if multiple jars are registered one at a time in Hive (Ratandeep Ratti
via Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master 688a3e17f -> db5a52548
HIVE-11878: ClassNotFoundException can possibly occur if multiple jars are registered one at a time in Hive (Ratandeep Ratti via Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db5a5254
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db5a5254
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db5a5254
Branch: refs/heads/master
Commit: db5a52548356312aa1c18c6f5f2ea9a84ec905bd
Parents: 688a3e1
Author: Jason Dere <jd...@hortonworks.com>
Authored: Thu Dec 10 17:17:29 2015 -0800
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Thu Dec 10 17:17:29 2015 -0800
----------------------------------------------------------------------
conf/ivysettings.xml | 12 +++-
itests/custom-udfs/pom.xml | 62 +++++++++++++++++
itests/custom-udfs/udf-classloader-udf1/pom.xml | 43 ++++++++++++
.../src/main/java/hive/it/custom/udfs/UDF1.java | 58 ++++++++++++++++
itests/custom-udfs/udf-classloader-udf2/pom.xml | 43 ++++++++++++
.../src/main/java/hive/it/custom/udfs/UDF2.java | 60 +++++++++++++++++
itests/custom-udfs/udf-classloader-util/pom.xml | 35 ++++++++++
.../src/main/java/hive/it/custom/udfs/Util.java | 25 +++++++
.../plugin/TestHiveAuthorizerShowFilters.java | 36 +++++-----
itests/pom.xml | 1 +
itests/qtest/pom.xml | 1 -
.../hadoop/hive/ql/exec/UDFClassLoader.java | 70 ++++++++++++++++++++
.../apache/hadoop/hive/ql/exec/Utilities.java | 50 +++++++++-----
.../hadoop/hive/ql/session/SessionState.java | 9 ++-
.../queries/clientpositive/udf_classloader.q | 16 +++++
..._classloader_dynamic_dependency_resolution.q | 16 +++++
.../clientpositive/udf_classloader.q.out | 43 ++++++++++++
...ssloader_dynamic_dependency_resolution.q.out | 43 ++++++++++++
18 files changed, 585 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/conf/ivysettings.xml
----------------------------------------------------------------------
diff --git a/conf/ivysettings.xml b/conf/ivysettings.xml
index bda842a..aa10f43 100644
--- a/conf/ivysettings.xml
+++ b/conf/ivysettings.xml
@@ -1,4 +1,3 @@
-
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -23,15 +22,24 @@
<ivysettings>
<!--name of the defaultResolver should always be 'downloadGrapes'. -->
<settings defaultResolver="downloadGrapes"/>
+ <!-- Only set maven.local.repository if not already set -->
+ <property name="maven.local.repository" value="${user.home}/.m2/repository" override="false" />
+ <property name="m2-pattern"
+ value="file:${maven.local.repository}/[organisation]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]"
+ override="false"/>
<resolvers>
<!-- more resolvers can be added here -->
<chain name="downloadGrapes">
<!-- This resolver uses ibiblio to find artifacts, compatible with maven2 repository -->
<ibiblio name="central" m2compatible="true"/>
+ <url name="local-maven2" m2compatible="true">
+ <artifact pattern="${m2-pattern}"/>
+ </url>
<!-- File resolver to add jars from the local system. -->
<filesystem name="test" checkmodified="true">
- <artifact pattern="/tmp/[module]-[revision](-[classifier]).jar" />
+ <artifact pattern="/tmp/[module]-[revision](-[classifier]).jar"/>
</filesystem>
+
</chain>
</resolvers>
</ivysettings>
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/pom.xml
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/pom.xml b/itests/custom-udfs/pom.xml
new file mode 100644
index 0000000..69e1f70
--- /dev/null
+++ b/itests/custom-udfs/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>hive-it-custom-udfs</artifactId>
+ <packaging>pom</packaging>
+ <name>Hive Integration - Custom udfs</name>
+ <description>Custom udfs used in hive itest can be defined under this module</description>
+
+ <properties>
+ <hive.path.to.root>../..</hive.path.to.root>
+ </properties>
+
+ <modules>
+ <!--
+ The following three modules are defined for itests udf_classloader.q
+ and udf_classloader_dynamic_dependency_resolution.q . They need to be
+ defined in separate modules instead of a single module to correctly
+ test the said itests
+ -->
+ <module>udf-classloader-util</module>
+ <module>udf-classloader-udf1</module>
+ <module>udf-classloader-udf2</module>
+ </modules>
+
+ <dependencies>
+ <!-- intra-project -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${project.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <optional>true</optional>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-udf1/pom.xml
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-udf1/pom.xml b/itests/custom-udfs/udf-classloader-udf1/pom.xml
new file mode 100644
index 0000000..5f35c63
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-udf1/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it-custom-udfs</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+ <artifactId>udf-classloader-udf1</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive Integration - Custom UDFs - udf-classloader-udf1</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+ <artifactId>udf-classloader-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <properties>
+ <hive.path.to.root>../../..</hive.path.to.root>
+ </properties>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-udf1/src/main/java/hive/it/custom/udfs/UDF1.java
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-udf1/src/main/java/hive/it/custom/udfs/UDF1.java b/itests/custom-udfs/udf-classloader-udf1/src/main/java/hive/it/custom/udfs/UDF1.java
new file mode 100644
index 0000000..6816190
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-udf1/src/main/java/hive/it/custom/udfs/UDF1.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hive.it.custom.udfs;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+
+public class UDF1 extends GenericUDF {
+ public UDF1() {
+ Util util = new Util();
+ System.out.println(
+ "constructor: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader() + ", " + util + " classloader: "
+ + util.getClass().getClassLoader());
+ }
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ Util util = new Util();
+ System.out.println(
+ "initialize: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader() + ", " + util + " classloader: "
+ + util.getClass().getClassLoader());
+ return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ Util util = new Util();
+ System.out.println(
+ "evaluate: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader() + ", " + util + " classloader: "
+ + util.getClass().getClassLoader());
+ return getClass().getSimpleName();
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return getClass().getName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-udf2/pom.xml
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-udf2/pom.xml b/itests/custom-udfs/udf-classloader-udf2/pom.xml
new file mode 100644
index 0000000..8edca63
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-udf2/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it-custom-udfs</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+ <artifactId>udf-classloader-udf2</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive Integration - Custom UDFs - udf-classloader-udf2</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+ <artifactId>udf-classloader-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <properties>
+ <hive.path.to.root>../../..</hive.path.to.root>
+ </properties>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-udf2/src/main/java/hive/it/custom/udfs/UDF2.java
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-udf2/src/main/java/hive/it/custom/udfs/UDF2.java b/itests/custom-udfs/udf-classloader-udf2/src/main/java/hive/it/custom/udfs/UDF2.java
new file mode 100644
index 0000000..4bb8772
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-udf2/src/main/java/hive/it/custom/udfs/UDF2.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hive.it.custom.udfs;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+
+public class UDF2 extends GenericUDF {
+ public UDF2() {
+ Util util = new Util();
+ System.out.println(
+ "constructor: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader() + ", " + util
+ + " classloader: "
+ + util.getClass().getClassLoader());
+ }
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ Util util = new Util();
+ System.out.println(
+ "initialize: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader() + ", " + util
+ + " classloader: "
+ + util.getClass().getClassLoader());
+ return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ Util util = new Util();
+ System.out.println(
+ "evaluate: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader() + ", " + util
+ + " classloader: "
+ + util.getClass().getClassLoader());
+ return getClass().getSimpleName();
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return getClass().getName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-util/pom.xml
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-util/pom.xml b/itests/custom-udfs/udf-classloader-util/pom.xml
new file mode 100644
index 0000000..b6475f5
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-util/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it-custom-udfs</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+ <artifactId>udf-classloader-util</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive Integration - Custom UDFs - udf-classloader-util</name>
+
+ <properties>
+ <hive.path.to.root>../../..</hive.path.to.root>
+ </properties>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-util/src/main/java/hive/it/custom/udfs/Util.java
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-util/src/main/java/hive/it/custom/udfs/Util.java b/itests/custom-udfs/udf-classloader-util/src/main/java/hive/it/custom/udfs/Util.java
new file mode 100644
index 0000000..8881ec7
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-util/src/main/java/hive/it/custom/udfs/Util.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hive.it.custom.udfs;
+
+public class Util {
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
index 0c03a00..5922a8c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
@@ -58,7 +58,7 @@ public class TestHiveAuthorizerShowFilters {
private static final String dbName2 = (TestHiveAuthorizerShowFilters.class.getSimpleName() + "db2")
.toLowerCase();
- static HiveAuthorizer mockedAuthorizer;
+ protected static HiveAuthorizer mockedAuthorizer;
static final List<String> AllTables = getSortedList(tableName1, tableName2);
static final List<String> AllDbs = getSortedList("default", dbName1, dbName2);
@@ -73,28 +73,28 @@ public class TestHiveAuthorizerShowFilters {
* HiveAuthorizer.filterListCmdObjects, and stores the list argument in
* filterArguments
*/
- static class MockedHiveAuthorizerFactory implements HiveAuthorizerFactory {
+ protected static class MockedHiveAuthorizerFactory implements HiveAuthorizerFactory {
+ protected abstract class AuthorizerWithFilterCmdImpl implements HiveAuthorizer {
+ @Override
+ public List<HivePrivilegeObject> filterListCmdObjects(List<HivePrivilegeObject> listObjs,
+ HiveAuthzContext context) throws HiveAuthzPluginException, HiveAccessControlException {
+ // capture arguments in static
+ filterArguments = listObjs;
+ // return static variable with results, if it is set to some set of
+ // values
+ // otherwise return the arguments
+ if (filteredResults.size() == 0) {
+ return filterArguments;
+ }
+ return filteredResults;
+ }
+ }
+
@Override
public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreClientFactory,
HiveConf conf, HiveAuthenticationProvider authenticator, HiveAuthzSessionContext ctx) {
Mockito.validateMockitoUsage();
- abstract class AuthorizerWithFilterCmdImpl implements HiveAuthorizer {
- @Override
- public List<HivePrivilegeObject> filterListCmdObjects(List<HivePrivilegeObject> listObjs,
- HiveAuthzContext context) throws HiveAuthzPluginException, HiveAccessControlException {
- // capture arguments in static
- filterArguments = listObjs;
- // return static variable with results, if it is set to some set of
- // values
- // otherwise return the arguments
- if (filteredResults.size() == 0) {
- return filterArguments;
- }
- return filteredResults;
- }
- }
-
mockedAuthorizer = Mockito.mock(AuthorizerWithFilterCmdImpl.class, Mockito.withSettings()
.verboseLogging());
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index 5d8249f..fb0a613 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -33,6 +33,7 @@
<modules>
<module>custom-serde</module>
+ <module>custom-udfs</module>
<module>hcatalog-unit</module>
<module>hive-unit</module>
<module>util</module>
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 8f6807a..ec2fc85 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -656,7 +656,6 @@
initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
-
</target>
</configuration>
<goals>
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFClassLoader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFClassLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFClassLoader.java
new file mode 100644
index 0000000..c363803
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFClassLoader.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+
+/**
+ * {@link UDFClassLoader} is used to dynamically register
+ * udf (and related) jars
+ *
+ * This was introducted to fix HIVE-11878
+ *
+ * Each session will have its own instance of {@link UDFClassLoader}
+ * This is to support HiveServer2 where there can be multiple
+ * active sessions. Addition/removal of jars/resources in one
+ * session should not affect other sessions.
+ */
+public class UDFClassLoader extends URLClassLoader {
+ private boolean isClosed;
+
+ public UDFClassLoader(URL[] urls) {
+ super(urls);
+ isClosed = false;
+ }
+
+ public UDFClassLoader(URL[] urls, ClassLoader parent) {
+ super(urls, parent);
+ isClosed = false;
+ }
+
+ @Override
+ public void addURL(URL url) {
+ Preconditions.checkState(!isClosed, getClass().getSimpleName() + " is already closed");
+ super.addURL(url);
+ }
+
+ /**
+ * See {@link URLClassLoader#close}
+ */
+ public boolean isClosed() {
+ return isClosed;
+ }
+
+ @Override
+ public void close() throws IOException {
+ isClosed = true;
+ super.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index c01994f..54abdef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.hive.ql.exec;
+import java.util.ArrayList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import java.beans.DefaultPersistenceDelegate;
import java.beans.Encoder;
import java.beans.Expression;
@@ -1748,6 +1751,19 @@ public final class Utilities {
return result;
}
+ private static boolean useExistingClassLoader(ClassLoader cl) {
+ if (!(cl instanceof UDFClassLoader)) {
+ // Cannot use the same classloader if it is not an instance of {@code UDFClassLoader}
+ return false;
+ }
+ final UDFClassLoader udfClassLoader = (UDFClassLoader) cl;
+ if (udfClassLoader.isClosed()) {
+ // The classloader may have been closed, Cannot add to the same instance
+ return false;
+ }
+ return true;
+ }
+
/**
* Add new elements to the classpath.
*
@@ -1755,24 +1771,28 @@ public final class Utilities {
* Array of classpath elements
*/
public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) throws Exception {
- URLClassLoader loader = (URLClassLoader) cloader;
- List<URL> curPath = Arrays.asList(loader.getURLs());
- ArrayList<URL> newPath = new ArrayList<URL>();
-
- // get a list with the current classpath components
- for (URL onePath : curPath) {
- newPath.add(onePath);
+ final URLClassLoader loader = (URLClassLoader) cloader;
+ if (useExistingClassLoader(cloader)) {
+ final UDFClassLoader udfClassLoader = (UDFClassLoader) loader;
+ for (String path : newPaths) {
+ udfClassLoader.addURL(urlFromPathString(path));
+ }
+ return udfClassLoader;
+ } else {
+ return createUDFClassLoader(loader, newPaths);
}
- curPath = newPath;
+ }
+ public static ClassLoader createUDFClassLoader(URLClassLoader loader, String[] newPaths) {
+ final Set<URL> curPathsSet = Sets.newHashSet(loader.getURLs());
+ final List<URL> curPaths = Lists.newArrayList(curPathsSet);
for (String onestr : newPaths) {
- URL oneurl = urlFromPathString(onestr);
- if (oneurl != null && !curPath.contains(oneurl)) {
- curPath.add(oneurl);
+ final URL oneurl = urlFromPathString(onestr);
+ if (oneurl != null && !curPathsSet.contains(oneurl)) {
+ curPaths.add(oneurl);
}
}
-
- return new URLClassLoader(curPath.toArray(new URL[0]), loader);
+ return new UDFClassLoader(curPaths.toArray(new URL[0]), loader);
}
/**
@@ -1793,13 +1813,13 @@ public final class Utilities {
}
}
JavaUtils.closeClassLoader(loader);
-//this loader is closed, remove it from cached registry loaders to avoid remove it again.
+ // This loader is closed, remove it from cached registry loaders to avoid removing it again.
Registry reg = SessionState.getRegistry();
if(reg != null) {
reg.removeFromUDFLoaders(loader);
}
- loader = new URLClassLoader(newPath.toArray(new URL[0]));
+ loader = new UDFClassLoader(newPath.toArray(new URL[0]));
curThread.setContextClassLoader(loader);
SessionState.get().getConf().setClassLoader(loader);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 5c69fb6..c066c7a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -362,7 +362,12 @@ public class SessionState {
if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) {
conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId());
}
- parentLoader = JavaUtils.getClassLoader();
+ // Using system classloader as the parent. Using thread context
+ // classloader as parent can pollute the session. See HIVE-11878
+ parentLoader = SessionState.class.getClassLoader();
+ // Make sure that each session has its own UDFClassloader. For details see {@link UDFClassLoader}
+ final ClassLoader currentLoader = Utilities.createUDFClassLoader((URLClassLoader) parentLoader, new String[]{});
+ this.conf.setClassLoader(currentLoader);
}
public void setCmd(String cmdString) {
@@ -1273,7 +1278,7 @@ public class SessionState {
return scheme;
}
- List<URI> resolveAndDownload(ResourceType t, String value, boolean convertToUnix) throws URISyntaxException,
+ protected List<URI> resolveAndDownload(ResourceType t, String value, boolean convertToUnix) throws URISyntaxException,
IOException {
URI uri = createURI(value);
if (getURLType(value).equals("file")) {
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/test/queries/clientpositive/udf_classloader.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_classloader.q b/ql/src/test/queries/clientpositive/udf_classloader.q
new file mode 100644
index 0000000..82678c2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_classloader.q
@@ -0,0 +1,16 @@
+ADD JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf1/${system:hive.version}/udf-classloader-udf1-${system:hive.version}.jar;
+ADD JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-util/${system:hive.version}/udf-classloader-util-${system:hive.version}.jar;
+ADD JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf2/${system:hive.version}/udf-classloader-udf2-${system:hive.version}.jar;
+
+CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1';
+CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2';
+
+-- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1;
+
+DELETE JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf2/${system:hive.version}/udf-classloader-udf2-${system:hive.version}.jar;
+SELECT f1(*) FROM SRC limit 1;
+
+ADD JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf2/${system:hive.version}/udf-classloader-udf2-${system:hive.version}.jar;
+SELECT f2(*) FROM SRC limit 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q b/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
new file mode 100644
index 0000000..2ceeaa3
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
@@ -0,0 +1,16 @@
+ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf1:+;
+ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-util:+;
+ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf2:+;
+
+CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1';
+CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2';
+
+-- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1;
+
+DELETE JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf2:+;
+SELECT f1(*) FROM SRC limit 1;
+
+ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf2:+;
+SELECT f2(*) FROM SRC limit 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/test/results/clientpositive/udf_classloader.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_classloader.q.out b/ql/src/test/results/clientpositive/udf_classloader.q.out
new file mode 100644
index 0000000..031d88a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udf_classloader.q.out
@@ -0,0 +1,43 @@
+PREHOOK: query: CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: f1
+POSTHOOK: query: CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: f1
+PREHOOK: query: CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: f2
+POSTHOOK: query: CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: f2
+PREHOOK: query: -- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF1 UDF2
+PREHOOK: query: SELECT f1(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT f1(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF1
+PREHOOK: query: SELECT f2(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT f2(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF2
http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/test/results/clientpositive/udf_classloader_dynamic_dependency_resolution.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_classloader_dynamic_dependency_resolution.q.out b/ql/src/test/results/clientpositive/udf_classloader_dynamic_dependency_resolution.q.out
new file mode 100644
index 0000000..031d88a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udf_classloader_dynamic_dependency_resolution.q.out
@@ -0,0 +1,43 @@
+PREHOOK: query: CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: f1
+POSTHOOK: query: CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: f1
+PREHOOK: query: CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: f2
+POSTHOOK: query: CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: f2
+PREHOOK: query: -- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF1 UDF2
+PREHOOK: query: SELECT f1(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT f1(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF1
+PREHOOK: query: SELECT f2(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT f2(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF2