You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ai...@apache.org on 2016/07/18 19:42:55 UTC
hive git commit: HIVE-14198: Refactor aux jar related code to make
them more consistent (Aihua Xu, reviewed by Mohit Sabharwal)
Repository: hive
Updated Branches:
refs/heads/master 879026e93 -> 8d966ed1e
HIVE-14198: Refactor aux jar related code to make them more consistent (Aihua Xu, reviewed by Mohit Sabharwal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8d966ed1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8d966ed1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8d966ed1
Branch: refs/heads/master
Commit: 8d966ed1ee7f934ec8d27756b3e3c8d174aff222
Parents: 879026e
Author: Aihua Xu <ai...@apache.org>
Authored: Fri Jul 8 16:44:59 2016 -0400
Committer: Aihua Xu <ai...@apache.org>
Committed: Mon Jul 18 15:41:53 2016 -0400
----------------------------------------------------------------------
.../apache/hadoop/hive/common/FileUtils.java | 56 +++++++++++++++++
.../org/apache/hadoop/hive/conf/HiveConf.java | 6 +-
.../hadoop/hive/common/TestFileUtils.java | 66 ++++++++++++++++++++
.../hadoop/hive/hbase/HBaseTestSetup.java | 7 ++-
.../apache/hadoop/hive/ql/exec/Utilities.java | 55 ----------------
.../ql/exec/spark/LocalHiveSparkClient.java | 3 +-
.../ql/exec/spark/RemoteHiveSparkClient.java | 10 +--
.../hive/ql/exec/spark/SparkUtilities.java | 14 -----
.../hadoop/hive/ql/session/SessionState.java | 2 +-
.../hadoop/hive/ql/exec/TestUtilities.java | 35 -----------
10 files changed, 138 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index 26ce26f..23fcc8a 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -27,6 +27,7 @@ import java.security.AccessControlException;
import java.security.PrivilegedExceptionAction;
import java.util.BitSet;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
@@ -35,6 +36,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
@@ -911,4 +913,58 @@ public final class FileUtils {
return false;
}
+ /**
+ * Get the URI of the path. Assume to be local file system if no scheme.
+ */
+ public static URI getURI(String path) throws URISyntaxException {
+ if (path == null) {
+ return null;
+ }
+
+ URI uri = new URI(path);
+ if (uri.getScheme() == null) {
+ // if no scheme in the path, we assume it's file on local fs.
+ uri = new File(path).toURI();
+ }
+
+ return uri;
+ }
+
+ /**
+ * Given a path string, get all the jars from the folder or the files themselves.
+ *
+ * @param pathString the path string is the comma-separated path list
+ * @return the list of the file names in the format of URI formats.
+ */
+ public static Set<String> getJarFilesByPath(String pathString, Configuration conf) {
+ Set<String> result = new HashSet<String>();
+ if (pathString == null || org.apache.commons.lang.StringUtils.isBlank(pathString)) {
+ return result;
+ }
+
+ String[] paths = pathString.split(",");
+ for(String path : paths) {
+ try {
+ Path p = new Path(getURI(path));
+ FileSystem fs = p.getFileSystem(conf);
+ if (!fs.exists(p)) {
+ LOG.error("The jar file path " + path + " doesn't exist");
+ continue;
+ }
+ if (fs.isDirectory(p)) {
+ // add all jar files under the folder
+ FileStatus[] files = fs.listStatus(p, new GlobFilter("*.jar"));
+ for(FileStatus file : files) {
+ result.add(file.getPath().toUri().toString());
+ }
+ } else {
+ result.add(p.toUri().toString());
+ }
+ } catch(URISyntaxException | IOException e) {
+ LOG.error("Invalid file path " + path, e);
+ }
+ }
+ return result;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 42f7d88..66203a5 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -22,6 +22,7 @@ import com.google.common.base.Joiner;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.hive.conf.Validator.PatternSet;
@@ -3756,7 +3757,7 @@ public class HiveConf extends Configuration {
}
if (auxJars == null) {
- auxJars = this.get(ConfVars.HIVEAUXJARS.varname);
+ auxJars = StringUtils.join(FileUtils.getJarFilesByPath(this.get(ConfVars.HIVEAUXJARS.varname), this), ',');
}
if (getBoolVar(ConfVars.METASTORE_SCHEMA_VERIFICATION)) {
@@ -4036,7 +4037,8 @@ public class HiveConf extends Configuration {
}
/**
- * @param auxJars the auxJars to set
+ * Set the auxiliary jars. Used for unit tests only.
+ * @param auxJars the auxJars to set.
*/
public void setAuxJars(String auxJars) {
this.auxJars = auxJars;
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
new file mode 100644
index 0000000..e9fcc13
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+import com.google.common.io.Files;
+
+import junit.framework.TestCase;
+
+public class TestFileUtils extends TestCase {
+ public static final Logger LOG = LoggerFactory.getLogger(TestFileUtils.class);
+
+ @Test
+ public void testGetJarFilesByPath() {
+ HiveConf conf = new HiveConf(this.getClass());
+ File tmpDir = Files.createTempDir();
+ String jarFileName1 = tmpDir.getAbsolutePath() + File.separator + "a.jar";
+ String jarFileName2 = tmpDir.getAbsolutePath() + File.separator + "b.jar";
+ File jarFile1 = new File(jarFileName1);
+ try {
+ org.apache.commons.io.FileUtils.touch(jarFile1);
+ Set<String> jars = FileUtils.getJarFilesByPath(tmpDir.getAbsolutePath(), conf);
+ Assert.assertEquals(Sets.newHashSet("file://" + jarFileName1),jars);
+
+ jars = FileUtils.getJarFilesByPath("/folder/not/exist", conf);
+ Assert.assertTrue(jars.isEmpty());
+
+ File jarFile2 = new File(jarFileName2);
+ org.apache.commons.io.FileUtils.touch(jarFile2);
+ String newPath = "file://" + jarFileName1 + "," + "file://" + jarFileName2 + ",/file/not/exist";
+ jars = FileUtils.getJarFilesByPath(newPath, conf);
+ Assert.assertEquals(Sets.newHashSet("file://" + jarFileName1, "file://" + jarFileName2), jars);
+ } catch (IOException e) {
+ LOG.error("failed to copy file to reloading folder", e);
+ Assert.fail(e.getMessage());
+ } finally {
+ org.apache.commons.io.FileUtils.deleteQuietly(tmpDir);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
index cee7158..4f8fa05 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.net.ServerSocket;
import java.util.Arrays;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -63,10 +64,10 @@ public class HBaseTestSetup {
conf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort());
conf.set("hbase.zookeeper.property.clientPort", Integer.toString(zooKeeperPort));
String auxJars = conf.getAuxJars();
- auxJars = ((auxJars == null) ? "" : (auxJars + ",")) + "file:///"
+ auxJars = (StringUtils.isBlank(auxJars) ? "" : (auxJars + ",")) + "file://"
+ new JobConf(conf, HBaseConfiguration.class).getJar();
- auxJars += ",file:///" + new JobConf(conf, HBaseSerDe.class).getJar();
- auxJars += ",file:///" + new JobConf(conf, Watcher.class).getJar();
+ auxJars += ",file://" + new JobConf(conf, HBaseSerDe.class).getJar();
+ auxJars += ",file://" + new JobConf(conf, Watcher.class).getJar();
conf.setAuxJars(auxJars);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index bc977a0..a376023 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -85,7 +85,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.GlobFilter;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -1742,60 +1741,6 @@ public final class Utilities {
return oneurl;
}
- /**
- * Get the URI of the path. Assume to be local file system if no scheme.
- */
- public static URI getURI(String path) throws URISyntaxException {
- if (path == null) {
- return null;
- }
-
- URI uri = new URI(path);
- if (uri.getScheme() == null) {
- // if no scheme in the path, we assume it's file on local fs.
- uri = new File(path).toURI();
- }
-
- return uri;
- }
-
- /**
- * Given a path string, get all the jars from the folder or the files themselves.
- *
- * @param pathString the path string is the comma-separated path list
- * @return the list of the file names in the format of URI formats.
- */
- public static Set<String> getJarFilesByPath(String pathString, Configuration conf) {
- Set<String> result = new HashSet<String>();
- if (pathString == null || StringUtils.isBlank(pathString)) {
- return result;
- }
-
- String[] paths = pathString.split(",");
- for(String path : paths) {
- try {
- Path p = new Path(getURI(path));
- FileSystem fs = p.getFileSystem(conf);
- if (!fs.exists(p)) {
- LOG.error("The jar file path " + path + " doesn't exist");
- continue;
- }
- if (fs.isDirectory(p)) {
- // add all jar files under the folder
- FileStatus[] files = fs.listStatus(p, new GlobFilter("*.jar"));
- for(FileStatus file : files) {
- result.add(file.getPath().toUri().toString());
- }
- } else {
- result.add(p.toUri().toString());
- }
- } catch(URISyntaxException | IOException e) {
- LOG.error("Invalid file path " + path, e);
- }
- }
- return result;
- }
-
private static boolean useExistingClassLoader(ClassLoader cl) {
if (!(cl instanceof UDFClassLoader)) {
// Cannot use the same classloader if it is not an instance of {@code UDFClassLoader}
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
index 0c0fe95..c75333d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
@@ -152,7 +152,8 @@ public class LocalHiveSparkClient implements HiveSparkClient {
addJars((new JobConf(this.getClass())).getJar());
// add aux jars
- addJars(HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS));
+ addJars(conf.getAuxJars());
+ addJars(SessionState.get() == null ? null : SessionState.get().getReloadableAuxJars());
// add added jars
String addedJars = Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR);
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
index 2b06a04..a9f70c4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
@@ -28,7 +28,6 @@ import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -39,6 +38,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
@@ -211,8 +211,8 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
addJars((new JobConf(this.getClass())).getJar());
// add aux jars
- addJars(HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS));
- addJars(SessionState.get().getReloadableAuxJars());
+ addJars(conf.getAuxJars());
+ addJars(SessionState.get() == null ? null : SessionState.get().getReloadableAuxJars());
// add added jars
String addedJars = Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR);
@@ -242,7 +242,7 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
private void addResources(String addedFiles) throws IOException {
for (String addedFile : CSV_SPLITTER.split(Strings.nullToEmpty(addedFiles))) {
try {
- URI fileUri = SparkUtilities.getURI(addedFile);
+ URI fileUri = FileUtils.getURI(addedFile);
if (fileUri != null && !localFiles.contains(fileUri)) {
localFiles.add(fileUri);
if (SparkUtilities.needUploadToHDFS(fileUri, sparkConf)) {
@@ -259,7 +259,7 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
private void addJars(String addedJars) throws IOException {
for (String addedJar : CSV_SPLITTER.split(Strings.nullToEmpty(addedJars))) {
try {
- URI jarUri = SparkUtilities.getURI(addedJar);
+ URI jarUri = FileUtils.getURI(addedJar);
if (jarUri != null && !localJars.contains(jarUri)) {
localJars.add(jarUri);
if (SparkUtilities.needUploadToHDFS(jarUri, sparkConf)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
index 5a6bef9..630b598 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkUtilities.java
@@ -65,20 +65,6 @@ public class SparkUtilities {
return copy;
}
- public static URI getURI(String path) throws URISyntaxException {
- if (path == null) {
- return null;
- }
-
- URI uri = new URI(path);
- if (uri.getScheme() == null) {
- // if no file schema in path, we assume it's file on local fs.
- uri = new File(path).toURI();
- }
-
- return uri;
- }
-
/**
* Uploads a local file to HDFS
*
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index f983b82..03010ea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -1109,7 +1109,7 @@ public class SessionState {
return;
}
- Set<String> jarPaths = Utilities.getJarFilesByPath(renewableJarPath, sessionConf);
+ Set<String> jarPaths = FileUtils.getJarFilesByPath(renewableJarPath, sessionConf);
// load jars under the hive.reloadable.aux.jars.path
if(!jarPaths.isEmpty()){
http://git-wip-us.apache.org/repos/asf/hive/blob/8d966ed1/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
index 7150424..0786f9b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
@@ -32,11 +32,8 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Properties;
-import java.util.Set;
import java.util.UUID;
-import org.apache.commons.io.FileUtils;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -59,19 +56,15 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapred.JobConf;
-
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import com.google.common.io.Files;
public class TestUtilities {
@Rule
@@ -151,34 +144,6 @@ public class TestUtilities {
}
@Test
- public void testGetJarFilesByPath() {
- HiveConf conf = new HiveConf(this.getClass());
- File f = Files.createTempDir();
- String jarFileName1 = f.getAbsolutePath() + File.separator + "a.jar";
- String jarFileName2 = f.getAbsolutePath() + File.separator + "b.jar";
- File jarFile = new File(jarFileName1);
- try {
- FileUtils.touch(jarFile);
- Set<String> jars = Utilities.getJarFilesByPath(f.getAbsolutePath(), conf);
- Assert.assertEquals(Sets.newHashSet("file://" + jarFileName1),jars);
-
- jars = Utilities.getJarFilesByPath("/folder/not/exist", conf);
- Assert.assertTrue(jars.isEmpty());
-
- File jarFile2 = new File(jarFileName2);
- FileUtils.touch(jarFile2);
- String newPath = "file://" + jarFileName1 + "," + "file://" + jarFileName2 + ",/file/not/exist";
- jars = Utilities.getJarFilesByPath(newPath, conf);
- Assert.assertEquals(Sets.newHashSet("file://" + jarFileName1, "file://" + jarFileName2), jars);
- } catch (IOException e) {
- LOG.error("failed to copy file to reloading folder", e);
- Assert.fail(e.getMessage());
- } finally {
- FileUtils.deleteQuietly(f);
- }
- }
-
- @Test
public void testReplaceTaskId() {
String taskID = "000000";
int bucketNum = 1;