You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2014/04/10 21:58:40 UTC
svn commit: r1586453 - in
/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez: DagUtils.java
TezSessionState.java
Author: sershe
Date: Thu Apr 10 19:58:40 2014
New Revision: 1586453
URL: http://svn.apache.org/r1586453
Log:
HIVE-6816 : jar upload path w/o schema is not handled correctly (Sergey Shelukhin, reviewed by Vikram Dixit K)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1586453&r1=1586452&r2=1586453&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Thu Apr 10 19:58:40 2014
@@ -676,6 +676,8 @@ public class DagUtils {
String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS);
+ // need to localize the additional jars and files
+ // we need the directory on hdfs to which we shall put all these files
String allFiles = auxJars + "," + addedJars + "," + addedFiles + "," + addedArchives;
addTempFiles(conf, tmpResources, hdfsDirPathStr, allFiles.split(","));
return tmpResources;
@@ -712,27 +714,38 @@ public class DagUtils {
}
}
- public String getHiveJarDirectory(Configuration conf) throws IOException, LoginException {
+ public FileStatus getHiveJarDirectory(Configuration conf) throws IOException, LoginException {
FileStatus fstatus = null;
String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, null);
if (hdfsDirPathStr != null) {
- Path hdfsDirPath = new Path(hdfsDirPathStr);
- FileSystem fs = hdfsDirPath.getFileSystem(conf);
- if (!(fs instanceof DistributedFileSystem)) {
- throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(hdfsDirPathStr));
- }
- try {
- fstatus = fs.getFileStatus(hdfsDirPath);
- } catch (FileNotFoundException fe) {
- // do nothing
- }
+ LOG.info("Hive jar directory is " + hdfsDirPathStr);
+ fstatus = validateTargetDir(new Path(hdfsDirPathStr), conf);
}
- if ((fstatus == null) || (!fstatus.isDir())) {
+ if (fstatus == null) {
Path destDir = getDefaultDestDir(conf);
- hdfsDirPathStr = destDir.toString();
+ LOG.info("Jar dir is null/directory doesn't exist. Choosing HIVE_INSTALL_DIR - " + destDir);
+ fstatus = validateTargetDir(destDir, conf);
+ }
+
+ if (fstatus == null) {
+ throw new IOException(ErrorMsg.NO_VALID_LOCATIONS.getMsg());
+ }
+ return fstatus;
+ }
+
+ public static FileStatus validateTargetDir(Path path, Configuration conf) throws IOException {
+ FileSystem fs = path.getFileSystem(conf);
+ if (!(fs instanceof DistributedFileSystem)) {
+ throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(path.toString()));
+ }
+ FileStatus fstatus = null;
+ try {
+ fstatus = fs.getFileStatus(path);
+ } catch (FileNotFoundException fe) {
+ // do nothing
}
- return hdfsDirPathStr;
+ return (fstatus != null && fstatus.isDir()) ? fstatus : null;
}
// the api that finds the jar being used by this class on disk
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java?rev=1586453&r1=1586452&r2=1586453&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java Thu Apr 10 19:58:40 2014
@@ -286,13 +286,13 @@ public class TezSessionState {
throws IOException {
// tez needs its own scratch dir (per session)
- Path tezDir = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR),
- TEZ_DIR);
+ Path tezDir = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR), TEZ_DIR);
tezDir = new Path(tezDir, sessionId);
FileSystem fs = tezDir.getFileSystem(conf);
FsPermission fsPermission = new FsPermission((short)00777);
Utilities.createDirsWithPermission(conf, tezDir, fsPermission, true);
-
+ // Make sure the path is normalized (we expect validation to pass since we just created it).
+ tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath();
// don't keep the directory around on non-clean exit
fs.deleteOnExit(tezDir);
@@ -311,39 +311,9 @@ public class TezSessionState {
private LocalResource createJarLocalResource(String localJarPath)
throws IOException, LoginException, IllegalArgumentException,
FileNotFoundException {
- Path destDirPath = null;
- FileSystem destFs = null;
- FileStatus destDirStatus = null;
-
- {
- String hiveJarDir = utils.getHiveJarDirectory(conf);
- if (hiveJarDir != null) {
- LOG.info("Hive jar directory is " + hiveJarDir);
- // check if it is a valid directory in HDFS
- destDirPath = new Path(hiveJarDir);
- destFs = destDirPath.getFileSystem(conf);
- destDirStatus = validateTargetDir(destDirPath, destFs);
- }
- }
-
- /*
- * Specified location does not exist or is not a directory.
- * Try to push the jar to the hdfs location pointed by config variable HIVE_INSTALL_DIR.
- * Path will be HIVE_INSTALL_DIR/{username}/.hiveJars/
- * This will probably never ever happen.
- */
- if (destDirStatus == null || !destDirStatus.isDir()) {
- destDirPath = utils.getDefaultDestDir(conf);
- LOG.info("Jar dir is null/directory doesn't exist. Choosing HIVE_INSTALL_DIR - "
- + destDirPath);
- destFs = destDirPath.getFileSystem(conf);
- destDirStatus = validateTargetDir(destDirPath, destFs);
- }
-
- // we couldn't find any valid locations. Throw exception
- if (destDirStatus == null || !destDirStatus.isDir()) {
- throw new IOException(ErrorMsg.NO_VALID_LOCATIONS.getMsg());
- }
+ FileStatus destDirStatus = utils.getHiveJarDirectory(conf);
+ assert destDirStatus != null;
+ Path destDirPath = destDirStatus.getPath();
Path localFile = new Path(localJarPath);
String sha = getSha(localFile);
@@ -365,17 +335,6 @@ public class TezSessionState {
return utils.localizeResource(localFile, destFile, conf);
}
- private FileStatus validateTargetDir(Path hiveJarDirPath, FileSystem fs) throws IOException {
- if (!(fs instanceof DistributedFileSystem)) {
- throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(hiveJarDirPath.toString()));
- }
- try {
- return fs.getFileStatus(hiveJarDirPath);
- } catch (FileNotFoundException fe) {
- // do nothing
- }
- return null;
- }
private String getSha(Path localFile) throws IOException, IllegalArgumentException {
InputStream is = null;