You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/11/22 02:46:42 UTC
[33/35] hive git commit: HIVE-14990 : run all tests for MM tables and
fix the issues that are found (Sergey Shelukhin)
HIVE-14990 : run all tests for MM tables and fix the issues that are found (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/13ea883c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/13ea883c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/13ea883c
Branch: refs/heads/hive-14535
Commit: 13ea883ce1f1a377b6ca07333f80912b75ed5534
Parents: a5ba17d
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Nov 17 19:49:58 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Nov 21 18:06:07 2016 -0800
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/metadata/Hive.java | 58 +++++++++++++++-----
.../formatting/TextMetaDataFormatter.java | 7 ++-
2 files changed, 48 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/13ea883c/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 7dabd78..7514ec4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1601,7 +1601,8 @@ public class Hive {
Utilities.LOG14535.info("maybe deleting stuff from " + oldPartPath + " (new " + newPartPath + ") for replace");
if (replace && oldPartPath != null) {
deleteOldPathForReplace(newPartPath, oldPartPath, getConf(),
- new ValidWriteIds.IdPathFilter(mmWriteId, false, true), mmWriteId != null);
+ new ValidWriteIds.IdPathFilter(mmWriteId, false, true), mmWriteId != null,
+ tbl.isStoredAsSubDirectories() ? tbl.getSkewedColNames().size() : 0);
}
} else {
// Either a non-MM query, or a load into MM table from an external source.
@@ -2080,7 +2081,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
if (replace) {
Path tableDest = tbl.getPath();
deleteOldPathForReplace(tableDest, tableDest, sessionConf,
- new ValidWriteIds.IdPathFilter(mmWriteId, false, true), mmWriteId != null);
+ new ValidWriteIds.IdPathFilter(mmWriteId, false, true), mmWriteId != null,
+ tbl.isStoredAsSubDirectories() ? tbl.getSkewedColNames().size() : 0);
}
newFiles = listFilesCreatedByQuery(loadPath, mmWriteId);
} else {
@@ -3490,7 +3492,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
if (oldPath != null) {
- deleteOldPathForReplace(destf, oldPath, conf, deletePathFilter, isMmTable);
+ // TODO: we assume lbLevels is 0 here. Same as old code for non-MM.
+ // For MM tables, this can only be a LOAD command. Does LOAD even support LB?
+ deleteOldPathForReplace(destf, oldPath, conf, deletePathFilter, isMmTable, 0);
}
// first call FileUtils.mkdir to make sure that destf directory exists, if not, it creates
@@ -3526,7 +3530,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
private void deleteOldPathForReplace(Path destPath, Path oldPath, HiveConf conf,
- PathFilter pathFilter, boolean isMmTable) throws HiveException {
+ PathFilter pathFilter, boolean isMmTable, int lbLevels) throws HiveException {
Utilities.LOG14535.info("Deleting old paths for replace in " + destPath + " and old path " + oldPath);
boolean isOldPathUnderDestf = false;
try {
@@ -3538,16 +3542,27 @@ private void constructOneLBLocationMap(FileStatus fSta,
// not the destf or its subdir?
isOldPathUnderDestf = isSubDir(oldPath, destPath, oldFs, destFs, false);
if (isOldPathUnderDestf || isMmTable) {
- FileStatus[] statuses = oldFs.listStatus(oldPath, pathFilter);
- if (statuses == null || statuses.length == 0) return;
- String s = "Deleting files under " + oldPath + " for replace: ";
- for (FileStatus file : statuses) {
- s += file.getPath().getName() + ", ";
- }
- Utilities.LOG14535.info(s);
- if (!trashFiles(oldFs, statuses, conf)) {
- throw new HiveException("Destination directory " + destPath
- + " has not been cleaned up.");
+ if (lbLevels == 0 || !isMmTable) {
+ cleanUpOneDirectoryForReplace(oldPath, oldFs, pathFilter, conf);
+ } else {
+ // We need to clean up different MM IDs from each LB directory separately.
+ // Avoid temporary directories in the immediate table/part dir.
+ // TODO: we could just find directories with any MM directories inside?
+ // the rest doesn't have to be cleaned up.
+ String mask = "[^._]*";
+ for (int i = 0; i < lbLevels - 1; ++i) {
+ mask += Path.SEPARATOR + "*";
+ }
+ Path glob = new Path(oldPath, mask);
+ FileStatus[] lbDirs = oldFs.globStatus(glob);
+ for (FileStatus lbDir : lbDirs) {
+ Path lbPath = lbDir.getPath();
+ if (!lbDir.isDirectory()) {
+ throw new HiveException("Unexpected path during overwrite: " + lbPath);
+ }
+ Utilities.LOG14535.info("Cleaning up LB directory " + lbPath);
+ cleanUpOneDirectoryForReplace(lbPath, oldFs, pathFilter, conf);
+ }
}
}
} catch (IOException e) {
@@ -3563,6 +3578,21 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
+ private void cleanUpOneDirectoryForReplace(Path path, FileSystem fs,
+ PathFilter pathFilter, HiveConf conf) throws IOException, HiveException {
+ FileStatus[] statuses = fs.listStatus(path, pathFilter);
+ if (statuses == null || statuses.length == 0) return;
+ String s = "Deleting files under " + path + " for replace: ";
+ for (FileStatus file : statuses) {
+ s += file.getPath().getName() + ", ";
+ }
+ Utilities.LOG14535.info(s);
+ if (!trashFiles(fs, statuses, conf)) {
+ throw new HiveException("Old path " + path + " has not been cleaned up.");
+ }
+ }
+
+
/**
* Trashes or deletes all files under a directory. Leaves the directory as is.
* @param fs FileSystem to use
http://git-wip-us.apache.org/repos/asf/hive/blob/13ea883c/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
index 22908d8..eac6963 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
@@ -318,10 +318,10 @@ class TextMetaDataFormatter implements MetaDataFormatter {
if (!unknown) {
for (Path loc : locations) {
try {
- FileStatus status = fs.getFileStatus(tblPath);
+ FileStatus status = fs.getFileStatus(loc);
// no matter loc is the table location or part location, it must be a
// directory.
- if (!status.isDir()) {
+ if (!status.isDirectory()) {
continue;
}
processDir(status, fs, fd);
@@ -381,6 +381,7 @@ class TextMetaDataFormatter implements MetaDataFormatter {
}
private void processDir(FileStatus status, FileSystem fs, FileData fd) throws IOException {
+ Utilities.LOG14535.info("Processing dir for status: " + status.getPath());
long accessTime = status.getAccessTime();
long updateTime = status.getModificationTime();
if (accessTime > fd.lastAccessTime) {
@@ -391,7 +392,7 @@ class TextMetaDataFormatter implements MetaDataFormatter {
}
FileStatus[] files = fs.listStatus(status.getPath());
for (FileStatus currentStatus : files) {
- if (currentStatus.isDir()) {
+ if (currentStatus.isDirectory()) {
processDir(currentStatus, fs, fd);
continue;
}