You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2016/01/15 18:10:32 UTC
hive git commit: HIVE-12724 ACID: Major compaction fails to include
the original bucket files into MR job(Wei Zheng via Eugene Koifman)(ADDENDUM)
Repository: hive
Updated Branches:
refs/heads/master 4935cfda7 -> ef706b76b
HIVE-12724 ACID: Major compaction fails to include the original bucket files into MR job(Wei Zheng via Eugene Koifman)(ADDENDUM)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ef706b76
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ef706b76
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ef706b76
Branch: refs/heads/master
Commit: ef706b76bf17578fdc4a95b94f2abfc866010fd7
Parents: 4935cfd
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Fri Jan 15 09:09:48 2016 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Fri Jan 15 09:09:48 2016 -0800
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java | 4 ++++
.../org/apache/hadoop/hive/ql/TestTxnCommands2.java | 12 ++++++++++++
2 files changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ef706b76/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 3437fc8..14f7374 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -497,12 +497,16 @@ public class AcidUtils {
// If we have a base, the original files are obsolete.
if (bestBase.status != null) {
+ // Add original files to obsolete list if any
for (HdfsFileStatusWithId fswid : original) {
obsolete.add(fswid.getFileStatus());
}
+ // Add original direcotries to obsolete list if any
+ obsolete.addAll(originalDirectories);
// remove the entries so we don't get confused later and think we should
// use them.
original.clear();
+ originalDirectories.clear();
} else {
// Okay, we're going to need these originals. Recurse through them and figure out what we
// really need.
http://git-wip-us.apache.org/repos/asf/hive/blob/ef706b76/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 890012a..b784585 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -348,6 +348,18 @@ public class TestTxnCommands2 {
Assert.assertEquals(resultCount, Integer.parseInt(rs.get(0)));
// 5. Let Cleaner delete obsolete files/dirs
+ // Note, here we create a fake directory along with fake files as original directories/files
+ String fakeFile0 = TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase() +
+ "/subdir/000000_0";
+ String fakeFile1 = TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase() +
+ "/subdir/000000_1";
+ fs.create(new Path(fakeFile0));
+ fs.create(new Path(fakeFile1));
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ // Before Cleaner, there should be 5 items:
+ // 2 original files, 1 original directory, 1 base directory and 1 delta directory
+ Assert.assertEquals(5, status.length);
Cleaner c = new Cleaner();
c.setThreadId((int) c.getId());
c.setHiveConf(hiveConf);