You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/09/22 07:22:27 UTC
svn commit: r999758 - in /hadoop/hive/trunk: CHANGES.txt
metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
Author: namit
Date: Wed Sep 22 05:22:27 2010
New Revision: 999758
URL: http://svn.apache.org/viewvc?rev=999758&view=rev
Log:
HIVE-1655. Adding consistency check at jobClose() when committing dynamic
partitions (Ning Zhang via namit)
Removed:
hadoop/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=999758&r1=999757&r2=999758&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Sep 22 05:22:27 2010
@@ -288,6 +288,9 @@ Trunk - Unreleased
HIVE-1534. Join filters do not work correctly with outer joins
(Amareshwari Sriramadasu via namit)
+ HIVE-1655. Adding consistency check at jobClose() when committing dynamic
+ partitions (Ning Zhang via namit)
+
TESTS
HIVE-1464. improve test query performance
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=999758&r1=999757&r2=999758&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Sep 22 05:22:27 2010
@@ -1134,6 +1134,16 @@ public final class Utilities {
for (int i = 0; i < parts.length; ++i) {
assert parts[i].isDir(): "dynamic partition " + parts[i].getPath() + " is not a direcgtory";
FileStatus[] items = fs.listStatus(parts[i].getPath());
+
+ // remove empty directory since DP insert should not generate empty partitions.
+ // empty directories could be generated by crashed Task/ScriptOperator
+ if (items.length == 0) {
+ if (!fs.delete(parts[i].getPath(), true)) {
+ LOG.error("Cannot delete empty directory " + parts[i].getPath());
+ throw new IOException("Cannot delete empty directory " + parts[i].getPath());
+ }
+ }
+
taskIDToFile = removeTempOrDuplicateFiles(items, fs);
// if the table is bucketed and enforce bucketing, we should check and generate all buckets
if (dpCtx.getNumBuckets() > 0 && taskIDToFile != null) {