You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2014/04/08 17:02:09 UTC
svn commit: r1585751 - in /hive/branches/branch-0.13: ./
ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
Author: omalley
Date: Tue Apr 8 15:02:08 2014
New Revision: 1585751
URL: http://svn.apache.org/r1585751
Log:
HIVE-6830. Remove restriction that ACID base directories have to be completely
covered. (omalley)
Modified:
hive/branches/branch-0.13/ (props changed)
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
Propchange: hive/branches/branch-0.13/
------------------------------------------------------------------------------
Merged /hive/trunk:r1585748
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java?rev=1585751&r1=1585750&r2=1585751&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java Tue Apr 8 15:02:08 2014
@@ -306,7 +306,6 @@ public class AcidUtils {
List<ParsedDelta> working = new ArrayList<ParsedDelta>();
final List<FileStatus> original = new ArrayList<FileStatus>();
final List<FileStatus> obsolete = new ArrayList<FileStatus>();
- Path ignoredBase = null;
List<FileStatus> children = SHIMS.listLocatedStatus(fs, directory,
hiddenFileFilter);
for(FileStatus child: children) {
@@ -314,20 +313,15 @@ public class AcidUtils {
String fn = p.getName();
if (fn.startsWith(BASE_PREFIX) && child.isDir()) {
long txn = parseBase(p);
- if (txnList.isTxnRangeCommitted(0, txn) !=
- ValidTxnList.RangeResponse.ALL) {
- ignoredBase = p;
+ if (bestBase == null) {
+ bestBase = child;
+ bestBaseTxn = txn;
+ } else if (bestBaseTxn < txn) {
+ obsolete.add(bestBase);
+ bestBase = child;
+ bestBaseTxn = txn;
} else {
- if (bestBase == null) {
- bestBase = child;
- bestBaseTxn = txn;
- } else if (bestBaseTxn < txn) {
- obsolete.add(bestBase);
- bestBase = child;
- bestBaseTxn = txn;
- } else {
- obsolete.add(child);
- }
+ obsolete.add(child);
}
} else if (fn.startsWith(DELTA_PREFIX) && child.isDir()) {
ParsedDelta delta = parseDelta(child);
@@ -341,13 +335,6 @@ public class AcidUtils {
}
}
- // Complain if all of the bases were too recent for the minimum excluded
- // transaction.
- if (bestBase == null && ignoredBase != null) {
- throw new IllegalArgumentException("All base directories were ignored," +
- " such as " + ignoredBase + " by " + txnList);
- }
-
// if we have a base, the original files are obsolete.
if (bestBase != null) {
obsolete.addAll(original);
Modified: hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java?rev=1585751&r1=1585750&r2=1585751&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java (original)
+++ hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java Tue Apr 8 15:02:08 2014
@@ -192,31 +192,18 @@ public class TestAcidUtils {
Path part = new MockPath(fs, "/tbl/part1");
AcidUtils.Directory dir =
AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("150:"));
- assertEquals("mock:/tbl/part1/base_100", dir.getBaseDirectory().toString());
+ assertEquals("mock:/tbl/part1/base_200", dir.getBaseDirectory().toString());
List<FileStatus> obsoletes = dir.getObsolete();
- assertEquals(3, obsoletes.size());
+ assertEquals(4, obsoletes.size());
assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString());
- assertEquals("mock:/tbl/part1/base_25", obsoletes.get(1).getPath().toString());
- assertEquals("mock:/tbl/part1/base_5", obsoletes.get(2).getPath().toString());
+ assertEquals("mock:/tbl/part1/base_100", obsoletes.get(1).getPath().toString());
+ assertEquals("mock:/tbl/part1/base_25", obsoletes.get(2).getPath().toString());
+ assertEquals("mock:/tbl/part1/base_5", obsoletes.get(3).getPath().toString());
assertEquals(0, dir.getOriginalFiles().size());
assertEquals(0, dir.getCurrentDirectories().size());
- dir = AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("150:99"));
- assertEquals("mock:/tbl/part1/base_25", dir.getBaseDirectory().toString());
- obsoletes = dir.getObsolete();
- assertEquals(2, obsoletes.size());
- assertEquals("mock:/tbl/part1/base_10", obsoletes.get(0).getPath().toString());
- assertEquals("mock:/tbl/part1/base_5", obsoletes.get(1).getPath().toString());
- dir = AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("150:25"));
- assertEquals("mock:/tbl/part1/base_10", dir.getBaseDirectory().toString());
- obsoletes = dir.getObsolete();
- assertEquals(1, obsoletes.size());
- assertEquals("mock:/tbl/part1/base_5", obsoletes.get(0).getPath().toString());
- try {
- AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("150:2"));
- fail("should not reach here.");
- } catch (IllegalArgumentException iae) {
- //expected
- }
+ // we should always get the latest base
+ dir = AcidUtils.getAcidState(part, conf, new ValidTxnListImpl("10:"));
+ assertEquals("mock:/tbl/part1/base_200", dir.getBaseDirectory().toString());
}
@Test