You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2013/09/09 19:40:10 UTC
svn commit: r1521219 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/filter/
test/java/org/apache/hadoop/hbase/regionserver/
Author: larsh
Date: Mon Sep 9 17:40:10 2013
New Revision: 1521219
URL: http://svn.apache.org/r1521219
Log:
HBASE-8930 REVERT due to test issues
Removed:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java?rev=1521219&r1=1521218&r2=1521219&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java Mon Sep 9 17:40:10 2013
@@ -47,46 +47,25 @@ import org.apache.hadoop.hbase.regionser
* This class is NOT thread-safe as queries are never multi-threaded
*/
public interface ColumnTracker {
-
/**
- * Checks if the column is present in the list of requested columns by returning the match code
- * instance. It does not check against the number of versions for the columns asked for. To do the
- * version check, one has to call {@link #checkVersions(byte[], int, int, long, byte, boolean)}
- * method based on the return type (INCLUDE) of this method. The values that can be returned by
- * this method are {@link MatchCode#INCLUDE}, {@link MatchCode#SEEK_NEXT_COL} and
- * {@link MatchCode#SEEK_NEXT_ROW}.
+ * Keeps track of the number of versions for the columns asked for
* @param bytes
* @param offset
* @param length
+ * @param ttl The timeToLive to enforce.
* @param type The type of the KeyValue
+ * @param ignoreCount indicates if the KV needs to be excluded while counting
+ * (used during compactions. We only count KV's that are older than all the
+ * scanners' read points.)
* @return The match code instance.
- * @throws IOException in case there is an internal consistency problem caused by a data
- * corruption.
+ * @throws IOException in case there is an internal consistency problem
+ * caused by a data corruption.
*/
- ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset, int length, byte type)
+ public ScanQueryMatcher.MatchCode checkColumn(byte[] bytes, int offset,
+ int length, long ttl, byte type, boolean ignoreCount)
throws IOException;
/**
- * Keeps track of the number of versions for the columns asked for. It assumes that the user has
- * already checked if the keyvalue needs to be included by calling the
- * {@link #checkColumn(byte[], int, int, byte)} method. The enum values returned by this method
- * are {@link MatchCode#SKIP}, {@link MatchCode#INCLUDE},
- * {@link MatchCode#INCLUDE_AND_SEEK_NEXT_COL} and {@link MatchCode#INCLUDE_AND_SEEK_NEXT_ROW}.
- * Implementations which include all the columns could just return {@link MatchCode#INCLUDE} in
- * the {@link #checkColumn(byte[], int, int, byte)} method and perform all the operations in this
- * checkVersions method.
- * @param type the type of the key value (Put/Delete)
- * @param ttl The timeToLive to enforce.
- * @param ignoreCount indicates if the KV needs to be excluded while counting (used during
- * compactions. We only count KV's that are older than all the scanners' read points.)
- * @return the scan query matcher match code instance
- * @throws IOException in case there is an internal consistency problem caused by a data
- * corruption.
- */
- ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length, long ttl,
- byte type, boolean ignoreCount) throws IOException;
-
- /**
* Resets the Matcher
*/
public void reset();
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java?rev=1521219&r1=1521218&r2=1521219&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java Mon Sep 9 17:40:10 2013
@@ -106,7 +106,7 @@ public class ExplicitColumnTracker imple
*/
@Override
public ScanQueryMatcher.MatchCode checkColumn(byte [] bytes, int offset,
- int length, byte type) {
+ int length, long timestamp, byte type, boolean ignoreCount) {
// delete markers should never be passed to an
// *Explicit*ColumnTracker
assert !KeyValue.isDelete(type);
@@ -125,9 +125,34 @@ public class ExplicitColumnTracker imple
int ret = Bytes.compareTo(column.getBuffer(), column.getOffset(),
column.getLength(), bytes, offset, length);
- // Column Matches. Return include code. The caller would call checkVersions
- // to limit the number of versions.
+ // Column Matches. If it is not a duplicate key, increment the version count
+ // and include.
if(ret == 0) {
+ if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
+
+ //If column matches, check if it is a duplicate timestamp
+ if (sameAsPreviousTS(timestamp)) {
+ //If duplicate, skip this Key
+ return ScanQueryMatcher.MatchCode.SKIP;
+ }
+ int count = this.column.increment();
+ if(count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
+ // Done with versions for this column
+ ++this.index;
+ resetTS();
+ if (done()) {
+ // We have served all the requested columns.
+ this.column = null;
+ return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
+ } else {
+ // We are done with current column; advance to next column
+ // of interest.
+ this.column = this.columns.get(this.index);
+ return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
+ }
+ } else {
+ setTS(timestamp);
+ }
return ScanQueryMatcher.MatchCode.INCLUDE;
}
@@ -155,35 +180,6 @@ public class ExplicitColumnTracker imple
} while(true);
}
- @Override
- public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length,
- long timestamp, byte type, boolean ignoreCount) {
- assert !KeyValue.isDelete(type);
- if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE;
- // Check if it is a duplicate timestamp
- if (sameAsPreviousTS(timestamp)) {
- // If duplicate, skip this Key
- return ScanQueryMatcher.MatchCode.SKIP;
- }
- int count = this.column.increment();
- if (count >= maxVersions || (count >= minVersions && isExpired(timestamp))) {
- // Done with versions for this column
- ++this.index;
- resetTS();
- if (done()) {
- // We have served all the requested columns.
- this.column = null;
- return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
- }
- // We are done with current column; advance to next column
- // of interest.
- this.column = this.columns.get(this.index);
- return ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
- }
- setTS(timestamp);
- return ScanQueryMatcher.MatchCode.INCLUDE;
- }
-
// Called between every row.
public void reset() {
this.index = 0;
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1521219&r1=1521218&r2=1521219&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java Mon Sep 9 17:40:10 2013
@@ -340,60 +340,42 @@ public class ScanQueryMatcher {
return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
}
- // STEP 1: Check if the column is part of the requested columns
- MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength, type);
- if (colChecker == MatchCode.INCLUDE) {
- ReturnCode filterResponse = ReturnCode.SKIP;
- // STEP 2: Yes, the column is part of the requested columns. Check if filter is present
- if (filter != null) {
- // STEP 3: Filter the key value and return if it filters out
- filterResponse = filter.filterKeyValue(kv);
- switch (filterResponse) {
- case SKIP:
- return MatchCode.SKIP;
- case NEXT_COL:
- return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
- case NEXT_ROW:
- stickyNextRow = true;
- return MatchCode.SEEK_NEXT_ROW;
- case SEEK_NEXT_USING_HINT:
- return MatchCode.SEEK_NEXT_USING_HINT;
- default:
- //It means it is either include or include and seek next
- break;
- }
+ /**
+ * Filters should be checked before checking column trackers. If we do
+ * otherwise, as was previously being done, ColumnTracker may increment its
+ * counter for even that KV which may be discarded later on by Filter. This
+ * would lead to incorrect results in certain cases.
+ */
+ ReturnCode filterResponse = ReturnCode.SKIP;
+ if (filter != null) {
+ filterResponse = filter.filterKeyValue(kv);
+ if (filterResponse == ReturnCode.SKIP) {
+ return MatchCode.SKIP;
+ } else if (filterResponse == ReturnCode.NEXT_COL) {
+ return columns.getNextRowOrNextColumn(bytes, offset, qualLength);
+ } else if (filterResponse == ReturnCode.NEXT_ROW) {
+ stickyNextRow = true;
+ return MatchCode.SEEK_NEXT_ROW;
+ } else if (filterResponse == ReturnCode.SEEK_NEXT_USING_HINT) {
+ return MatchCode.SEEK_NEXT_USING_HINT;
}
- /*
- * STEP 4: Reaching this step means the column is part of the requested columns and either
- * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response.
- * Now check the number of versions needed. This method call returns SKIP, INCLUDE,
- * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL.
- *
- * FilterResponse ColumnChecker Desired behavior
- * INCLUDE SKIP row has already been included, SKIP.
- * INCLUDE INCLUDE INCLUDE
- * INCLUDE INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL
- * INCLUDE INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW
- * INCLUDE_AND_SEEK_NEXT_COL SKIP row has already been included, SKIP.
- * INCLUDE_AND_SEEK_NEXT_COL INCLUDE INCLUDE_AND_SEEK_NEXT_COL
- * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL
- * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW INCLUDE_AND_SEEK_NEXT_ROW
- *
- * In all the above scenarios, we return the column checker return value except for
- * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE)
- */
- colChecker =
- columns.checkVersions(bytes, offset, qualLength, timestamp, type,
- kv.getMemstoreTS() > maxReadPointToTrackVersions);
- //Optimize with stickyNextRow
- stickyNextRow = colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW ? true : stickyNextRow;
- return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL &&
- colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL
- : colChecker;
}
- stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true
- : stickyNextRow;
+
+ MatchCode colChecker = columns.checkColumn(bytes, offset, qualLength,
+ timestamp, type, kv.getMemstoreTS() > maxReadPointToTrackVersions);
+ /*
+ * According to current implementation, colChecker can only be
+ * SEEK_NEXT_COL, SEEK_NEXT_ROW, SKIP or INCLUDE. Therefore, always return
+ * the MatchCode. If it is SEEK_NEXT_ROW, also set stickyNextRow.
+ */
+ if (colChecker == MatchCode.SEEK_NEXT_ROW) {
+ stickyNextRow = true;
+ } else if (filter != null && colChecker == MatchCode.INCLUDE &&
+ filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL) {
+ return MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
+ }
return colChecker;
+
}
public boolean moreRowsMayExistAfter(KeyValue kv) {
@@ -472,16 +454,6 @@ public class ScanQueryMatcher {
null, 0, 0);
}
- // Used only for testing purposes
- static MatchCode checkColumn(ColumnTracker columnTracker, byte[] bytes, int offset, int length,
- long ttl, byte type, boolean ignoreCount) throws IOException {
- MatchCode matchCode = columnTracker.checkColumn(bytes, offset, length, type);
- if (matchCode == MatchCode.INCLUDE) {
- return columnTracker.checkVersions(bytes, offset, length, ttl, type, ignoreCount);
- }
- return matchCode;
- }
-
/**
* {@link #match} return codes. These instruct the scanner moving through
* memstores and StoreFiles what to do with the current KeyValue.
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java?rev=1521219&r1=1521218&r2=1521219&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java Mon Sep 9 17:40:10 2013
@@ -61,22 +61,13 @@ public class ScanWildcardColumnTracker i
/**
* {@inheritDoc}
* This receives puts *and* deletes.
+ * Deletes do not count as a version, but rather take the version
+ * of the previous put (so eventually all but the last can be reclaimed).
*/
@Override
- public MatchCode checkColumn(byte[] bytes, int offset, int length, byte type)
- throws IOException {
- return MatchCode.INCLUDE;
- }
-
- /**
- * {@inheritDoc}
- * This receives puts *and* deletes. Deletes do not count as a version, but rather
- * take the version of the previous put (so eventually all but the last can be reclaimed).
- */
- @Override
- public MatchCode checkVersions(byte[] bytes, int offset, int length, long timestamp, byte type,
- boolean ignoreCount) throws IOException {
-
+ public MatchCode checkColumn(byte[] bytes, int offset, int length,
+ long timestamp, byte type, boolean ignoreCount) throws IOException {
+
if (columnBuffer == null) {
// first iteration.
resetBuffer(bytes, offset, length);
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java?rev=1521219&r1=1521218&r2=1521219&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java Mon Sep 9 17:40:10 2013
@@ -56,7 +56,7 @@ public class TestExplicitColumnTracker e
long timestamp = 0;
//"Match"
for(byte [] col : scannerColumns){
- result.add(ScanQueryMatcher.checkColumn(exp, col, 0, col.length, ++timestamp,
+ result.add(exp.checkColumn(col, 0, col.length, ++timestamp,
KeyValue.Type.Put.getCode(), false));
}
@@ -169,14 +169,14 @@ public class TestExplicitColumnTracker e
Long.MIN_VALUE);
for (int i = 0; i < 100000; i+=2) {
byte [] col = Bytes.toBytes("col"+i);
- ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
+ explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
false);
}
explicit.reset();
for (int i = 1; i < 100000; i+=2) {
byte [] col = Bytes.toBytes("col"+i);
- ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
+ explicit.checkColumn(col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
false);
}
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java?rev=1521219&r1=1521218&r2=1521219&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java Mon Sep 9 17:40:10 2013
@@ -55,7 +55,7 @@ public class TestScanWildcardColumnTrack
List<ScanQueryMatcher.MatchCode> actual = new ArrayList<MatchCode>();
for(byte [] qualifier : qualifiers) {
- ScanQueryMatcher.MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0,
+ ScanQueryMatcher.MatchCode mc = tracker.checkColumn(qualifier, 0,
qualifier.length, 1, KeyValue.Type.Put.getCode(), false);
actual.add(mc);
}
@@ -88,7 +88,7 @@ public class TestScanWildcardColumnTrack
long timestamp = 0;
for(byte [] qualifier : qualifiers) {
- MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length,
+ MatchCode mc = tracker.checkColumn(qualifier, 0, qualifier.length,
++timestamp, KeyValue.Type.Put.getCode(), false);
actual.add(mc);
}
@@ -112,7 +112,7 @@ public class TestScanWildcardColumnTrack
try {
for(byte [] qualifier : qualifiers) {
- ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length, 1,
+ tracker.checkColumn(qualifier, 0, qualifier.length, 1,
KeyValue.Type.Put.getCode(), false);
}
} catch (Exception e) {