You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/10/14 21:52:46 UTC
svn commit: r825242 - in /hadoop/hbase/branches/0.20: ./
src/java/org/apache/hadoop/hbase/filter/
src/java/org/apache/hadoop/hbase/regionserver/
src/test/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/client/
src/test/org/apache/hadoop/hbase...
Author: stack
Date: Wed Oct 14 19:52:46 2009
New Revision: 825242
URL: http://svn.apache.org/viewvc?rev=825242&view=rev
Log:
HBASE-1906 FilterList of prefix and columnvalue not working properly with deletes and multiple values
Modified:
hadoop/hbase/branches/0.20/CHANGES.txt
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java
hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Wed Oct 14 19:52:46 2009
@@ -5,6 +5,8 @@
BUG FIXES
HBASE-1905 Remove unused config. hbase.hstore.blockCache.blockSize
+ HBASE-1906 FilterList of prefix and columnvalue not working properly
+ with deletes and multiple values
IMPROVEMENTS
HBASE-1899 Use scanner caching in shell count
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java Wed Oct 14 19:52:46 2009
@@ -25,10 +25,8 @@
import java.io.IOException;
import java.util.Arrays;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.HbaseObjectWritable;
-import org.apache.hadoop.io.ObjectWritable;
/**
* This is a generic filter to be used to filter by comparison. It takes an
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java Wed Oct 14 19:52:46 2009
@@ -33,13 +33,14 @@
/**
* Implementation of {@link Filter} that represents an ordered List of Filters
- * which will be evaluated with a specified boolean operator MUST_PASS_ALL
- * (!AND) or MUST_PASS_ONE (!OR). Since you can use Filter Lists as children
- * of Filter Lists, you can create a hierarchy of filters to be evaluated.
+ * which will be evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL}
+ * (<code>!AND</code>) or {@link Operator#MUST_PASS_ONE} (<code>!OR</code>).
+ * Since you can use Filter Lists as children of Filter Lists, you can create a
+ * hierarchy of filters to be evaluated.
+ * Defaults to {@link Operator#MUST_PASS_ALL}.
* <p>TODO: Fix creation of Configuration on serialization and deserialization.
*/
public class FilterList implements Filter {
-
/** set operator */
public static enum Operator {
/** !AND */
@@ -70,6 +71,15 @@
}
/**
+ * Constructor that takes an operator.
+ *
+ * @param operator Operator to process filter set with.
+ */
+ public FilterList(final Operator operator) {
+ this.operator = operator;
+ }
+
+ /**
* Constructor that takes a set of {@link Filter}s and an operator.
*
* @param operator Operator to process filter set with.
@@ -115,19 +125,19 @@
public boolean filterRowKey(byte[] rowKey, int offset, int length) {
for (Filter filter : filters) {
- if (operator == Operator.MUST_PASS_ALL) {
- if (filter.filterAllRemaining()
- || filter.filterRowKey(rowKey, offset, length)) {
+ if (this.operator == Operator.MUST_PASS_ALL) {
+ if (filter.filterAllRemaining() ||
+ filter.filterRowKey(rowKey, offset, length)) {
return true;
}
- } else if (operator == Operator.MUST_PASS_ONE) {
- if (!filter.filterAllRemaining()
- && !filter.filterRowKey(rowKey, offset, length)) {
+ } else if (this.operator == Operator.MUST_PASS_ONE) {
+ if (!filter.filterAllRemaining() &&
+ !filter.filterRowKey(rowKey, offset, length)) {
return false;
}
}
}
- return operator == Operator.MUST_PASS_ONE;
+ return this.operator == Operator.MUST_PASS_ONE;
}
public boolean filterAllRemaining() {
@@ -179,8 +189,7 @@
public boolean filterRow() {
for (Filter filter : filters) {
if (operator == Operator.MUST_PASS_ALL) {
- if (filter.filterAllRemaining()
- || filter.filterRow()) {
+ if (filter.filterAllRemaining() || filter.filterRow()) {
return true;
}
} else if (operator == Operator.MUST_PASS_ONE) {
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java Wed Oct 14 19:52:46 2009
@@ -109,16 +109,19 @@
this.compareOp = compareOp;
this.comparator = comparator;
}
-
+
public boolean filterRowKey(byte[] rowKey, int offset, int length) {
+ // We don't filter on the row key... we filter later on column value so
+ // always return false.
return false;
}
public ReturnCode filterKeyValue(KeyValue keyValue) {
- if(matchedColumn) {
+ // System.out.println("REMOVE KEY=" + keyValue.toString() + ", value=" + Bytes.toString(keyValue.getValue()));
+ if (this.matchedColumn) {
// We already found and matched the single column, all keys now pass
return ReturnCode.INCLUDE;
- } else if(foundColumn) {
+ } else if (this.foundColumn) {
// We found but did not match the single column, skip to next row
return ReturnCode.NEXT_ROW;
}
@@ -130,16 +133,18 @@
keyValue.getValueOffset(), keyValue.getValueLength())) {
return ReturnCode.NEXT_ROW;
}
- matchedColumn = true;
+ this.matchedColumn = true;
return ReturnCode.INCLUDE;
}
- private boolean filterColumnValue(final byte[] data, final int offset,
+ private boolean filterColumnValue(final byte [] data, final int offset,
final int length) {
- int compareResult = comparator.compareTo(Arrays.copyOfRange(data, offset,
- offset + length));
-
- switch (compareOp) {
+ // TODO: Can this filter take a rawcomparator so don't have to make this
+ // byte array copy?
+ int compareResult =
+ this.comparator.compareTo(Arrays.copyOfRange(data, offset, offset + length));
+ LOG.debug("compareResult=" + compareResult + " " + Bytes.toString(data, offset, length));
+ switch (this.compareOp) {
case LESS:
return compareResult <= 0;
case LESS_OR_EQUAL:
@@ -164,23 +169,23 @@
public boolean filterRow() {
// If column was found, return false if it was matched, true if it was not
// If column not found, return true if we filter if missing, false if not
- return foundColumn ? !matchedColumn : filterIfMissing;
+ return this.foundColumn? !this.matchedColumn: this.filterIfMissing;
}
public void reset() {
foundColumn = false;
matchedColumn = false;
}
-
+
/**
* Get whether entire row should be filtered if column is not found.
- * @return filterIfMissing true if row should be skipped if column not found,
- * false if row should be let through anyways
+ * @return true if row should be skipped if column not found, false if row
+ * should be let through anyways
*/
public boolean getFilterIfMissing() {
return filterIfMissing;
}
-
+
/**
* Set whether entire row should be filtered if column is not found.
* <p>
@@ -201,12 +206,12 @@
if(this.columnQualifier.length == 0) {
this.columnQualifier = null;
}
- compareOp = CompareOp.valueOf(in.readUTF());
- comparator = (WritableByteArrayComparable) HbaseObjectWritable.readObject(in,
- null);
- foundColumn = in.readBoolean();
- matchedColumn = in.readBoolean();
- filterIfMissing = in.readBoolean();
+ this.compareOp = CompareOp.valueOf(in.readUTF());
+ this.comparator =
+ (WritableByteArrayComparable)HbaseObjectWritable.readObject(in, null);
+ this.foundColumn = in.readBoolean();
+ this.matchedColumn = in.readBoolean();
+ this.filterIfMissing = in.readBoolean();
}
public void write(final DataOutput out) throws IOException {
@@ -219,4 +224,4 @@
out.writeBoolean(matchedColumn);
out.writeBoolean(filterIfMissing);
}
-}
+}
\ No newline at end of file
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed Oct 14 19:52:46 2009
@@ -1740,58 +1740,58 @@
* @throws IOException
*/
private boolean nextInternal() throws IOException {
- // This method should probably be reorganized a bit... has gotten messy
- KeyValue kv;
- byte[] currentRow = null;
+ byte [] currentRow = null;
boolean filterCurrentRow = false;
while (true) {
- kv = this.storeHeap.peek();
- if (kv == null) {
- return false;
- }
+ KeyValue kv = this.storeHeap.peek();
+ if (kv == null) return false;
byte [] row = kv.getRow();
- if (filterCurrentRow && Bytes.equals(currentRow, row)) {
- // filter all columns until row changes
+ boolean samerow = Bytes.equals(currentRow, row);
+ if (samerow && filterCurrentRow) {
+ // Filter all columns until row changes
this.storeHeap.next(results);
results.clear();
continue;
}
- // see if current row should be filtered based on row key
- if ((filter != null && filter.filterRowKey(row, 0, row.length)) ||
- (oldFilter != null && oldFilter.filterRowKey(row, 0, row.length))) {
- if(!results.isEmpty() && !Bytes.equals(currentRow, row)) {
- return true;
- }
- this.storeHeap.next(results);
- results.clear();
- resetFilters();
- filterCurrentRow = true;
- currentRow = row;
- continue;
- }
- if(!Bytes.equals(currentRow, row)) {
+ if (!samerow) {
// Continue on the next row:
currentRow = row;
filterCurrentRow = false;
// See if we passed stopRow
- if(stopRow != null &&
- comparator.compareRows(stopRow, 0, stopRow.length,
- currentRow, 0, currentRow.length) <= 0) {
+ if (this.stopRow != null &&
+ comparator.compareRows(this.stopRow, 0, this.stopRow.length,
+ currentRow, 0, currentRow.length) <= 0) {
return false;
}
- // if there are _no_ results or current row should be filtered
- if (results.isEmpty() || filter != null && filter.filterRow()) {
- // make sure results is empty
- results.clear();
- resetFilters();
- continue;
- }
- return true;
+ if (hasResults()) return true;
+ }
+ // See if current row should be filtered based on row key
+ if (this.filter != null && this.filter.filterRowKey(row, 0, row.length)) {
+ resetFilters();
+ filterCurrentRow = true;
+ currentRow = row;
}
this.storeHeap.next(results);
}
}
+ /*
+ * Do we have results to return or should we continue. Call when we get to
+ * the end of a row. Does house cleaning -- clearing results and resetting
+ * filters -- if we are to continue.
+ * @return True if we should return else false if need to keep going.
+ */
+ private boolean hasResults() {
+ if (this.results.isEmpty() ||
+ this.filter != null && this.filter.filterRow()) {
+ // Make sure results is empty, reset filters
+ results.clear();
+ resetFilters();
+ return false;
+ }
+ return true;
+ }
+
public void close() {
storeHeap.close();
}
@@ -2323,7 +2323,6 @@
store.get(get, qualifiers, results);
if (!results.isEmpty()) {
- byte [] oldValue = results.get(0).getValue();
KeyValue kv = results.get(0);
byte [] buffer = kv.getBuffer();
int valueOffset = kv.getValueOffset();
@@ -2514,4 +2513,4 @@
if (bc != null) bc.shutdown();
}
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java Wed Oct 14 19:52:46 2009
@@ -224,8 +224,7 @@
if (startKeyBytes == null || startKeyBytes.length == 0) {
startKeyBytes = START_KEY_BYTES;
}
- return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily),
- null,
+ return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
startKeyBytes, endKey, -1);
}
Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java (original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java Wed Oct 14 19:52:46 2009
@@ -20,10 +20,6 @@
package org.apache.hadoop.hbase.client;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Map;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseClusterTestCase;
@@ -35,14 +31,23 @@
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.util.Bytes;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.UUID;
+
/**
* Tests from client-side of a cluster.
*/
@@ -63,6 +68,135 @@
}
/**
+ * Test from client side of an involved filter against a multi family that
+ * involves deletes.
+ *
+ * @throws Exception
+ */
+ public void testWeirdCacheBehaviour() throws Exception {
+ byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
+ byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
+ Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
+ Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
+ HTable ht = createTable(TABLE, FAMILIES);
+ String value = "this is the value";
+ String value2 = "this is some other value";
+ String keyPrefix1 = UUID.randomUUID().toString();
+ String keyPrefix2 = UUID.randomUUID().toString();
+ String keyPrefix3 = UUID.randomUUID().toString();
+ putRows(ht, 3, value, keyPrefix1);
+ putRows(ht, 3, value, keyPrefix2);
+ putRows(ht, 3, value, keyPrefix3);
+ ht.flushCommits();
+ putRows(ht, 3, value2, keyPrefix1);
+ putRows(ht, 3, value2, keyPrefix2);
+ putRows(ht, 3, value2, keyPrefix3);
+ HTable table = new HTable(conf, Bytes.toBytes("testWeirdCacheBehaviour"));
+ System.out.println("Checking values for key: " + keyPrefix1);
+ assertEquals("Got back incorrect number of rows from scan", 3,
+ getNumberOfRows(keyPrefix1, value2, table));
+ System.out.println("Checking values for key: " + keyPrefix2);
+ assertEquals("Got back incorrect number of rows from scan", 3,
+ getNumberOfRows(keyPrefix2, value2, table));
+ System.out.println("Checking values for key: " + keyPrefix3);
+ assertEquals("Got back incorrect number of rows from scan", 3,
+ getNumberOfRows(keyPrefix3, value2, table));
+ deleteColumns(ht, value2, keyPrefix1);
+ deleteColumns(ht, value2, keyPrefix2);
+ deleteColumns(ht, value2, keyPrefix3);
+ System.out.println("Starting important checks.....");
+ assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
+ 0, getNumberOfRows(keyPrefix1, value2, table));
+ assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
+ 0, getNumberOfRows(keyPrefix2, value2, table));
+ assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
+ 0, getNumberOfRows(keyPrefix3, value2, table));
+ ht.setScannerCaching(0);
+ assertEquals("Got back incorrect number of rows from scan", 0,
+ getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100);
+ assertEquals("Got back incorrect number of rows from scan", 0,
+ getNumberOfRows(keyPrefix2, value2, table));
+ }
+
+ private void deleteColumns(HTable ht, String value, String keyPrefix)
+ throws IOException {
+ ResultScanner scanner = buildScanner(keyPrefix, value, ht);
+ Iterator<Result> it = scanner.iterator();
+ int count = 0;
+ while (it.hasNext()) {
+ Result result = it.next();
+ Delete delete = new Delete(result.getRow());
+ delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
+ ht.delete(delete);
+ count++;
+ }
+ assertEquals("Did not perform correct number of deletes", 3, count);
+ }
+
+ private int getNumberOfRows(String keyPrefix, String value, HTable ht)
+ throws Exception {
+ ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
+ Iterator<Result> scanner = resultScanner.iterator();
+ int numberOfResults = 0;
+ while (scanner.hasNext()) {
+ Result result = scanner.next();
+ System.out.println("Got back key: " + Bytes.toString(result.getRow()));
+ for (KeyValue kv : result.raw()) {
+ System.out.println("kv=" + kv.toString() + ", "
+ + Bytes.toString(kv.getValue()));
+ }
+ numberOfResults++;
+ }
+ return numberOfResults;
+ }
+
+ private ResultScanner buildScanner(String keyPrefix, String value, HTable ht)
+ throws IOException {
+ // OurFilterList allFilters = new OurFilterList();
+ FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
+ allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
+ SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
+ .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes
+ .toBytes(value));
+ filter.setFilterIfMissing(true);
+ allFilters.addFilter(filter);
+
+ // allFilters.addFilter(new
+ // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
+ // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
+
+ Scan scan = new Scan();
+ scan.addFamily(Bytes.toBytes("trans-blob"));
+ scan.addFamily(Bytes.toBytes("trans-type"));
+ scan.addFamily(Bytes.toBytes("trans-date"));
+ scan.addFamily(Bytes.toBytes("trans-tags"));
+ scan.addFamily(Bytes.toBytes("trans-group"));
+ scan.setFilter(allFilters);
+
+ return ht.getScanner(scan);
+ }
+
+ private void putRows(HTable ht, int numRows, String value, String key)
+ throws IOException {
+ for (int i = 0; i < numRows; i++) {
+ String row = key + "_" + UUID.randomUUID().toString();
+ System.out.println(String.format("Saving row: %s, with value %s", row,
+ value));
+ Put put = new Put(Bytes.toBytes(row));
+ put.add(Bytes.toBytes("trans-blob"), null, Bytes
+ .toBytes("value for blob"));
+ put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
+ put.add(Bytes.toBytes("trans-date"), null, Bytes
+ .toBytes("20090921010101999"));
+ put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
+ .toBytes(value));
+ put.add(Bytes.toBytes("trans-group"), null, Bytes
+ .toBytes("adhocTransactionGroupId"));
+ ht.put(put);
+ }
+ }
+
+ /**
* Test filters when multiple regions. It does counts. Needs eye-balling of
* logs to ensure that we're not scanning more regions that we're supposed to.
* Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
@@ -249,6 +383,7 @@
scanner.close();
System.out.println("Done.");
}
+
public void testFilters() throws Exception {
byte [] TABLE = Bytes.toBytes("testFilters");
HTable ht = createTable(TABLE, FAMILY);
@@ -2671,7 +2806,7 @@
}
private byte [][] makeN(byte [] base, int n) {
- if(n > 256) {
+ if (n > 256) {
return makeNBig(base, n);
}
byte [][] ret = new byte[n][];
Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java Wed Oct 14 19:52:46 2009
@@ -40,6 +40,10 @@
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.regionserver.HRegion.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
@@ -79,6 +83,127 @@
// individual code pieces in the HRegion. Putting files locally in
// /tmp/testtable
//////////////////////////////////////////////////////////////////////////////
+
+
+ /**
+ * An involved filter test. Has multiple column families and deletes in mix.
+ */
+ public void testWeirdCacheBehaviour() throws Exception {
+ byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
+ byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
+ Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
+ Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
+ initHRegion(TABLE, getName(), FAMILIES);
+ String value = "this is the value";
+ String value2 = "this is some other value";
+ String keyPrefix1 = "prefix1"; // UUID.randomUUID().toString();
+ String keyPrefix2 = "prefix2"; // UUID.randomUUID().toString();
+ String keyPrefix3 = "prefix3"; // UUID.randomUUID().toString();
+ putRows(this.region, 3, value, keyPrefix1);
+ putRows(this.region, 3, value, keyPrefix2);
+ putRows(this.region, 3, value, keyPrefix3);
+ // this.region.flushCommits();
+ putRows(this.region, 3, value2, keyPrefix1);
+ putRows(this.region, 3, value2, keyPrefix2);
+ putRows(this.region, 3, value2, keyPrefix3);
+ System.out.println("Checking values for key: " + keyPrefix1);
+ assertEquals("Got back incorrect number of rows from scan", 3,
+ getNumberOfRows(keyPrefix1, value2, this.region));
+ System.out.println("Checking values for key: " + keyPrefix2);
+ assertEquals("Got back incorrect number of rows from scan", 3,
+ getNumberOfRows(keyPrefix2, value2, this.region));
+ System.out.println("Checking values for key: " + keyPrefix3);
+ assertEquals("Got back incorrect number of rows from scan", 3,
+ getNumberOfRows(keyPrefix3, value2, this.region));
+ deleteColumns(this.region, value2, keyPrefix1);
+ deleteColumns(this.region, value2, keyPrefix2);
+ deleteColumns(this.region, value2, keyPrefix3);
+ System.out.println("Starting important checks.....");
+ assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
+ 0, getNumberOfRows(keyPrefix1, value2, this.region));
+ assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
+ 0, getNumberOfRows(keyPrefix2, value2, this.region));
+ assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
+ 0, getNumberOfRows(keyPrefix3, value2, this.region));
+ }
+
+ private void deleteColumns(HRegion r, String value, String keyPrefix)
+ throws IOException {
+ InternalScanner scanner = buildScanner(keyPrefix, value, r);
+ int count = 0;
+ boolean more = false;
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ do {
+ more = scanner.next(results);
+ if (results != null && !results.isEmpty())
+ count++;
+ else
+ break;
+ Delete delete = new Delete(results.get(0).getRow());
+ delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
+ r.delete(delete, null, false);
+ results.clear();
+ } while (more);
+ assertEquals("Did not perform correct number of deletes", 3, count);
+ }
+
+ private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception {
+ InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
+ int numberOfResults = 0;
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ boolean more = false;
+ do {
+ more = resultScanner.next(results);
+ if (results != null && !results.isEmpty()) numberOfResults++;
+ else break;
+ for (KeyValue kv: results) {
+ System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue()));
+ }
+ results.clear();
+ } while(more);
+ return numberOfResults;
+ }
+
+ private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
+ throws IOException {
+ // Defaults FilterList.Operator.MUST_PASS_ALL.
+ FilterList allFilters = new FilterList();
+ allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
+ // Only return rows where this column value exists in the row.
+ SingleColumnValueFilter filter =
+ new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
+ Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
+ filter.setFilterIfMissing(true);
+ allFilters.addFilter(filter);
+ Scan scan = new Scan();
+ scan.addFamily(Bytes.toBytes("trans-blob"));
+ scan.addFamily(Bytes.toBytes("trans-type"));
+ scan.addFamily(Bytes.toBytes("trans-date"));
+ scan.addFamily(Bytes.toBytes("trans-tags"));
+ scan.addFamily(Bytes.toBytes("trans-group"));
+ scan.setFilter(allFilters);
+ return r.getScanner(scan);
+ }
+
+ private void putRows(HRegion r, int numRows, String value, String key)
+ throws IOException {
+ for (int i = 0; i < numRows; i++) {
+ String row = key + "_" + i/* UUID.randomUUID().toString() */;
+ System.out.println(String.format("Saving row: %s, with value %s", row,
+ value));
+ Put put = new Put(Bytes.toBytes(row));
+ put.add(Bytes.toBytes("trans-blob"), null,
+ Bytes.toBytes("value for blob"));
+ put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
+ put.add(Bytes.toBytes("trans-date"), null,
+ Bytes.toBytes("20090921010101999"));
+ put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"),
+ Bytes.toBytes(value));
+ put.add(Bytes.toBytes("trans-group"), null,
+ Bytes.toBytes("adhocTransactionGroupId"));
+ r.put(put);
+ }
+ }
public void testFamilyWithAndWithoutColon() throws Exception {
byte [] b = Bytes.toBytes(getName());
@@ -1764,7 +1889,8 @@
}
private void initHRegion (byte [] tableName, String callingMethod,
- HBaseConfiguration conf, byte [] ... families) throws IOException{
+ HBaseConfiguration conf, byte [] ... families)
+ throws IOException{
HTableDescriptor htd = new HTableDescriptor(tableName);
for(byte [] family : families) {
htd.addFamily(new HColumnDescriptor(family));
Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java (original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java Wed Oct 14 19:52:46 2009
@@ -180,9 +180,11 @@
try {
this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
addContent(this.r, HConstants.CATALOG_FAMILY);
- Filter newFilter = new PrefixFilter(Bytes.toBytes("ab"));
+ byte [] prefix = Bytes.toBytes("ab");
+ Filter newFilter = new PrefixFilter(prefix);
Scan scan = new Scan();
scan.setFilter(newFilter);
+ scan.setStartRow(prefix);
rowPrefixFilter(scan);
RowFilterInterface oldFilter = new PrefixRowFilter(Bytes.toBytes("ab"));
scan = new Scan();