You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:18:51 UTC
svn commit: r1181548 - in /hbase/branches/0.89/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/client/
test/java/org/apache/hadoop/hbase/regionserver/
Author: nspiegelberg
Date: Tue Oct 11 02:18:51 2011
New Revision: 1181548
URL: http://svn.apache.org/viewvc?rev=1181548&view=rev
Log:
Porting patch for HBASE-3416 to hbase-trunk.
Summary:
Don't call setRow when scanning over same row. The patch fixes one case, setRow
is also called from next. Fixed it there too.
Test Plan:
Unit test. When batch is used with scan, in some cases deletes were getting
ignored because of this bug.
Reviewed By: kannan
Reviewers: kannan, jgray
CC: , madhuvaidya, kannan
Differential Revision: 261692
Added:
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1181548&r1=1181547&r2=1181548&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java Tue Oct 11 02:18:51 2011
@@ -1066,6 +1066,15 @@ public class KeyValue implements Writabl
this.bytes, o, l) == 0;
}
+ public boolean matchingRow(final byte [] row) {
+ return matchingRow(row, 0, row.length);
+ }
+
+ public boolean matchingRow(final byte[] row, int offset, int length) {
+ return Bytes.compareTo(row, offset, length,
+ this.bytes, getRowOffset(), getRowLength()) == 0;
+ }
+
/**
* @param column Column minus its delimiter
* @return True if column matches.
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1181548&r1=1181547&r2=1181548&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java Tue Oct 11 02:18:51 2011
@@ -235,7 +235,9 @@ class StoreScanner implements KeyValueSc
return false;
}
- matcher.setRow(peeked.getRow());
+ if ((matcher.row == null) || !peeked.matchingRow(matcher.row)) {
+ matcher.setRow(peeked.getRow());
+ }
KeyValue kv;
List<KeyValue> results = new ArrayList<KeyValue>();
LOOP: while((kv = this.heap.peek()) != null) {
@@ -361,10 +363,17 @@ class StoreScanner implements KeyValueSc
// Combine all seeked scanners with a heap
heap = new KeyValueHeap(scanners, store.comparator);
- // Reset the state of the Query Matcher and set to top row
- matcher.reset();
+ // Reset the state of the Query Matcher and set to top row.
+ // Only reset and call setRow if the row changes; avoids confusing the
+ // query matcher if scanning intra-row.
KeyValue kv = heap.peek();
- matcher.setRow((kv == null ? lastTopKey : kv).getRow());
+ if (kv == null) {
+ kv = lastTopKey;
+ }
+ if ((matcher.row == null) || !kv.matchingRow(matcher.row)) {
+ matcher.reset();
+ matcher.setRow(kv.getRow());
+ }
}
@Override
Added: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java?rev=1181548&view=auto
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java (added)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java Tue Oct 11 02:18:51 2011
@@ -0,0 +1,172 @@
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
+import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
+import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+
+public class TestFromClientSide2 {
+ final Log LOG = LogFactory.getLog(getClass());
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static byte [] ROW = Bytes.toBytes("testRow");
+ private static byte [] FAMILY = Bytes.toBytes("testFamily");
+ private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
+ private static byte [] VALUE = Bytes.toBytes("testValue");
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.startMiniCluster(3);
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before
+ public void setUp() throws Exception {
+ // Nothing to do.
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @After
+ public void tearDown() throws Exception {
+ // Nothing to do.
+ }
+
+
+ /**
+ * Test from client side for batch of scan
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testScanBatch() throws Exception {
+ byte [] TABLE = Bytes.toBytes("testScanBatch");
+ byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 8);
+
+ HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
+
+ Put put;
+ Scan scan;
+ Delete delete;
+ Result result;
+ ResultScanner scanner;
+ boolean toLog = true;
+ List<KeyValue> kvListExp;
+
+ // table: row, family, c0:0, c1:1, ... , c7:7
+ put = new Put(ROW);
+ for (int i=0; i < QUALIFIERS.length; i++) {
+ KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
+ put.add(kv);
+ }
+ ht.put(put);
+
+ // table: row, family, c0:0, c1:1, ..., c6:2, c6:6 , c7:7
+ put = new Put(ROW);
+ KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
+ put.add(kv);
+ ht.put(put);
+
+ // delete upto ts: 3
+ delete = new Delete(ROW);
+ delete.deleteFamily(FAMILY, 3);
+ ht.delete(delete);
+
+ // without batch
+ scan = new Scan(ROW);
+ scan.setMaxVersions();
+ scanner = ht.getScanner(scan);
+
+ // c4:4, c5:5, c6:6, c7:7
+ kvListExp = new ArrayList<KeyValue>();
+ kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
+ kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
+ kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
+ kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
+ result = scanner.next();
+ verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
+
+ // with batch
+ scan = new Scan(ROW);
+ scan.setMaxVersions();
+ scan.setBatch(2);
+ scanner = ht.getScanner(scan);
+
+ // First batch: c4:4, c5:5
+ kvListExp = new ArrayList<KeyValue>();
+ kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
+ kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
+ result = scanner.next();
+ verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
+
+ // Second batch: c6:6, c7:7
+ kvListExp = new ArrayList<KeyValue>();
+ kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
+ kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
+ result = scanner.next();
+ verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
+
+ }
+
+ private void verifyResult(Result result, List<KeyValue> kvList, boolean toLog, String msg) {
+ int i =0;
+
+ LOG.info(msg);
+ LOG.info("Exp cnt: " + kvList.size());
+ LOG.info("True cnt is: " + result.size());
+ assertEquals(kvList.size(), result.size());
+
+ for (KeyValue kv : result.sorted()) {
+ KeyValue kvExp = kvList.get(i++);
+ if (toLog) {
+ LOG.info("get kv is: " + kv.toString());
+ LOG.info("exp kv is: " + kvExp.toString());
+ }
+ assertTrue("Not equal", kvExp.equals(kv));
+ }
+
+ }
+
+ private byte [][] makeNAscii(byte [] base, int n) {
+ byte [][] ret = new byte[n][];
+ for(int i=0;i<n;i++) {
+ byte [] tail = Bytes.toBytes(Integer.toString(i));
+ ret[i] = Bytes.add(base, tail);
+ }
+ return ret;
+ }
+
+}
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java?rev=1181548&r1=1181547&r2=1181548&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java Tue Oct 11 02:18:51 2011
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.regionse
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.Random;
@@ -51,17 +52,17 @@ public class TestWideScanner extends HBa
new HTableDescriptor("testwidescan");
static {
TESTTABLEDESC.addFamily(new HColumnDescriptor(A,
- 10, // Ten is arbitrary number. Keep versions to help debuggging.
+ 100, // Keep versions to help debuggging.
Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
TESTTABLEDESC.addFamily(new HColumnDescriptor(B,
- 10, // Ten is arbitrary number. Keep versions to help debuggging.
+ 100, // Keep versions to help debuggging.
Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
TESTTABLEDESC.addFamily(new HColumnDescriptor(C,
- 10, // Ten is arbitrary number. Keep versions to help debuggging.
+ 100, // Keep versions to help debuggging.
Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
@@ -88,19 +89,23 @@ public class TestWideScanner extends HBa
int count = 0;
for (char c = 'a'; c <= 'c'; c++) {
byte[] row = Bytes.toBytes("ab" + c);
- int i;
- for (i = 0; i < 2500; i++) {
+ int i, j;
+ long ts = System.currentTimeMillis();
+ for (i = 0; i < 100; i++) {
byte[] b = Bytes.toBytes(String.format("%10d", i));
- Put put = new Put(row);
- put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, b);
- region.put(put);
- count++;
+ for (j = 0; j < 100; j++) {
+ Put put = new Put(row);
+ put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, ++ts, b);
+ region.put(put);
+ count++;
+ }
}
}
return count;
}
public void testWideScanBatching() throws IOException {
+ final int batch = 256;
try {
this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
int inserted = addWideContent(this.r);
@@ -109,7 +114,8 @@ public class TestWideScanner extends HBa
scan.addFamily(A);
scan.addFamily(B);
scan.addFamily(C);
- scan.setBatch(1000);
+ scan.setMaxVersions(100);
+ scan.setBatch(batch);
InternalScanner s = r.getScanner(scan);
int total = 0;
int i = 0;
@@ -120,7 +126,7 @@ public class TestWideScanner extends HBa
LOG.info("iteration #" + i + ", results.size=" + results.size());
// assert that the result set is no larger than 1000
- assertTrue(results.size() <= 1000);
+ assertTrue(results.size() <= batch);
total += results.size();
@@ -133,11 +139,19 @@ public class TestWideScanner extends HBa
}
results.clear();
+
+ // trigger ChangedReadersObservers
+ Iterator<KeyValueScanner> scanners =
+ ((HRegion.RegionScanner)s).storeHeap.getHeap().iterator();
+ while (scanners.hasNext()) {
+ StoreScanner ss = (StoreScanner)scanners.next();
+ ss.updateReaders();
+ }
} while (more);
// assert that the scanner returned all values
LOG.info("inserted " + inserted + ", scanned " + total);
- assertTrue(total == inserted);
+ assertEquals(total, inserted);
s.close();
} finally {