You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2009/06/06 03:26:27 UTC
svn commit: r782178 [15/16] - in /hadoop/hbase/trunk: bin/
src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/client/tableindexed/
src/java/org/apache/hadoop/hbase/client/transactional/ src/java/...
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java?rev=782178&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java Sat Jun 6 01:26:21 2009
@@ -0,0 +1,602 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.rmi.UnexpectedException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import junit.framework.TestCase;
+
+/** memcache test case */
+public class TestMemcache extends TestCase {
+
+ private Memcache memcache;
+
+ private static final int ROW_COUNT = 10;
+
+ private static final int QUALIFIER_COUNT = 10;
+
+ private static final byte [] FAMILY = Bytes.toBytes("column");
+
+ private static final int FIRST_ROW = 1;
+ private static final int NUM_VALS = 1000;
+ private static final byte [] CONTENTS_BASIC = Bytes.toBytes("contents:basic");
+ private static final String CONTENTSTR = "contentstr";
+ private static final String ANCHORNUM = "anchor:anchornum-";
+ private static final String ANCHORSTR = "anchorstr";
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ this.memcache = new Memcache();
+ }
+
+ public void testMultipleVersionsSimple() throws Exception {
+ Memcache m = new Memcache(HConstants.FOREVER, KeyValue.COMPARATOR);
+ byte [] row = Bytes.toBytes("testRow");
+ byte [] family = Bytes.toBytes("testFamily");
+ byte [] qf = Bytes.toBytes("testQualifier");
+ long [] stamps = {1,2,3};
+ byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
+ Bytes.toBytes("value2")};
+ KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
+ KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
+ KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
+
+ m.add(key0);
+ m.add(key1);
+ m.add(key2);
+
+ assertTrue("Expected memcache to hold 3 values, actually has " +
+ m.memcache.size(), m.memcache.size() == 3);
+ }
+
+ public void testBinary() throws IOException {
+ Memcache mc = new Memcache(HConstants.FOREVER, KeyValue.ROOT_COMPARATOR);
+ final int start = 43;
+ final int end = 46;
+ for (int k = start; k <= end; k++) {
+ byte [] kk = Bytes.toBytes(k);
+ byte [] row =
+ Bytes.toBytes(".META.,table," + Bytes.toString(kk) + ",1," + k);
+ KeyValue key = new KeyValue(row, CONTENTS_BASIC,
+ System.currentTimeMillis(),
+ (CONTENTSTR + k).getBytes(HConstants.UTF8_ENCODING));
+ mc.add(key);
+ System.out.println(key);
+// key = new KeyValue(row, Bytes.toBytes(ANCHORNUM + k),
+// System.currentTimeMillis(),
+// (ANCHORSTR + k).getBytes(HConstants.UTF8_ENCODING));
+// mc.add(key);
+// System.out.println(key);
+ }
+ int index = start;
+ for (KeyValue kv: mc.memcache) {
+ System.out.println(kv);
+ byte [] b = kv.getRow();
+ // Hardcoded offsets into String
+ String str = Bytes.toString(b, 13, 4);
+ byte [] bb = Bytes.toBytes(index);
+ String bbStr = Bytes.toString(bb);
+ assertEquals(str, bbStr);
+ index++;
+ }
+ }
+
+
+ /**
+ * Test memcache snapshots
+ * @throws IOException
+ */
+ public void testSnapshotting() throws IOException {
+ final int snapshotCount = 5;
+ // Add some rows, run a snapshot. Do it a few times.
+ for (int i = 0; i < snapshotCount; i++) {
+ addRows(this.memcache);
+ runSnapshot(this.memcache);
+ Set<KeyValue> ss = this.memcache.getSnapshot();
+ assertEquals("History not being cleared", 0, ss.size());
+ }
+ }
+
+
+ //////////////////////////////////////////////////////////////////////////////
+ // Get tests
+ //////////////////////////////////////////////////////////////////////////////
+ /** For HBASE-528 */
+ public void testGetRowKeyAtOrBefore() {
+ // set up some test data
+ byte [] t10 = Bytes.toBytes("010");
+ byte [] t20 = Bytes.toBytes("020");
+ byte [] t30 = Bytes.toBytes("030");
+ byte [] t35 = Bytes.toBytes("035");
+ byte [] t40 = Bytes.toBytes("040");
+
+ memcache.add(getKV(t10, "t10 bytes".getBytes()));
+ memcache.add(getKV(t20, "t20 bytes".getBytes()));
+ memcache.add(getKV(t30, "t30 bytes".getBytes()));
+ memcache.add(getKV(t35, "t35 bytes".getBytes()));
+ // write a delete in there to see if things still work ok
+ memcache.add(getDeleteKV(t35));
+ memcache.add(getKV(t40, "t40 bytes".getBytes()));
+
+ NavigableSet<KeyValue> results = null;
+
+ // try finding "015"
+ results =
+ new TreeSet<KeyValue>(this.memcache.comparator.getComparatorIgnoringType());
+ KeyValue t15 = new KeyValue(Bytes.toBytes("015"),
+ System.currentTimeMillis());
+ memcache.getRowKeyAtOrBefore(t15, results);
+ KeyValue kv = results.last();
+ assertTrue(KeyValue.COMPARATOR.compareRows(kv, t10) == 0);
+
+ // try "020", we should get that row exactly
+ results =
+ new TreeSet<KeyValue>(this.memcache.comparator.getComparatorIgnoringType());
+ memcache.getRowKeyAtOrBefore(new KeyValue(t20, System.currentTimeMillis()),
+ results);
+ assertTrue(KeyValue.COMPARATOR.compareRows(results.last(), t20) == 0);
+
+ // try "030", we should get that row exactly
+ results =
+ new TreeSet<KeyValue>(this.memcache.comparator.getComparatorIgnoringType());
+ memcache.getRowKeyAtOrBefore(new KeyValue(t30, System.currentTimeMillis()),
+ results);
+ assertTrue(KeyValue.COMPARATOR.compareRows(results.last(), t30) == 0);
+
+ // try "038", should skip the deleted "035" and give "030"
+ results =
+ new TreeSet<KeyValue>(this.memcache.comparator.getComparatorIgnoringType());
+ byte [] t38 = Bytes.toBytes("038");
+ memcache.getRowKeyAtOrBefore(new KeyValue(t38, System.currentTimeMillis()),
+ results);
+ assertTrue(KeyValue.COMPARATOR.compareRows(results.last(), t30) == 0);
+
+ // try "050", should get stuff from "040"
+ results =
+ new TreeSet<KeyValue>(this.memcache.comparator.getComparatorIgnoringType());
+ byte [] t50 = Bytes.toBytes("050");
+ memcache.getRowKeyAtOrBefore(new KeyValue(t50, System.currentTimeMillis()),
+ results);
+ assertTrue(KeyValue.COMPARATOR.compareRows(results.last(), t40) == 0);
+ }
+
+
+ /** Test getNextRow from memcache
+ * @throws InterruptedException
+ */
+ public void testGetNextRow() throws Exception {
+ addRows(this.memcache);
+ // Add more versions to make it a little more interesting.
+ Thread.sleep(1);
+ addRows(this.memcache);
+ KeyValue closestToEmpty = this.memcache.getNextRow(KeyValue.LOWESTKEY);
+ assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
+ new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ KeyValue nr = this.memcache.getNextRow(new KeyValue(Bytes.toBytes(i),
+ System.currentTimeMillis()));
+ if (i + 1 == ROW_COUNT) {
+ assertEquals(nr, null);
+ } else {
+ assertTrue(KeyValue.COMPARATOR.compareRows(nr,
+ new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
+ }
+ }
+ //starting from each row, validate results should contain the starting row
+ for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
+ InternalScanner scanner =
+ new StoreScanner(new Scan(Bytes.toBytes(startRowId)), FAMILY,
+ Integer.MAX_VALUE, this.memcache.comparator, null,
+ new KeyValueScanner[]{memcache.getScanner()});
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ for (int i = 0; scanner.next(results); i++) {
+ int rowId = startRowId + i;
+ assertTrue("Row name",
+ KeyValue.COMPARATOR.compareRows(results.get(0),
+ Bytes.toBytes(rowId)) == 0);
+ assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
+ List<KeyValue> row = new ArrayList<KeyValue>();
+ for (KeyValue kv : results) {
+ row.add(kv);
+ }
+ isExpectedRowWithoutTimestamps(rowId, row);
+ // Clear out set. Otherwise row results accumulate.
+ results.clear();
+ }
+ }
+ }
+
+ public void testGet_Basic_Found() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier1");
+ byte [] qf2 = Bytes.toBytes("testqualifier2");
+ byte [] qf3 = Bytes.toBytes("testqualifier3");
+ byte [] val = Bytes.toBytes("testval");
+
+ //Setting up memcache
+ KeyValue add1 = new KeyValue(row, fam ,qf1, val);
+ KeyValue add2 = new KeyValue(row, fam ,qf2, val);
+ KeyValue add3 = new KeyValue(row, fam ,qf3, val);
+ memcache.add(add1);
+ memcache.add(add2);
+ memcache.add(add3);
+
+ //test
+ Get get = new Get(row);
+ NavigableSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ columns.add(qf2);
+ long ttl = Long.MAX_VALUE;
+
+ QueryMatcher matcher =
+ new QueryMatcher(get, row, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
+
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ boolean res = memcache.get(matcher, result);
+ assertEquals(true, res);
+ }
+
+ public void testGet_Basic_NotFound() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier1");
+ byte [] qf2 = Bytes.toBytes("testqualifier2");
+ byte [] qf3 = Bytes.toBytes("testqualifier3");
+ byte [] val = Bytes.toBytes("testval");
+
+ //Setting up memcache
+ KeyValue add1 = new KeyValue(row, fam ,qf1, val);
+ KeyValue add3 = new KeyValue(row, fam ,qf3, val);
+ memcache.add(add1);
+ memcache.add(add3);
+
+ //test
+ Get get = new Get(row);
+ NavigableSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ columns.add(qf2);
+ long ttl = Long.MAX_VALUE;
+
+ QueryMatcher matcher =
+ new QueryMatcher(get, row, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
+
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ boolean res = memcache.get(matcher, result);
+ assertEquals(false, res);
+ }
+
+ public void testGet_MemcacheAndSnapShot() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier1");
+ byte [] qf2 = Bytes.toBytes("testqualifier2");
+ byte [] qf3 = Bytes.toBytes("testqualifier3");
+ byte [] qf4 = Bytes.toBytes("testqualifier4");
+ byte [] qf5 = Bytes.toBytes("testqualifier5");
+ byte [] val = Bytes.toBytes("testval");
+
+ //Creating get
+ Get get = new Get(row);
+ NavigableSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ columns.add(qf2);
+ columns.add(qf4);
+ long ttl = Long.MAX_VALUE;
+
+ QueryMatcher matcher =
+ new QueryMatcher(get, row, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
+
+ //Setting up memcache
+ memcache.add(new KeyValue(row, fam ,qf1, val));
+ memcache.add(new KeyValue(row, fam ,qf2, val));
+ memcache.add(new KeyValue(row, fam ,qf3, val));
+ //Creating a snapshot
+ memcache.snapshot();
+ assertEquals(3, memcache.snapshot.size());
+ //Adding value to "new" memcache
+ assertEquals(0, memcache.memcache.size());
+ memcache.add(new KeyValue(row, fam ,qf4, val));
+ memcache.add(new KeyValue(row, fam ,qf5, val));
+ assertEquals(2, memcache.memcache.size());
+
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ boolean res = memcache.get(matcher, result);
+ assertEquals(true, res);
+ }
+
+ public void testGet_SpecificTimeStamp() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier1");
+ byte [] qf2 = Bytes.toBytes("testqualifier2");
+ byte [] qf3 = Bytes.toBytes("testqualifier3");
+ byte [] val = Bytes.toBytes("testval");
+
+ long ts1 = System.currentTimeMillis();
+ long ts2 = ts1++;
+ long ts3 = ts2++;
+
+ //Creating get
+ Get get = new Get(row);
+ get.setTimeStamp(ts2);
+ NavigableSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ columns.add(qf1);
+ columns.add(qf2);
+ columns.add(qf3);
+ long ttl = Long.MAX_VALUE;
+
+ QueryMatcher matcher =
+ new QueryMatcher(get, row, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
+
+ //Setting up expected
+ List<KeyValue> expected = new ArrayList<KeyValue>();
+ KeyValue kv1 = new KeyValue(row, fam ,qf1, ts2, val);
+ KeyValue kv2 = new KeyValue(row, fam ,qf2, ts2, val);
+ KeyValue kv3 = new KeyValue(row, fam ,qf3, ts2, val);
+ expected.add(kv1);
+ expected.add(kv2);
+ expected.add(kv3);
+
+ //Setting up memcache
+ memcache.add(new KeyValue(row, fam ,qf1, ts1, val));
+ memcache.add(new KeyValue(row, fam ,qf2, ts1, val));
+ memcache.add(new KeyValue(row, fam ,qf3, ts1, val));
+ memcache.add(kv1);
+ memcache.add(kv2);
+ memcache.add(kv3);
+ memcache.add(new KeyValue(row, fam ,qf1, ts3, val));
+ memcache.add(new KeyValue(row, fam ,qf2, ts3, val));
+ memcache.add(new KeyValue(row, fam ,qf3, ts3, val));
+
+ //Get
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ memcache.get(matcher, result);
+
+ assertEquals(expected.size(), result.size());
+ for(int i=0; i<expected.size(); i++){
+ assertEquals(expected.get(i), result.get(i));
+ }
+ }
+
+
+ //////////////////////////////////////////////////////////////////////////////
+ // Delete tests
+ //////////////////////////////////////////////////////////////////////////////
+ public void testGetWithDelete() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier");
+ byte [] val = Bytes.toBytes("testval");
+
+ long ts1 = System.nanoTime();
+ KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
+ long ts2 = ts1 + 1;
+ KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
+ long ts3 = ts2 +1;
+ KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
+ memcache.add(put1);
+ memcache.add(put2);
+ memcache.add(put3);
+
+ assertEquals(3, memcache.memcache.size());
+
+ KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
+ memcache.delete(del2);
+
+ List<KeyValue> expected = new ArrayList<KeyValue>();
+ expected.add(put3);
+ expected.add(del2);
+ expected.add(put1);
+
+ assertEquals(3, memcache.memcache.size());
+ int i=0;
+ for(KeyValue actual : memcache.memcache) {
+ assertEquals(expected.get(i++), actual);
+ }
+ }
+
+ public void testGetWithDeleteColumn() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier");
+ byte [] val = Bytes.toBytes("testval");
+
+ long ts1 = System.nanoTime();
+ KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
+ long ts2 = ts1 + 1;
+ KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
+ long ts3 = ts2 +1;
+ KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
+ memcache.add(put1);
+ memcache.add(put2);
+ memcache.add(put3);
+
+ assertEquals(3, memcache.memcache.size());
+
+ KeyValue del2 =
+ new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
+ memcache.delete(del2);
+
+ List<KeyValue> expected = new ArrayList<KeyValue>();
+ expected.add(put3);
+ expected.add(del2);
+
+ assertEquals(2, memcache.memcache.size());
+ int i=0;
+ for(KeyValue actual : memcache.memcache) {
+ assertEquals(expected.get(i++), actual);
+ }
+ }
+
+
+ public void testGetWithDeleteFamily() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier1");
+ byte [] qf2 = Bytes.toBytes("testqualifier2");
+ byte [] qf3 = Bytes.toBytes("testqualifier3");
+ byte [] val = Bytes.toBytes("testval");
+ long ts = System.nanoTime();
+
+ KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
+ KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
+ KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
+ memcache.add(put1);
+ memcache.add(put2);
+ memcache.add(put3);
+
+ KeyValue del =
+ new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
+ memcache.delete(del);
+
+ List<KeyValue> expected = new ArrayList<KeyValue>();
+ expected.add(del);
+
+ assertEquals(1, memcache.memcache.size());
+ int i=0;
+ for(KeyValue actual : memcache.memcache) {
+ assertEquals(expected.get(i++), actual);
+ }
+ }
+
+ public void testKeepDeleteInMemcache() {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf = Bytes.toBytes("testqualifier");
+ byte [] val = Bytes.toBytes("testval");
+ long ts = System.nanoTime();
+ memcache.add(new KeyValue(row, fam, qf, ts, val));
+ KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
+ memcache.delete(delete);
+ assertEquals(1, memcache.memcache.size());
+ assertEquals(delete, memcache.memcache.first());
+ }
+
+ public void testRetainsDeleteVersion() throws IOException {
+ // add a put to memcache
+ memcache.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
+
+ // now process a specific delete:
+ KeyValue delete = KeyValueTestUtil.create(
+ "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
+ memcache.delete(delete);
+
+ assertEquals(1, memcache.memcache.size());
+ assertEquals(delete, memcache.memcache.first());
+ }
+ public void testRetainsDeleteColumn() throws IOException {
+ // add a put to memcache
+ memcache.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
+
+ // now process a specific delete:
+ KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
+ KeyValue.Type.DeleteColumn, "dont-care");
+ memcache.delete(delete);
+
+ assertEquals(1, memcache.memcache.size());
+ assertEquals(delete, memcache.memcache.first());
+ }
+ public void testRetainsDeleteFamily() throws IOException {
+ // add a put to memcache
+ memcache.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
+
+ // now process a specific delete:
+ KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
+ KeyValue.Type.DeleteFamily, "dont-care");
+ memcache.delete(delete);
+
+ assertEquals(1, memcache.memcache.size());
+ assertEquals(delete, memcache.memcache.first());
+ }
+
+
+ //////////////////////////////////////////////////////////////////////////////
+ // Helpers
+ //////////////////////////////////////////////////////////////////////////////
+ private byte [] makeQualifier(final int i1, final int i2){
+ return Bytes.toBytes(Integer.toString(i1) + ";" +
+ Integer.toString(i2));
+ }
+
+ /**
+ * Adds {@link #ROW_COUNT} rows and {@link #COLUMNS_COUNT}
+ * @param hmc Instance to add rows to.
+ * @throws IOException
+ */
+ private void addRows(final Memcache hmc) {
+ for (int i = 0; i < ROW_COUNT; i++) {
+ long timestamp = System.currentTimeMillis();
+ for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
+ byte [] row = Bytes.toBytes(i);
+ byte [] qf = makeQualifier(i, ii);
+ hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
+ }
+ }
+ }
+
+ private void runSnapshot(final Memcache hmc) throws UnexpectedException {
+ // Save off old state.
+ int oldHistorySize = hmc.getSnapshot().size();
+ hmc.snapshot();
+ Set<KeyValue> ss = hmc.getSnapshot();
+ // Make some assertions about what just happened.
+ assertTrue("History size has not increased", oldHistorySize < ss.size());
+ hmc.clearSnapshot(ss);
+ }
+
+ private void isExpectedRowWithoutTimestamps(final int rowIndex,
+ List<KeyValue> kvs) {
+ int i = 0;
+ for (KeyValue kv: kvs) {
+ String expectedColname = Bytes.toString(makeQualifier(rowIndex, i++));
+ String colnameStr = Bytes.toString(kv.getQualifier());
+ assertEquals("Column name", colnameStr, expectedColname);
+ // Value is column name as bytes. Usually result is
+ // 100 bytes in size at least. This is the default size
+ // for BytesWriteable. For comparison, convert bytes to
+ // String and trim to remove trailing null bytes.
+ String colvalueStr = Bytes.toString(kv.getBuffer(), kv.getValueOffset(),
+ kv.getValueLength());
+ assertEquals("Content", colnameStr, colvalueStr);
+ }
+ }
+
+ private KeyValue getDeleteKV(byte [] row) {
+ return new KeyValue(row, Bytes.toBytes("test_col:"),
+ HConstants.LATEST_TIMESTAMP, KeyValue.Type.Delete, null);
+ }
+
+ private KeyValue getKV(byte [] row, byte [] value) {
+ return new KeyValue(row, Bytes.toBytes("test_col:"),
+ HConstants.LATEST_TIMESTAMP, value);
+ }
+}
\ No newline at end of file
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMinorCompactingStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMinorCompactingStoreScanner.java?rev=782178&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMinorCompactingStoreScanner.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMinorCompactingStoreScanner.java Sat Jun 6 01:26:21 2009
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestMinorCompactingStoreScanner extends TestCase {
+
+ public void testDeleteFamiliy() throws IOException {
+ KeyValue[] kvs = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ InternalScanner scan =
+ new MinorCompactingStoreScanner("cf", KeyValue.COMPARATOR, scanners);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertTrue(scan.next(results));
+ assertEquals(3, results.size());
+ assertEquals(kvs[0], results.get(0));
+ assertEquals(kvs[5], results.get(1));
+ assertEquals(kvs[8], results.get(2));
+
+ results.clear();
+ assertFalse(scan.next(results));
+ assertEquals(1, results.size());
+ assertEquals(kvs[kvs.length-1], results.get(0));
+ }
+
+ public void testDeleteVersion() throws IOException {
+ KeyValue[] kvs = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", 15, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.Delete, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.Put, "dont-care")
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ InternalScanner scan =
+ new MinorCompactingStoreScanner("cf", KeyValue.COMPARATOR, scanners);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertFalse(scan.next(results));
+ assertEquals(2, results.size());
+ assertEquals(kvs[0], results.get(0));
+ assertEquals(kvs[1], results.get(1));
+ }
+}
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java?rev=782178&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java Sat Jun 6 01:26:21 2009
@@ -0,0 +1,170 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KeyComparator;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
+
+
+public class TestQueryMatcher extends HBaseTestCase
+implements HConstants {
+ private final boolean PRINT = false;
+
+ private byte [] row1;
+ private byte [] row2;
+ private byte [] fam1;
+ private byte [] fam2;
+ private byte [] col1;
+ private byte [] col2;
+ private byte [] col3;
+ private byte [] col4;
+ private byte [] col5;
+ private byte [] col6;
+
+ private byte [] data;
+
+ private Get get;
+
+ long ttl = Long.MAX_VALUE;
+ KeyComparator rowComparator;
+
+ public void setUp(){
+ row1 = Bytes.toBytes("row1");
+ row2 = Bytes.toBytes("row2");
+ fam1 = Bytes.toBytes("fam1");
+ fam2 = Bytes.toBytes("fam2");
+ col1 = Bytes.toBytes("col1");
+ col2 = Bytes.toBytes("col2");
+ col3 = Bytes.toBytes("col3");
+ col4 = Bytes.toBytes("col4");
+ col5 = Bytes.toBytes("col5");
+ col6 = Bytes.toBytes("col6");
+
+ data = Bytes.toBytes("data");
+
+ //Create Get
+ get = new Get(row1);
+ get.addFamily(fam1);
+ get.addColumn(fam2, col2);
+ get.addColumn(fam2, col4);
+ get.addColumn(fam2, col5);
+
+ rowComparator = KeyValue.KEY_COMPARATOR;
+
+ }
+
+ public void testMatch_ExplicitColumns()
+ throws IOException {
+ //Moving up from the Tracker by using Gets and List<KeyValue> instead
+ //of just byte []
+
+ //Expected result
+ List<MatchCode> expected = new ArrayList<MatchCode>();
+ expected.add(MatchCode.SKIP);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.SKIP);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.DONE);
+
+ QueryMatcher qm = new QueryMatcher(get, get.getRow(), fam2,
+ get.getFamilyMap().get(fam2), ttl, rowComparator, 1);
+
+ List<KeyValue> memCache = new ArrayList<KeyValue>();
+ memCache.add(new KeyValue(row1, fam2, col1, data));
+ memCache.add(new KeyValue(row1, fam2, col2, data));
+ memCache.add(new KeyValue(row1, fam2, col3, data));
+ memCache.add(new KeyValue(row1, fam2, col4, data));
+ memCache.add(new KeyValue(row1, fam2, col5, data));
+
+ memCache.add(new KeyValue(row2, fam1, col1, data));
+
+ List<MatchCode> actual = new ArrayList<MatchCode>();
+
+ for(KeyValue kv : memCache){
+ actual.add(qm.match(kv));
+ }
+
+ assertEquals(expected.size(), actual.size());
+ for(int i=0; i< expected.size(); i++){
+ assertEquals(expected.get(i), actual.get(i));
+ if(PRINT){
+ System.out.println("expected "+expected.get(i)+
+ ", actual " +actual.get(i));
+ }
+ }
+ }
+
+
+ public void testMatch_Wildcard()
+ throws IOException {
+ //Moving up from the Tracker by using Gets and List<KeyValue> instead
+ //of just byte []
+
+ //Expected result
+ List<MatchCode> expected = new ArrayList<MatchCode>();
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.NEXT);
+
+ QueryMatcher qm = new QueryMatcher(get, get.getRow(), fam2, null,
+ ttl, rowComparator, 1);
+
+ List<KeyValue> memCache = new ArrayList<KeyValue>();
+ memCache.add(new KeyValue(row1, fam2, col1, data));
+ memCache.add(new KeyValue(row1, fam2, col2, data));
+ memCache.add(new KeyValue(row1, fam2, col3, data));
+ memCache.add(new KeyValue(row1, fam2, col4, data));
+ memCache.add(new KeyValue(row1, fam2, col5, data));
+ memCache.add(new KeyValue(row2, fam1, col1, data));
+
+ List<MatchCode> actual = new ArrayList<MatchCode>();
+
+ for(KeyValue kv : memCache){
+ actual.add(qm.match(kv));
+ }
+
+ assertEquals(expected.size(), actual.size());
+ for(int i=0; i< expected.size(); i++){
+ assertEquals(expected.get(i), actual.get(i));
+ if(PRINT){
+ System.out.println("expected "+expected.get(i)+
+ ", actual " +actual.get(i));
+ }
+ }
+ }
+
+}
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java?rev=782178&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanDeleteTracker.java Sat Jun 6 01:26:21 2009
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+
+
+public class TestScanDeleteTracker extends HBaseTestCase implements HConstants {
+
+ private ScanDeleteTracker sdt;
+ private long timestamp = 10L;
+ private byte deleteType = 0;
+
+ public void setUp(){
+ sdt = new ScanDeleteTracker(KeyValue.KEY_COMPARATOR);
+ }
+
+ public void testDeletedBy_Delete() {
+ byte [] qualifier = Bytes.toBytes("qualifier");
+ deleteType = KeyValue.Type.Delete.getCode();
+
+ sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType);
+ boolean ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp);
+ assertEquals(true, ret);
+ }
+
+ public void testDeletedBy_DeleteColumn() {
+ byte [] qualifier = Bytes.toBytes("qualifier");
+ deleteType = KeyValue.Type.DeleteColumn.getCode();
+
+ sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType);
+ timestamp -= 5;
+ boolean ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp);
+ assertEquals(true, ret);
+ }
+
+ public void testDeletedBy_DeleteFamily() {
+ byte [] qualifier = Bytes.toBytes("qualifier");
+ deleteType = KeyValue.Type.DeleteFamily.getCode();
+
+ sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType);
+
+ timestamp -= 5;
+ boolean ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp);
+ assertEquals(true, ret);
+ }
+
+ public void testDelete_DeleteColumn() {
+ byte [] qualifier = Bytes.toBytes("qualifier");
+ deleteType = KeyValue.Type.Delete.getCode();
+
+ sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType);
+
+ timestamp -= 5;
+ deleteType = KeyValue.Type.DeleteColumn.getCode();
+ sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType);
+
+ timestamp -= 5;
+ boolean ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp);
+ assertEquals(true, ret);
+ }
+
+
+ public void testDeleteColumn_Delete() {
+ byte [] qualifier = Bytes.toBytes("qualifier");
+ deleteType = KeyValue.Type.DeleteColumn.getCode();
+
+ sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType);
+
+ qualifier = Bytes.toBytes("qualifier1");
+ deleteType = KeyValue.Type.Delete.getCode();
+ sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType);
+
+ boolean ret = sdt.isDeleted(qualifier, 0, qualifier.length, timestamp);
+ assertEquals(true, ret);
+ }
+
+ //Testing new way where we save the Delete in case of a Delete for specific
+ //ts, could have just added the last line to the first test, but rather keep
+ //them separated
+ public void testDelete_KeepDelete(){
+ byte [] qualifier = Bytes.toBytes("qualifier");
+ deleteType = KeyValue.Type.Delete.getCode();
+
+ sdt.add(qualifier, 0, qualifier.length, timestamp, deleteType);
+ sdt.isDeleted(qualifier, 0, qualifier.length, timestamp);
+ assertEquals(false ,sdt.isEmpty());
+ }
+
+
+}
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java?rev=782178&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java Sat Jun 6 01:26:21 2009
@@ -0,0 +1,101 @@
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
+
+public class TestScanWildcardColumnTracker extends HBaseTestCase {
+
+ final int VERSIONS = 2;
+
+ public void testCheckColumn_Ok() {
+ //Create a WildcardColumnTracker
+ ScanWildcardColumnTracker tracker =
+ new ScanWildcardColumnTracker(VERSIONS);
+
+ //Create list of qualifiers
+ List<byte[]> qualifiers = new ArrayList<byte[]>();
+ qualifiers.add(Bytes.toBytes("qualifer1"));
+ qualifiers.add(Bytes.toBytes("qualifer2"));
+ qualifiers.add(Bytes.toBytes("qualifer3"));
+ qualifiers.add(Bytes.toBytes("qualifer4"));
+
+ //Setting up expected result
+ List<MatchCode> expected = new ArrayList<MatchCode>();
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.INCLUDE);
+
+ List<MatchCode> actual = new ArrayList<MatchCode>();
+
+ for(byte [] qualifier : qualifiers) {
+ MatchCode mc = tracker.checkColumn(qualifier, 0, qualifier.length);
+ actual.add(mc);
+ }
+
+ //Compare actual with expected
+ for(int i=0; i<expected.size(); i++) {
+ assertEquals(expected.get(i), actual.get(i));
+ }
+ }
+
+ public void testCheckColumn_EnforceVersions() {
+ //Create a WildcardColumnTracker
+ ScanWildcardColumnTracker tracker =
+ new ScanWildcardColumnTracker(VERSIONS);
+
+ //Create list of qualifiers
+ List<byte[]> qualifiers = new ArrayList<byte[]>();
+ qualifiers.add(Bytes.toBytes("qualifer1"));
+ qualifiers.add(Bytes.toBytes("qualifer1"));
+ qualifiers.add(Bytes.toBytes("qualifer1"));
+ qualifiers.add(Bytes.toBytes("qualifer2"));
+
+ //Setting up expected result
+ List<MatchCode> expected = new ArrayList<MatchCode>();
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.INCLUDE);
+ expected.add(MatchCode.SKIP);
+ expected.add(MatchCode.INCLUDE);
+
+ List<MatchCode> actual = new ArrayList<MatchCode>();
+
+ for(byte [] qualifier : qualifiers) {
+ MatchCode mc = tracker.checkColumn(qualifier, 0, qualifier.length);
+ actual.add(mc);
+ }
+
+ //Compare actual with expected
+ for(int i=0; i<expected.size(); i++) {
+ assertEquals(expected.get(i), actual.get(i));
+ }
+ }
+
+ public void testCheckColumn_WrongOrder() {
+ //Create a WildcardColumnTracker
+ ScanWildcardColumnTracker tracker =
+ new ScanWildcardColumnTracker(VERSIONS);
+
+ //Create list of qualifiers
+ List<byte[]> qualifiers = new ArrayList<byte[]>();
+ qualifiers.add(Bytes.toBytes("qualifer2"));
+ qualifiers.add(Bytes.toBytes("qualifer1"));
+
+ boolean ok = false;
+
+ try {
+ for(byte [] qualifier : qualifiers) {
+ MatchCode mc = tracker.checkColumn(qualifier, 0, qualifier.length);
+ }
+ } catch (Exception e) {
+ ok = true;
+ }
+
+ assertEquals(true, ok);
+ }
+
+}
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=782178&r1=782177&r2=782178&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java Sat Jun 6 01:26:21 2009
@@ -34,6 +34,10 @@
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.StopRowFilter;
import org.apache.hadoop.hbase.filter.WhileMatchRowFilter;
import org.apache.hadoop.hbase.io.BatchUpdate;
@@ -48,24 +52,21 @@
public class TestScanner extends HBaseTestCase {
private final Log LOG = LogFactory.getLog(this.getClass());
- private static final byte [] FIRST_ROW =
- HConstants.EMPTY_START_ROW;
- private static final byte [][] COLS = {
- HConstants.COLUMN_FAMILY
- };
+ private static final byte [] FIRST_ROW = HConstants.EMPTY_START_ROW;
+ private static final byte [][] COLS = { HConstants.CATALOG_FAMILY };
private static final byte [][] EXPLICIT_COLS = {
- HConstants.COL_REGIONINFO,
- HConstants.COL_SERVER,
- HConstants.COL_STARTCODE
+ HConstants.REGIONINFO_QUALIFIER, HConstants.SERVER_QUALIFIER,
+ // TODO ryan
+ //HConstants.STARTCODE_QUALIFIER
};
static final HTableDescriptor TESTTABLEDESC =
new HTableDescriptor("testscanner");
static {
- TESTTABLEDESC.addFamily(new HColumnDescriptor(HConstants.COLUMN_FAMILY,
+ TESTTABLEDESC.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY,
10, // Ten is arbitrary number. Keep versions to help debuggging.
Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
- Integer.MAX_VALUE, HConstants.FOREVER, false));
+ HConstants.FOREVER, false));
}
/** HRegionInfo for root region */
public static final HRegionInfo REGION_INFO =
@@ -99,12 +100,13 @@
byte [] stoprow = Bytes.toBytes("ccc");
try {
this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
- addContent(this.r, HConstants.COLUMN_FAMILY);
+ addContent(this.r, HConstants.CATALOG_FAMILY);
List<KeyValue> results = new ArrayList<KeyValue>();
// Do simple test of getting one row only first.
- InternalScanner s = r.getScanner(HConstants.COLUMN_FAMILY_ARRAY,
- Bytes.toBytes("abc"), HConstants.LATEST_TIMESTAMP,
- new WhileMatchRowFilter(new StopRowFilter(Bytes.toBytes("abd"))));
+ Scan scan = new Scan(Bytes.toBytes("abc"), Bytes.toBytes("abd"));
+ scan.addFamily(HConstants.CATALOG_FAMILY);
+
+ InternalScanner s = r.getScanner(scan);
int count = 0;
while (s.next(results)) {
count++;
@@ -112,9 +114,10 @@
s.close();
assertEquals(1, count);
// Now do something a bit more imvolved.
- s = r.getScanner(HConstants.COLUMN_FAMILY_ARRAY,
- startrow, HConstants.LATEST_TIMESTAMP,
- new WhileMatchRowFilter(new StopRowFilter(stoprow)));
+ scan = new Scan(startrow, stoprow);
+ scan.addFamily(HConstants.CATALOG_FAMILY);
+
+ s = r.getScanner(scan);
count = 0;
KeyValue kv = null;
results = new ArrayList<KeyValue>();
@@ -147,14 +150,15 @@
// Write information to the meta table
- BatchUpdate batchUpdate =
- new BatchUpdate(ROW_KEY, System.currentTimeMillis());
+ Put put = new Put(ROW_KEY);
+ put.setTimeStamp(System.currentTimeMillis());
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutputStream s = new DataOutputStream(byteStream);
REGION_INFO.write(s);
- batchUpdate.put(HConstants.COL_REGIONINFO, byteStream.toByteArray());
- region.commit(batchUpdate);
+ put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ byteStream.toByteArray());
+ region.put(put);
// What we just committed is in the memcache. Verify that we can get
// it back both with scanning and get
@@ -177,13 +181,14 @@
HServerAddress address = new HServerAddress("foo.bar.com:1234");
- batchUpdate = new BatchUpdate(ROW_KEY, System.currentTimeMillis());
+ put = new Put(ROW_KEY);
+ put.setTimeStamp(System.currentTimeMillis());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+ Bytes.toBytes(address.toString()));
- batchUpdate.put(HConstants.COL_SERVER, Bytes.toBytes(address.toString()));
+// put.add(HConstants.COL_STARTCODE, Bytes.toBytes(START_CODE));
- batchUpdate.put(HConstants.COL_STARTCODE, Bytes.toBytes(START_CODE));
-
- region.commit(batchUpdate);
+ region.put(put);
// Validate that we can still get the HRegionInfo, even though it is in
// an older row on disk and there is a newer row in the memcache
@@ -215,12 +220,12 @@
address = new HServerAddress("bar.foo.com:4321");
- batchUpdate = new BatchUpdate(ROW_KEY, System.currentTimeMillis());
-
- batchUpdate.put(HConstants.COL_SERVER,
- Bytes.toBytes(address.toString()));
+ put = new Put(ROW_KEY);
+ put.setTimeStamp(System.currentTimeMillis());
- region.commit(batchUpdate);
+ put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+ Bytes.toBytes(address.toString()));
+ region.put(put);
// Validate again
@@ -273,6 +278,7 @@
private void scan(boolean validateStartcode, String serverName)
throws IOException {
InternalScanner scanner = null;
+ Scan scan = null;
List<KeyValue> results = new ArrayList<KeyValue>();
byte [][][] scanColumns = {
COLS,
@@ -281,15 +287,20 @@
for(int i = 0; i < scanColumns.length; i++) {
try {
- scanner = r.getScanner(scanColumns[i], FIRST_ROW,
- System.currentTimeMillis(), null);
+ scan = new Scan(FIRST_ROW);
+ scan.addColumns(scanColumns[i]);
+ scanner = r.getScanner(scan);
while (scanner.next(results)) {
- assertTrue(hasColumn(results, HConstants.COL_REGIONINFO));
- byte [] val = getColumn(results, HConstants.COL_REGIONINFO).getValue();
+ assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER));
+ byte [] val = getColumn(results, HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER).getValue();
validateRegionInfo(val);
if(validateStartcode) {
- assertTrue(hasColumn(results, HConstants.COL_STARTCODE));
- val = getColumn(results, HConstants.COL_STARTCODE).getValue();
+// assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
+// HConstants.STARTCODE_QUALIFIER));
+// val = getColumn(results, HConstants.CATALOG_FAMILY,
+// HConstants.STARTCODE_QUALIFIER).getValue();
assertNotNull(val);
assertFalse(val.length == 0);
long startCode = Bytes.toLong(val);
@@ -297,8 +308,10 @@
}
if(serverName != null) {
- assertTrue(hasColumn(results, HConstants.COL_SERVER));
- val = getColumn(results, HConstants.COL_SERVER).getValue();
+ assertTrue(hasColumn(results, HConstants.CATALOG_FAMILY,
+ HConstants.SERVER_QUALIFIER));
+ val = getColumn(results, HConstants.CATALOG_FAMILY,
+ HConstants.SERVER_QUALIFIER).getValue();
assertNotNull(val);
assertFalse(val.length == 0);
String server = Bytes.toString(val);
@@ -317,27 +330,33 @@
}
}
- private boolean hasColumn(final List<KeyValue> kvs, final byte [] column) {
+ private boolean hasColumn(final List<KeyValue> kvs, final byte [] family,
+ final byte [] qualifier) {
for (KeyValue kv: kvs) {
- if (kv.matchingColumn(column)) {
+ if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) {
return true;
}
}
- return false;
+ return false;
}
-
- private KeyValue getColumn(final List<KeyValue> kvs, final byte [] column) {
+
+ private KeyValue getColumn(final List<KeyValue> kvs, final byte [] family,
+ final byte [] qualifier) {
for (KeyValue kv: kvs) {
- if (kv.matchingColumn(column)) {
+ if (kv.matchingFamily(family) && kv.matchingQualifier(qualifier)) {
return kv;
}
}
return null;
}
-
+
+
/** Use get to retrieve the HRegionInfo and validate it */
private void getRegionInfo() throws IOException {
- byte [] bytes = region.get(ROW_KEY, HConstants.COL_REGIONINFO).getValue();
+ Get get = new Get(ROW_KEY);
+ get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+ Result result = region.get(get, null);
+ byte [] bytes = result.value();
validateRegionInfo(bytes);
}
@@ -349,8 +368,9 @@
this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
HRegionIncommon hri = new HRegionIncommon(r);
try {
- LOG.info("Added: " +
- addContent(hri, Bytes.toString(HConstants.COL_REGIONINFO)));
+ String columnString = Bytes.toString(HConstants.CATALOG_FAMILY) + ':' +
+ Bytes.toString(HConstants.REGIONINFO_QUALIFIER);
+ LOG.info("Added: " + addContent(hri, columnString));
int count = count(hri, -1);
assertEquals(count, count(hri, 100));
assertEquals(count, count(hri, 0));
@@ -374,7 +394,7 @@
private int count(final HRegionIncommon hri, final int flushIndex)
throws IOException {
LOG.info("Taking out counting scan");
- ScannerIncommon s = hri.getScanner(EXPLICIT_COLS,
+ ScannerIncommon s = hri.getScanner(HConstants.CATALOG_FAMILY, EXPLICIT_COLS,
HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP);
List<KeyValue> values = new ArrayList<KeyValue>();
int count = 0;
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java?rev=782178&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java Sat Jun 6 01:26:21 2009
@@ -0,0 +1,325 @@
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.Progressable;
+
+import junit.framework.TestCase;
+
+/**
+ * Test class fosr the Store
+ */
+public class TestStore extends TestCase {
+ Store store;
+ byte [] table = Bytes.toBytes("table");
+ byte [] family = Bytes.toBytes("family");
+
+ byte [] row = Bytes.toBytes("row");
+ byte [] qf1 = Bytes.toBytes("qf1");
+ byte [] qf2 = Bytes.toBytes("qf2");
+ byte [] qf3 = Bytes.toBytes("qf3");
+ byte [] qf4 = Bytes.toBytes("qf4");
+ byte [] qf5 = Bytes.toBytes("qf5");
+ byte [] qf6 = Bytes.toBytes("qf6");
+
+ NavigableSet<byte[]> qualifiers =
+ new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
+
+ List<KeyValue> expected = new ArrayList<KeyValue>();
+ List<KeyValue> result = new ArrayList<KeyValue>();
+
+ long id = System.currentTimeMillis();
+ Get get = new Get(row);
+
+ private final String DIR = "test/build/data/TestStore/";
+
+ /**
+ * Setup
+ * @throws IOException
+ */
+ @Override
+ public void setUp() throws IOException {
+ qualifiers.add(qf1);
+ qualifiers.add(qf3);
+ qualifiers.add(qf5);
+
+ Iterator<byte[]> iter = qualifiers.iterator();
+ while(iter.hasNext()){
+ byte [] next = iter.next();
+ expected.add(new KeyValue(row, family, next, null));
+ get.addColumn(family, next);
+ }
+ }
+
+ private void init(String methodName) throws IOException {
+ //Setting up a Store
+ Path basedir = new Path(DIR+methodName);
+ HColumnDescriptor hcd = new HColumnDescriptor(family);
+ HBaseConfiguration conf = new HBaseConfiguration();
+ FileSystem fs = FileSystem.get(conf);
+ Path reconstructionLog = null;
+ Progressable reporter = null;
+
+ HTableDescriptor htd = new HTableDescriptor(table);
+ htd.addFamily(hcd);
+ HRegionInfo info = new HRegionInfo(htd, null, null, false);
+
+ store = new Store(basedir, info, hcd, fs, reconstructionLog, conf,
+ reporter);
+ }
+
+
+ //////////////////////////////////////////////////////////////////////////////
+ // Get tests
+ //////////////////////////////////////////////////////////////////////////////
+ /**
+ * Getting data from memcache only
+ * @throws IOException
+ */
+ public void testGet_FromMemCacheOnly() throws IOException {
+ init(this.getName());
+
+ //Put data in memcache
+ this.store.add(new KeyValue(row, family, qf1, null));
+ this.store.add(new KeyValue(row, family, qf2, null));
+ this.store.add(new KeyValue(row, family, qf3, null));
+ this.store.add(new KeyValue(row, family, qf4, null));
+ this.store.add(new KeyValue(row, family, qf5, null));
+ this.store.add(new KeyValue(row, family, qf6, null));
+
+ //Get
+ this.store.get(get, qualifiers, result);
+
+ //Compare
+ assertCheck();
+ }
+
+ /**
+ * Getting data from files only
+ * @throws IOException
+ */
+ public void testGet_FromFilesOnly() throws IOException {
+ init(this.getName());
+
+ //Put data in memcache
+ this.store.add(new KeyValue(row, family, qf1, null));
+ this.store.add(new KeyValue(row, family, qf2, null));
+ //flush
+ flush(1);
+
+ //Add more data
+ this.store.add(new KeyValue(row, family, qf3, null));
+ this.store.add(new KeyValue(row, family, qf4, null));
+ //flush
+ flush(2);
+
+ //Add more data
+ this.store.add(new KeyValue(row, family, qf5, null));
+ this.store.add(new KeyValue(row, family, qf6, null));
+ //flush
+ flush(3);
+
+ //Get
+ this.store.get(get, qualifiers, result);
+
+ //Need to sort the result since multiple files
+ Collections.sort(result, KeyValue.COMPARATOR);
+
+ //Compare
+ assertCheck();
+ }
+
+ /**
+ * Getting data from memcache and files
+ * @throws IOException
+ */
+ public void testGet_FromMemCacheAndFiles() throws IOException {
+ init(this.getName());
+
+ //Put data in memcache
+ this.store.add(new KeyValue(row, family, qf1, null));
+ this.store.add(new KeyValue(row, family, qf2, null));
+ //flush
+ flush(1);
+
+ //Add more data
+ this.store.add(new KeyValue(row, family, qf3, null));
+ this.store.add(new KeyValue(row, family, qf4, null));
+ //flush
+ flush(2);
+
+ //Add more data
+ this.store.add(new KeyValue(row, family, qf5, null));
+ this.store.add(new KeyValue(row, family, qf6, null));
+
+ //Get
+ this.store.get(get, qualifiers, result);
+
+ //Need to sort the result since multiple files
+ Collections.sort(result, KeyValue.COMPARATOR);
+
+ //Compare
+ assertCheck();
+ }
+
+ private void flush(int storeFilessize) throws IOException{
+ this.store.snapshot();
+ this.store.flushCache(id++);
+ assertEquals(storeFilessize, this.store.getStorefiles().size());
+ assertEquals(0, this.store.memcache.memcache.size());
+ }
+
+ private void assertCheck() {
+ assertEquals(expected.size(), result.size());
+ for(int i=0; i<expected.size(); i++) {
+ assertEquals(expected.get(i), result.get(i));
+ }
+ }
+
+ //////////////////////////////////////////////////////////////////////////////
+ // IncrementColumnValue tests
+ //////////////////////////////////////////////////////////////////////////////
+ /**
+ * Testing if the update in place works. When you want to update a value that
+ * is already in memcache, you don't delete it and put a new one, but just
+ * update the value in the original KeyValue
+ * @throws IOException
+ */
+ public void testIncrementColumnValue_UpdatingInPlace() throws IOException {
+ init(this.getName());
+
+ //Put data in memcache
+ long value = 1L;
+ long amount = 3L;
+ this.store.add(new KeyValue(row, family, qf1, Bytes.toBytes(value)));
+
+ this.store.incrementColumnValue(row, family, qf1, amount);
+ Get get = new Get(row);
+ get.addColumn(family, qf1);
+ NavigableSet<byte[]> qualifiers =
+ new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ qualifiers.add(qf1);
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ this.store.get(get, qualifiers, result);
+ assertEquals(value + amount, Bytes.toLong(result.get(0).getValue()));
+ }
+
+ /**
+ * Same as above but for a negative number
+ * @throws IOException
+ */
+ public void testIncrementColumnValue_UpdatingInPlace_Negative()
+ throws IOException {
+ init(this.getName());
+
+ //Put data in memcache
+ long value = 3L;
+ long amount = -1L;
+ this.store.add(new KeyValue(row, family, qf1, Bytes.toBytes(value)));
+
+ this.store.incrementColumnValue(row, family, qf1, amount);
+ Get get = new Get(row);
+ get.addColumn(family, qf1);
+ NavigableSet<byte[]> qualifiers =
+ new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ qualifiers.add(qf1);
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ this.store.get(get, qualifiers, result);
+ assertEquals(value + amount, Bytes.toLong(result.get(0).getValue()));
+ }
+
+ /**
+ * When there is no mathing key already, adding a new.
+ * @throws IOException
+ */
+ public void testIncrementColumnValue_AddingNew() throws IOException {
+ init(this.getName());
+
+ //Put data in memcache
+ long value = 1L;
+ long amount = 3L;
+ this.store.add(new KeyValue(row, family, qf1, Bytes.toBytes(value)));
+ this.store.add(new KeyValue(row, family, qf2, Bytes.toBytes(value)));
+
+ this.store.incrementColumnValue(row, family, qf3, amount);
+ Get get = new Get(row);
+ get.addColumn(family, qf3);
+ NavigableSet<byte[]> qualifiers =
+ new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ qualifiers.add(qf3);
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ this.store.get(get, qualifiers, result);
+ assertEquals(amount, Bytes.toLong(result.get(0).getValue()));
+ }
+
+ /**
+ * When we have the key in a file add a new key + value to memcache with the
+ * updates value.
+ * @throws IOException
+ */
+ public void testIncrementColumnValue_UpdatingFromSF() throws IOException {
+ init(this.getName());
+
+ //Put data in memcache
+ long value = 1L;
+ long amount = 3L;
+ this.store.add(new KeyValue(row, family, qf1, Bytes.toBytes(value)));
+ this.store.add(new KeyValue(row, family, qf2, Bytes.toBytes(value)));
+
+ flush(1);
+
+ this.store.incrementColumnValue(row, family, qf1, amount);
+ Get get = new Get(row);
+ get.addColumn(family, qf1);
+ NavigableSet<byte[]> qualifiers =
+ new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ qualifiers.add(qf1);
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ this.store.get(get, qualifiers, result);
+ assertEquals(value + amount, Bytes.toLong(result.get(0).getValue()));
+ }
+
+ /**
+ * Same as testIncrementColumnValue_AddingNew() except that the keys are
+ * checked in file not in memcache
+ * @throws IOException
+ */
+ public void testIncrementColumnValue_AddingNewAfterSFCheck()
+ throws IOException {
+ init(this.getName());
+
+ //Put data in memcache
+ long value = 1L;
+ long amount = 3L;
+ this.store.add(new KeyValue(row, family, qf1, Bytes.toBytes(value)));
+ this.store.add(new KeyValue(row, family, qf2, Bytes.toBytes(value)));
+
+ flush(1);
+
+ this.store.incrementColumnValue(row, family, qf3, amount);
+ Get get = new Get(row);
+ get.addColumn(family, qf3);
+ NavigableSet<byte[]> qualifiers =
+ new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ qualifiers.add(qf3);
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ this.store.get(get, qualifiers, result);
+ assertEquals(amount, Bytes.toLong(result.get(0).getValue()));
+ }
+
+}
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=782178&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java Sat Jun 6 01:26:21 2009
@@ -0,0 +1,480 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.RowInclusiveStopFilter;
+import org.apache.hadoop.hbase.filter.RowWhileMatchFilter;
+import org.apache.hadoop.hbase.filter.RowPrefixFilter;
+import org.apache.hadoop.hbase.filter.RowFilterInterface;
+import org.apache.hadoop.hbase.filter.WhileMatchRowFilter;
+import org.apache.hadoop.hbase.filter.PrefixRowFilter;
+import org.apache.hadoop.hbase.filter.InclusiveStopRowFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+
+
+public class TestStoreScanner extends TestCase {
+
+ final byte [] CF = Bytes.toBytes("cf");
+
+ /**
+ * Test utility for building a NavigableSet for scanners.
+ * @param strCols
+ * @return
+ */
+ NavigableSet<byte[]> getCols(String ...strCols) {
+ NavigableSet<byte[]> cols = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ for (String col : strCols) {
+ byte[] bytes = Bytes.toBytes(col);
+ cols.add(bytes);
+ }
+ return cols;
+ }
+
+ /**
+ * Test test shows exactly how the matcher's return codes confuses the StoreScanner
+ * and prevent it from doing the right thing. Seeking once, then nexting twice
+ * should return R1, then R2, but in this case it doesnt.
+ * @throws IOException
+ */
+ public void testWontNextToNext() throws IOException {
+ // build the scan file:
+ KeyValue [] kvs = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR,
+ kvs)
+ };
+
+ Scan scanSpec = new Scan(Bytes.toBytes("R1"));
+ // this only uses maxVersions (default=1) and TimeRange (default=all)
+ StoreScanner scan =
+ new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+ KeyValue.COMPARATOR, getCols("a"),
+ scanners);
+
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ scan.next(results);
+ assertEquals(1, results.size());
+ assertEquals(kvs[0], results.get(0));
+ // should be ok...
+ // now scan _next_ again.
+ results.clear();
+ scan.next(results);
+ assertEquals(1, results.size());
+ assertEquals(kvs[2], results.get(0));
+
+ results.clear();
+ scan.next(results);
+ assertEquals(0, results.size());
+
+ }
+
+
+ public void testDeleteVersionSameTimestamp() throws IOException {
+ KeyValue [] kvs = new KeyValue [] {
+ KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ Scan scanSpec = new Scan(Bytes.toBytes("R1"));
+ StoreScanner scan =
+ new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ getCols("a"), scanners);
+
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertFalse(scan.next(results));
+ assertEquals(0, results.size());
+ }
+
+ /**
+ * Test the case where there is a delete row 'in front of' the next row, the scanner
+ * will move to the next row.
+ */
+ public void testDeletedRowThenGoodRow() throws IOException {
+ KeyValue [] kvs = new KeyValue [] {
+ KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Delete, "dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "a", 20, KeyValue.Type.Put, "dont-care")
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ Scan scanSpec = new Scan(Bytes.toBytes("R1"));
+ StoreScanner scan =
+ new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ getCols("a"), scanners);
+
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(1, results.size());
+ assertEquals(kvs[2], results.get(0));
+ }
+
+ public void testDeleteVersionMaskingMultiplePuts() throws IOException {
+ long now = System.currentTimeMillis();
+ KeyValue [] kvs1 = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
+ };
+ KeyValue [] kvs2 = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care")
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs1),
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs2)
+ };
+ StoreScanner scan =
+ new StoreScanner(new Scan(Bytes.toBytes("R1")), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ getCols("a"), scanners);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ // the two put at ts=now will be masked by the 1 delete, and
+ // since the scan default returns 1 version we'll return the newest
+ // key, which is kvs[2], now-100.
+ assertEquals(true, scan.next(results));
+ assertEquals(1, results.size());
+ assertEquals(kvs2[1], results.get(0));
+ }
+ public void testDeleteVersionsMixedAndMultipleVersionReturn() throws IOException {
+ long now = System.currentTimeMillis();
+ KeyValue [] kvs1 = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care")
+ };
+ KeyValue [] kvs2 = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", now+500, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "z", now, KeyValue.Type.Put, "dont-care")
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs1),
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs2)
+ };
+ Scan scanSpec = new Scan(Bytes.toBytes("R1")).setMaxVersions(2);
+ StoreScanner scan =
+ new StoreScanner(scanSpec, CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ getCols("a"), scanners);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(2, results.size());
+ assertEquals(kvs2[1], results.get(0));
+ assertEquals(kvs2[0], results.get(1));
+ }
+
+ public void testWildCardOneVersionScan() throws IOException {
+ KeyValue [] kvs = new KeyValue [] {
+ KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ StoreScanner scan =
+ new StoreScanner(new Scan(Bytes.toBytes("R1")), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ null, scanners);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(2, results.size());
+ assertEquals(kvs[0], results.get(0));
+ assertEquals(kvs[1], results.get(1));
+ }
+ public void testWildCardScannerUnderDeletes() throws IOException {
+ KeyValue [] kvs = new KeyValue [] {
+ KeyValueTestUtil.create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), // inc
+ // orphaned delete column.
+ KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"),
+ // column b
+ KeyValueTestUtil.create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), // inc
+ KeyValueTestUtil.create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), // inc
+ // column c
+ KeyValueTestUtil.create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), // no
+ KeyValueTestUtil.create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"), // inc
+ // column d
+ KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), // inc
+ KeyValueTestUtil.create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"), // no
+ KeyValueTestUtil.create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), // no
+
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ StoreScanner scan =
+ new StoreScanner(new Scan().setMaxVersions(2), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ null, scanners);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(5, results.size());
+ assertEquals(kvs[0], results.get(0));
+ assertEquals(kvs[2], results.get(1));
+ assertEquals(kvs[3], results.get(2));
+ assertEquals(kvs[6], results.get(3));
+ assertEquals(kvs[7], results.get(4));
+ }
+ public void testDeleteFamily() throws IOException {
+ KeyValue [] kvs = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ StoreScanner scan =
+ new StoreScanner(new Scan().setMaxVersions(Integer.MAX_VALUE), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ null, scanners);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(1, results.size());
+ assertEquals(kvs[kvs.length-1], results.get(0));
+ }
+
+ public void testDeleteColumn() throws IOException {
+ KeyValue [] kvs = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", 10, KeyValue.Type.DeleteColumn, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", 9, KeyValue.Type.Delete, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "a", 8, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "b", 5, KeyValue.Type.Put, "dont-care")
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs),
+ };
+ StoreScanner scan =
+ new StoreScanner(new Scan(), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ null, scanners);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(1, results.size());
+ assertEquals(kvs[3], results.get(0));
+ }
+
+ public void testSkipColumn() throws IOException {
+ KeyValue [] kvs = new KeyValue[] {
+ KeyValueTestUtil.create("R1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ };
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+ };
+ StoreScanner scan =
+ new StoreScanner(new Scan(), CF, Long.MAX_VALUE, KeyValue.COMPARATOR,
+ getCols("a", "d"), scanners);
+
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertEquals(true, scan.next(results));
+ assertEquals(2, results.size());
+ assertEquals(kvs[0], results.get(0));
+ assertEquals(kvs[3], results.get(1));
+ results.clear();
+
+ assertEquals(true, scan.next(results));
+ assertEquals(1, results.size());
+ assertEquals(kvs[kvs.length-1], results.get(0));
+
+ results.clear();
+ assertEquals(false, scan.next(results));
+ }
+
+ KeyValue [] stdKvs = new KeyValue[] {
+ KeyValueTestUtil.create("R:1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"),
+
+ // 9...
+ KeyValueTestUtil.create("R:2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:2", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:2", "cf", "c", 10, KeyValue.Type.Put, "dont-care"),
+
+ // 12...
+ KeyValueTestUtil.create("R:3", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:3", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:3", "cf", "c", 10, KeyValue.Type.Put, "dont-care"),
+
+ // 15 ...
+ KeyValueTestUtil.create("R:4", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:4", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:4", "cf", "c", 10, KeyValue.Type.Put, "dont-care"),
+
+ // 18 ..
+ KeyValueTestUtil.create("R:5", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:5", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+
+ // 20...
+ KeyValueTestUtil.create("R:6", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:6", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+
+ // 22...
+ KeyValueTestUtil.create("R:7", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:7", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+
+ // 24...
+ KeyValueTestUtil.create("R:8", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ KeyValueTestUtil.create("R:8", "cf", "c", 11, KeyValue.Type.Put, "dont-care"),
+
+ // 26 ..
+ KeyValueTestUtil.create("RA:1", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+
+ // 27...
+ KeyValueTestUtil.create("RA:2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+
+ // 28..
+ KeyValueTestUtil.create("RA:3", "cf", "a", 11, KeyValue.Type.Put, "dont-care"),
+ };
+ private StoreScanner getTestScanner(Scan s, NavigableSet<byte[]> cols) {
+ KeyValueScanner [] scanners = new KeyValueScanner[] {
+ new KeyValueScanFixture(KeyValue.COMPARATOR, stdKvs)
+ };
+
+ return new StoreScanner(s, CF, Long.MAX_VALUE, KeyValue.COMPARATOR, cols,
+ scanners);
+ }
+
+
+ // Test new and old row prefix filters.
+ public void testNewRowPrefixFilter() throws IOException {
+ Filter f = new RowWhileMatchFilter(
+ new RowPrefixFilter(Bytes.toBytes("R:")));
+ Scan s = new Scan(Bytes.toBytes("R:7"));
+ s.setFilter(f);
+
+ rowPrefixFilter(s);
+ }
+
+ public void testOldRowPrefixFilter() throws IOException {
+ RowFilterInterface f = new WhileMatchRowFilter(
+ new PrefixRowFilter(Bytes.toBytes("R:")));
+ Scan s = new Scan(Bytes.toBytes("R:7"));
+ s.setOldFilter(f);
+
+ rowPrefixFilter(s);
+
+ }
+ public void rowPrefixFilter(Scan s) throws IOException {
+
+ StoreScanner scan = getTestScanner(s, null);
+
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertTrue(scan.next(results));
+ assertEquals(2, results.size());
+ assertEquals(stdKvs[22], results.get(0));
+ assertEquals(stdKvs[23], results.get(1));
+ results.clear();
+
+ assertTrue(scan.next(results));
+ assertEquals(2, results.size());
+ assertEquals(stdKvs[24], results.get(0));
+ assertEquals(stdKvs[25], results.get(1));
+ results.clear();
+
+ assertFalse(scan.next(results));
+ assertEquals(0, results.size());
+ }
+
+ // Test new and old row-inclusive stop filter.
+ public void testNewRowInclusiveStopFilter() throws IOException {
+ Filter f = new RowWhileMatchFilter(new RowInclusiveStopFilter(Bytes.toBytes("R:3")));
+ Scan scan = new Scan();
+ scan.setFilter(f);
+
+ rowInclusiveStopFilter(scan);
+ }
+
+ public void testOldRowInclusiveTopFilter() throws IOException {
+ RowFilterInterface f = new WhileMatchRowFilter(
+ new InclusiveStopRowFilter(Bytes.toBytes("R:3")));
+ Scan scan = new Scan();
+ scan.setOldFilter(f);
+
+ rowInclusiveStopFilter(scan);
+ }
+
+ public void rowInclusiveStopFilter(Scan scan) throws IOException {
+ StoreScanner s = getTestScanner(scan, getCols("a"));
+
+ // read crap.
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ assertTrue(s.next(results));
+ assertEquals(1, results.size());
+ assertEquals(stdKvs[0], results.get(0));
+ results.clear();
+
+ assertTrue(s.next(results));
+ assertEquals(1, results.size());
+ assertEquals(stdKvs[9], results.get(0));
+ results.clear();
+
+ assertTrue(s.next(results));
+ assertEquals(1, results.size());
+ assertEquals(stdKvs[12], results.get(0));
+ results.clear();
+
+ // without aggressive peeking, the scanner doesnt know if the next row is good or not
+ // under the affects of a filter.
+ assertFalse(s.next(results));
+ assertEquals(0, results.size());
+ }
+
+
+
+}