You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by an...@apache.org on 2014/03/14 15:07:10 UTC
svn commit: r1577541 [3/3] - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/io/
test/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/regionserver/wal/
Added: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java?rev=1577541&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java (added)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java Fri Mar 14 14:07:10 2014
@@ -0,0 +1,1036 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.rmi.UnexpectedException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdge;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+/** memstore test case */
+@Category(MediumTests.class)
+public class TestDefaultMemStore extends TestCase {
+ private final Log LOG = LogFactory.getLog(this.getClass());
+ private DefaultMemStore memstore;
+ private static final int ROW_COUNT = 10;
+ private static final int QUALIFIER_COUNT = ROW_COUNT;
+ private static final byte [] FAMILY = Bytes.toBytes("column");
+ private MultiVersionConsistencyControl mvcc;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ this.mvcc = new MultiVersionConsistencyControl();
+ this.memstore = new DefaultMemStore();
+ }
+
+ public void testPutSameKey() {
+ byte [] bytes = Bytes.toBytes(getName());
+ KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
+ this.memstore.add(kv);
+ byte [] other = Bytes.toBytes("somethingelse");
+ KeyValue samekey = new KeyValue(bytes, bytes, bytes, other);
+ this.memstore.add(samekey);
+ KeyValue found = this.memstore.kvset.first();
+ assertEquals(1, this.memstore.kvset.size());
+ assertTrue(Bytes.toString(found.getValue()), CellUtil.matchingValue(samekey, found));
+ }
+
+ /**
+ * Test memstore snapshot happening while scanning.
+ * @throws IOException
+ */
+ public void testScanAcrossSnapshot() throws IOException {
+ int rowCount = addRows(this.memstore);
+ List<KeyValueScanner> memstorescanners = this.memstore.getScanners(0);
+ Scan scan = new Scan();
+ List<Cell> result = new ArrayList<Cell>();
+ ScanInfo scanInfo = new ScanInfo(null, 0, 1, HConstants.LATEST_TIMESTAMP, false,
+ 0, this.memstore.comparator);
+ ScanType scanType = ScanType.USER_SCAN;
+ StoreScanner s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
+ int count = 0;
+ try {
+ while (s.next(result)) {
+ LOG.info(result);
+ count++;
+ // Row count is same as column count.
+ assertEquals(rowCount, result.size());
+ result.clear();
+ }
+ } finally {
+ s.close();
+ }
+ assertEquals(rowCount, count);
+ for (KeyValueScanner scanner : memstorescanners) {
+ scanner.close();
+ }
+
+ memstorescanners = this.memstore.getScanners(mvcc.memstoreReadPoint());
+ // Now assert can count same number even if a snapshot mid-scan.
+ s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
+ count = 0;
+ try {
+ while (s.next(result)) {
+ LOG.info(result);
+ // Assert the stuff is coming out in right order.
+ assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
+ count++;
+ // Row count is same as column count.
+ assertEquals(rowCount, result.size());
+ if (count == 2) {
+ this.memstore.snapshot();
+ LOG.info("Snapshotted");
+ }
+ result.clear();
+ }
+ } finally {
+ s.close();
+ }
+ assertEquals(rowCount, count);
+ for (KeyValueScanner scanner : memstorescanners) {
+ scanner.close();
+ }
+ memstorescanners = this.memstore.getScanners(mvcc.memstoreReadPoint());
+ // Assert that new values are seen in kvset as we scan.
+ long ts = System.currentTimeMillis();
+ s = new StoreScanner(scan, scanInfo, scanType, null, memstorescanners);
+ count = 0;
+ int snapshotIndex = 5;
+ try {
+ while (s.next(result)) {
+ LOG.info(result);
+ // Assert the stuff is coming out in right order.
+ assertTrue(CellUtil.matchingRow(result.get(0), Bytes.toBytes(count)));
+ // Row count is same as column count.
+ assertEquals("count=" + count + ", result=" + result, rowCount, result.size());
+ count++;
+ if (count == snapshotIndex) {
+ MemStoreSnapshot snapshot = this.memstore.snapshot();
+ this.memstore.clearSnapshot(snapshot.getId());
+ // Added more rows into kvset. But the scanner wont see these rows.
+ addRows(this.memstore, ts);
+ LOG.info("Snapshotted, cleared it and then added values (which wont be seen)");
+ }
+ result.clear();
+ }
+ } finally {
+ s.close();
+ }
+ assertEquals(rowCount, count);
+ }
+
+ /**
+ * A simple test which verifies the 3 possible states when scanning across snapshot.
+ * @throws IOException
+ * @throws CloneNotSupportedException
+ */
+ public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException {
+ // we are going to the scanning across snapshot with two kvs
+ // kv1 should always be returned before kv2
+ final byte[] one = Bytes.toBytes(1);
+ final byte[] two = Bytes.toBytes(2);
+ final byte[] f = Bytes.toBytes("f");
+ final byte[] q = Bytes.toBytes("q");
+ final byte[] v = Bytes.toBytes(3);
+
+ final KeyValue kv1 = new KeyValue(one, f, q, v);
+ final KeyValue kv2 = new KeyValue(two, f, q, v);
+
+ // use case 1: both kvs in kvset
+ this.memstore.add(kv1.clone());
+ this.memstore.add(kv2.clone());
+ verifyScanAcrossSnapshot2(kv1, kv2);
+
+ // use case 2: both kvs in snapshot
+ this.memstore.snapshot();
+ verifyScanAcrossSnapshot2(kv1, kv2);
+
+ // use case 3: first in snapshot second in kvset
+ this.memstore = new DefaultMemStore();
+ this.memstore.add(kv1.clone());
+ this.memstore.snapshot();
+ this.memstore.add(kv2.clone());
+ verifyScanAcrossSnapshot2(kv1, kv2);
+ }
+
+ private void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
+ throws IOException {
+ List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.memstoreReadPoint());
+ assertEquals(1, memstorescanners.size());
+ final KeyValueScanner scanner = memstorescanners.get(0);
+ scanner.seek(KeyValue.createFirstOnRow(HConstants.EMPTY_START_ROW));
+ assertEquals(kv1, scanner.next());
+ assertEquals(kv2, scanner.next());
+ assertNull(scanner.next());
+ }
+
+ private void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
+ throws IOException {
+ scanner.seek(KeyValue.createFirstOnRow(new byte[]{}));
+ List<Cell> returned = Lists.newArrayList();
+
+ while (true) {
+ KeyValue next = scanner.next();
+ if (next == null) break;
+ returned.add(next);
+ }
+
+ assertTrue(
+ "Got:\n" + Joiner.on("\n").join(returned) +
+ "\nExpected:\n" + Joiner.on("\n").join(expected),
+ Iterables.elementsEqual(Arrays.asList(expected), returned));
+ assertNull(scanner.peek());
+ }
+
+ public void testMemstoreConcurrentControl() throws IOException {
+ final byte[] row = Bytes.toBytes(1);
+ final byte[] f = Bytes.toBytes("family");
+ final byte[] q1 = Bytes.toBytes("q1");
+ final byte[] q2 = Bytes.toBytes("q2");
+ final byte[] v = Bytes.toBytes("value");
+
+ MultiVersionConsistencyControl.WriteEntry w =
+ mvcc.beginMemstoreInsert();
+
+ KeyValue kv1 = new KeyValue(row, f, q1, v);
+ kv1.setMvccVersion(w.getWriteNumber());
+ memstore.add(kv1);
+
+ KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{});
+
+ mvcc.completeMemstoreInsert(w);
+
+ s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{kv1});
+
+ w = mvcc.beginMemstoreInsert();
+ KeyValue kv2 = new KeyValue(row, f, q2, v);
+ kv2.setMvccVersion(w.getWriteNumber());
+ memstore.add(kv2);
+
+ s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{kv1});
+
+ mvcc.completeMemstoreInsert(w);
+
+ s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{kv1, kv2});
+ }
+
+ /**
+ * Regression test for HBASE-2616, HBASE-2670.
+ * When we insert a higher-memstoreTS version of a cell but with
+ * the same timestamp, we still need to provide consistent reads
+ * for the same scanner.
+ */
+ public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
+ final byte[] row = Bytes.toBytes(1);
+ final byte[] f = Bytes.toBytes("family");
+ final byte[] q1 = Bytes.toBytes("q1");
+ final byte[] q2 = Bytes.toBytes("q2");
+ final byte[] v1 = Bytes.toBytes("value1");
+ final byte[] v2 = Bytes.toBytes("value2");
+
+ // INSERT 1: Write both columns val1
+ MultiVersionConsistencyControl.WriteEntry w =
+ mvcc.beginMemstoreInsert();
+
+ KeyValue kv11 = new KeyValue(row, f, q1, v1);
+ kv11.setMvccVersion(w.getWriteNumber());
+ memstore.add(kv11);
+
+ KeyValue kv12 = new KeyValue(row, f, q2, v1);
+ kv12.setMvccVersion(w.getWriteNumber());
+ memstore.add(kv12);
+ mvcc.completeMemstoreInsert(w);
+
+ // BEFORE STARTING INSERT 2, SEE FIRST KVS
+ KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{kv11, kv12});
+
+ // START INSERT 2: Write both columns val2
+ w = mvcc.beginMemstoreInsert();
+ KeyValue kv21 = new KeyValue(row, f, q1, v2);
+ kv21.setMvccVersion(w.getWriteNumber());
+ memstore.add(kv21);
+
+ KeyValue kv22 = new KeyValue(row, f, q2, v2);
+ kv22.setMvccVersion(w.getWriteNumber());
+ memstore.add(kv22);
+
+ // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
+ s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{kv11, kv12});
+
+ // COMPLETE INSERT 2
+ mvcc.completeMemstoreInsert(w);
+
+ // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
+ // See HBASE-1485 for discussion about what we should do with
+ // the duplicate-TS inserts
+ s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
+ }
+
+ /**
+ * When we insert a higher-memstoreTS deletion of a cell but with
+ * the same timestamp, we still need to provide consistent reads
+ * for the same scanner.
+ */
+ public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
+ final byte[] row = Bytes.toBytes(1);
+ final byte[] f = Bytes.toBytes("family");
+ final byte[] q1 = Bytes.toBytes("q1");
+ final byte[] q2 = Bytes.toBytes("q2");
+ final byte[] v1 = Bytes.toBytes("value1");
+ // INSERT 1: Write both columns val1
+ MultiVersionConsistencyControl.WriteEntry w =
+ mvcc.beginMemstoreInsert();
+
+ KeyValue kv11 = new KeyValue(row, f, q1, v1);
+ kv11.setMvccVersion(w.getWriteNumber());
+ memstore.add(kv11);
+
+ KeyValue kv12 = new KeyValue(row, f, q2, v1);
+ kv12.setMvccVersion(w.getWriteNumber());
+ memstore.add(kv12);
+ mvcc.completeMemstoreInsert(w);
+
+ // BEFORE STARTING INSERT 2, SEE FIRST KVS
+ KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{kv11, kv12});
+
+ // START DELETE: Insert delete for one of the columns
+ w = mvcc.beginMemstoreInsert();
+ KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
+ KeyValue.Type.DeleteColumn);
+ kvDel.setMvccVersion(w.getWriteNumber());
+ memstore.add(kvDel);
+
+ // BEFORE COMPLETING DELETE, SEE FIRST KVS
+ s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{kv11, kv12});
+
+ // COMPLETE DELETE
+ mvcc.completeMemstoreInsert(w);
+
+ // NOW WE SHOULD SEE DELETE
+ s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
+ }
+
+
+ private static class ReadOwnWritesTester extends Thread {
+ static final int NUM_TRIES = 1000;
+
+ final byte[] row;
+
+ final byte[] f = Bytes.toBytes("family");
+ final byte[] q1 = Bytes.toBytes("q1");
+
+ final MultiVersionConsistencyControl mvcc;
+ final MemStore memstore;
+
+ AtomicReference<Throwable> caughtException;
+
+
+ public ReadOwnWritesTester(int id,
+ MemStore memstore,
+ MultiVersionConsistencyControl mvcc,
+ AtomicReference<Throwable> caughtException)
+ {
+ this.mvcc = mvcc;
+ this.memstore = memstore;
+ this.caughtException = caughtException;
+ row = Bytes.toBytes(id);
+ }
+
+ public void run() {
+ try {
+ internalRun();
+ } catch (Throwable t) {
+ caughtException.compareAndSet(null, t);
+ }
+ }
+
+ private void internalRun() throws IOException {
+ for (long i = 0; i < NUM_TRIES && caughtException.get() == null; i++) {
+ MultiVersionConsistencyControl.WriteEntry w =
+ mvcc.beginMemstoreInsert();
+
+ // Insert the sequence value (i)
+ byte[] v = Bytes.toBytes(i);
+
+ KeyValue kv = new KeyValue(row, f, q1, i, v);
+ kv.setMvccVersion(w.getWriteNumber());
+ memstore.add(kv);
+ mvcc.completeMemstoreInsert(w);
+
+ // Assert that we can read back
+ KeyValueScanner s = this.memstore.getScanners(mvcc.memstoreReadPoint()).get(0);
+ s.seek(kv);
+
+ KeyValue ret = s.next();
+ assertNotNull("Didnt find own write at all", ret);
+ assertEquals("Didnt read own writes",
+ kv.getTimestamp(), ret.getTimestamp());
+ }
+ }
+ }
+
+ public void testReadOwnWritesUnderConcurrency() throws Throwable {
+
+ int NUM_THREADS = 8;
+
+ ReadOwnWritesTester threads[] = new ReadOwnWritesTester[NUM_THREADS];
+ AtomicReference<Throwable> caught = new AtomicReference<Throwable>();
+
+ for (int i = 0; i < NUM_THREADS; i++) {
+ threads[i] = new ReadOwnWritesTester(i, memstore, mvcc, caught);
+ threads[i].start();
+ }
+
+ for (int i = 0; i < NUM_THREADS; i++) {
+ threads[i].join();
+ }
+
+ if (caught.get() != null) {
+ throw caught.get();
+ }
+ }
+
+ /**
+ * Test memstore snapshots
+ * @throws IOException
+ */
+ public void testSnapshotting() throws IOException {
+ final int snapshotCount = 5;
+ // Add some rows, run a snapshot. Do it a few times.
+ for (int i = 0; i < snapshotCount; i++) {
+ addRows(this.memstore);
+ runSnapshot(this.memstore);
+ assertEquals("History not being cleared", 0, this.memstore.snapshot.size());
+ }
+ }
+
+ public void testMultipleVersionsSimple() throws Exception {
+ DefaultMemStore m = new DefaultMemStore(new Configuration(), KeyValue.COMPARATOR);
+ byte [] row = Bytes.toBytes("testRow");
+ byte [] family = Bytes.toBytes("testFamily");
+ byte [] qf = Bytes.toBytes("testQualifier");
+ long [] stamps = {1,2,3};
+ byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
+ Bytes.toBytes("value2")};
+ KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
+ KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
+ KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);
+
+ m.add(key0);
+ m.add(key1);
+ m.add(key2);
+
+ assertTrue("Expected memstore to hold 3 values, actually has " +
+ m.kvset.size(), m.kvset.size() == 3);
+ }
+
+ //////////////////////////////////////////////////////////////////////////////
+ // Get tests
+ //////////////////////////////////////////////////////////////////////////////
+
+ /** Test getNextRow from memstore
+ * @throws InterruptedException
+ */
+ public void testGetNextRow() throws Exception {
+ addRows(this.memstore);
+ // Add more versions to make it a little more interesting.
+ Thread.sleep(1);
+ addRows(this.memstore);
+ KeyValue closestToEmpty = this.memstore.getNextRow(KeyValue.LOWESTKEY);
+ assertTrue(KeyValue.COMPARATOR.compareRows(closestToEmpty,
+ new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
+ for (int i = 0; i < ROW_COUNT; i++) {
+ KeyValue nr = this.memstore.getNextRow(new KeyValue(Bytes.toBytes(i),
+ System.currentTimeMillis()));
+ if (i + 1 == ROW_COUNT) {
+ assertEquals(nr, null);
+ } else {
+ assertTrue(KeyValue.COMPARATOR.compareRows(nr,
+ new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
+ }
+ }
+ //starting from each row, validate results should contain the starting row
+ for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
+ ScanInfo scanInfo = new ScanInfo(FAMILY, 0, 1, Integer.MAX_VALUE, false,
+ 0, this.memstore.comparator);
+ ScanType scanType = ScanType.USER_SCAN;
+ InternalScanner scanner = new StoreScanner(new Scan(
+ Bytes.toBytes(startRowId)), scanInfo, scanType, null,
+ memstore.getScanners(0));
+ List<Cell> results = new ArrayList<Cell>();
+ for (int i = 0; scanner.next(results); i++) {
+ int rowId = startRowId + i;
+ Cell left = results.get(0);
+ byte[] row1 = Bytes.toBytes(rowId);
+ assertTrue(
+ "Row name",
+ KeyValue.COMPARATOR.compareRows(left.getRowArray(), left.getRowOffset(),
+ (int) left.getRowLength(), row1, 0, row1.length) == 0);
+ assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
+ List<Cell> row = new ArrayList<Cell>();
+ for (Cell kv : results) {
+ row.add(kv);
+ }
+ isExpectedRowWithoutTimestamps(rowId, row);
+ // Clear out set. Otherwise row results accumulate.
+ results.clear();
+ }
+ }
+ }
+
+ public void testGet_memstoreAndSnapShot() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier1");
+ byte [] qf2 = Bytes.toBytes("testqualifier2");
+ byte [] qf3 = Bytes.toBytes("testqualifier3");
+ byte [] qf4 = Bytes.toBytes("testqualifier4");
+ byte [] qf5 = Bytes.toBytes("testqualifier5");
+ byte [] val = Bytes.toBytes("testval");
+
+ //Setting up memstore
+ memstore.add(new KeyValue(row, fam ,qf1, val));
+ memstore.add(new KeyValue(row, fam ,qf2, val));
+ memstore.add(new KeyValue(row, fam ,qf3, val));
+ //Creating a snapshot
+ memstore.snapshot();
+ assertEquals(3, memstore.snapshot.size());
+ //Adding value to "new" memstore
+ assertEquals(0, memstore.kvset.size());
+ memstore.add(new KeyValue(row, fam ,qf4, val));
+ memstore.add(new KeyValue(row, fam ,qf5, val));
+ assertEquals(2, memstore.kvset.size());
+ }
+
+ //////////////////////////////////////////////////////////////////////////////
+ // Delete tests
+ //////////////////////////////////////////////////////////////////////////////
+ public void testGetWithDelete() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier");
+ byte [] val = Bytes.toBytes("testval");
+
+ long ts1 = System.nanoTime();
+ KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
+ long ts2 = ts1 + 1;
+ KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
+ long ts3 = ts2 +1;
+ KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
+ memstore.add(put1);
+ memstore.add(put2);
+ memstore.add(put3);
+
+ assertEquals(3, memstore.kvset.size());
+
+ KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val);
+ memstore.delete(del2);
+
+ List<Cell> expected = new ArrayList<Cell>();
+ expected.add(put3);
+ expected.add(del2);
+ expected.add(put2);
+ expected.add(put1);
+
+ assertEquals(4, memstore.kvset.size());
+ int i = 0;
+ for(KeyValue kv : memstore.kvset) {
+ assertEquals(expected.get(i++), kv);
+ }
+ }
+
+ public void testGetWithDeleteColumn() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier");
+ byte [] val = Bytes.toBytes("testval");
+
+ long ts1 = System.nanoTime();
+ KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val);
+ long ts2 = ts1 + 1;
+ KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val);
+ long ts3 = ts2 +1;
+ KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val);
+ memstore.add(put1);
+ memstore.add(put2);
+ memstore.add(put3);
+
+ assertEquals(3, memstore.kvset.size());
+
+ KeyValue del2 =
+ new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val);
+ memstore.delete(del2);
+
+ List<Cell> expected = new ArrayList<Cell>();
+ expected.add(put3);
+ expected.add(del2);
+ expected.add(put2);
+ expected.add(put1);
+
+
+ assertEquals(4, memstore.kvset.size());
+ int i = 0;
+ for (KeyValue kv: memstore.kvset) {
+ assertEquals(expected.get(i++), kv);
+ }
+ }
+
+
+ public void testGetWithDeleteFamily() throws IOException {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf1 = Bytes.toBytes("testqualifier1");
+ byte [] qf2 = Bytes.toBytes("testqualifier2");
+ byte [] qf3 = Bytes.toBytes("testqualifier3");
+ byte [] val = Bytes.toBytes("testval");
+ long ts = System.nanoTime();
+
+ KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
+ KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
+ KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
+ KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
+
+ memstore.add(put1);
+ memstore.add(put2);
+ memstore.add(put3);
+ memstore.add(put4);
+
+ KeyValue del =
+ new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
+ memstore.delete(del);
+
+ List<Cell> expected = new ArrayList<Cell>();
+ expected.add(del);
+ expected.add(put1);
+ expected.add(put2);
+ expected.add(put4);
+ expected.add(put3);
+
+
+
+ assertEquals(5, memstore.kvset.size());
+ int i = 0;
+ for (KeyValue kv: memstore.kvset) {
+ assertEquals(expected.get(i++), kv);
+ }
+ }
+
+ public void testKeepDeleteInmemstore() {
+ byte [] row = Bytes.toBytes("testrow");
+ byte [] fam = Bytes.toBytes("testfamily");
+ byte [] qf = Bytes.toBytes("testqualifier");
+ byte [] val = Bytes.toBytes("testval");
+ long ts = System.nanoTime();
+ memstore.add(new KeyValue(row, fam, qf, ts, val));
+ KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val);
+ memstore.delete(delete);
+ assertEquals(2, memstore.kvset.size());
+ assertEquals(delete, memstore.kvset.first());
+ }
+
+ public void testRetainsDeleteVersion() throws IOException {
+ // add a put to memstore
+ memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
+
+ // now process a specific delete:
+ KeyValue delete = KeyValueTestUtil.create(
+ "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care");
+ memstore.delete(delete);
+
+ assertEquals(2, memstore.kvset.size());
+ assertEquals(delete, memstore.kvset.first());
+ }
+ public void testRetainsDeleteColumn() throws IOException {
+ // add a put to memstore
+ memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
+
+ // now process a specific delete:
+ KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
+ KeyValue.Type.DeleteColumn, "dont-care");
+ memstore.delete(delete);
+
+ assertEquals(2, memstore.kvset.size());
+ assertEquals(delete, memstore.kvset.first());
+ }
+ public void testRetainsDeleteFamily() throws IOException {
+ // add a put to memstore
+ memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"));
+
+ // now process a specific delete:
+ KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100,
+ KeyValue.Type.DeleteFamily, "dont-care");
+ memstore.delete(delete);
+
+ assertEquals(2, memstore.kvset.size());
+ assertEquals(delete, memstore.kvset.first());
+ }
+
+ ////////////////////////////////////
+ //Test for timestamps
+ ////////////////////////////////////
+
+ /**
+ * Test to ensure correctness when using Memstore with multiple timestamps
+ */
+ public void testMultipleTimestamps() throws IOException {
+ long[] timestamps = new long[] {20,10,5,1};
+ Scan scan = new Scan();
+
+ for (long timestamp: timestamps)
+ addRows(memstore,timestamp);
+
+ scan.setTimeRange(0, 2);
+ assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
+
+ scan.setTimeRange(20, 82);
+ assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
+
+ scan.setTimeRange(10, 20);
+ assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
+
+ scan.setTimeRange(8, 12);
+ assertTrue(memstore.shouldSeek(scan, Long.MIN_VALUE));
+
+ /*This test is not required for correctness but it should pass when
+ * timestamp range optimization is on*/
+ //scan.setTimeRange(28, 42);
+ //assertTrue(!memstore.shouldSeek(scan));
+ }
+
+ ////////////////////////////////////
+ //Test for upsert with MSLAB
+ ////////////////////////////////////
+
+ /**
+ * Test a pathological pattern that shows why we can't currently
+ * use the MSLAB for upsert workloads. This test inserts data
+ * in the following pattern:
+ *
+ * - row0001 through row1000 (fills up one 2M Chunk)
+ * - row0002 through row1001 (fills up another 2M chunk, leaves one reference
+ * to the first chunk
+ * - row0003 through row1002 (another chunk, another dangling reference)
+ *
+ * This causes OOME pretty quickly if we use MSLAB for upsert
+ * since each 2M chunk is held onto by a single reference.
+ */
+ public void testUpsertMSLAB() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true);
+ memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR);
+
+ int ROW_SIZE = 2048;
+ byte[] qualifier = new byte[ROW_SIZE - 4];
+
+ MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
+ for (int i = 0; i < 3; i++) { System.gc(); }
+ long usageBefore = bean.getHeapMemoryUsage().getUsed();
+
+ long size = 0;
+ long ts=0;
+
+ for (int newValue = 0; newValue < 1000; newValue++) {
+ for (int row = newValue; row < newValue + 1000; row++) {
+ byte[] rowBytes = Bytes.toBytes(row);
+ size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
+ }
+ }
+ System.out.println("Wrote " + ts + " vals");
+ for (int i = 0; i < 3; i++) { System.gc(); }
+ long usageAfter = bean.getHeapMemoryUsage().getUsed();
+ System.out.println("Memory used: " + (usageAfter - usageBefore)
+ + " (heapsize: " + memstore.heapSize() +
+ " size: " + size + ")");
+ }
+
+ //////////////////////////////////////////////////////////////////////////////
+ // Helpers
+ //////////////////////////////////////////////////////////////////////////////
+ private static byte [] makeQualifier(final int i1, final int i2){
+ return Bytes.toBytes(Integer.toString(i1) + ";" +
+ Integer.toString(i2));
+ }
+
+ /**
+ * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
+ * as older keyvalues are deleted from the memstore.
+ * @throws Exception
+ */
+ public void testUpsertMemstoreSize() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR);
+ long oldSize = memstore.size.get();
+
+ List<Cell> l = new ArrayList<Cell>();
+ KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
+ KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
+ KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");
+
+ kv1.setMvccVersion(1); kv2.setMvccVersion(1);kv3.setMvccVersion(1);
+ l.add(kv1); l.add(kv2); l.add(kv3);
+
+ this.memstore.upsert(l, 2);// readpoint is 2
+ long newSize = this.memstore.size.get();
+ assert(newSize > oldSize);
+
+ KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
+ kv4.setMvccVersion(1);
+ l.clear(); l.add(kv4);
+ this.memstore.upsert(l, 3);
+ assertEquals(newSize, this.memstore.size.get());
+ //this.memstore = null;
+ }
+
+ ////////////////////////////////////
+ // Test for periodic memstore flushes
+ // based on time of oldest edit
+ ////////////////////////////////////
+
+ /**
+ * Tests that the timeOfOldestEdit is updated correctly for the
+ * various edit operations in memstore.
+ * @throws Exception
+ */
+ public void testUpdateToTimeOfOldestEdit() throws Exception {
+ try {
+ EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
+ EnvironmentEdgeManager.injectEdge(edge);
+ DefaultMemStore memstore = new DefaultMemStore();
+ long t = memstore.timeOfOldestEdit();
+ assertEquals(t, Long.MAX_VALUE);
+
+ // test the case that the timeOfOldestEdit is updated after a KV add
+ memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
+ t = memstore.timeOfOldestEdit();
+ assertTrue(t == 1234);
+ // snapshot() will reset timeOfOldestEdit. The method will also assert the
+ // value is reset to Long.MAX_VALUE
+ t = runSnapshot(memstore);
+
+ // test the case that the timeOfOldestEdit is updated after a KV delete
+ memstore.delete(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
+ t = memstore.timeOfOldestEdit();
+ assertTrue(t == 1234);
+ t = runSnapshot(memstore);
+
+ // test the case that the timeOfOldestEdit is updated after a KV upsert
+ List<Cell> l = new ArrayList<Cell>();
+ KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
+ kv1.setMvccVersion(100);
+ l.add(kv1);
+ memstore.upsert(l, 1000);
+ t = memstore.timeOfOldestEdit();
+ assertTrue(t == 1234);
+ } finally {
+ EnvironmentEdgeManager.reset();
+ }
+ }
+
+ /**
+ * Tests the HRegion.shouldFlush method - adds an edit in the memstore
+ * and checks that shouldFlush returns true, and another where it disables
+ * the periodic flush functionality and tests whether shouldFlush returns
+ * false.
+ * @throws Exception
+ */
+ public void testShouldFlush() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 1000);
+ checkShouldFlush(conf, true);
+ // test disable flush
+ conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, 0);
+ checkShouldFlush(conf, false);
+ }
+
+ private void checkShouldFlush(Configuration conf, boolean expected) throws Exception {
+ try {
+ EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
+ EnvironmentEdgeManager.injectEdge(edge);
+ HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf);
+ HRegion region = hbaseUtility.createTestRegion("foobar", new HColumnDescriptor("foo"));
+
+ Map<byte[], Store> stores = region.getStores();
+ assertTrue(stores.size() == 1);
+
+ Store s = stores.entrySet().iterator().next().getValue();
+ edge.setCurrentTimeMillis(1234);
+ s.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"));
+ edge.setCurrentTimeMillis(1234 + 100);
+ assertTrue(region.shouldFlush() == false);
+ edge.setCurrentTimeMillis(1234 + 10000);
+ assertTrue(region.shouldFlush() == expected);
+ } finally {
+ EnvironmentEdgeManager.reset();
+ }
+ }
+
+ private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge {
+ long t = 1234;
+ @Override
+ public long currentTimeMillis() {
+ return t;
+ }
+ public void setCurrentTimeMillis(long t) {
+ this.t = t;
+ }
+ }
+
+ /**
+ * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
+ * @param hmc Instance to add rows to.
+ * @return How many rows we added.
+ * @throws IOException
+ */
+ private int addRows(final MemStore hmc) {
+ return addRows(hmc, HConstants.LATEST_TIMESTAMP);
+ }
+
+ /**
+ * Adds {@link #ROW_COUNT} rows and {@link #QUALIFIER_COUNT}
+ * @param hmc Instance to add rows to.
+ * @return How many rows we added.
+ * @throws IOException
+ */
+ private int addRows(final MemStore hmc, final long ts) {
+ for (int i = 0; i < ROW_COUNT; i++) {
+ long timestamp = ts == HConstants.LATEST_TIMESTAMP?
+ System.currentTimeMillis(): ts;
+ for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
+ byte [] row = Bytes.toBytes(i);
+ byte [] qf = makeQualifier(i, ii);
+ hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
+ }
+ }
+ return ROW_COUNT;
+ }
+
+ private long runSnapshot(final DefaultMemStore hmc) throws UnexpectedException {
+ // Save off old state.
+ int oldHistorySize = hmc.snapshot.size();
+ MemStoreSnapshot snapshot = hmc.snapshot();
+ // Make some assertions about what just happened.
+ assertTrue("History size has not increased", oldHistorySize < hmc.snapshot.size());
+ long t = memstore.timeOfOldestEdit();
+ assertTrue("Time of oldest edit is not Long.MAX_VALUE", t == Long.MAX_VALUE);
+ hmc.clearSnapshot(snapshot.getId());
+ return t;
+ }
+
+ private void isExpectedRowWithoutTimestamps(final int rowIndex,
+ List<Cell> kvs) {
+ int i = 0;
+ for (Cell kv: kvs) {
+ byte[] expectedColname = makeQualifier(rowIndex, i++);
+ assertTrue("Column name", CellUtil.matchingQualifier(kv, expectedColname));
+ // Value is column name as bytes. Usually result is
+ // 100 bytes in size at least. This is the default size
+ // for BytesWriteable. For comparison, convert bytes to
+ // String and trim to remove trailing null bytes.
+ assertTrue("Content", CellUtil.matchingValue(kv, expectedColname));
+ }
+ }
+
+ private static void addRows(int count, final MemStore mem) {
+ long nanos = System.nanoTime();
+
+ for (int i = 0 ; i < count ; i++) {
+ if (i % 1000 == 0) {
+
+ System.out.println(i + " Took for 1k usec: " + (System.nanoTime() - nanos)/1000);
+ nanos = System.nanoTime();
+ }
+ long timestamp = System.currentTimeMillis();
+
+ for (int ii = 0; ii < QUALIFIER_COUNT ; ii++) {
+ byte [] row = Bytes.toBytes(i);
+ byte [] qf = makeQualifier(i, ii);
+ mem.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
+ }
+ }
+ }
+
+
+ static void doScan(MemStore ms, int iteration) throws IOException {
+ long nanos = System.nanoTime();
+ KeyValueScanner s = ms.getScanners(0).get(0);
+ s.seek(KeyValue.createFirstOnRow(new byte[]{}));
+
+ System.out.println(iteration + " create/seek took: " + (System.nanoTime() - nanos)/1000);
+ int cnt=0;
+ while(s.next() != null) ++cnt;
+
+ System.out.println(iteration + " took usec: " + (System.nanoTime() - nanos) / 1000 + " for: "
+ + cnt);
+
+ }
+
+ public static void main(String [] args) throws IOException {
+ MemStore ms = new DefaultMemStore();
+
+ long n1 = System.nanoTime();
+ addRows(25000, ms);
+ System.out.println("Took for insert: " + (System.nanoTime()-n1)/1000);
+
+ System.out.println("foo");
+
+ for (int i = 0 ; i < 50 ; i++)
+ doScan(ms, i);
+ }
+}
+
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1577541&r1=1577540&r2=1577541&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Fri Mar 14 14:07:10 2014
@@ -1315,13 +1315,9 @@ public class TestHRegion {
put.add(kv);
// checkAndPut with wrong value
- HStore store = (HStore) region.getStore(fam1);
- store.memstore.kvset.size();
-
boolean res = region.checkAndMutate(row1, fam1, qf1, CompareOp.EQUAL, new BinaryComparator(
val1), put, true);
assertEquals(true, res);
- store.memstore.kvset.size();
Get get = new Get(row1);
get.addColumn(fam2, qf1);
@@ -1834,12 +1830,13 @@ public class TestHRegion {
// extract the key values out the memstore:
// This is kinda hacky, but better than nothing...
long now = System.currentTimeMillis();
- KeyValue firstKv = ((HStore) region.getStore(fam1)).memstore.kvset.first();
+ DefaultMemStore memstore = (DefaultMemStore) ((HStore) region.getStore(fam1)).memstore;
+ KeyValue firstKv = memstore.kvset.first();
assertTrue(firstKv.getTimestamp() <= now);
now = firstKv.getTimestamp();
- for (KeyValue kv : ((HStore) region.getStore(fam1)).memstore.kvset) {
- assertTrue(kv.getTimestamp() <= now);
- now = kv.getTimestamp();
+ for (Cell cell : memstore.kvset) {
+ assertTrue(cell.getTimestamp() <= now);
+ now = cell.getTimestamp();
}
} finally {
HRegion.closeHRegion(this.region);
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java?rev=1577541&r1=1577540&r2=1577541&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java Fri Mar 14 14:07:10 2014
@@ -47,7 +47,7 @@ public class TestMemStoreChunkPool {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- conf.setBoolean(MemStore.USEMSLAB_KEY, true);
+ conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true);
conf.setFloat(MemStoreChunkPool.CHUNK_POOL_MAXSIZE_KEY, 0.2f);
chunkPoolDisabledBeforeTest = MemStoreChunkPool.chunkPoolDisabled;
MemStoreChunkPool.chunkPoolDisabled = false;
@@ -107,7 +107,7 @@ public class TestMemStoreChunkPool {
byte[] qf5 = Bytes.toBytes("testqualifier5");
byte[] val = Bytes.toBytes("testval");
- MemStore memstore = new MemStore();
+ DefaultMemStore memstore = new DefaultMemStore();
// Setting up memstore
memstore.add(new KeyValue(row, fam, qf1, val));
@@ -115,8 +115,7 @@ public class TestMemStoreChunkPool {
memstore.add(new KeyValue(row, fam, qf3, val));
// Creating a snapshot
- memstore.snapshot();
- KeyValueSkipListSet snapshot = memstore.getSnapshot();
+ MemStoreSnapshot snapshot = memstore.snapshot();
assertEquals(3, memstore.snapshot.size());
// Adding value to "new" memstore
@@ -124,7 +123,7 @@ public class TestMemStoreChunkPool {
memstore.add(new KeyValue(row, fam, qf4, val));
memstore.add(new KeyValue(row, fam, qf5, val));
assertEquals(2, memstore.kvset.size());
- memstore.clearSnapshot(snapshot);
+ memstore.clearSnapshot(snapshot.getId());
int chunkCount = chunkPool.getPoolSize();
assertTrue(chunkCount > 0);
@@ -145,7 +144,7 @@ public class TestMemStoreChunkPool {
byte[] qf7 = Bytes.toBytes("testqualifier7");
byte[] val = Bytes.toBytes("testval");
- MemStore memstore = new MemStore();
+ DefaultMemStore memstore = new DefaultMemStore();
// Setting up memstore
memstore.add(new KeyValue(row, fam, qf1, val));
@@ -153,8 +152,7 @@ public class TestMemStoreChunkPool {
memstore.add(new KeyValue(row, fam, qf3, val));
// Creating a snapshot
- memstore.snapshot();
- KeyValueSkipListSet snapshot = memstore.getSnapshot();
+ MemStoreSnapshot snapshot = memstore.snapshot();
assertEquals(3, memstore.snapshot.size());
// Adding value to "new" memstore
@@ -167,7 +165,7 @@ public class TestMemStoreChunkPool {
List<KeyValueScanner> scanners = memstore.getScanners(0);
// Shouldn't putting back the chunks to pool,since some scanners are opening
// based on their data
- memstore.clearSnapshot(snapshot);
+ memstore.clearSnapshot(snapshot.getId());
assertTrue(chunkPool.getPoolSize() == 0);
@@ -181,8 +179,7 @@ public class TestMemStoreChunkPool {
chunkPool.clearChunks();
// Creating another snapshot
- memstore.snapshot();
- snapshot = memstore.getSnapshot();
+ snapshot = memstore.snapshot();
// Adding more value
memstore.add(new KeyValue(row, fam, qf6, val));
memstore.add(new KeyValue(row, fam, qf7, val));
@@ -194,7 +191,7 @@ public class TestMemStoreChunkPool {
}
// Since no opening scanner, the chunks of snapshot should be put back to
// pool
- memstore.clearSnapshot(snapshot);
+ memstore.clearSnapshot(snapshot.getId());
assertTrue(chunkPool.getPoolSize() > 0);
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java?rev=1577541&r1=1577540&r2=1577541&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestReversibleScanners.java Fri Mar 14 14:07:10 2014
@@ -28,7 +28,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
-import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -114,7 +113,7 @@ public class TestReversibleScanners {
@Test
public void testReversibleMemstoreScanner() throws IOException {
- MemStore memstore = new MemStore();
+ MemStore memstore = new DefaultMemStore();
writeMemstore(memstore);
List<KeyValueScanner> scanners = memstore.getScanners(Long.MAX_VALUE);
seekTestOfReversibleKeyValueScanner(scanners.get(0));
@@ -144,7 +143,7 @@ public class TestReversibleScanners {
TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(
hfilePath).withFileContext(hFileContext).build();
- MemStore memstore = new MemStore();
+ MemStore memstore = new DefaultMemStore();
writeMemstoreAndStoreFiles(memstore, new StoreFile.Writer[] { writer1,
writer2 });
@@ -234,7 +233,7 @@ public class TestReversibleScanners {
TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(
hfilePath).withFileContext(hFileContext).build();
- MemStore memstore = new MemStore();
+ MemStore memstore = new DefaultMemStore();
writeMemstoreAndStoreFiles(memstore, new StoreFile.Writer[] { writer1,
writer2 });
@@ -627,7 +626,6 @@ public class TestReversibleScanners {
private static void writeMemstoreAndStoreFiles(MemStore memstore,
final StoreFile.Writer[] writers) throws IOException {
- Random rand = new Random();
try {
for (int i = 0; i < ROWSIZE; i++) {
for (int j = 0; j < QUALSIZE; j++) {
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java?rev=1577541&r1=1577540&r2=1577541&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java Fri Mar 14 14:07:10 2014
@@ -515,7 +515,7 @@ public class TestStore {
this.store.snapshot();
flushStore(store, id++);
Assert.assertEquals(storeFilessize, this.store.getStorefiles().size());
- Assert.assertEquals(0, this.store.memstore.kvset.size());
+ Assert.assertEquals(0, ((DefaultMemStore)this.store.memstore).kvset.size());
}
private void assertCheck() {
@@ -560,7 +560,7 @@ public class TestStore {
flushStore(store, id++);
Assert.assertEquals(1, this.store.getStorefiles().size());
// from the one we inserted up there, and a new one
- Assert.assertEquals(2, this.store.memstore.kvset.size());
+ Assert.assertEquals(2, ((DefaultMemStore)this.store.memstore).kvset.size());
// how many key/values for this row are there?
Get get = new Get(row);
@@ -634,8 +634,8 @@ public class TestStore {
}
long computedSize=0;
- for (KeyValue kv : this.store.memstore.kvset) {
- long kvsize = MemStore.heapSizeChange(kv, true);
+ for (KeyValue kv : ((DefaultMemStore)this.store.memstore).kvset) {
+ long kvsize = DefaultMemStore.heapSizeChange(kv, true);
//System.out.println(kv + " size= " + kvsize + " kvsize= " + kv.heapSize());
computedSize += kvsize;
}
@@ -666,7 +666,7 @@ public class TestStore {
// then flush.
flushStore(store, id++);
Assert.assertEquals(1, this.store.getStorefiles().size());
- Assert.assertEquals(1, this.store.memstore.kvset.size());
+ Assert.assertEquals(1, ((DefaultMemStore)this.store.memstore).kvset.size());
// now increment again:
newValue += 1;
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java?rev=1577541&r1=1577540&r2=1577541&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java Fri Mar 14 14:07:10 2014
@@ -26,7 +26,6 @@ import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
-import java.util.SortedSet;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -69,10 +68,10 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.FlushRequester;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.MemStoreSnapshot;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdge;
@@ -560,14 +559,12 @@ public class TestWALReplay {
super(conf, store);
}
@Override
- public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
- TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
- throws IOException {
+ public List<Path> flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId,
+ MonitoredTask status) throws IOException {
if (throwExceptionWhenFlushing.get()) {
throw new IOException("Simulated exception by tests");
}
- return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
- flushedSize, status);
+ return super.flushSnapshot(snapshot, cacheFlushId, status);
}
};