You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2015/04/18 11:26:17 UTC

[14/19] hbase git commit: HBASE-13090 Progress heartbeats for long running scanners

http://git-wip-us.apache.org/repos/asf/hbase/blob/abe3796a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
new file mode 100644
index 0000000..495be30
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
@@ -0,0 +1,538 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CoordinatedStateManager;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HTestConst;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.KVComparator;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.ScannerCallable;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
+import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+/**
+ * Here we test to make sure that scans return the expected Results when the server is sending the
+ * Client heartbeat messages. Heartbeat messages are essentially keep-alive messages (they prevent
+ * the scanner on the client side from timing out). A heartbeat message is sent from the server to
+ * the client when the server has exceeded the time limit during the processing of the scan. When
+ * the time limit is reached, the server will return to the Client whatever Results it has
+ * accumulated (potentially empty).
+ */
+@Category(MediumTests.class)
+public class TestScannerHeartbeatMessages {
+  final Log LOG = LogFactory.getLog(getClass());
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  private static Table TABLE = null;
+
+  /**
+   * Table configuration
+   */
+  private static TableName TABLE_NAME = TableName.valueOf("testScannerHeartbeatMessagesTable");
+
+  private static int NUM_ROWS = 10;
+  private static byte[] ROW = Bytes.toBytes("testRow");
+  private static byte[][] ROWS = HTestConst.makeNAscii(ROW, NUM_ROWS);
+
+  private static int NUM_FAMILIES = 3;
+  private static byte[] FAMILY = Bytes.toBytes("testFamily");
+  private static byte[][] FAMILIES = HTestConst.makeNAscii(FAMILY, NUM_FAMILIES);
+
+  private static int NUM_QUALIFIERS = 3;
+  private static byte[] QUALIFIER = Bytes.toBytes("testQualifier");
+  private static byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, NUM_QUALIFIERS);
+
+  private static int VALUE_SIZE = 128;
+  private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE);
+
+  // Time, in milliseconds, that the client will wait for a response from the server before timing
+  // out. This value is used server side to determine when it is necessary to send a heartbeat
+  // message to the client
+  private static int CLIENT_TIMEOUT = 500;
+
+  // The server limits itself to running for half of the CLIENT_TIMEOUT value.
+  private static int SERVER_TIME_LIMIT = CLIENT_TIMEOUT / 2;
+
+  // By default, at most one row's worth of cells will be retrieved before the time limit is reached
+  private static int DEFAULT_ROW_SLEEP_TIME = SERVER_TIME_LIMIT / 2;
+  // By default, at most cells for two column families are retrieved before the time limit is
+  // reached
+  private static int DEFAULT_CF_SLEEP_TIME = SERVER_TIME_LIMIT / 2;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    ((Log4JLogger) ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
+    ((Log4JLogger) HeartbeatRPCServices.LOG).getLogger().setLevel(Level.ALL);
+    Configuration conf = TEST_UTIL.getConfiguration();
+
+    conf.setStrings(HConstants.REGION_IMPL, HeartbeatHRegion.class.getName());
+    conf.setStrings(HConstants.REGION_SERVER_IMPL, HeartbeatHRegionServer.class.getName());
+    conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, CLIENT_TIMEOUT);
+    conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, CLIENT_TIMEOUT);
+    conf.setInt(HConstants.HBASE_CLIENT_PAUSE, 1);
+
+    // Check the timeout condition after every cell
+    conf.setLong(StoreScanner.HBASE_CELLS_SCANNED_PER_HEARTBEAT_CHECK, 1);
+    TEST_UTIL.startMiniCluster(1);
+
+    TABLE = createTestTable(TABLE_NAME, ROWS, FAMILIES, QUALIFIERS, VALUE);
+  }
+
+  static Table createTestTable(TableName name, byte[][] rows, byte[][] families,
+      byte[][] qualifiers, byte[] cellValue) throws IOException {
+    Table ht = TEST_UTIL.createTable(name, families);
+    List<Put> puts = createPuts(rows, families, qualifiers, cellValue);
+    ht.put(puts);
+
+    return ht;
+  }
+
+  /**
+   * Make puts to put the input value into each combination of row, family, and qualifier
+   * @param rows
+   * @param families
+   * @param qualifiers
+   * @param value
+   * @return
+   * @throws IOException
+   */
+  static ArrayList<Put> createPuts(byte[][] rows, byte[][] families, byte[][] qualifiers,
+      byte[] value) throws IOException {
+    Put put;
+    ArrayList<Put> puts = new ArrayList<>();
+
+    for (int row = 0; row < rows.length; row++) {
+      put = new Put(rows[row]);
+      for (int fam = 0; fam < families.length; fam++) {
+        for (int qual = 0; qual < qualifiers.length; qual++) {
+          KeyValue kv = new KeyValue(rows[row], families[fam], qualifiers[qual], qual, value);
+          put.add(kv);
+        }
+      }
+      puts.add(put);
+    }
+
+    return puts;
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Before
+  public void setupBeforeTest() throws Exception {
+    disableSleeping();
+  }
+
+  @After
+  public void teardownAfterTest() throws Exception {
+    disableSleeping();
+  }
+
+  /**
+   * Test a variety of scan configurations to ensure that they return the expected Results when
+   * heartbeat messages are necessary. These tests are accumulated under one test case to ensure
+   * that they don't run in parallel. If the tests ran in parallel, they may conflict with each
+   * other due to changing static variables
+   */
+  @Test
+  public void testScannerHeartbeatMessages() throws Exception {
+    testImportanceOfHeartbeats(testHeartbeatBetweenRows());
+    testImportanceOfHeartbeats(testHeartbeatBetweenColumnFamilies());
+  }
+
+  /**
+   * Run the test callable when heartbeats are enabled/disabled. We expect all tests to only pass
+   * when heartbeat messages are enabled (otherwise the test is pointless). When heartbeats are
+   * disabled, the test should throw an exception.
+   * @param testCallable
+   * @throws InterruptedException
+   */
+  public void testImportanceOfHeartbeats(Callable<Void> testCallable) throws InterruptedException {
+    HeartbeatRPCServices.heartbeatsEnabled = true;
+
+    try {
+      testCallable.call();
+    } catch (Exception e) {
+      fail("Heartbeat messages are enabled, exceptions should NOT be thrown. Exception trace:"
+          + ExceptionUtils.getStackTrace(e));
+    }
+
+    HeartbeatRPCServices.heartbeatsEnabled = false;
+    try {
+      testCallable.call();
+    } catch (Exception e) {
+      return;
+    } finally {
+      HeartbeatRPCServices.heartbeatsEnabled = true;
+    }
+    fail("Heartbeats messages are disabled, an exception should be thrown. If an exception "
+        + " is not thrown, the test case is not testing the importance of heartbeat messages");
+  }
+
+  /**
+   * Test the case that the time limit for the scan is reached after each full row of cells is
+   * fetched.
+   * @throws Exception
+   */
+  public Callable<Void> testHeartbeatBetweenRows() throws Exception {
+    return new Callable<Void>() {
+
+      @Override
+      public Void call() throws Exception {
+        // Configure the scan so that it can read the entire table in a single RPC. We want to test
+        // the case where a scan stops on the server side due to a time limit
+        Scan scan = new Scan();
+        scan.setMaxResultSize(Long.MAX_VALUE);
+        scan.setCaching(Integer.MAX_VALUE);
+
+        testEquivalenceOfScanWithHeartbeats(scan, DEFAULT_ROW_SLEEP_TIME, -1, false);
+        return null;
+      }
+    };
+  }
+
+  /**
+   * Test the case that the time limit for scans is reached in between column families
+   * @throws Exception
+   */
+  public Callable<Void> testHeartbeatBetweenColumnFamilies() throws Exception {
+    return new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        // Configure the scan so that it can read the entire table in a single RPC. We want to test
+        // the case where a scan stops on the server side due to a time limit
+        Scan baseScan = new Scan();
+        baseScan.setMaxResultSize(Long.MAX_VALUE);
+        baseScan.setCaching(Integer.MAX_VALUE);
+
+        // Copy the scan before each test. When a scan object is used by a scanner, some of its
+        // fields may be changed such as start row
+        Scan scanCopy = new Scan(baseScan);
+        testEquivalenceOfScanWithHeartbeats(scanCopy, -1, DEFAULT_CF_SLEEP_TIME, false);
+        scanCopy = new Scan(baseScan);
+        testEquivalenceOfScanWithHeartbeats(scanCopy, -1, DEFAULT_CF_SLEEP_TIME, true);
+        return null;
+      }
+    };
+  }
+
+  /**
+   * Test the equivalence of a scan versus the same scan executed when heartbeat messages are
+   * necessary
+   * @param scan The scan configuration being tested
+   * @param rowSleepTime The time to sleep between fetches of row cells
+   * @param cfSleepTime The time to sleep between fetches of column family cells
+   * @param sleepBeforeCf set to true when column family sleeps should occur before the cells for
+   *          that column family are fetched
+   * @throws Exception
+   */
+  public void testEquivalenceOfScanWithHeartbeats(final Scan scan, int rowSleepTime,
+      int cfSleepTime, boolean sleepBeforeCf) throws Exception {
+    disableSleeping();
+    final ResultScanner scanner = TABLE.getScanner(scan);
+    final ResultScanner scannerWithHeartbeats = TABLE.getScanner(scan);
+
+    Result r1 = null;
+    Result r2 = null;
+
+    while ((r1 = scanner.next()) != null) {
+      // Enforce the specified sleep conditions during calls to the heartbeat scanner
+      configureSleepTime(rowSleepTime, cfSleepTime, sleepBeforeCf);
+      r2 = scannerWithHeartbeats.next();
+      disableSleeping();
+
+      assertTrue(r2 != null);
+      try {
+        Result.compareResults(r1, r2);
+      } catch (Exception e) {
+        fail(e.getMessage());
+      }
+    }
+
+    assertTrue(scannerWithHeartbeats.next() == null);
+    scanner.close();
+    scannerWithHeartbeats.close();
+  }
+
+  /**
+   * Helper method for setting the time to sleep between rows and column families. If a sleep time
+   * is negative then that sleep will be disabled
+   * @param rowSleepTime
+   * @param cfSleepTime
+   */
+  private static void configureSleepTime(int rowSleepTime, int cfSleepTime, boolean sleepBeforeCf) {
+    HeartbeatHRegion.sleepBetweenRows = rowSleepTime > 0;
+    HeartbeatHRegion.rowSleepTime = rowSleepTime;
+
+    HeartbeatHRegion.sleepBetweenColumnFamilies = cfSleepTime > 0;
+    HeartbeatHRegion.columnFamilySleepTime = cfSleepTime;
+    HeartbeatHRegion.sleepBeforeColumnFamily = sleepBeforeCf;
+  }
+
+  /**
+   * Disable the sleeping mechanism server side.
+   */
+  private static void disableSleeping() {
+    HeartbeatHRegion.sleepBetweenRows = false;
+    HeartbeatHRegion.sleepBetweenColumnFamilies = false;
+  }
+
+  /**
+   * Custom HRegionServer instance that instantiates {@link HeartbeatRPCServices} in place of
+   * {@link RSRpcServices} to allow us to toggle support for heartbeat messages
+   */
+  private static class HeartbeatHRegionServer extends HRegionServer {
+    public HeartbeatHRegionServer(Configuration conf) throws IOException, InterruptedException {
+      super(conf);
+    }
+
+    public HeartbeatHRegionServer(Configuration conf, CoordinatedStateManager csm)
+        throws IOException {
+      super(conf, csm);
+    }
+
+    @Override
+    protected RSRpcServices createRpcServices() throws IOException {
+      return new HeartbeatRPCServices(this);
+    }
+  }
+
+  /**
+   * Custom RSRpcServices instance that allows heartbeat support to be toggled
+   */
+  private static class HeartbeatRPCServices extends RSRpcServices {
+    private static boolean heartbeatsEnabled = true;
+
+    public HeartbeatRPCServices(HRegionServer rs) throws IOException {
+      super(rs);
+    }
+
+    @Override
+    public ScanResponse scan(RpcController controller, ScanRequest request) 
+        throws ServiceException {
+      ScanRequest.Builder builder = ScanRequest.newBuilder(request);
+      builder.setClientHandlesHeartbeats(heartbeatsEnabled);
+      return super.scan(controller, builder.build());
+    }
+  }
+
+  /**
+   * Custom HRegion class that instantiates {@link RegionScanner}s with configurable sleep times
+   * between fetches of row Results and/or column family cells. Useful for emulating an instance
+   * where the server is taking a long time to process a client's scan request
+   */
+  private static class HeartbeatHRegion extends HRegion {
+    // Row sleeps occur AFTER each row worth of cells is retrieved.
+    private static int rowSleepTime = DEFAULT_ROW_SLEEP_TIME;
+    private static boolean sleepBetweenRows = false;
+
+    // The sleep for column families can be initiated before or after we fetch the cells for the
+    // column family. If the sleep occurs BEFORE then the time limits will be reached inside
+    // StoreScanner while we are fetching individual cells. If the sleep occurs AFTER then the time
+    // limit will be reached inside RegionScanner after all the cells for a column family have been
+    // retrieved.
+    private static boolean sleepBeforeColumnFamily = false;
+    private static int columnFamilySleepTime = DEFAULT_CF_SLEEP_TIME;
+    private static boolean sleepBetweenColumnFamilies = false;
+
+    public HeartbeatHRegion(Path tableDir, WAL wal, FileSystem fs, Configuration confParam,
+        HRegionInfo regionInfo, HTableDescriptor htd, RegionServerServices rsServices) {
+      super(tableDir, wal, fs, confParam, regionInfo, htd, rsServices);
+    }
+
+    public HeartbeatHRegion(HRegionFileSystem fs, WAL wal, Configuration confParam,
+        HTableDescriptor htd, RegionServerServices rsServices) {
+      super(fs, wal, confParam, htd, rsServices);
+    }
+
+    private static void columnFamilySleep() {
+      if (HeartbeatHRegion.sleepBetweenColumnFamilies) {
+        try {
+          Thread.sleep(HeartbeatHRegion.columnFamilySleepTime);
+        } catch (InterruptedException e) {
+        }
+      }
+    }
+
+    private static void rowSleep() {
+      try {
+        if (HeartbeatHRegion.sleepBetweenRows) {
+          Thread.sleep(HeartbeatHRegion.rowSleepTime);
+        }
+      } catch (InterruptedException e) {
+      }
+    }
+
+    // Instantiate the custom heartbeat region scanners
+    @Override
+    protected RegionScanner instantiateRegionScanner(Scan scan,
+        List<KeyValueScanner> additionalScanners) throws IOException {
+      if (scan.isReversed()) {
+        if (scan.getFilter() != null) {
+          scan.getFilter().setReversed(true);
+        }
+        return new HeartbeatReversedRegionScanner(scan, additionalScanners, this);
+      }
+      return new HeartbeatRegionScanner(scan, additionalScanners, this);
+    }
+  }
+
+  /**
+   * Custom ReversedRegionScanner that can be configured to sleep between retrievals of row Results
+   * and/or column family cells
+   */
+  private static class HeartbeatReversedRegionScanner extends ReversedRegionScannerImpl {
+    HeartbeatReversedRegionScanner(Scan scan, List<KeyValueScanner> additionalScanners,
+        HRegion region) throws IOException {
+      super(scan, additionalScanners, region);
+    }
+
+    @Override
+    public boolean nextRaw(List<Cell> outResults, ScannerContext context)
+        throws IOException {
+      boolean moreRows = super.nextRaw(outResults, context);
+      HeartbeatHRegion.rowSleep();
+      return moreRows;
+    }
+
+    @Override
+    protected void initializeKVHeap(List<KeyValueScanner> scanners,
+        List<KeyValueScanner> joinedScanners, HRegion region) throws IOException {
+      this.storeHeap = new HeartbeatReversedKVHeap(scanners, region.getComparator());
+      if (!joinedScanners.isEmpty()) {
+        this.joinedHeap = new HeartbeatReversedKVHeap(joinedScanners, region.getComparator());
+      }
+    }
+  }
+
+  /**
+   * Custom RegionScanner that can be configured to sleep between retrievals of row Results and/or
+   * column family cells
+   */
+  private static class HeartbeatRegionScanner extends RegionScannerImpl {
+    HeartbeatRegionScanner(Scan scan, List<KeyValueScanner> additionalScanners, HRegion region)
+        throws IOException {
+      region.super(scan, additionalScanners, region);
+    }
+
+    @Override
+    public boolean nextRaw(List<Cell> outResults, ScannerContext context)
+        throws IOException {
+      boolean moreRows = super.nextRaw(outResults, context);
+      HeartbeatHRegion.rowSleep();
+      return moreRows;
+    }
+
+    @Override
+    protected void initializeKVHeap(List<KeyValueScanner> scanners,
+        List<KeyValueScanner> joinedScanners, HRegion region) throws IOException {
+      this.storeHeap = new HeartbeatKVHeap(scanners, region.getComparator());
+      if (!joinedScanners.isEmpty()) {
+        this.joinedHeap = new HeartbeatKVHeap(joinedScanners, region.getComparator());
+      }
+    }
+  }
+
+  /**
+   * Custom KV Heap that can be configured to sleep/wait in between retrievals of column family
+   * cells. Useful for testing
+   */
+  private static final class HeartbeatKVHeap extends KeyValueHeap {
+    public HeartbeatKVHeap(List<? extends KeyValueScanner> scanners, KVComparator comparator)
+        throws IOException {
+      super(scanners, comparator);
+    }
+
+    HeartbeatKVHeap(List<? extends KeyValueScanner> scanners, KVScannerComparator comparator)
+        throws IOException {
+      super(scanners, comparator);
+    }
+
+    @Override
+    public boolean next(List<Cell> result, ScannerContext context)
+        throws IOException {
+      if (HeartbeatHRegion.sleepBeforeColumnFamily) HeartbeatHRegion.columnFamilySleep();
+      boolean moreRows = super.next(result, context);
+      if (!HeartbeatHRegion.sleepBeforeColumnFamily) HeartbeatHRegion.columnFamilySleep();
+      return moreRows;
+    }
+  }
+
+  /**
+   * Custom reversed KV Heap that can be configured to sleep in between retrievals of column family
+   * cells.
+   */
+  private static final class HeartbeatReversedKVHeap extends ReversedKeyValueHeap {
+    public HeartbeatReversedKVHeap(List<? extends KeyValueScanner> scanners, 
+        KVComparator comparator) throws IOException {
+      super(scanners, comparator);
+    }
+
+    @Override
+    public boolean next(List<Cell> result, ScannerContext context)
+        throws IOException {
+      if (HeartbeatHRegion.sleepBeforeColumnFamily) HeartbeatHRegion.columnFamilySleep();
+      boolean moreRows = super.next(result, context);
+      if (!HeartbeatHRegion.sleepBeforeColumnFamily) HeartbeatHRegion.columnFamilySleep();
+      return moreRows;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/abe3796a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
index 9350ce4..3d42809 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
@@ -789,7 +789,6 @@ public class TestStripeCompactionPolicy {
     public boolean next(List<Cell> results) throws IOException {
       if (kvs.isEmpty()) return false;
       results.add(kvs.remove(0));
-
       return !kvs.isEmpty();
     }