You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2015/03/10 11:20:13 UTC

[1/2] hbase git commit: HBASE-13063 Allow to turn off memstore replication for region replicas

Repository: hbase
Updated Branches:
  refs/heads/branch-1 52ff5485a -> eb2193afa
  refs/heads/master eeb8c846d -> cbf03787e


HBASE-13063 Allow to turn off memstore replication for region replicas


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cbf03787
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cbf03787
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cbf03787

Branch: refs/heads/master
Commit: cbf03787ea04da8d1c2ceef8fbb4ccbabb7d67ed
Parents: eeb8c84
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Tue Mar 10 10:02:04 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Tue Mar 10 10:02:04 2015 +0000

----------------------------------------------------------------------
 .../apache/hadoop/hbase/HTableDescriptor.java   | 31 ++++++++
 .../hbase/regionserver/HRegionServer.java       |  7 ++
 .../hadoop/hbase/regionserver/wal/WALEdit.java  | 13 +++-
 .../hbase/replication/ReplicationEndpoint.java  |  9 ++-
 .../RegionReplicaReplicationEndpoint.java       | 74 ++++++++++++++++++--
 .../regionserver/ReplicationSourceManager.java  |  9 ++-
 .../hadoop/hbase/HBaseTestingUtility.java       | 10 +++
 .../TestRegionReplicaReplicationEndpoint.java   | 39 ++++++++++-
 8 files changed, 180 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf03787/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 51352bb..2c0bd30 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -176,6 +176,14 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> {
   private static final Bytes REGION_REPLICATION_KEY =
       new Bytes(Bytes.toBytes(REGION_REPLICATION));
 
+  /**
+   * <em>INTERNAL</em> flag to indicate whether or not the memstore should be replicated
+   * for read-replicas (CONSISTENCY => TIMELINE).
+   */
+  public static final String REGION_MEMSTORE_REPLICATION = "REGION_MEMSTORE_REPLICATION";
+  private static final Bytes REGION_MEMSTORE_REPLICATION_KEY =
+      new Bytes(Bytes.toBytes(REGION_MEMSTORE_REPLICATION));
+
   /** Default durability for HTD is USE_DEFAULT, which defaults to HBase-global default value */
   private static final Durability DEFAULT_DURABLITY = Durability.USE_DEFAULT;
 
@@ -210,6 +218,8 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> {
 
   public static final int DEFAULT_REGION_REPLICATION = 1;
 
+  public static final boolean DEFAULT_REGION_MEMSTORE_REPLICATION = true;
+
   private final static Map<String, String> DEFAULT_VALUES
     = new HashMap<String, String>();
   private final static Set<Bytes> RESERVED_KEYWORDS
@@ -1074,6 +1084,27 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> {
   }
 
   /**
+   * @return true if the read-replicas memstore replication is enabled.
+   */
+  public boolean hasRegionMemstoreReplication() {
+    return isSomething(REGION_MEMSTORE_REPLICATION_KEY, DEFAULT_REGION_MEMSTORE_REPLICATION);
+  }
+
+  /**
+   * Enable or Disable the memstore replication from the primary region to the replicas.
+   * The replication will be used only for meta operations (e.g. flush, compaction, ...)
+   *
+   * @param memstoreReplication true if the new data written to the primary region
+   *                                 should be replicated.
+   *                            false if the secondaries can tollerate to have new
+   *                                  data only when the primary flushes the memstore.
+   */
+  public HTableDescriptor setRegionMemstoreReplication(boolean memstoreReplication) {
+    setValue(REGION_MEMSTORE_REPLICATION_KEY, memstoreReplication ? TRUE : FALSE);
+    return this;
+  }
+
+  /**
    * Returns all the column family names of the current table. The map of
    * HTableDescriptor contains mapping of family name to HColumnDescriptors.
    * This returns all the keys of the family map which represents the column

http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf03787/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index d4bb931..4b8939a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3217,6 +3217,13 @@ public class HRegionServer extends HasThread implements
   }
 
   /**
+   * @return Return table descriptors implementation.
+   */
+  public TableDescriptors getTableDescriptors() {
+    return this.tableDescriptors;
+  }
+
+  /**
    * Reload the configuration from disk.
    */
   public void updateConfiguration() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf03787/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index 39d0536..5d0573f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -126,6 +126,15 @@ public class WALEdit implements Writable, HeapSize {
     return CellUtil.matchingFamily(cell, METAFAMILY);
   }
 
+  public boolean isMetaEdit() {
+    for (Cell cell: cells) {
+      if (!isMetaEditFamily(cell)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   /**
    * @return True when current WALEdit is created by log replay. Replication skips WALEdits from
    *         replay.
@@ -345,7 +354,7 @@ public class WALEdit implements Writable, HeapSize {
         bulkLoadDescriptor.toByteArray());
     return new WALEdit().add(kv);
   }
-  
+
   /**
    * Deserialized and returns a BulkLoadDescriptor from the passed in Cell
    * @param cell the key value
@@ -357,4 +366,4 @@ public class WALEdit implements Writable, HeapSize {
     }
     return null;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf03787/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
index c3ec976..978e853 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.wal.WAL.Entry;
 import org.apache.hadoop.hbase.replication.regionserver.MetricsSource;
 
@@ -51,6 +52,7 @@ public interface ReplicationEndpoint extends Service {
   class Context {
     private final Configuration conf;
     private final FileSystem fs;
+    private final TableDescriptors tableDescriptors;
     private final ReplicationPeerConfig peerConfig;
     private final ReplicationPeer replicationPeer;
     private final String peerId;
@@ -65,7 +67,8 @@ public interface ReplicationEndpoint extends Service {
         final String peerId,
         final UUID clusterId,
         final ReplicationPeer replicationPeer,
-        final MetricsSource metrics) {
+        final MetricsSource metrics,
+        final TableDescriptors tableDescriptors) {
       this.peerConfig = peerConfig;
       this.conf = conf;
       this.fs = fs;
@@ -73,6 +76,7 @@ public interface ReplicationEndpoint extends Service {
       this.peerId = peerId;
       this.replicationPeer = replicationPeer;
       this.metrics = metrics;
+      this.tableDescriptors = tableDescriptors;
     }
     public Configuration getConfiguration() {
       return conf;
@@ -95,6 +99,9 @@ public interface ReplicationEndpoint extends Service {
     public MetricsSource getMetrics() {
       return metrics;
     }
+    public TableDescriptors getTableDescriptors() {
+      return tableDescriptors;
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf03787/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
index 559b7bf..c75f81f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
@@ -43,9 +43,11 @@ import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.RegionAdminServiceCallable;
 import org.apache.hadoop.hbase.client.ClusterConnection;
@@ -94,6 +96,7 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
 
   private Configuration conf;
   private ClusterConnection connection;
+  private TableDescriptors tableDescriptors;
 
   // Reuse WALSplitter constructs as a WAL pipe
   private PipelineController controller;
@@ -150,6 +153,7 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
     super.init(context);
 
     this.conf = HBaseConfiguration.create(context.getConfiguration());
+    this.tableDescriptors = context.getTableDescriptors();
 
     // HRS multiplies client retries by 10 globally for meta operations, but we do not want this.
     // We are resetting it here because we want default number of retries (35) rather than 10 times
@@ -182,8 +186,8 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
     try {
       connection = (ClusterConnection) ConnectionFactory.createConnection(this.conf);
       this.pool = getDefaultThreadPool(conf);
-      outputSink = new RegionReplicaOutputSink(controller, entryBuffers, connection, pool,
-        numWriterThreads, operationTimeout);
+      outputSink = new RegionReplicaOutputSink(controller, tableDescriptors, entryBuffers,
+        connection, pool, numWriterThreads, operationTimeout);
       outputSink.startWriterThreads();
       super.doStart();
     } catch (IOException ex) {
@@ -311,12 +315,28 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
   }
 
   static class RegionReplicaOutputSink extends OutputSink {
-    private RegionReplicaSinkWriter sinkWriter;
+    private final RegionReplicaSinkWriter sinkWriter;
+    private final TableDescriptors tableDescriptors;
+    private final Cache<TableName, Boolean> memstoreReplicationEnabled;
 
-    public RegionReplicaOutputSink(PipelineController controller, EntryBuffers entryBuffers,
-        ClusterConnection connection, ExecutorService pool, int numWriters, int operationTimeout) {
+    public RegionReplicaOutputSink(PipelineController controller, TableDescriptors tableDescriptors,
+        EntryBuffers entryBuffers, ClusterConnection connection, ExecutorService pool,
+        int numWriters, int operationTimeout) {
       super(controller, entryBuffers, numWriters);
       this.sinkWriter = new RegionReplicaSinkWriter(this, connection, pool, operationTimeout);
+      this.tableDescriptors = tableDescriptors;
+
+      // A cache for the table "memstore replication enabled" flag.
+      // It has a default expiry of 5 sec. This means that if the table is altered
+      // with a different flag value, we might miss to replicate for that amount of
+      // time. But this cache avoid the slow lookup and parsing of the TableDescriptor.
+      int memstoreReplicationEnabledCacheExpiryMs = connection.getConfiguration()
+        .getInt("hbase.region.replica.replication.cache.memstoreReplicationEnabled.expiryMs", 5000);
+      this.memstoreReplicationEnabled = CacheBuilder.newBuilder()
+        .expireAfterWrite(memstoreReplicationEnabledCacheExpiryMs, TimeUnit.MILLISECONDS)
+        .initialCapacity(10)
+        .maximumSize(1000)
+        .build();
     }
 
     @Override
@@ -327,6 +347,12 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
         return;
       }
 
+      // meta edits (e.g. flush) are always replicated.
+      // data edits (e.g. put) are replicated if the table requires them.
+      if (!requiresReplication(buffer.getTableName(), entries)) {
+        return;
+      }
+
       sinkWriter.append(buffer.getTableName(), buffer.getEncodedRegionName(),
         entries.get(0).getEdit().getCells().get(0).getRow(), entries);
     }
@@ -358,6 +384,44 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
     AtomicLong getSkippedEditsCounter() {
       return skippedEdits;
     }
+
+    /**
+     * returns true if the specified entry must be replicated.
+     * We should always replicate meta operations (e.g. flush)
+     * and use the user HTD flag to decide whether or not replicate the memstore.
+     */
+    private boolean requiresReplication(final TableName tableName, final List<Entry> entries)
+        throws IOException {
+      // unit-tests may not the TableDescriptors, bypass the check and always replicate
+      if (tableDescriptors == null) return true;
+
+      Boolean requiresReplication = memstoreReplicationEnabled.getIfPresent(tableName);
+      if (requiresReplication == null) {
+        // check if the table requires memstore replication
+        // some unit-test drop the table, so we should do a bypass check and always replicate.
+        HTableDescriptor htd = tableDescriptors.get(tableName);
+        requiresReplication = htd == null || htd.hasRegionMemstoreReplication();
+        memstoreReplicationEnabled.put(tableName, requiresReplication);
+      }
+
+      // if memstore replication is not required, check the entries.
+      // meta edits (e.g. flush) must be always replicated.
+      if (!requiresReplication) {
+        int skipEdits = 0;
+        java.util.Iterator<Entry> it = entries.iterator();
+        while (it.hasNext()) {
+          Entry entry = it.next();
+          if (entry.getEdit().isMetaEdit()) {
+            requiresReplication = true;
+          } else {
+            it.remove();
+            skipEdits++;
+          }
+        }
+        skippedEdits.addAndGet(skipEdits);
+      }
+      return requiresReplication;
+    }
   }
 
   static class RegionReplicaSinkWriter extends SinkWriter {

http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf03787/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 4908ebc..4d97257 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
@@ -200,7 +201,7 @@ public class ReplicationSourceManager implements ReplicationListener {
       }
     }
  }
-  
+
   private void cleanOldLogs(SortedSet<String> wals, String key, String id) {
     SortedSet<String> walSet = wals.headSet(key);
     LOG.debug("Removing " + walSet.size() + " logs in the list: " + walSet);
@@ -302,7 +303,7 @@ public class ReplicationSourceManager implements ReplicationListener {
   protected Map<String, SortedSet<String>> getWALs() {
     return Collections.unmodifiableMap(walsById);
   }
-  
+
   /**
    * Get a copy of the wals of the recovered sources on this rs
    * @return a sorted set of wal names
@@ -375,8 +376,10 @@ public class ReplicationSourceManager implements ReplicationListener {
       final ReplicationPeerConfig peerConfig, final ReplicationPeer replicationPeer)
           throws IOException {
     RegionServerCoprocessorHost rsServerHost = null;
+    TableDescriptors tableDescriptors = null;
     if (server instanceof HRegionServer) {
       rsServerHost = ((HRegionServer) server).getRegionServerCoprocessorHost();
+      tableDescriptors = ((HRegionServer) server).getTableDescriptors();
     }
     ReplicationSourceInterface src;
     try {
@@ -420,7 +423,7 @@ public class ReplicationSourceManager implements ReplicationListener {
 
     // init replication endpoint
     replicationEndpoint.init(new ReplicationEndpoint.Context(replicationPeer.getConfiguration(),
-      fs, peerConfig, peerId, clusterId, replicationPeer, metrics));
+      fs, peerConfig, peerId, clusterId, replicationPeer, metrics, tableDescriptors));
 
     return src;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf03787/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 84d38ad..26290d6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2203,10 +2203,20 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
 
   public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow)
       throws IOException {
+    verifyNumericRows(region, f, startRow, endRow, true);
+  }
+
+  public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow,
+      final boolean present) throws IOException {
     for (int i = startRow; i < endRow; i++) {
       String failMsg = "Failed verification of row :" + i;
       byte[] data = Bytes.toBytes(String.valueOf(i));
       Result result = region.get(new Get(data));
+
+      boolean hasResult = result != null && !result.isEmpty();
+      assertEquals(failMsg + result, present, hasResult);
+      if (!present) continue;
+
       assertTrue(failMsg, result.containsColumn(f, null));
       assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
       Cell cell = result.getColumnLatestCell(f, null);

http://git-wip-us.apache.org/repos/asf/hbase/blob/cbf03787/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
index 886bd6e..1da2897 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
@@ -172,6 +172,11 @@ public class TestRegionReplicaReplicationEndpoint {
 
   private void verifyReplication(TableName tableName, int regionReplication,
       final int startRow, final int endRow) throws Exception {
+    verifyReplication(tableName, regionReplication, startRow, endRow, true);
+  }
+
+  private void verifyReplication(TableName tableName, int regionReplication,
+      final int startRow, final int endRow, final boolean present) throws Exception {
     // find the regions
     final HRegion[] regions = new HRegion[regionReplication];
 
@@ -195,7 +200,7 @@ public class TestRegionReplicaReplicationEndpoint {
         public boolean evaluate() throws Exception {
           LOG.info("verifying replication for region replica:" + region.getRegionInfo());
           try {
-            HTU.verifyNumericRows(region, HBaseTestingUtility.fam1, startRow, endRow);
+            HTU.verifyNumericRows(region, HBaseTestingUtility.fam1, startRow, endRow, present);
           } catch(Throwable ex) {
             LOG.warn("Verification from secondary region is not complete yet. Got:" + ex
               + " " + ex.getMessage());
@@ -224,6 +229,38 @@ public class TestRegionReplicaReplicationEndpoint {
   }
 
   @Test (timeout = 240000)
+  public void testRegionReplicaWithoutMemstoreReplication() throws Exception {
+    int regionReplication = 3;
+    TableName tableName = TableName.valueOf("testRegionReplicaWithoutMemstoreReplication");
+    HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString());
+    htd.setRegionReplication(regionReplication);
+    htd.setRegionMemstoreReplication(false);
+    HTU.getHBaseAdmin().createTable(htd);
+
+    Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
+    Table table = connection.getTable(tableName);
+    try {
+      // write data to the primary. The replicas should not receive the data
+      final int STEP = 100;
+      for (int i = 0; i < 3; ++i) {
+        final int startRow = i * STEP;
+        final int endRow = (i + 1) * STEP;
+        LOG.info("Writing data from " + startRow + " to " + endRow);
+        HTU.loadNumericRows(table, HBaseTestingUtility.fam1, startRow, endRow);
+        verifyReplication(tableName, regionReplication, startRow, endRow, false);
+
+        // Flush the table, now the data should show up in the replicas
+        LOG.info("flushing table");
+        HTU.flush(tableName);
+        verifyReplication(tableName, regionReplication, 0, endRow, true);
+      }
+    } finally {
+      table.close();
+      connection.close();
+    }
+  }
+
+  @Test (timeout = 240000)
   public void testRegionReplicaReplicationForFlushAndCompaction() throws Exception {
     // Tests a table with region replication 3. Writes some data, and causes flushes and
     // compactions. Verifies that the data is readable from the replicas. Note that this


[2/2] hbase git commit: HBASE-13063 Allow to turn off memstore replication for region replicas

Posted by mb...@apache.org.
HBASE-13063 Allow to turn off memstore replication for region replicas


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eb2193af
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eb2193af
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eb2193af

Branch: refs/heads/branch-1
Commit: eb2193afa0d266063b705d8743a940a06a5c2e9f
Parents: 52ff548
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Tue Mar 10 10:02:04 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Tue Mar 10 10:17:15 2015 +0000

----------------------------------------------------------------------
 .../apache/hadoop/hbase/HTableDescriptor.java   | 31 ++++++++
 .../hbase/regionserver/HRegionServer.java       |  7 ++
 .../hadoop/hbase/regionserver/wal/WALEdit.java  | 13 +++-
 .../hbase/replication/ReplicationEndpoint.java  |  9 ++-
 .../RegionReplicaReplicationEndpoint.java       | 74 ++++++++++++++++++--
 .../regionserver/ReplicationSourceManager.java  |  9 ++-
 .../hadoop/hbase/HBaseTestingUtility.java       | 10 +++
 .../TestRegionReplicaReplicationEndpoint.java   | 39 ++++++++++-
 8 files changed, 180 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/eb2193af/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index eb050fc..17956ff 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -192,6 +192,14 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
   private static final ImmutableBytesWritable REGION_REPLICATION_KEY =
       new ImmutableBytesWritable(Bytes.toBytes(REGION_REPLICATION));
 
+  /**
+   * <em>INTERNAL</em> flag to indicate whether or not the memstore should be replicated
+   * for read-replicas (CONSISTENCY => TIMELINE).
+   */
+  public static final String REGION_MEMSTORE_REPLICATION = "REGION_MEMSTORE_REPLICATION";
+  private static final ImmutableBytesWritable REGION_MEMSTORE_REPLICATION_KEY =
+      new ImmutableBytesWritable(Bytes.toBytes(REGION_MEMSTORE_REPLICATION));
+
   /** Default durability for HTD is USE_DEFAULT, which defaults to HBase-global default value */
   private static final Durability DEFAULT_DURABLITY = Durability.USE_DEFAULT;
 
@@ -226,6 +234,8 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
 
   public static final int DEFAULT_REGION_REPLICATION = 1;
 
+  public static final boolean DEFAULT_REGION_MEMSTORE_REPLICATION = true;
+
   private final static Map<String, String> DEFAULT_VALUES
     = new HashMap<String, String>();
   private final static Set<ImmutableBytesWritable> RESERVED_KEYWORDS
@@ -1168,6 +1178,27 @@ public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
   }
 
   /**
+   * @return true if the read-replicas memstore replication is enabled.
+   */
+  public boolean hasRegionMemstoreReplication() {
+    return isSomething(REGION_MEMSTORE_REPLICATION_KEY, DEFAULT_REGION_MEMSTORE_REPLICATION);
+  }
+
+  /**
+   * Enable or Disable the memstore replication from the primary region to the replicas.
+   * The replication will be used only for meta operations (e.g. flush, compaction, ...)
+   *
+   * @param memstoreReplication true if the new data written to the primary region
+   *                                 should be replicated.
+   *                            false if the secondaries can tollerate to have new
+   *                                  data only when the primary flushes the memstore.
+   */
+  public HTableDescriptor setRegionMemstoreReplication(boolean memstoreReplication) {
+    setValue(REGION_MEMSTORE_REPLICATION_KEY, memstoreReplication ? TRUE : FALSE);
+    return this;
+  }
+
+  /**
    * Returns all the column family names of the current table. The map of
    * HTableDescriptor contains mapping of family name to HColumnDescriptors.
    * This returns all the keys of the family map which represents the column

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb2193af/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index a2f4a22..d5f694a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3173,6 +3173,13 @@ public class HRegionServer extends HasThread implements
   }
 
   /**
+   * @return Return table descriptors implementation.
+   */
+  public TableDescriptors getTableDescriptors() {
+    return this.tableDescriptors;
+  }
+
+  /**
    * Reload the configuration from disk.
    */
   public void updateConfiguration() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb2193af/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index 39d0536..5d0573f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -126,6 +126,15 @@ public class WALEdit implements Writable, HeapSize {
     return CellUtil.matchingFamily(cell, METAFAMILY);
   }
 
+  public boolean isMetaEdit() {
+    for (Cell cell: cells) {
+      if (!isMetaEditFamily(cell)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   /**
    * @return True when current WALEdit is created by log replay. Replication skips WALEdits from
    *         replay.
@@ -345,7 +354,7 @@ public class WALEdit implements Writable, HeapSize {
         bulkLoadDescriptor.toByteArray());
     return new WALEdit().add(kv);
   }
-  
+
   /**
    * Deserialized and returns a BulkLoadDescriptor from the passed in Cell
    * @param cell the key value
@@ -357,4 +366,4 @@ public class WALEdit implements Writable, HeapSize {
     }
     return null;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb2193af/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
index c3ec976..978e853 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.wal.WAL.Entry;
 import org.apache.hadoop.hbase.replication.regionserver.MetricsSource;
 
@@ -51,6 +52,7 @@ public interface ReplicationEndpoint extends Service {
   class Context {
     private final Configuration conf;
     private final FileSystem fs;
+    private final TableDescriptors tableDescriptors;
     private final ReplicationPeerConfig peerConfig;
     private final ReplicationPeer replicationPeer;
     private final String peerId;
@@ -65,7 +67,8 @@ public interface ReplicationEndpoint extends Service {
         final String peerId,
         final UUID clusterId,
         final ReplicationPeer replicationPeer,
-        final MetricsSource metrics) {
+        final MetricsSource metrics,
+        final TableDescriptors tableDescriptors) {
       this.peerConfig = peerConfig;
       this.conf = conf;
       this.fs = fs;
@@ -73,6 +76,7 @@ public interface ReplicationEndpoint extends Service {
       this.peerId = peerId;
       this.replicationPeer = replicationPeer;
       this.metrics = metrics;
+      this.tableDescriptors = tableDescriptors;
     }
     public Configuration getConfiguration() {
       return conf;
@@ -95,6 +99,9 @@ public interface ReplicationEndpoint extends Service {
     public MetricsSource getMetrics() {
       return metrics;
     }
+    public TableDescriptors getTableDescriptors() {
+      return tableDescriptors;
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb2193af/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
index a008456..db3b87d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
@@ -43,9 +43,11 @@ import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.RegionLocations;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.RegionAdminServiceCallable;
 import org.apache.hadoop.hbase.client.ClusterConnection;
@@ -96,6 +98,7 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
 
   private Configuration conf;
   private ClusterConnection connection;
+  private TableDescriptors tableDescriptors;
 
   // Reuse WALSplitter constructs as a WAL pipe
   private PipelineController controller;
@@ -152,6 +155,7 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
     super.init(context);
 
     this.conf = HBaseConfiguration.create(context.getConfiguration());
+    this.tableDescriptors = context.getTableDescriptors();
 
     // HRS multiplies client retries by 10 globally for meta operations, but we do not want this.
     // We are resetting it here because we want default number of retries (35) rather than 10 times
@@ -184,8 +188,8 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
     try {
       connection = (ClusterConnection) ConnectionFactory.createConnection(this.conf);
       this.pool = getDefaultThreadPool(conf);
-      outputSink = new RegionReplicaOutputSink(controller, entryBuffers, connection, pool,
-        numWriterThreads, operationTimeout);
+      outputSink = new RegionReplicaOutputSink(controller, tableDescriptors, entryBuffers,
+        connection, pool, numWriterThreads, operationTimeout);
       outputSink.startWriterThreads();
       super.doStart();
     } catch (IOException ex) {
@@ -313,12 +317,28 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
   }
 
   static class RegionReplicaOutputSink extends OutputSink {
-    private RegionReplicaSinkWriter sinkWriter;
+    private final RegionReplicaSinkWriter sinkWriter;
+    private final TableDescriptors tableDescriptors;
+    private final Cache<TableName, Boolean> memstoreReplicationEnabled;
 
-    public RegionReplicaOutputSink(PipelineController controller, EntryBuffers entryBuffers,
-        ClusterConnection connection, ExecutorService pool, int numWriters, int operationTimeout) {
+    public RegionReplicaOutputSink(PipelineController controller, TableDescriptors tableDescriptors,
+        EntryBuffers entryBuffers, ClusterConnection connection, ExecutorService pool,
+        int numWriters, int operationTimeout) {
       super(controller, entryBuffers, numWriters);
       this.sinkWriter = new RegionReplicaSinkWriter(this, connection, pool, operationTimeout);
+      this.tableDescriptors = tableDescriptors;
+
+      // A cache for the table "memstore replication enabled" flag.
+      // It has a default expiry of 5 sec. This means that if the table is altered
+      // with a different flag value, we might miss to replicate for that amount of
+      // time. But this cache avoid the slow lookup and parsing of the TableDescriptor.
+      int memstoreReplicationEnabledCacheExpiryMs = connection.getConfiguration()
+        .getInt("hbase.region.replica.replication.cache.memstoreReplicationEnabled.expiryMs", 5000);
+      this.memstoreReplicationEnabled = CacheBuilder.newBuilder()
+        .expireAfterWrite(memstoreReplicationEnabledCacheExpiryMs, TimeUnit.MILLISECONDS)
+        .initialCapacity(10)
+        .maximumSize(1000)
+        .build();
     }
 
     @Override
@@ -329,6 +349,12 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
         return;
       }
 
+      // meta edits (e.g. flush) are always replicated.
+      // data edits (e.g. put) are replicated if the table requires them.
+      if (!requiresReplication(buffer.getTableName(), entries)) {
+        return;
+      }
+
       sinkWriter.append(buffer.getTableName(), buffer.getEncodedRegionName(),
         entries.get(0).getEdit().getCells().get(0).getRow(), entries);
     }
@@ -360,6 +386,44 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
     AtomicLong getSkippedEditsCounter() {
       return skippedEdits;
     }
+
+    /**
+     * returns true if the specified entry must be replicated.
+     * We should always replicate meta operations (e.g. flush)
+     * and use the user HTD flag to decide whether or not replicate the memstore.
+     */
+    private boolean requiresReplication(final TableName tableName, final List<Entry> entries)
+        throws IOException {
+      // unit-tests may not the TableDescriptors, bypass the check and always replicate
+      if (tableDescriptors == null) return true;
+
+      Boolean requiresReplication = memstoreReplicationEnabled.getIfPresent(tableName);
+      if (requiresReplication == null) {
+        // check if the table requires memstore replication
+        // some unit-test drop the table, so we should do a bypass check and always replicate.
+        HTableDescriptor htd = tableDescriptors.get(tableName);
+        requiresReplication = htd == null || htd.hasRegionMemstoreReplication();
+        memstoreReplicationEnabled.put(tableName, requiresReplication);
+      }
+
+      // if memstore replication is not required, check the entries.
+      // meta edits (e.g. flush) must be always replicated.
+      if (!requiresReplication) {
+        int skipEdits = 0;
+        java.util.Iterator<Entry> it = entries.iterator();
+        while (it.hasNext()) {
+          Entry entry = it.next();
+          if (entry.getEdit().isMetaEdit()) {
+            requiresReplication = true;
+          } else {
+            it.remove();
+            skipEdits++;
+          }
+        }
+        skippedEdits.addAndGet(skipEdits);
+      }
+      return requiresReplication;
+    }
   }
 
   static class RegionReplicaSinkWriter extends SinkWriter {

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb2193af/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 4908ebc..4d97257 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableDescriptors;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
@@ -200,7 +201,7 @@ public class ReplicationSourceManager implements ReplicationListener {
       }
     }
  }
-  
+
   private void cleanOldLogs(SortedSet<String> wals, String key, String id) {
     SortedSet<String> walSet = wals.headSet(key);
     LOG.debug("Removing " + walSet.size() + " logs in the list: " + walSet);
@@ -302,7 +303,7 @@ public class ReplicationSourceManager implements ReplicationListener {
   protected Map<String, SortedSet<String>> getWALs() {
     return Collections.unmodifiableMap(walsById);
   }
-  
+
   /**
    * Get a copy of the wals of the recovered sources on this rs
    * @return a sorted set of wal names
@@ -375,8 +376,10 @@ public class ReplicationSourceManager implements ReplicationListener {
       final ReplicationPeerConfig peerConfig, final ReplicationPeer replicationPeer)
           throws IOException {
     RegionServerCoprocessorHost rsServerHost = null;
+    TableDescriptors tableDescriptors = null;
     if (server instanceof HRegionServer) {
       rsServerHost = ((HRegionServer) server).getRegionServerCoprocessorHost();
+      tableDescriptors = ((HRegionServer) server).getTableDescriptors();
     }
     ReplicationSourceInterface src;
     try {
@@ -420,7 +423,7 @@ public class ReplicationSourceManager implements ReplicationListener {
 
     // init replication endpoint
     replicationEndpoint.init(new ReplicationEndpoint.Context(replicationPeer.getConfiguration(),
-      fs, peerConfig, peerId, clusterId, replicationPeer, metrics));
+      fs, peerConfig, peerId, clusterId, replicationPeer, metrics, tableDescriptors));
 
     return src;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb2193af/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 04f3676..5ff3418 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2144,10 +2144,20 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
 
   public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow)
       throws IOException {
+    verifyNumericRows(region, f, startRow, endRow, true);
+  }
+
+  public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow,
+      final boolean present) throws IOException {
     for (int i = startRow; i < endRow; i++) {
       String failMsg = "Failed verification of row :" + i;
       byte[] data = Bytes.toBytes(String.valueOf(i));
       Result result = region.get(new Get(data));
+
+      boolean hasResult = result != null && !result.isEmpty();
+      assertEquals(failMsg + result, present, hasResult);
+      if (!present) continue;
+
       assertTrue(failMsg, result.containsColumn(f, null));
       assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
       Cell cell = result.getColumnLatestCell(f, null);

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb2193af/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
index a89d4f4..9205f35 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
@@ -172,6 +172,11 @@ public class TestRegionReplicaReplicationEndpoint {
 
   private void verifyReplication(TableName tableName, int regionReplication,
       final int startRow, final int endRow) throws Exception {
+    verifyReplication(tableName, regionReplication, startRow, endRow, true);
+  }
+
+  private void verifyReplication(TableName tableName, int regionReplication,
+      final int startRow, final int endRow, final boolean present) throws Exception {
     // find the regions
     final HRegion[] regions = new HRegion[regionReplication];
 
@@ -195,7 +200,7 @@ public class TestRegionReplicaReplicationEndpoint {
         public boolean evaluate() throws Exception {
           LOG.info("verifying replication for region replica:" + region.getRegionInfo());
           try {
-            HTU.verifyNumericRows(region, HBaseTestingUtility.fam1, startRow, endRow);
+            HTU.verifyNumericRows(region, HBaseTestingUtility.fam1, startRow, endRow, present);
           } catch(Throwable ex) {
             LOG.warn("Verification from secondary region is not complete yet. Got:" + ex
               + " " + ex.getMessage());
@@ -224,6 +229,38 @@ public class TestRegionReplicaReplicationEndpoint {
   }
 
   @Test (timeout = 240000)
+  public void testRegionReplicaWithoutMemstoreReplication() throws Exception {
+    int regionReplication = 3;
+    TableName tableName = TableName.valueOf("testRegionReplicaWithoutMemstoreReplication");
+    HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString());
+    htd.setRegionReplication(regionReplication);
+    htd.setRegionMemstoreReplication(false);
+    HTU.getHBaseAdmin().createTable(htd);
+
+    Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
+    Table table = connection.getTable(tableName);
+    try {
+      // write data to the primary. The replicas should not receive the data
+      final int STEP = 100;
+      for (int i = 0; i < 3; ++i) {
+        final int startRow = i * STEP;
+        final int endRow = (i + 1) * STEP;
+        LOG.info("Writing data from " + startRow + " to " + endRow);
+        HTU.loadNumericRows(table, HBaseTestingUtility.fam1, startRow, endRow);
+        verifyReplication(tableName, regionReplication, startRow, endRow, false);
+
+        // Flush the table, now the data should show up in the replicas
+        LOG.info("flushing table");
+        HTU.flush(tableName);
+        verifyReplication(tableName, regionReplication, 0, endRow, true);
+      }
+    } finally {
+      table.close();
+      connection.close();
+    }
+  }
+
+  @Test (timeout = 240000)
   public void testRegionReplicaReplicationForFlushAndCompaction() throws Exception {
     // Tests a table with region replication 3. Writes some data, and causes flushes and
     // compactions. Verifies that the data is readable from the replicas. Note that this