You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/01/19 13:24:35 UTC
[01/31] hbase git commit: HBASE-19793 Minor improvements on Master/RS
startup [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-19064 e47ba21cd -> b56bf0dbb (forced update)
HBASE-19793 Minor improvements on Master/RS startup
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b638f7e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b638f7e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b638f7e
Branch: refs/heads/HBASE-19064
Commit: 3b638f7ea1ee050cde1dc0c66d622a3dd96e98cc
Parents: e546034
Author: zhangduo <zh...@apache.org>
Authored: Sun Jan 14 20:45:31 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Mon Jan 15 17:53:28 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 13 +-
.../hadoop/hbase/master/MasterFileSystem.java | 7 +-
.../hbase/regionserver/HRegionServer.java | 130 +++++++++----------
.../master/assignment/MockMasterServices.java | 2 +-
4 files changed, 74 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b638f7e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 510cfb8..0e626ce 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -52,6 +52,7 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ClusterId;
@@ -540,6 +541,11 @@ public class HMaster extends HRegionServer implements MasterServices {
}
}
+ @Override
+ protected String getUseThisHostnameInstead(Configuration conf) {
+ return conf.get(MASTER_HOSTNAME_KEY);
+ }
+
// Main run loop. Calls through to the regionserver run loop AFTER becoming active Master; will
// block in here until then.
@Override
@@ -612,7 +618,8 @@ public class HMaster extends HRegionServer implements MasterServices {
masterJettyServer.addConnector(connector);
masterJettyServer.setStopAtShutdown(true);
- final String redirectHostname = shouldUseThisHostnameInstead() ? useThisHostnameInstead : null;
+ final String redirectHostname =
+ StringUtils.isBlank(useThisHostnameInstead) ? null : useThisHostnameInstead;
final RedirectServlet redirect = new RedirectServlet(infoServer, redirectHostname);
final WebAppContext context = new WebAppContext(null, "/", null, null, null, null, WebAppContext.NO_SESSIONS);
@@ -788,7 +795,7 @@ public class HMaster extends HRegionServer implements MasterServices {
// TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
// Initialize the chunkCreator
initializeMemStoreChunkCreator();
- this.fileSystemManager = new MasterFileSystem(this);
+ this.fileSystemManager = new MasterFileSystem(conf);
this.walManager = new MasterWalManager(this);
// enable table descriptors cache
@@ -806,7 +813,7 @@ public class HMaster extends HRegionServer implements MasterServices {
ClusterId clusterId = fileSystemManager.getClusterId();
status.setStatus("Publishing Cluster ID " + clusterId + " in ZooKeeper");
ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
- this.clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);
+ this.clusterId = clusterId.toString();
this.serverManager = createServerManager(this);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b638f7e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 8c2c9fd..a37fd4e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -98,11 +98,8 @@ public class MasterFileSystem {
private boolean isSecurityEnabled;
- private final MasterServices services;
-
- public MasterFileSystem(MasterServices services) throws IOException {
- this.conf = services.getConfiguration();
- this.services = services;
+ public MasterFileSystem(Configuration conf) throws IOException {
+ this.conf = conf;
// Set filesystem to be that of this.rootdir else we get complaints about
// mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
// default localfs. Presumption is that rootdir is fully-qualified before
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b638f7e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index a81c27f..516a77e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -25,6 +25,7 @@ import java.lang.reflect.Constructor;
import java.net.BindException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -42,8 +43,6 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
@@ -51,6 +50,7 @@ import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
import javax.servlet.http.HttpServlet;
import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -169,7 +169,6 @@ import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -179,6 +178,7 @@ import sun.misc.SignalHandler;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
import org.apache.hbase.thirdparty.com.google.protobuf.BlockingRpcChannel;
import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
@@ -385,13 +385,13 @@ public class HRegionServer extends HasThread implements
final AtomicBoolean online = new AtomicBoolean(false);
// zookeeper connection and watcher
- protected ZKWatcher zooKeeper;
+ protected final ZKWatcher zooKeeper;
// master address tracker
- private MasterAddressTracker masterAddressTracker;
+ private final MasterAddressTracker masterAddressTracker;
// Cluster Status Tracker
- protected ClusterStatusTracker clusterStatusTracker;
+ protected final ClusterStatusTracker clusterStatusTracker;
// Log Splitting Worker
private SplitLogWorker splitLogWorker;
@@ -524,7 +524,6 @@ public class HRegionServer extends HasThread implements
private final boolean masterless;
static final String MASTERLESS_CONFIG_NAME = "hbase.masterless";
-
/**
* Starts a HRegionServer at the default location
*/
@@ -571,23 +570,10 @@ public class HRegionServer extends HasThread implements
this.stopped = false;
rpcServices = createRpcServices();
- if (this instanceof HMaster) {
- useThisHostnameInstead = conf.get(MASTER_HOSTNAME_KEY);
- } else {
- useThisHostnameInstead = conf.get(RS_HOSTNAME_KEY);
- if (conf.getBoolean(RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY, false)) {
- if (shouldUseThisHostnameInstead()) {
- String msg = RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " and " + RS_HOSTNAME_KEY +
- " are mutually exclusive. Do not set " + RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY +
- " to true while " + RS_HOSTNAME_KEY + " is used";
- throw new IOException(msg);
- } else {
- useThisHostnameInstead = rpcServices.isa.getHostName();
- }
- }
- }
- String hostName = shouldUseThisHostnameInstead() ?
- this.useThisHostnameInstead : this.rpcServices.isa.getHostName();
+ useThisHostnameInstead = getUseThisHostnameInstead(conf);
+ String hostName =
+ StringUtils.isBlank(useThisHostnameInstead) ? this.rpcServices.isa.getHostName()
+ : this.useThisHostnameInstead;
serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode);
rpcControllerFactory = RpcControllerFactory.instantiate(this.conf);
@@ -623,7 +609,6 @@ public class HRegionServer extends HasThread implements
// Open connection to zookeeper and set primary watcher
zooKeeper = new ZKWatcher(conf, getProcessName() + ":" +
rpcServices.isa.getPort(), this, canCreateBaseZNode());
-
// If no master in cluster, skip trying to track one or look for a cluster status.
if (!this.masterless) {
this.csm = new ZkCoordinatedStateManager(this);
@@ -633,7 +618,14 @@ public class HRegionServer extends HasThread implements
clusterStatusTracker = new ClusterStatusTracker(zooKeeper, this);
clusterStatusTracker.start();
+ } else {
+ masterAddressTracker = null;
+ clusterStatusTracker = null;
}
+ } else {
+ zooKeeper = null;
+ masterAddressTracker = null;
+ clusterStatusTracker = null;
}
// This violates 'no starting stuff in Constructor' but Master depends on the below chore
// and executor being created and takes a different startup route. Lots of overlap between HRS
@@ -652,6 +644,23 @@ public class HRegionServer extends HasThread implements
}
}
+ // HMaster should override this method to load the specific config for master
+ protected String getUseThisHostnameInstead(Configuration conf) throws IOException {
+ String hostname = conf.get(RS_HOSTNAME_KEY);
+ if (conf.getBoolean(RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY, false)) {
+ if (!StringUtils.isBlank(hostname)) {
+ String msg = RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY + " and " + RS_HOSTNAME_KEY +
+ " are mutually exclusive. Do not set " + RS_HOSTNAME_DISABLE_MASTER_REVERSEDNS_KEY +
+ " to true while " + RS_HOSTNAME_KEY + " is used";
+ throw new IOException(msg);
+ } else {
+ return rpcServices.isa.getHostName();
+ }
+ } else {
+ return hostname;
+ }
+ }
+
/**
* If running on Windows, do windows-specific setup.
*/
@@ -701,13 +710,6 @@ public class HRegionServer extends HasThread implements
return null;
}
- /*
- * Returns true if configured hostname should be used
- */
- protected boolean shouldUseThisHostnameInstead() {
- return useThisHostnameInstead != null && !useThisHostnameInstead.isEmpty();
- }
-
protected void login(UserProvider user, String host) throws IOException {
user.login("hbase.regionserver.keytab.file",
"hbase.regionserver.kerberos.principal", host);
@@ -810,17 +812,14 @@ public class HRegionServer extends HasThread implements
}
/**
- * All initialization needed before we go register with Master.
- * Do bare minimum. Do bulk of initializations AFTER we've connected to the Master.
+ * All initialization needed before we go register with Master.<br>
+ * Do bare minimum. Do bulk of initializations AFTER we've connected to the Master.<br>
* In here we just put up the RpcServer, setup Connection, and ZooKeeper.
- *
- * @throws IOException
- * @throws InterruptedException
*/
private void preRegistrationInitialization() {
try {
- setupClusterConnection();
initializeZooKeeper();
+ setupClusterConnection();
// Setup RPC client for master communication
this.rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress(
this.rpcServices.isa.getAddress(), 0), clusterConnection.getConnectionMetrics());
@@ -833,18 +832,18 @@ public class HRegionServer extends HasThread implements
}
/**
- * Bring up connection to zk ensemble and then wait until a master for this
- * cluster and then after that, wait until cluster 'up' flag has been set.
- * This is the order in which master does things.
+ * Bring up connection to zk ensemble and then wait until a master for this cluster and then after
+ * that, wait until cluster 'up' flag has been set. This is the order in which master does things.
+ * <p>
* Finally open long-living server short-circuit connection.
- * @throws IOException
- * @throws InterruptedException
*/
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE",
justification="cluster Id znode read would give us correct response")
private void initializeZooKeeper() throws IOException, InterruptedException {
// Nothing to do in here if no Master in the mix.
- if (this.masterless) return;
+ if (this.masterless) {
+ return;
+ }
// Create the master address tracker, register with zk, and start it. Then
// block until a master is available. No point in starting up if no master
@@ -855,17 +854,20 @@ public class HRegionServer extends HasThread implements
// when ready.
blockAndCheckIfStopped(this.clusterStatusTracker);
- // Retrieve clusterId
- // Since cluster status is now up
- // ID should have already been set by HMaster
- try {
- clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);
- if (clusterId == null) {
- this.abort("Cluster ID has not been set");
+ // If we are HMaster then the cluster id should have already been set.
+ if (clusterId == null) {
+ // Retrieve clusterId
+ // Since cluster status is now up
+ // ID should have already been set by HMaster
+ try {
+ clusterId = ZKClusterId.readClusterIdZNode(this.zooKeeper);
+ if (clusterId == null) {
+ this.abort("Cluster ID has not been set");
+ }
+ LOG.info("ClusterId : " + clusterId);
+ } catch (KeeperException e) {
+ this.abort("Failed to retrieve Cluster ID", e);
}
- LOG.info("ClusterId : "+clusterId);
- } catch (KeeperException e) {
- this.abort("Failed to retrieve Cluster ID",e);
}
// In case colocated master, wait here till it's active.
@@ -887,16 +889,6 @@ public class HRegionServer extends HasThread implements
}
}
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="RV_RETURN_VALUE_IGNORED",
- justification="We don't care about the return")
- private void doLatch(final CountDownLatch latch) throws InterruptedException {
- if (latch != null) {
- // Result is ignored intentionally but if I remove the below, findbugs complains (the
- // above justification on this method doesn't seem to suppress it).
- boolean result = latch.await(20, TimeUnit.SECONDS);
- }
- }
-
/**
* Utilty method to wait indefinitely on a znode availability while checking
* if the region server is shut down
@@ -1467,14 +1459,14 @@ public class HRegionServer extends HasThread implements
String hostnameFromMasterPOV = e.getValue();
this.serverName = ServerName.valueOf(hostnameFromMasterPOV, rpcServices.isa.getPort(),
this.startcode);
- if (shouldUseThisHostnameInstead() &&
+ if (!StringUtils.isBlank(useThisHostnameInstead) &&
!hostnameFromMasterPOV.equals(useThisHostnameInstead)) {
String msg = "Master passed us a different hostname to use; was=" +
this.useThisHostnameInstead + ", but now=" + hostnameFromMasterPOV;
LOG.error(msg);
throw new IOException(msg);
}
- if (!shouldUseThisHostnameInstead() &&
+ if (StringUtils.isBlank(useThisHostnameInstead) &&
!hostnameFromMasterPOV.equals(rpcServices.isa.getHostName())) {
String msg = "Master passed us a different hostname to use; was=" +
rpcServices.isa.getHostName() + ", but now=" + hostnameFromMasterPOV;
@@ -1691,7 +1683,7 @@ public class HRegionServer extends HasThread implements
CompactionChecker(final HRegionServer h, final int sleepTime, final Stoppable stopper) {
super("CompactionChecker", stopper, sleepTime);
this.instance = h;
- LOG.info(this.getName() + " runs every " + StringUtils.formatTime(sleepTime));
+ LOG.info(this.getName() + " runs every " + Duration.ofMillis(sleepTime));
/* MajorCompactPriority is configurable.
* If not set, the compaction will use default priority.
@@ -2386,7 +2378,7 @@ public class HRegionServer extends HasThread implements
// Do our best to report our abort to the master, but this may not work
try {
if (cause != null) {
- msg += "\nCause:\n" + StringUtils.stringifyException(cause);
+ msg += "\nCause:\n" + Throwables.getStackTraceAsString(cause);
}
// Report to the master but only if we have already registered with the master.
if (rssStub != null && this.serverName != null) {
@@ -2614,7 +2606,7 @@ public class HRegionServer extends HasThread implements
long now = EnvironmentEdgeManager.currentTime();
int port = rpcServices.isa.getPort();
RegionServerStartupRequest.Builder request = RegionServerStartupRequest.newBuilder();
- if (shouldUseThisHostnameInstead()) {
+ if (!StringUtils.isBlank(useThisHostnameInstead)) {
request.setUseThisHostnameInstead(useThisHostnameInstead);
}
request.setPort(port);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b638f7e/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
index 45d5b08..83fafff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
@@ -103,7 +103,7 @@ public class MockMasterServices extends MockNoopMasterServices {
super(conf);
this.regionsToRegionServers = regionsToRegionServers;
Superusers.initialize(conf);
- this.fileSystemManager = new MasterFileSystem(this);
+ this.fileSystemManager = new MasterFileSystem(conf);
this.walManager = new MasterWalManager(this);
// Mock an AM.
this.assignmentManager = new AssignmentManager(this, new MockRegionStateStore(this)) {
[03/31] hbase git commit: HBASE-19732 Replica regions does not return
back the MSLAB chunks to pool (Ram)
Posted by zh...@apache.org.
HBASE-19732 Replica regions does not return back the MSLAB chunks to pool
(Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d155a4e3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d155a4e3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d155a4e3
Branch: refs/heads/HBASE-19064
Commit: d155a4e33d279bdf4b57b08ebc8220fe5d97f4cc
Parents: 3a66f15
Author: Vasudevan <ra...@intel.com>
Authored: Mon Jan 15 22:15:01 2018 +0530
Committer: Vasudevan <ra...@intel.com>
Committed: Mon Jan 15 22:20:27 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/regionserver/HStore.java | 1 +
.../hadoop/hbase/regionserver/MemStoreSnapshot.java | 12 +++++++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d155a4e3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 2ccc8a0..01121dd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -2283,6 +2283,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
long snapshotId = -1; // -1 means do not drop
if (dropMemstoreSnapshot && snapshot != null) {
snapshotId = snapshot.getId();
+ snapshot.close();
}
HStore.this.updateStorefiles(storeFiles, snapshotId);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d155a4e3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java
index 0d4e5f1..1a0317d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.regionserver;
import org.apache.yetus.audience.InterfaceAudience;
+import java.io.Closeable;
import java.util.List;
/**
* Holds details of the snapshot taken on a MemStore. Details include the snapshot's identifier,
@@ -26,7 +27,7 @@ import java.util.List;
* all the cells and a scanner to read all cells in it.
*/
@InterfaceAudience.Private
-public class MemStoreSnapshot {
+public class MemStoreSnapshot implements Closeable {
private final long id;
private final int cellsCount;
private final long dataSize;
@@ -90,4 +91,13 @@ public class MemStoreSnapshot {
public boolean isTagsPresent() {
return this.tagsPresent;
}
+
+ @Override
+ public void close() {
+ if (this.scanners != null) {
+ for (KeyValueScanner scanner : scanners) {
+ scanner.close();
+ }
+ }
+ }
}
[27/31] hbase git commit: HBASE-19757 System table gets stuck after
enabling region server group feature in secure cluster - addendum removes
unused import
Posted by zh...@apache.org.
HBASE-19757 System table gets stuck after enabling region server group feature in secure cluster - addendum removes unused import
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2018ffc9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2018ffc9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2018ffc9
Branch: refs/heads/HBASE-19064
Commit: 2018ffc943fd93c409a1a267a7dd785439e33bd6
Parents: fd43507
Author: tedyu <yu...@gmail.com>
Authored: Thu Jan 18 18:13:56 2018 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jan 18 18:13:56 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2018ffc9/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
index 956f8ba..25f833d 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.MetaTableAccessor.DefaultVisitorBase;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ClusterConnection;
[14/31] hbase git commit: HBASE-19815 Flakey
TestAssignmentManager.testAssignWithRandExec
Posted by zh...@apache.org.
HBASE-19815 Flakey TestAssignmentManager.testAssignWithRandExec
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/53d0c238
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/53d0c238
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/53d0c238
Branch: refs/heads/HBASE-19064
Commit: 53d0c2388d9eb564c20e4231d3cb7ae7be360478
Parents: c1a8dc0
Author: Michael Stack <st...@apache.org>
Authored: Wed Jan 17 10:56:50 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Wed Jan 17 13:52:39 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/master/procedure/RecoverMetaProcedure.java | 6 +++---
.../apache/hadoop/hbase/master/MockNoopMasterServices.java | 7 ++++---
2 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/53d0c238/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java
index 6aa7fff..50ef3e0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
-import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.assignment.AssignProcedure;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
@@ -58,7 +58,7 @@ public class RecoverMetaProcedure
private int replicaId;
private final ProcedurePrepareLatch syncLatch;
- private HMaster master;
+ private MasterServices master;
/**
* Call this constructor to queue up a {@link RecoverMetaProcedure} in response to meta
@@ -250,7 +250,7 @@ public class RecoverMetaProcedure
*/
private void prepare(MasterProcedureEnv env) {
if (master == null) {
- master = (HMaster) env.getMasterServices();
+ master = env.getMasterServices();
Preconditions.checkArgument(master != null);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/53d0c238/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index 7196b7c..48e9e8d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
@@ -58,7 +57,9 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-public class MockNoopMasterServices implements MasterServices, Server {
+import com.google.protobuf.Service;
+
+public class MockNoopMasterServices implements MasterServices {
private final Configuration conf;
private final MetricsMaster metricsMaster;
@@ -480,4 +481,4 @@ public class MockNoopMasterServices implements MasterServices, Server {
public ReplicationPeerManager getReplicationPeerManager() {
return null;
}
-}
\ No newline at end of file
+}
[22/31] hbase git commit: HBASE-19527 Make ExecutorService threads
daemon=true
Posted by zh...@apache.org.
HBASE-19527 Make ExecutorService threads daemon=true
Set the ProcedureExcecutor worker threads as daemon.
Ditto for the timeout thread.
Remove hack from TestRegionsOnMasterOptions that was
put in place because the test would not go down.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/646770dd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/646770dd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/646770dd
Branch: refs/heads/HBASE-19064
Commit: 646770dd51aefab1174ae949230e084402ea8336
Parents: b3eff31
Author: Michael Stack <st...@apache.org>
Authored: Fri Dec 15 13:57:16 2017 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Thu Jan 18 11:30:15 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/procedure2/ProcedureExecutor.java | 2 ++
.../hadoop/hbase/executor/ExecutorService.java | 2 +-
.../master/balancer/TestRegionsOnMasterOptions.java | 16 +++++-----------
3 files changed, 8 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/646770dd/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 069bf28..7a964a8 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -1710,6 +1710,7 @@ public class ProcedureExecutor<TEnvironment> {
public WorkerThread(final ThreadGroup group) {
super(group, "ProcExecWrkr-" + workerId.incrementAndGet());
+ setDaemon(true);
}
@Override
@@ -1784,6 +1785,7 @@ public class ProcedureExecutor<TEnvironment> {
public TimeoutExecutorThread(final ThreadGroup group) {
super(group, "ProcExecTimeout");
+ setDaemon(true);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/646770dd/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
index d04956b..342d441 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorService.java
@@ -178,7 +178,7 @@ public class ExecutorService {
// name the threads for this threadpool
ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
tfb.setNameFormat(this.name + "-%d");
- // TODO: Should these executors be daemon threads?
+ tfb.setDaemon(true);
this.threadPoolExecutor.setThreadFactory(tfb.build());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/646770dd/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java
index 8f06886..f7d10dc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java
@@ -18,12 +18,15 @@
package org.apache.hadoop.hbase.master.balancer;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.CategoryBasedTimeout;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.LoadBalancer;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.apache.hadoop.hbase.util.Threads;
@@ -196,15 +199,6 @@ public class TestRegionsOnMasterOptions {
// still only carry system regions post crash.
assertEquals(masterCount, mNewActualCount);
}
- // Disable balancer and wait till RIT done else cluster won't go down.
- TEST_UTIL.getAdmin().balancerSwitch(false, true);
- while (true) {
- if (!TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().
- isMetaRegionInTransition()) {
- break;
- }
- Threads.sleep(10);
- }
} finally {
LOG.info("Running shutdown of cluster");
TEST_UTIL.shutdownMiniCluster();
[06/31] hbase git commit: HBASE-19739 Include thrift IDL files in
hbase-thrift.jar and .proto files in hbase-protocol-shaded.jar respectively
Posted by zh...@apache.org.
HBASE-19739 Include thrift IDL files in hbase-thrift.jar and .proto files in hbase-protocol-shaded.jar respectively
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8b6b2b0b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8b6b2b0b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8b6b2b0b
Branch: refs/heads/HBASE-19064
Commit: 8b6b2b0b227f9b16d9c85ab69afa1f968ce72400
Parents: d3a306d
Author: Umesh Agashe <ua...@cloudera.com>
Authored: Wed Jan 10 14:03:30 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Tue Jan 16 12:35:00 2018 -0800
----------------------------------------------------------------------
hbase-protocol-shaded/pom.xml | 8 ++++++++
hbase-thrift/pom.xml | 6 ++++++
2 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b6b2b0b/hbase-protocol-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/pom.xml b/hbase-protocol-shaded/pom.xml
index 03030b8..1da0a32 100644
--- a/hbase-protocol-shaded/pom.xml
+++ b/hbase-protocol-shaded/pom.xml
@@ -38,6 +38,14 @@
<internal.protobuf.version>3.3.0</internal.protobuf.version>
</properties>
<build>
+ <resources>
+ <resource>
+ <directory>src/main/</directory>
+ <includes>
+ <include>**/*.proto</include>
+ </includes>
+ </resource>
+ </resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b6b2b0b/hbase-thrift/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml
index 891676a..7410d99 100644
--- a/hbase-thrift/pom.xml
+++ b/hbase-thrift/pom.xml
@@ -43,6 +43,12 @@
<include>hbase-webapps/**</include>
</includes>
</resource>
+ <resource>
+ <directory>src/main/resources/</directory>
+ <includes>
+ <include>**/*.thrift</include>
+ </includes>
+ </resource>
</resources>
<testResources>
<testResource>
[07/31] hbase git commit: HBASE-19792
TestReplicationSmallTests.testDisableEnable fails
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8d6ecda/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
new file mode 100644
index 0000000..cb47827
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -0,0 +1,436 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.client.replication.TableCFs;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
+import org.apache.hadoop.hbase.replication.regionserver.Replication;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALEdit;
+import org.apache.hadoop.hbase.wal.WALKeyImpl;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
+
+@Category({ ReplicationTests.class, LargeTests.class })
+public class TestReplicationSmallTests extends TestReplicationBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestReplicationSmallTests.class);
+ private static final String PEER_ID = "2";
+
+ @Before
+ public void setUp() throws Exception {
+ cleanUp();
+ }
+
+ /**
+ * Verify that version and column delete marker types are replicated correctly.
+ */
+ @Test
+ public void testDeleteTypes() throws Exception {
+ LOG.info("testDeleteTypes");
+ final byte[] v1 = Bytes.toBytes("v1");
+ final byte[] v2 = Bytes.toBytes("v2");
+ final byte[] v3 = Bytes.toBytes("v3");
+ htable1 = utility1.getConnection().getTable(tableName);
+
+ long t = EnvironmentEdgeManager.currentTime();
+ // create three versions for "row"
+ Put put = new Put(row);
+ put.addColumn(famName, row, t, v1);
+ htable1.put(put);
+
+ put = new Put(row);
+ put.addColumn(famName, row, t + 1, v2);
+ htable1.put(put);
+
+ put = new Put(row);
+ put.addColumn(famName, row, t + 2, v3);
+ htable1.put(put);
+
+ Get get = new Get(row);
+ get.readAllVersions();
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for put replication");
+ }
+ Result res = htable2.get(get);
+ if (res.size() < 3) {
+ LOG.info("Rows not available");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ assertArrayEquals(CellUtil.cloneValue(res.rawCells()[0]), v3);
+ assertArrayEquals(CellUtil.cloneValue(res.rawCells()[1]), v2);
+ assertArrayEquals(CellUtil.cloneValue(res.rawCells()[2]), v1);
+ break;
+ }
+ }
+ // place a version delete marker (delete last version)
+ Delete d = new Delete(row);
+ d.addColumn(famName, row, t);
+ htable1.delete(d);
+
+ get = new Get(row);
+ get.readAllVersions();
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for put replication");
+ }
+ Result res = htable2.get(get);
+ if (res.size() > 2) {
+ LOG.info("Version not deleted");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ assertArrayEquals(CellUtil.cloneValue(res.rawCells()[0]), v3);
+ assertArrayEquals(CellUtil.cloneValue(res.rawCells()[1]), v2);
+ break;
+ }
+ }
+
+ // place a column delete marker
+ d = new Delete(row);
+ d.addColumns(famName, row, t + 2);
+ htable1.delete(d);
+
+ // now *both* of the remaining version should be deleted
+ // at the replica
+ get = new Get(row);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for del replication");
+ }
+ Result res = htable2.get(get);
+ if (res.size() >= 1) {
+ LOG.info("Rows not deleted");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Add a row, check it's replicated, delete it, check's gone
+ */
+ @Test
+ public void testSimplePutDelete() throws Exception {
+ LOG.info("testSimplePutDelete");
+ runSimplePutDeleteTest();
+ }
+
+ /**
+ * Try a small batch upload using the write buffer, check it's replicated
+ */
+ @Test
+ public void testSmallBatch() throws Exception {
+ LOG.info("testSmallBatch");
+ runSmallBatchTest();
+ }
+
+ /**
+ * Test disable/enable replication, trying to insert, make sure nothing's replicated, enable it,
+ * the insert should be replicated
+ */
+ @Test
+ public void testDisableEnable() throws Exception {
+ // Test disabling replication
+ hbaseAdmin.disableReplicationPeer(PEER_ID);
+
+ byte[] rowkey = Bytes.toBytes("disable enable");
+ Put put = new Put(rowkey);
+ put.addColumn(famName, row, row);
+ htable1.put(put);
+
+ Get get = new Get(rowkey);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ Result res = htable2.get(get);
+ if (res.size() >= 1) {
+ fail("Replication wasn't disabled");
+ } else {
+ LOG.info("Row not replicated, let's wait a bit more...");
+ Thread.sleep(SLEEP_TIME);
+ }
+ }
+
+ // Test enable replication
+ hbaseAdmin.enableReplicationPeer(PEER_ID);
+
+ for (int i = 0; i < NB_RETRIES; i++) {
+ Result res = htable2.get(get);
+ if (res.isEmpty()) {
+ LOG.info("Row not available");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ assertArrayEquals(res.value(), row);
+ return;
+ }
+ }
+ fail("Waited too much time for put replication");
+ }
+
+ /**
+ * Integration test for TestReplicationAdmin, removes and re-add a peer cluster
+ */
+ @Test
+ public void testAddAndRemoveClusters() throws Exception {
+ LOG.info("testAddAndRemoveClusters");
+ hbaseAdmin.removeReplicationPeer(PEER_ID);
+ Thread.sleep(SLEEP_TIME);
+ byte[] rowKey = Bytes.toBytes("Won't be replicated");
+ Put put = new Put(rowKey);
+ put.addColumn(famName, row, row);
+ htable1.put(put);
+
+ Get get = new Get(rowKey);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ break;
+ }
+ Result res = htable2.get(get);
+ if (res.size() >= 1) {
+ fail("Not supposed to be replicated");
+ } else {
+ LOG.info("Row not replicated, let's wait a bit more...");
+ Thread.sleep(SLEEP_TIME);
+ }
+ }
+ ReplicationPeerConfig rpc =
+ ReplicationPeerConfig.newBuilder().setClusterKey(utility2.getClusterKey()).build();
+ hbaseAdmin.addReplicationPeer(PEER_ID, rpc);
+ Thread.sleep(SLEEP_TIME);
+ rowKey = Bytes.toBytes("do rep");
+ put = new Put(rowKey);
+ put.addColumn(famName, row, row);
+ LOG.info("Adding new row");
+ htable1.put(put);
+
+ get = new Get(rowKey);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for put replication");
+ }
+ Result res = htable2.get(get);
+ if (res.isEmpty()) {
+ LOG.info("Row not available");
+ Thread.sleep(SLEEP_TIME * i);
+ } else {
+ assertArrayEquals(res.value(), row);
+ break;
+ }
+ }
+ }
+
+ /**
+ * Do a more intense version testSmallBatch, one that will trigger wal rolling and other
+ * non-trivial code paths
+ */
+ @Test
+ public void testLoading() throws Exception {
+ LOG.info("Writing out rows to table1 in testLoading");
+ List<Put> puts = new ArrayList<>(NB_ROWS_IN_BIG_BATCH);
+ for (int i = 0; i < NB_ROWS_IN_BIG_BATCH; i++) {
+ Put put = new Put(Bytes.toBytes(i));
+ put.addColumn(famName, row, row);
+ puts.add(put);
+ }
+ // The puts will be iterated through and flushed only when the buffer
+ // size is reached.
+ htable1.put(puts);
+
+ Scan scan = new Scan();
+
+ ResultScanner scanner = htable1.getScanner(scan);
+ Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH);
+ scanner.close();
+
+ assertEquals(NB_ROWS_IN_BIG_BATCH, res.length);
+
+ LOG.info("Looking in table2 for replicated rows in testLoading");
+ long start = System.currentTimeMillis();
+ // Retry more than NB_RETRIES. As it was, retries were done in 5 seconds and we'd fail
+ // sometimes.
+ final long retries = NB_RETRIES * 10;
+ for (int i = 0; i < retries; i++) {
+ scan = new Scan();
+ scanner = htable2.getScanner(scan);
+ res = scanner.next(NB_ROWS_IN_BIG_BATCH);
+ scanner.close();
+ if (res.length != NB_ROWS_IN_BIG_BATCH) {
+ if (i == retries - 1) {
+ int lastRow = -1;
+ for (Result result : res) {
+ int currentRow = Bytes.toInt(result.getRow());
+ for (int row = lastRow + 1; row < currentRow; row++) {
+ LOG.error("Row missing: " + row);
+ }
+ lastRow = currentRow;
+ }
+ LOG.error("Last row: " + lastRow);
+ fail("Waited too much time for normal batch replication, " + res.length + " instead of " +
+ NB_ROWS_IN_BIG_BATCH + "; waited=" + (System.currentTimeMillis() - start) + "ms");
+ } else {
+ LOG.info("Only got " + res.length + " rows... retrying");
+ Thread.sleep(SLEEP_TIME);
+ }
+ } else {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out the
+ * compaction WALEdit.
+ */
+ @Test
+ public void testCompactionWALEdits() throws Exception {
+ WALProtos.CompactionDescriptor compactionDescriptor =
+ WALProtos.CompactionDescriptor.getDefaultInstance();
+ RegionInfo hri = RegionInfoBuilder.newBuilder(htable1.getName())
+ .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).build();
+ WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor);
+ Replication.scopeWALEdits(new WALKeyImpl(), edit, htable1.getConfiguration(), null);
+ }
+
+ /**
+ * Test for HBASE-8663
+ * <p>
+ * Create two new Tables with colfamilies enabled for replication then run
+ * ReplicationAdmin.listReplicated(). Finally verify the table:colfamilies. Note:
+ * TestReplicationAdmin is a better place for this testing but it would need mocks.
+ */
+ @Test
+ public void testVerifyListReplicatedTable() throws Exception {
+ LOG.info("testVerifyListReplicatedTable");
+
+ final String tName = "VerifyListReplicated_";
+ final String colFam = "cf1";
+ final int numOfTables = 3;
+
+ Admin hadmin = utility1.getAdmin();
+
+ // Create Tables
+ for (int i = 0; i < numOfTables; i++) {
+ hadmin.createTable(TableDescriptorBuilder.newBuilder(TableName.valueOf(tName + i))
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(colFam))
+ .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
+ .build());
+ }
+
+ // verify the result
+ List<TableCFs> replicationColFams = hbaseAdmin.listReplicatedTableCFs();
+ int[] match = new int[numOfTables]; // array of 3 with init value of zero
+
+ for (int i = 0; i < replicationColFams.size(); i++) {
+ TableCFs replicationEntry = replicationColFams.get(i);
+ String tn = replicationEntry.getTable().getNameAsString();
+ if (tn.startsWith(tName) && replicationEntry.getColumnFamilyMap().containsKey(colFam)) {
+ int m = Integer.parseInt(tn.substring(tn.length() - 1)); // get the last digit
+ match[m]++; // should only increase once
+ }
+ }
+
+ // check the matching result
+ for (int i = 0; i < match.length; i++) {
+ assertTrue("listReplicated() does not match table " + i, (match[i] == 1));
+ }
+
+ // drop tables
+ for (int i = 0; i < numOfTables; i++) {
+ TableName tableName = TableName.valueOf(tName + i);
+ hadmin.disableTable(tableName);
+ hadmin.deleteTable(tableName);
+ }
+
+ hadmin.close();
+ }
+
+ /**
+ * Test for HBase-15259 WALEdits under replay will also be replicated
+ */
+ @Test
+ public void testReplicationInReplay() throws Exception {
+ final TableName tableName = htable1.getName();
+
+ HRegion region = utility1.getMiniHBaseCluster().getRegions(tableName).get(0);
+ RegionInfo hri = region.getRegionInfo();
+ NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ for (byte[] fam : htable1.getDescriptor().getColumnFamilyNames()) {
+ scopes.put(fam, 1);
+ }
+ final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
+ int index = utility1.getMiniHBaseCluster().getServerWith(hri.getRegionName());
+ WAL wal = utility1.getMiniHBaseCluster().getRegionServer(index).getWAL(region.getRegionInfo());
+ final byte[] rowName = Bytes.toBytes("testReplicationInReplay");
+ final byte[] qualifier = Bytes.toBytes("q");
+ final byte[] value = Bytes.toBytes("v");
+ WALEdit edit = new WALEdit(true);
+ long now = EnvironmentEdgeManager.currentTime();
+ edit.add(new KeyValue(rowName, famName, qualifier, now, value));
+ WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes);
+ wal.append(hri, walKey, edit, true);
+ wal.sync();
+
+ Get get = new Get(rowName);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ break;
+ }
+ Result res = htable2.get(get);
+ if (res.size() >= 1) {
+ fail("Not supposed to be replicated for " + Bytes.toString(res.getRow()));
+ } else {
+ LOG.info("Row not replicated, let's wait a bit more...");
+ Thread.sleep(SLEEP_TIME);
+ }
+ }
+ }
+}
[20/31] hbase git commit: HBASE-19820 Restore public constructor of
MiniHBaseCluster (API compat).
Posted by zh...@apache.org.
HBASE-19820 Restore public constructor of MiniHBaseCluster (API compat).
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/09ffbb5b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/09ffbb5b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/09ffbb5b
Branch: refs/heads/HBASE-19064
Commit: 09ffbb5b68caa395407241eb891d567c98aa49b7
Parents: 7573e07
Author: Apekshit Sharma <ap...@apache.org>
Authored: Wed Jan 17 18:45:58 2018 -0800
Committer: Apekshit Sharma <ap...@apache.org>
Committed: Thu Jan 18 09:05:30 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/MiniHBaseCluster.java | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/09ffbb5b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index e7f8496..3a40477 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -78,7 +78,20 @@ public class MiniHBaseCluster extends HBaseCluster {
*/
public MiniHBaseCluster(Configuration conf, int numMasters, int numRegionServers)
throws IOException, InterruptedException {
- this(conf, numMasters, numRegionServers, null, null, null);
+ this(conf, numMasters, numRegionServers, null, null);
+ }
+
+ /**
+ * Start a MiniHBaseCluster.
+ * @param conf Configuration to be used for cluster
+ * @param numMasters initial number of masters to start.
+ * @param numRegionServers initial number of region servers to start.
+ */
+ public MiniHBaseCluster(Configuration conf, int numMasters, int numRegionServers,
+ Class<? extends HMaster> masterClass,
+ Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)
+ throws IOException, InterruptedException {
+ this(conf, numMasters, numRegionServers, null, masterClass, regionserverClass);
}
/**
[11/31] hbase git commit: HBASE-19770 Separate command return values
from interactive shells
Posted by zh...@apache.org.
HBASE-19770 Separate command return values from interactive shells
Uses a new option to the shell to specify that return values are
unwanted instead of overloading the interactive option. Enable
return_values when the shell is non-interactive.
Signed-off-by: Apekshit Sharma <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7224546b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7224546b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7224546b
Branch: refs/heads/HBASE-19064
Commit: 7224546b1ed99faf41ecef82043971c7d44a5836
Parents: a3c98b2
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 11 14:47:13 2018 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Jan 17 13:14:46 2018 -0500
----------------------------------------------------------------------
bin/hirb.rb | 8 +++++++-
hbase-shell/src/main/ruby/shell.rb | 18 +++++++++---------
.../src/test/ruby/shell/noninteractive_test.rb | 2 +-
hbase-shell/src/test/ruby/test_helper.rb | 2 +-
4 files changed, 18 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7224546b/bin/hirb.rb
----------------------------------------------------------------------
diff --git a/bin/hirb.rb b/bin/hirb.rb
index 35895f8..6e92d51 100644
--- a/bin/hirb.rb
+++ b/bin/hirb.rb
@@ -59,12 +59,15 @@ Usage: shell [OPTIONS] [SCRIPTFILE [ARGUMENTS]]
-n | --noninteractive Do not run within an IRB session
and exit with non-zero status on
first error.
+ -r | --return-values Include return values from commands
+ executed in the shell.
HERE
found = []
script2run = nil
log_level = org.apache.log4j.Level::ERROR
@shell_debug = false
interactive = true
+return_values = false
for arg in ARGV
if arg == '-h' || arg == '--help'
puts cmdline_help
@@ -78,6 +81,9 @@ for arg in ARGV
elsif arg == '-n' || arg == '--noninteractive'
interactive = false
found.push(arg)
+ elsif arg == '-r' || arg == '--return-values'
+ return_values = true
+ found.push(arg)
else
# Presume it a script. Save it off for running later below
# after we've set up some environment.
@@ -110,7 +116,7 @@ require 'shell/formatter'
@hbase = Hbase::Hbase.new
# Setup console
-@shell = Shell::Shell.new(@hbase, interactive)
+@shell = Shell::Shell.new(@hbase, interactive, return_values)
@shell.debug = @shell_debug
# Add commands to this namespace
http://git-wip-us.apache.org/repos/asf/hbase/blob/7224546b/hbase-shell/src/main/ruby/shell.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
index a01a890..4472021 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -71,14 +71,19 @@ module Shell
class Shell
attr_accessor :hbase
attr_accessor :interactive
+ attr_accessor :return_values
alias interactive? interactive
+ alias return_values? return_values
@debug = false
attr_accessor :debug
- def initialize(hbase, interactive = true)
+ def initialize(hbase, interactive = true, return_values = !interactive)
self.hbase = hbase
self.interactive = interactive
+ self.return_values = return_values
+ # If we're in non-interactive mode, force return_values
+ self.return_values = true if not self.interactive
end
# Returns Admin class from admin.rb
@@ -135,16 +140,11 @@ module Shell
end
# call the method 'command' on the specified command
- # If interactive is enabled, then we suppress the return value. The command should have
- # printed relevant output.
- # Return value is only useful in non-interactive mode, for e.g. tests.
+ # If return_values is false, then we suppress the return value. The command
+ # should have printed relevant output.
def command(command, *args)
ret = internal_command(command, :command, *args)
- if interactive
- return nil
- else
- return ret
- end
+ ret if return_values
end
# call a specific internal method in the command instance
http://git-wip-us.apache.org/repos/asf/hbase/blob/7224546b/hbase-shell/src/test/ruby/shell/noninteractive_test.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/shell/noninteractive_test.rb b/hbase-shell/src/test/ruby/shell/noninteractive_test.rb
index 0fae4cb..75a2c7d 100644
--- a/hbase-shell/src/test/ruby/shell/noninteractive_test.rb
+++ b/hbase-shell/src/test/ruby/shell/noninteractive_test.rb
@@ -20,7 +20,7 @@ require 'shell'
class NonInteractiveTest < Test::Unit::TestCase
def setup
@hbase = ::Hbase::Hbase.new($TEST_CLUSTER.getConfiguration)
- @shell = Shell::Shell.new(@hbase, false)
+ @shell = Shell::Shell.new(@hbase, false, true)
end
define_test "Shell::Shell noninteractive mode should throw" do
http://git-wip-us.apache.org/repos/asf/hbase/blob/7224546b/hbase-shell/src/test/ruby/test_helper.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/test_helper.rb b/hbase-shell/src/test/ruby/test_helper.rb
index ec6bb6a..874b554 100644
--- a/hbase-shell/src/test/ruby/test_helper.rb
+++ b/hbase-shell/src/test/ruby/test_helper.rb
@@ -43,7 +43,7 @@ module Hbase
def setup_hbase
hbase = ::Hbase::Hbase.new($TEST_CLUSTER.getConfiguration)
- @shell = ::Shell::Shell.new(hbase, interactive = false)
+ @shell = ::Shell::Shell.new(hbase, interactive = false, return_values = true)
end
def shutdown
[05/31] hbase git commit: HBASE-19791 TestZKAsyncRegistry hangs
Posted by zh...@apache.org.
HBASE-19791 TestZKAsyncRegistry hangs
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d3a306d8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d3a306d8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d3a306d8
Branch: refs/heads/HBASE-19064
Commit: d3a306d81d3f087696fc6d45dd8d6bda939378b2
Parents: 9917244
Author: Michael Stack <st...@apache.org>
Authored: Sat Jan 13 12:40:36 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Tue Jan 16 12:34:05 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a306d8/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
index 3478549..a0c4c25 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
@@ -47,6 +47,9 @@ import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * Test the ZooKeeper Async Registry.
+ */
@Category({ MediumTests.class, ClientTests.class })
public class TestZKAsyncRegistry {
private static final Logger LOG = LoggerFactory.getLogger(TestZKAsyncRegistry.class);
[12/31] hbase git commit: HBASE-19809 Fix findbugs and error-prone
warnings in hbase-procedure (branch-2)
Posted by zh...@apache.org.
HBASE-19809 Fix findbugs and error-prone warnings in hbase-procedure (branch-2)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c269e63a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c269e63a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c269e63a
Branch: refs/heads/HBASE-19064
Commit: c269e63a07360eeee6db57e65b06f3df12b91373
Parents: 7224546
Author: Peter Somogyi <ps...@cloudera.com>
Authored: Wed Jan 17 09:47:59 2018 +0100
Committer: Michael Stack <st...@apache.org>
Committed: Wed Jan 17 11:23:38 2018 -0800
----------------------------------------------------------------------
.../hbase/procedure2/RemoteProcedureDispatcher.java | 3 +++
.../procedure2/store/wal/ProcedureWALPrettyPrinter.java | 1 +
.../org/apache/hadoop/hbase/procedure2/util/ByteSlot.java | 2 ++
.../hadoop/hbase/procedure2/TestProcedureEvents.java | 2 +-
.../hadoop/hbase/procedure2/TestProcedureExecutor.java | 2 +-
.../hbase/procedure2/TestProcedureInMemoryChore.java | 10 +++++-----
.../hadoop/hbase/procedure2/TestProcedureReplayOrder.java | 5 +++--
.../hbase/procedure2/TestStateMachineProcedure.java | 8 +++++++-
.../hadoop/hbase/procedure2/TestYieldProcedures.java | 2 +-
.../wal/ProcedureWALLoaderPerformanceEvaluation.java | 5 ++---
.../store/wal/ProcedureWALPerformanceEvaluation.java | 2 +-
11 files changed, 27 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index bdff1ca..9da7748 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -347,10 +347,12 @@ public abstract class RemoteProcedureDispatcher<TEnv, TRemote extends Comparable
super(key, 0);
}
+ @Override
public TRemote getKey() {
return getObject();
}
+ @Override
public synchronized void add(final RemoteProcedure operation) {
if (this.operations == null) {
this.operations = new HashSet<>();
@@ -364,6 +366,7 @@ public abstract class RemoteProcedureDispatcher<TEnv, TRemote extends Comparable
}
}
+ @Override
public synchronized void dispatch() {
if (operations != null) {
remoteDispatch(getKey(), operations);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
index eb8994b..3f007c6 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
@@ -149,6 +149,7 @@ public class ProcedureWALPrettyPrinter extends Configured implements Tool {
* @throws IOException
* Thrown upon file system errors etc.
*/
+ @Override
public int run(final String[] args) throws IOException {
// create options
Options options = new Options();
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
index 068a823..3e95de5 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
@@ -81,11 +81,13 @@ public class ByteSlot extends OutputStream {
buf[offset] = (byte)b;
}
+ @Override
public void write(int b) {
ensureCapacity(size + 1);
buf[size++] = (byte)b;
}
+ @Override
public void write(byte[] b, int off, int len) {
ensureCapacity(size + len);
System.arraycopy(b, off, buf, size, len);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureEvents.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureEvents.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureEvents.java
index a0afe70..f36da82 100644
--- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureEvents.java
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureEvents.java
@@ -160,7 +160,7 @@ public class TestProcedureEvents {
}
@Override
- protected boolean setTimeoutFailure(final TestProcEnv env) {
+ protected synchronized boolean setTimeoutFailure(final TestProcEnv env) {
int n = ntimeouts.incrementAndGet();
LOG.info("HANDLE TIMEOUT " + this + " ntimeouts=" + n);
setState(ProcedureState.RUNNABLE);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecutor.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecutor.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecutor.java
index 29a0472..e5b5070 100644
--- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecutor.java
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecutor.java
@@ -186,5 +186,5 @@ public class TestProcedureExecutor {
}
}
- private class TestProcEnv { }
+ private static class TestProcEnv { }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureInMemoryChore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureInMemoryChore.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureInMemoryChore.java
index 6546ea3..33d93d5 100644
--- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureInMemoryChore.java
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureInMemoryChore.java
@@ -18,14 +18,17 @@
package org.apache.hadoop.hbase.procedure2;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.procedure2.store.NoopProcedureStore;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -33,10 +36,6 @@ import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
@Category({MasterTests.class, SmallTests.class})
public class TestProcedureInMemoryChore {
private static final Logger LOG = LoggerFactory.getLogger(TestProcedureInMemoryChore.class);
@@ -49,6 +48,7 @@ public class TestProcedureInMemoryChore {
private HBaseCommonTestingUtility htu;
+ @SuppressWarnings("rawtypes")
@Before
public void setUp() throws IOException {
htu = new HBaseCommonTestingUtility();
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java
index dc359a1..821901c 100644
--- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java
@@ -132,8 +132,9 @@ public class TestProcedureReplayOrder {
public void run() {
for (int i = 0; i < nprocPerThread; ++i) {
try {
- procExecutor.submitProcedure((Procedure)procClazz.newInstance());
- } catch (InstantiationException|IllegalAccessException e) {
+ procExecutor.submitProcedure((Procedure)
+ procClazz.getDeclaredConstructor().newInstance());
+ } catch (Exception e) {
LOG.error("unable to instantiate the procedure", e);
fail("failure during the proc.newInstance(): " + e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.java
index 43d47f2..563afbc 100644
--- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.java
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.java
@@ -153,6 +153,7 @@ public class TestStateMachineProcedure {
public static class TestSMProcedure
extends StateMachineProcedure<TestProcEnv, TestSMProcedureState> {
+ @Override
protected Flow executeFromState(TestProcEnv env, TestSMProcedureState state) {
LOG.info("EXEC " + state + " " + this);
env.execCount.incrementAndGet();
@@ -169,25 +170,30 @@ public class TestStateMachineProcedure {
return Flow.HAS_MORE_STATE;
}
+ @Override
protected void rollbackState(TestProcEnv env, TestSMProcedureState state) {
LOG.info("ROLLBACK " + state + " " + this);
env.rollbackCount.incrementAndGet();
}
+ @Override
protected TestSMProcedureState getState(int stateId) {
return TestSMProcedureState.values()[stateId];
}
+ @Override
protected int getStateId(TestSMProcedureState state) {
return state.ordinal();
}
+ @Override
protected TestSMProcedureState getInitialState() {
return TestSMProcedureState.STEP_1;
}
}
public static class SimpleChildProcedure extends NoopProcedure<TestProcEnv> {
+ @Override
protected Procedure[] execute(TestProcEnv env) {
LOG.info("EXEC " + this);
env.execCount.incrementAndGet();
@@ -204,7 +210,7 @@ public class TestStateMachineProcedure {
}
}
- public class TestProcEnv {
+ public static class TestProcEnv {
AtomicInteger execCount = new AtomicInteger(0);
AtomicInteger rollbackCount = new AtomicInteger(0);
boolean triggerChildRollback = false;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java
index 2023535..e6fc4bb 100644
--- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java
@@ -186,7 +186,7 @@ public class TestYieldProcedures {
extends StateMachineProcedure<TestProcEnv, TestStateMachineProcedure.State> {
enum State { STATE_1, STATE_2, STATE_3 }
- public class ExecutionInfo {
+ public static class ExecutionInfo {
private final boolean rollback;
private final long timestamp;
private final State step;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALLoaderPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALLoaderPerformanceEvaluation.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALLoaderPerformanceEvaluation.java
index 503850d..3c908fd 100644
--- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALLoaderPerformanceEvaluation.java
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALLoaderPerformanceEvaluation.java
@@ -70,7 +70,7 @@ public class ProcedureWALLoaderPerformanceEvaluation extends AbstractHBaseTool {
private WALProcedureStore store;
static byte[] serializedState;
- private class LoadCounter implements ProcedureStore.ProcedureLoader {
+ private static class LoadCounter implements ProcedureStore.ProcedureLoader {
public LoadCounter() {}
@Override
@@ -165,8 +165,7 @@ public class ProcedureWALLoaderPerformanceEvaluation extends AbstractHBaseTool {
private void writeWals() throws IOException {
List<Integer> procStates = shuffleProcWriteSequence();
TestProcedure[] procs = new TestProcedure[numProcs + 1]; // 0 is not used.
- int numProcsPerWal = numWals > 0 ? (int)Math.ceil(procStates.size() / numWals)
- : Integer.MAX_VALUE;
+ int numProcsPerWal = numWals > 0 ? procStates.size() / numWals : Integer.MAX_VALUE;
long startTime = currentTimeMillis();
long lastTime = startTime;
for (int i = 0; i < procStates.size(); ++i) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/c269e63a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPerformanceEvaluation.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPerformanceEvaluation.java
index 1a7fc80..12ed930 100644
--- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPerformanceEvaluation.java
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPerformanceEvaluation.java
@@ -243,7 +243,7 @@ public class ProcedureWALPerformanceEvaluation extends AbstractHBaseTool {
}
}
- private class NoSyncWalProcedureStore extends WALProcedureStore {
+ private static class NoSyncWalProcedureStore extends WALProcedureStore {
public NoSyncWalProcedureStore(final Configuration conf, final Path logDir) throws IOException {
super(conf, logDir, null, new WALProcedureStore.LeaseRecovery() {
@Override
[21/31] hbase git commit: HBASE-19821
TestCleanerChore#testOnConfigurationChange() requires at least 4 processors
to get passed
Posted by zh...@apache.org.
HBASE-19821 TestCleanerChore#testOnConfigurationChange() requires at least 4 processors to get passed
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b3eff314
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b3eff314
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b3eff314
Branch: refs/heads/HBASE-19064
Commit: b3eff314323d89dcce0ffe94875213a4679913cd
Parents: 09ffbb5
Author: lixiang <li...@freewheel.tv>
Authored: Thu Jan 18 04:41:05 2018 +0000
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jan 18 11:20:29 2018 -0800
----------------------------------------------------------------------
.../hbase/master/cleaner/TestCleanerChore.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3eff314/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
index 7b49848..3db22e4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
@@ -356,13 +356,22 @@ public class TestCleanerChore {
@Test
public void testOnConfigurationChange() throws Exception {
+ int availableProcessorNum = Runtime.getRuntime().availableProcessors();
+ if (availableProcessorNum == 1) { // no need to run this test
+ return;
+ }
+
+ // have at least 2 available processors/cores
+ int initPoolSize = availableProcessorNum / 2;
+ int changedPoolSize = availableProcessorNum;
+
Stoppable stop = new StoppableImplementation();
Configuration conf = UTIL.getConfiguration();
Path testDir = UTIL.getDataTestDir();
FileSystem fs = UTIL.getTestFileSystem();
String confKey = "hbase.test.cleaner.delegates";
conf.set(confKey, AlwaysDelete.class.getName());
- conf.set(CleanerChore.CHORE_POOL_SIZE, "2");
+ conf.set(CleanerChore.CHORE_POOL_SIZE, String.valueOf(initPoolSize));
AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
chore.setEnabled(true);
// Create subdirs under testDir
@@ -381,9 +390,9 @@ public class TestCleanerChore {
t.setDaemon(true);
t.start();
// Change size of chore's pool
- conf.set(CleanerChore.CHORE_POOL_SIZE, "4");
+ conf.set(CleanerChore.CHORE_POOL_SIZE, String.valueOf(changedPoolSize));
chore.onConfigurationChange(conf);
- assertEquals(4, chore.getChorePoolSize());
+ assertEquals(changedPoolSize, chore.getChorePoolSize());
// Stop chore
t.join();
}
[02/31] hbase git commit: HBASE-19797 Operator priority leads to
wrong logic in ReplicationSourceWALReader
Posted by zh...@apache.org.
HBASE-19797 Operator priority leads to wrong logic in ReplicationSourceWALReader
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3a66f155
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3a66f155
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3a66f155
Branch: refs/heads/HBASE-19064
Commit: 3a66f155577e517ad836f124271a303779e6037f
Parents: 3b638f7
Author: zhangduo <zh...@apache.org>
Authored: Mon Jan 15 15:20:13 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Mon Jan 15 18:10:50 2018 +0800
----------------------------------------------------------------------
.../replication/regionserver/ReplicationSourceWALReader.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3a66f155/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
index e2201e5..90a421d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
@@ -204,8 +204,8 @@ public class ReplicationSourceWALReader extends Thread {
// (highly likely we've closed the current log), we've hit the max retries, and autorecovery is
// enabled, then dump the log
private void handleEofException(IOException e) {
- if (e instanceof EOFException ||
- e.getCause() instanceof EOFException && logQueue.size() > 1 && this.eofAutoRecovery) {
+ if ((e instanceof EOFException || e.getCause() instanceof EOFException) &&
+ logQueue.size() > 1 && this.eofAutoRecovery) {
try {
if (fs.getFileStatus(logQueue.peek()).getLen() == 0) {
LOG.warn("Forcing removal of 0 length log in queue: " + logQueue.peek());
[08/31] hbase git commit: HBASE-19792
TestReplicationSmallTests.testDisableEnable fails
Posted by zh...@apache.org.
HBASE-19792 TestReplicationSmallTests.testDisableEnable fails
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d8d6ecda
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d8d6ecda
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d8d6ecda
Branch: refs/heads/HBASE-19064
Commit: d8d6ecdad1b360f877b290e462cf8bf9b717753b
Parents: 8b6b2b0
Author: zhangduo <zh...@apache.org>
Authored: Wed Jan 17 14:23:05 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed Jan 17 14:23:05 2018 +0800
----------------------------------------------------------------------
.../replication/TestReplicationSmallTests.java | 1079 ------------------
.../replication/TestVerifyReplication.java | 493 ++++++++
.../hbase/replication/TestReplicationBase.java | 163 ++-
.../TestReplicationEmptyWALRecovery.java | 117 ++
.../replication/TestReplicationSmallTests.java | 436 +++++++
5 files changed, 1190 insertions(+), 1098 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8d6ecda/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
deleted file mode 100644
index 3877f71..0000000
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ /dev/null
@@ -1,1079 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.replication;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.client.RegionInfoBuilder;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.client.replication.TableCFs;
-import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
-import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
-import org.apache.hadoop.hbase.replication.regionserver.Replication;
-import org.apache.hadoop.hbase.replication.regionserver.ReplicationSource;
-import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface;
-import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.wal.WALKeyImpl;
-import org.apache.hadoop.hbase.wal.WALEdit;
-import org.apache.hadoop.mapreduce.Job;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
-
-@Category({ReplicationTests.class, LargeTests.class})
-public class TestReplicationSmallTests extends TestReplicationBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestReplicationSmallTests.class);
- private static final String PEER_ID = "2";
-
- @Rule
- public TestName name = new TestName();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- // Starting and stopping replication can make us miss new logs,
- // rolling like this makes sure the most recent one gets added to the queue
- for ( JVMClusterUtil.RegionServerThread r :
- utility1.getHBaseCluster().getRegionServerThreads()) {
- utility1.getAdmin().rollWALWriter(r.getRegionServer().getServerName());
- }
- int rowCount = utility1.countRows(tableName);
- utility1.deleteTableData(tableName);
- // truncating the table will send one Delete per row to the slave cluster
- // in an async fashion, which is why we cannot just call deleteTableData on
- // utility2 since late writes could make it to the slave in some way.
- // Instead, we truncate the first table and wait for all the Deletes to
- // make it to the slave.
- Scan scan = new Scan();
- int lastCount = 0;
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for truncate");
- }
- ResultScanner scanner = htable2.getScanner(scan);
- Result[] res = scanner.next(rowCount);
- scanner.close();
- if (res.length != 0) {
- if (res.length < lastCount) {
- i--; // Don't increment timeout if we make progress
- }
- lastCount = res.length;
- LOG.info("Still got " + res.length + " rows");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- }
-
- /**
- * Verify that version and column delete marker types are replicated
- * correctly.
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testDeleteTypes() throws Exception {
- LOG.info("testDeleteTypes");
- final byte[] v1 = Bytes.toBytes("v1");
- final byte[] v2 = Bytes.toBytes("v2");
- final byte[] v3 = Bytes.toBytes("v3");
- htable1 = utility1.getConnection().getTable(tableName);
-
- long t = EnvironmentEdgeManager.currentTime();
- // create three versions for "row"
- Put put = new Put(row);
- put.addColumn(famName, row, t, v1);
- htable1.put(put);
-
- put = new Put(row);
- put.addColumn(famName, row, t + 1, v2);
- htable1.put(put);
-
- put = new Put(row);
- put.addColumn(famName, row, t + 2, v3);
- htable1.put(put);
-
- Get get = new Get(row);
- get.readAllVersions();
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = htable2.get(get);
- if (res.size() < 3) {
- LOG.info("Rows not available");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[0]), v3);
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[1]), v2);
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[2]), v1);
- break;
- }
- }
- // place a version delete marker (delete last version)
- Delete d = new Delete(row);
- d.addColumn(famName, row, t);
- htable1.delete(d);
-
- get = new Get(row);
- get.readAllVersions();
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = htable2.get(get);
- if (res.size() > 2) {
- LOG.info("Version not deleted");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[0]), v3);
- assertArrayEquals(CellUtil.cloneValue(res.rawCells()[1]), v2);
- break;
- }
- }
-
- // place a column delete marker
- d = new Delete(row);
- d.addColumns(famName, row, t+2);
- htable1.delete(d);
-
- // now *both* of the remaining version should be deleted
- // at the replica
- get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for del replication");
- }
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- LOG.info("Rows not deleted");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- }
-
- /**
- * Add a row, check it's replicated, delete it, check's gone
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testSimplePutDelete() throws Exception {
- LOG.info("testSimplePutDelete");
- Put put = new Put(row);
- put.addColumn(famName, row, row);
-
- htable1 = utility1.getConnection().getTable(tableName);
- htable1.put(put);
-
- Get get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = htable2.get(get);
- if (res.isEmpty()) {
- LOG.info("Row not available");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(res.value(), row);
- break;
- }
- }
-
- Delete del = new Delete(row);
- htable1.delete(del);
-
- get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for del replication");
- }
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- LOG.info("Row not deleted");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- }
-
- /**
- * Try a small batch upload using the write buffer, check it's replicated
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testSmallBatch() throws Exception {
- LOG.info("testSmallBatch");
- // normal Batch tests
- loadData("", row);
-
- Scan scan = new Scan();
-
- ResultScanner scanner1 = htable1.getScanner(scan);
- Result[] res1 = scanner1.next(NB_ROWS_IN_BATCH);
- scanner1.close();
- assertEquals(NB_ROWS_IN_BATCH, res1.length);
-
- waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
- }
-
- private void waitForReplication(int expectedRows, int retries) throws IOException, InterruptedException {
- Scan scan;
- for (int i = 0; i < retries; i++) {
- scan = new Scan();
- if (i== retries -1) {
- fail("Waited too much time for normal batch replication");
- }
- ResultScanner scanner = htable2.getScanner(scan);
- Result[] res = scanner.next(expectedRows);
- scanner.close();
- if (res.length != expectedRows) {
- LOG.info("Only got " + res.length + " rows");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- }
-
- private void loadData(String prefix, byte[] row) throws IOException {
- List<Put> puts = new ArrayList<>(NB_ROWS_IN_BATCH);
- for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
- Put put = new Put(Bytes.toBytes(prefix + Integer.toString(i)));
- put.addColumn(famName, row, row);
- puts.add(put);
- }
- htable1.put(puts);
- }
-
- /**
- * Test disable/enable replication, trying to insert, make sure nothing's
- * replicated, enable it, the insert should be replicated
- *
- * @throws Exception
- */
- @Test(timeout = 300000)
- public void testDisableEnable() throws Exception {
-
- // Test disabling replication
- hbaseAdmin.disableReplicationPeer(PEER_ID);
-
- byte[] rowkey = Bytes.toBytes("disable enable");
- Put put = new Put(rowkey);
- put.addColumn(famName, row, row);
- htable1.put(put);
-
- Get get = new Get(rowkey);
- for (int i = 0; i < NB_RETRIES; i++) {
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- fail("Replication wasn't disabled");
- } else {
- LOG.info("Row not replicated, let's wait a bit more...");
- Thread.sleep(SLEEP_TIME);
- }
- }
-
- // Test enable replication
- hbaseAdmin.enableReplicationPeer(PEER_ID);
-
- for (int i = 0; i < NB_RETRIES; i++) {
- Result res = htable2.get(get);
- if (res.isEmpty()) {
- LOG.info("Row not available");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(res.value(), row);
- return;
- }
- }
- fail("Waited too much time for put replication");
- }
-
- /**
- * Integration test for TestReplicationAdmin, removes and re-add a peer
- * cluster
- *
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testAddAndRemoveClusters() throws Exception {
- LOG.info("testAddAndRemoveClusters");
- hbaseAdmin.removeReplicationPeer(PEER_ID);
- Thread.sleep(SLEEP_TIME);
- byte[] rowKey = Bytes.toBytes("Won't be replicated");
- Put put = new Put(rowKey);
- put.addColumn(famName, row, row);
- htable1.put(put);
-
- Get get = new Get(rowKey);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i == NB_RETRIES-1) {
- break;
- }
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- fail("Not supposed to be replicated");
- } else {
- LOG.info("Row not replicated, let's wait a bit more...");
- Thread.sleep(SLEEP_TIME);
- }
- }
- ReplicationPeerConfig rpc = new ReplicationPeerConfig();
- rpc.setClusterKey(utility2.getClusterKey());
- hbaseAdmin.addReplicationPeer(PEER_ID, rpc);
- Thread.sleep(SLEEP_TIME);
- rowKey = Bytes.toBytes("do rep");
- put = new Put(rowKey);
- put.addColumn(famName, row, row);
- LOG.info("Adding new row");
- htable1.put(put);
-
- get = new Get(rowKey);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = htable2.get(get);
- if (res.isEmpty()) {
- LOG.info("Row not available");
- Thread.sleep(SLEEP_TIME*i);
- } else {
- assertArrayEquals(res.value(), row);
- break;
- }
- }
- }
-
-
- /**
- * Do a more intense version testSmallBatch, one that will trigger
- * wal rolling and other non-trivial code paths
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testLoading() throws Exception {
- LOG.info("Writing out rows to table1 in testLoading");
- List<Put> puts = new ArrayList<>(NB_ROWS_IN_BIG_BATCH);
- for (int i = 0; i < NB_ROWS_IN_BIG_BATCH; i++) {
- Put put = new Put(Bytes.toBytes(i));
- put.addColumn(famName, row, row);
- puts.add(put);
- }
- // The puts will be iterated through and flushed only when the buffer
- // size is reached.
- htable1.put(puts);
-
- Scan scan = new Scan();
-
- ResultScanner scanner = htable1.getScanner(scan);
- Result[] res = scanner.next(NB_ROWS_IN_BIG_BATCH);
- scanner.close();
-
- assertEquals(NB_ROWS_IN_BIG_BATCH, res.length);
-
- LOG.info("Looking in table2 for replicated rows in testLoading");
- long start = System.currentTimeMillis();
- // Retry more than NB_RETRIES. As it was, retries were done in 5 seconds and we'd fail
- // sometimes.
- final long retries = NB_RETRIES * 10;
- for (int i = 0; i < retries; i++) {
- scan = new Scan();
- scanner = htable2.getScanner(scan);
- res = scanner.next(NB_ROWS_IN_BIG_BATCH);
- scanner.close();
- if (res.length != NB_ROWS_IN_BIG_BATCH) {
- if (i == retries - 1) {
- int lastRow = -1;
- for (Result result : res) {
- int currentRow = Bytes.toInt(result.getRow());
- for (int row = lastRow+1; row < currentRow; row++) {
- LOG.error("Row missing: " + row);
- }
- lastRow = currentRow;
- }
- LOG.error("Last row: " + lastRow);
- fail("Waited too much time for normal batch replication, " +
- res.length + " instead of " + NB_ROWS_IN_BIG_BATCH + "; waited=" +
- (System.currentTimeMillis() - start) + "ms");
- } else {
- LOG.info("Only got " + res.length + " rows... retrying");
- Thread.sleep(SLEEP_TIME);
- }
- } else {
- break;
- }
- }
- }
-
- /**
- * Do a small loading into a table, make sure the data is really the same,
- * then run the VerifyReplication job to check the results. Do a second
- * comparison where all the cells are different.
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testVerifyRepJob() throws Exception {
- // Populate the tables, at the same time it guarantees that the tables are
- // identical since it does the check
- testSmallBatch();
-
- String[] args = new String[] {PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(args, NB_ROWS_IN_BATCH, 0);
-
- Scan scan = new Scan();
- ResultScanner rs = htable2.getScanner(scan);
- Put put = null;
- for (Result result : rs) {
- put = new Put(result.getRow());
- Cell firstVal = result.rawCells()[0];
- put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal),
- Bytes.toBytes("diff data"));
- htable2.put(put);
- }
- Delete delete = new Delete(put.getRow());
- htable2.delete(delete);
- runVerifyReplication(args, 0, NB_ROWS_IN_BATCH);
- }
-
- /**
- * Load a row into a table, make sure the data is really the same,
- * delete the row, make sure the delete marker is replicated,
- * run verify replication with and without raw to check the results.
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testVerifyRepJobWithRawOptions() throws Exception {
- LOG.info(name.getMethodName());
-
- final TableName tableName = TableName.valueOf(name.getMethodName());
- byte[] familyname = Bytes.toBytes("fam_raw");
- byte[] row = Bytes.toBytes("row_raw");
-
- Table lHtable1 = null;
- Table lHtable2 = null;
-
- try {
- ColumnFamilyDescriptor fam = ColumnFamilyDescriptorBuilder.newBuilder(familyname)
- .setMaxVersions(100).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build();
- TableDescriptor table = TableDescriptorBuilder.newBuilder(tableName).addColumnFamily(fam).build();
- scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (ColumnFamilyDescriptor f : table.getColumnFamilies()) {
- scopes.put(f.getName(), f.getScope());
- }
-
- Connection connection1 = ConnectionFactory.createConnection(conf1);
- Connection connection2 = ConnectionFactory.createConnection(conf2);
- try (Admin admin1 = connection1.getAdmin()) {
- admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
- }
- try (Admin admin2 = connection2.getAdmin()) {
- admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
- }
- utility1.waitUntilAllRegionsAssigned(tableName);
- utility2.waitUntilAllRegionsAssigned(tableName);
-
- lHtable1 = utility1.getConnection().getTable(tableName);
- lHtable2 = utility2.getConnection().getTable(tableName);
-
- Put put = new Put(row);
- put.addColumn(familyname, row, row);
- lHtable1.put(put);
-
- Get get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for put replication");
- }
- Result res = lHtable2.get(get);
- if (res.isEmpty()) {
- LOG.info("Row not available");
- Thread.sleep(SLEEP_TIME);
- } else {
- assertArrayEquals(res.value(), row);
- break;
- }
- }
-
- Delete del = new Delete(row);
- lHtable1.delete(del);
-
- get = new Get(row);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i==NB_RETRIES-1) {
- fail("Waited too much time for del replication");
- }
- Result res = lHtable2.get(get);
- if (res.size() >= 1) {
- LOG.info("Row not deleted");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
-
- // Checking verifyReplication for the default behavior.
- String[] argsWithoutRaw = new String[] {PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(argsWithoutRaw, 0, 0);
-
- // Checking verifyReplication with raw
- String[] argsWithRawAsTrue = new String[] {"--raw", PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(argsWithRawAsTrue, 1, 0);
- } finally {
- if (lHtable1 != null) {
- lHtable1.close();
- }
- if (lHtable2 != null) {
- lHtable2.close();
- }
- }
- }
-
- private void runVerifyReplication(String[] args, int expectedGoodRows, int expectedBadRows)
- throws IOException, InterruptedException, ClassNotFoundException {
- Job job = new VerifyReplication().createSubmittableJob(new Configuration(conf1), args);
- if (job == null) {
- fail("Job wasn't created, see the log");
- }
- if (!job.waitForCompletion(true)) {
- fail("Job failed, see the log");
- }
- assertEquals(expectedGoodRows, job.getCounters().
- findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
- assertEquals(expectedBadRows, job.getCounters().
- findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
- }
-
- @Test(timeout=300000)
- // VerifyReplication should honor versions option
- public void testHBase14905() throws Exception {
- // normal Batch tests
- byte[] qualifierName = Bytes.toBytes("f1");
- Put put = new Put(Bytes.toBytes("r1"));
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1002"));
- htable1.put(put);
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1001"));
- htable1.put(put);
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1112"));
- htable1.put(put);
-
- Scan scan = new Scan();
- scan.readVersions(100);
- ResultScanner scanner1 = htable1.getScanner(scan);
- Result[] res1 = scanner1.next(1);
- scanner1.close();
-
- assertEquals(1, res1.length);
- assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
-
- for (int i = 0; i < NB_RETRIES; i++) {
- scan = new Scan();
- scan.readVersions(100);
- scanner1 = htable2.getScanner(scan);
- res1 = scanner1.next(1);
- scanner1.close();
- if (res1.length != 1) {
- LOG.info("Only got " + res1.length + " rows");
- Thread.sleep(SLEEP_TIME);
- } else {
- int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size();
- if (cellNumber != 3) {
- LOG.info("Only got " + cellNumber + " cells");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- if (i == NB_RETRIES-1) {
- fail("Waited too much time for normal batch replication");
- }
- }
-
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1111"));
- htable2.put(put);
- put.addColumn(famName, qualifierName, Bytes.toBytes("v1112"));
- htable2.put(put);
-
- scan = new Scan();
- scan.readVersions(100);
- scanner1 = htable2.getScanner(scan);
- res1 = scanner1.next(NB_ROWS_IN_BATCH);
- scanner1.close();
-
- assertEquals(1, res1.length);
- assertEquals(5, res1[0].getColumnCells(famName, qualifierName).size());
-
- String[] args = new String[] {"--versions=100", PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(args, 0, 1);
- }
-
- @Test(timeout=300000)
- // VerifyReplication should honor versions option
- public void testVersionMismatchHBase14905() throws Exception {
- // normal Batch tests
- byte[] qualifierName = Bytes.toBytes("f1");
- Put put = new Put(Bytes.toBytes("r1"));
- long ts = System.currentTimeMillis();
- put.addColumn(famName, qualifierName, ts + 1, Bytes.toBytes("v1"));
- htable1.put(put);
- put.addColumn(famName, qualifierName, ts + 2, Bytes.toBytes("v2"));
- htable1.put(put);
- put.addColumn(famName, qualifierName, ts + 3, Bytes.toBytes("v3"));
- htable1.put(put);
-
- Scan scan = new Scan();
- scan.readVersions(100);
- ResultScanner scanner1 = htable1.getScanner(scan);
- Result[] res1 = scanner1.next(1);
- scanner1.close();
-
- assertEquals(1, res1.length);
- assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
-
- for (int i = 0; i < NB_RETRIES; i++) {
- scan = new Scan();
- scan.readVersions(100);
- scanner1 = htable2.getScanner(scan);
- res1 = scanner1.next(1);
- scanner1.close();
- if (res1.length != 1) {
- LOG.info("Only got " + res1.length + " rows");
- Thread.sleep(SLEEP_TIME);
- } else {
- int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size();
- if (cellNumber != 3) {
- LOG.info("Only got " + cellNumber + " cells");
- Thread.sleep(SLEEP_TIME);
- } else {
- break;
- }
- }
- if (i == NB_RETRIES-1) {
- fail("Waited too much time for normal batch replication");
- }
- }
-
- try {
- // Disabling replication and modifying the particular version of the cell to validate the feature.
- hbaseAdmin.disableReplicationPeer(PEER_ID);
- Put put2 = new Put(Bytes.toBytes("r1"));
- put2.addColumn(famName, qualifierName, ts +2, Bytes.toBytes("v99"));
- htable2.put(put2);
-
- scan = new Scan();
- scan.readVersions(100);
- scanner1 = htable2.getScanner(scan);
- res1 = scanner1.next(NB_ROWS_IN_BATCH);
- scanner1.close();
- assertEquals(1, res1.length);
- assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
-
- String[] args = new String[] {"--versions=100", PEER_ID, tableName.getNameAsString()};
- runVerifyReplication(args, 0, 1);
- }
- finally {
- hbaseAdmin.enableReplicationPeer(PEER_ID);
- }
- }
-
- /**
- * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out
- * the compaction WALEdit
- * @throws Exception
- */
- @Test(timeout=300000)
- public void testCompactionWALEdits() throws Exception {
- WALProtos.CompactionDescriptor compactionDescriptor =
- WALProtos.CompactionDescriptor.getDefaultInstance();
- RegionInfo hri = RegionInfoBuilder.newBuilder(htable1.getName())
- .setStartKey(HConstants.EMPTY_START_ROW)
- .setEndKey(HConstants.EMPTY_END_ROW)
- .build();
- WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor);
- Replication.scopeWALEdits(new WALKeyImpl(), edit,
- htable1.getConfiguration(), null);
- }
-
- /**
- * Test for HBASE-8663
- * Create two new Tables with colfamilies enabled for replication then run
- * ReplicationAdmin.listReplicated(). Finally verify the table:colfamilies. Note:
- * TestReplicationAdmin is a better place for this testing but it would need mocks.
- * @throws Exception
- */
- @Test(timeout = 300000)
- public void testVerifyListReplicatedTable() throws Exception {
- LOG.info("testVerifyListReplicatedTable");
-
- final String tName = "VerifyListReplicated_";
- final String colFam = "cf1";
- final int numOfTables = 3;
-
- Admin hadmin = utility1.getAdmin();
-
- // Create Tables
- for (int i = 0; i < numOfTables; i++) {
- hadmin.createTable(TableDescriptorBuilder.newBuilder(TableName.valueOf(tName + i))
- .addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(colFam))
- .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
- .build());
- }
-
- // verify the result
- List<TableCFs> replicationColFams = hbaseAdmin.listReplicatedTableCFs();
- int[] match = new int[numOfTables]; // array of 3 with init value of zero
-
- for (int i = 0; i < replicationColFams.size(); i++) {
- TableCFs replicationEntry = replicationColFams.get(i);
- String tn = replicationEntry.getTable().getNameAsString();
- if (tn.startsWith(tName) && replicationEntry.getColumnFamilyMap().containsKey(colFam)) {
- int m = Integer.parseInt(tn.substring(tn.length() - 1)); // get the last digit
- match[m]++; // should only increase once
- }
- }
-
- // check the matching result
- for (int i = 0; i < match.length; i++) {
- assertTrue("listReplicated() does not match table " + i, (match[i] == 1));
- }
-
- // drop tables
- for (int i = 0; i < numOfTables; i++) {
- TableName tableName = TableName.valueOf(tName + i);
- hadmin.disableTable(tableName);
- hadmin.deleteTable(tableName);
- }
-
- hadmin.close();
- }
-
- /**
- * Test for HBase-15259 WALEdits under replay will also be replicated
- * */
- @Test
- public void testReplicationInReplay() throws Exception {
- final TableName tableName = htable1.getName();
-
- HRegion region = utility1.getMiniHBaseCluster().getRegions(tableName).get(0);
- RegionInfo hri = region.getRegionInfo();
- NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : htable1.getDescriptor().getColumnFamilyNames()) {
- scopes.put(fam, 1);
- }
- final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
- int index = utility1.getMiniHBaseCluster().getServerWith(hri.getRegionName());
- WAL wal = utility1.getMiniHBaseCluster().getRegionServer(index).getWAL(region.getRegionInfo());
- final byte[] rowName = Bytes.toBytes("testReplicationInReplay");
- final byte[] qualifier = Bytes.toBytes("q");
- final byte[] value = Bytes.toBytes("v");
- WALEdit edit = new WALEdit(true);
- long now = EnvironmentEdgeManager.currentTime();
- edit.add(new KeyValue(rowName, famName, qualifier,
- now, value));
- WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, now, mvcc, scopes);
- wal.append(hri, walKey, edit, true);
- wal.sync();
-
- Get get = new Get(rowName);
- for (int i = 0; i < NB_RETRIES; i++) {
- if (i == NB_RETRIES-1) {
- break;
- }
- Result res = htable2.get(get);
- if (res.size() >= 1) {
- fail("Not supposed to be replicated for " + Bytes.toString(res.getRow()));
- } else {
- LOG.info("Row not replicated, let's wait a bit more...");
- Thread.sleep(SLEEP_TIME);
- }
- }
- }
-
- @Test(timeout=300000)
- public void testVerifyReplicationPrefixFiltering() throws Exception {
- final byte[] prefixRow = Bytes.toBytes("prefixrow");
- final byte[] prefixRow2 = Bytes.toBytes("secondrow");
- loadData("prefixrow", prefixRow);
- loadData("secondrow", prefixRow2);
- loadData("aaa", row);
- loadData("zzz", row);
- waitForReplication(NB_ROWS_IN_BATCH * 4, NB_RETRIES * 4);
- String[] args = new String[] {"--row-prefixes=prefixrow,secondrow", PEER_ID,
- tableName.getNameAsString()};
- runVerifyReplication(args, NB_ROWS_IN_BATCH *2, 0);
- }
-
- @Test(timeout = 300000)
- public void testVerifyReplicationSnapshotArguments() {
- String[] args =
- new String[] { "--sourceSnapshotName=snapshot1", "2", tableName.getNameAsString() };
- assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--sourceSnapshotTmpDir=tmp", "2", tableName.getNameAsString() };
- assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=tmp", "2",
- tableName.getNameAsString() };
- assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--peerSnapshotName=snapshot1", "2", tableName.getNameAsString() };
- assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--peerSnapshotTmpDir=/tmp/", "2", tableName.getNameAsString() };
- assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--peerSnapshotName=snapshot1", "--peerSnapshotTmpDir=/tmp/",
- "--peerFSAddress=tempfs", "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2",
- tableName.getNameAsString() };
- assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
-
- args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=/tmp/",
- "--peerSnapshotName=snapshot2", "--peerSnapshotTmpDir=/tmp/", "--peerFSAddress=tempfs",
- "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2", tableName.getNameAsString() };
-
- assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
- }
-
- private void checkRestoreTmpDir(Configuration conf, String restoreTmpDir, int expectedCount)
- throws IOException {
- FileSystem fs = FileSystem.get(conf);
- FileStatus[] subDirectories = fs.listStatus(new Path(restoreTmpDir));
- assertNotNull(subDirectories);
- assertEquals(subDirectories.length, expectedCount);
- for (int i = 0; i < expectedCount; i++) {
- assertTrue(subDirectories[i].isDirectory());
- }
- }
-
- @Test(timeout = 300000)
- public void testVerifyReplicationWithSnapshotSupport() throws Exception {
- // Populate the tables, at the same time it guarantees that the tables are
- // identical since it does the check
- testSmallBatch();
-
- // Take source and target tables snapshot
- Path rootDir = FSUtils.getRootDir(conf1);
- FileSystem fs = rootDir.getFileSystem(conf1);
- String sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis();
- SnapshotTestingUtils.createSnapshotAndValidate(utility1.getAdmin(), tableName,
- new String(famName), sourceSnapshotName, rootDir, fs, true);
-
- // Take target snapshot
- Path peerRootDir = FSUtils.getRootDir(conf2);
- FileSystem peerFs = peerRootDir.getFileSystem(conf2);
- String peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis();
- SnapshotTestingUtils.createSnapshotAndValidate(utility2.getAdmin(), tableName,
- new String(famName), peerSnapshotName, peerRootDir, peerFs, true);
-
- String peerFSAddress = peerFs.getUri().toString();
- String temPath1 = utility1.getRandomDir().toString();
- String temPath2 = "/tmp2";
-
- String[] args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName,
- "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName,
- "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress,
- "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() };
-
- Job job = new VerifyReplication().createSubmittableJob(conf1, args);
- if (job == null) {
- fail("Job wasn't created, see the log");
- }
- if (!job.waitForCompletion(true)) {
- fail("Job failed, see the log");
- }
- assertEquals(NB_ROWS_IN_BATCH,
- job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
- assertEquals(0,
- job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
-
- checkRestoreTmpDir(conf1, temPath1, 1);
- checkRestoreTmpDir(conf2, temPath2, 1);
-
- Scan scan = new Scan();
- ResultScanner rs = htable2.getScanner(scan);
- Put put = null;
- for (Result result : rs) {
- put = new Put(result.getRow());
- Cell firstVal = result.rawCells()[0];
- put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal),
- Bytes.toBytes("diff data"));
- htable2.put(put);
- }
- Delete delete = new Delete(put.getRow());
- htable2.delete(delete);
-
- sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis();
- SnapshotTestingUtils.createSnapshotAndValidate(utility1.getAdmin(), tableName,
- new String(famName), sourceSnapshotName, rootDir, fs, true);
-
- peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis();
- SnapshotTestingUtils.createSnapshotAndValidate(utility2.getAdmin(), tableName,
- new String(famName), peerSnapshotName, peerRootDir, peerFs, true);
-
- args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName,
- "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName,
- "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress,
- "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() };
-
- job = new VerifyReplication().createSubmittableJob(conf1, args);
- if (job == null) {
- fail("Job wasn't created, see the log");
- }
- if (!job.waitForCompletion(true)) {
- fail("Job failed, see the log");
- }
- assertEquals(0,
- job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
- assertEquals(NB_ROWS_IN_BATCH,
- job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
-
- checkRestoreTmpDir(conf1, temPath1, 2);
- checkRestoreTmpDir(conf2, temPath2, 2);
- }
-
- @Test
- public void testEmptyWALRecovery() throws Exception {
- final int numRs = utility1.getHBaseCluster().getRegionServerThreads().size();
-
- // for each RS, create an empty wal with same walGroupId
- final List<Path> emptyWalPaths = new ArrayList<>();
- long ts = System.currentTimeMillis();
- for (int i = 0; i < numRs; i++) {
- RegionInfo regionInfo =
- utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
- WAL wal = utility1.getHBaseCluster().getRegionServer(i).getWAL(regionInfo);
- Path currentWalPath = AbstractFSWALProvider.getCurrentFileName(wal);
- String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(currentWalPath.getName());
- Path emptyWalPath = new Path(utility1.getDataTestDir(), walGroupId + "." + ts);
- utility1.getTestFileSystem().create(emptyWalPath).close();
- emptyWalPaths.add(emptyWalPath);
- }
-
- // inject our empty wal into the replication queue, and then roll the original wal, which
- // enqueues a new wal behind our empty wal. We must roll the wal here as now we use the WAL to
- // determine if the file being replicated currently is still opened for write, so just inject a
- // new wal to the replication queue does not mean the previous file is closed.
- for (int i = 0; i < numRs; i++) {
- HRegionServer hrs = utility1.getHBaseCluster().getRegionServer(i);
- Replication replicationService = (Replication) hrs.getReplicationSourceService();
- replicationService.preLogRoll(null, emptyWalPaths.get(i));
- replicationService.postLogRoll(null, emptyWalPaths.get(i));
- RegionInfo regionInfo =
- utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
- WAL wal = hrs.getWAL(regionInfo);
- wal.rollWriter(true);
- }
-
- // ReplicationSource should advance past the empty wal, or else the test will fail
- waitForLogAdvance(numRs);
-
- // we're now writing to the new wal
- // if everything works, the source should've stopped reading from the empty wal, and start
- // replicating from the new wal
- testSimplePutDelete();
- }
-
- /**
- * Waits until there is only one log(the current writing one) in the replication queue
- * @param numRs number of regionservers
- */
- private void waitForLogAdvance(int numRs) throws Exception {
- Waiter.waitFor(conf1, 10000, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- for (int i = 0; i < numRs; i++) {
- HRegionServer hrs = utility1.getHBaseCluster().getRegionServer(i);
- RegionInfo regionInfo =
- utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
- WAL wal = hrs.getWAL(regionInfo);
- Path currentFile = ((AbstractFSWAL<?>) wal).getCurrentFileName();
- Replication replicationService = (Replication) utility1.getHBaseCluster()
- .getRegionServer(i).getReplicationSourceService();
- for (ReplicationSourceInterface rsi : replicationService.getReplicationManager()
- .getSources()) {
- ReplicationSource source = (ReplicationSource) rsi;
- if (!currentFile.equals(source.getCurrentPath())) {
- return false;
- }
- }
- }
- return true;
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8d6ecda/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestVerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestVerifyReplication.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestVerifyReplication.java
new file mode 100644
index 0000000..c0b2839
--- /dev/null
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/replication/TestVerifyReplication.java
@@ -0,0 +1,493 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.TreeMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+
+@Category({ ReplicationTests.class, LargeTests.class })
+public class TestVerifyReplication extends TestReplicationBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestVerifyReplication.class);
+
+ private static final String PEER_ID = "2";
+
+ @Rule
+ public TestName name = new TestName();
+
+ @Before
+ public void setUp() throws Exception {
+ cleanUp();
+ }
+
+ private void runVerifyReplication(String[] args, int expectedGoodRows, int expectedBadRows)
+ throws IOException, InterruptedException, ClassNotFoundException {
+ Job job = new VerifyReplication().createSubmittableJob(new Configuration(conf1), args);
+ if (job == null) {
+ fail("Job wasn't created, see the log");
+ }
+ if (!job.waitForCompletion(true)) {
+ fail("Job failed, see the log");
+ }
+ assertEquals(expectedGoodRows,
+ job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
+ assertEquals(expectedBadRows,
+ job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
+ }
+
+ /**
+ * Do a small loading into a table, make sure the data is really the same, then run the
+ * VerifyReplication job to check the results. Do a second comparison where all the cells are
+ * different.
+ */
+ @Test
+ public void testVerifyRepJob() throws Exception {
+ // Populate the tables, at the same time it guarantees that the tables are
+ // identical since it does the check
+ runSmallBatchTest();
+
+ String[] args = new String[] { PEER_ID, tableName.getNameAsString() };
+ runVerifyReplication(args, NB_ROWS_IN_BATCH, 0);
+
+ Scan scan = new Scan();
+ ResultScanner rs = htable2.getScanner(scan);
+ Put put = null;
+ for (Result result : rs) {
+ put = new Put(result.getRow());
+ Cell firstVal = result.rawCells()[0];
+ put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal),
+ Bytes.toBytes("diff data"));
+ htable2.put(put);
+ }
+ Delete delete = new Delete(put.getRow());
+ htable2.delete(delete);
+ runVerifyReplication(args, 0, NB_ROWS_IN_BATCH);
+ }
+
+ /**
+ * Load a row into a table, make sure the data is really the same, delete the row, make sure the
+ * delete marker is replicated, run verify replication with and without raw to check the results.
+ */
+ @Test
+ public void testVerifyRepJobWithRawOptions() throws Exception {
+ LOG.info(name.getMethodName());
+
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ byte[] familyname = Bytes.toBytes("fam_raw");
+ byte[] row = Bytes.toBytes("row_raw");
+
+ Table lHtable1 = null;
+ Table lHtable2 = null;
+
+ try {
+ ColumnFamilyDescriptor fam = ColumnFamilyDescriptorBuilder.newBuilder(familyname)
+ .setMaxVersions(100).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build();
+ TableDescriptor table =
+ TableDescriptorBuilder.newBuilder(tableName).addColumnFamily(fam).build();
+ scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ for (ColumnFamilyDescriptor f : table.getColumnFamilies()) {
+ scopes.put(f.getName(), f.getScope());
+ }
+
+ Connection connection1 = ConnectionFactory.createConnection(conf1);
+ Connection connection2 = ConnectionFactory.createConnection(conf2);
+ try (Admin admin1 = connection1.getAdmin()) {
+ admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+ }
+ try (Admin admin2 = connection2.getAdmin()) {
+ admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
+ }
+ utility1.waitUntilAllRegionsAssigned(tableName);
+ utility2.waitUntilAllRegionsAssigned(tableName);
+
+ lHtable1 = utility1.getConnection().getTable(tableName);
+ lHtable2 = utility2.getConnection().getTable(tableName);
+
+ Put put = new Put(row);
+ put.addColumn(familyname, row, row);
+ lHtable1.put(put);
+
+ Get get = new Get(row);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for put replication");
+ }
+ Result res = lHtable2.get(get);
+ if (res.isEmpty()) {
+ LOG.info("Row not available");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ assertArrayEquals(res.value(), row);
+ break;
+ }
+ }
+
+ Delete del = new Delete(row);
+ lHtable1.delete(del);
+
+ get = new Get(row);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for del replication");
+ }
+ Result res = lHtable2.get(get);
+ if (res.size() >= 1) {
+ LOG.info("Row not deleted");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ break;
+ }
+ }
+
+ // Checking verifyReplication for the default behavior.
+ String[] argsWithoutRaw = new String[] { PEER_ID, tableName.getNameAsString() };
+ runVerifyReplication(argsWithoutRaw, 0, 0);
+
+ // Checking verifyReplication with raw
+ String[] argsWithRawAsTrue = new String[] { "--raw", PEER_ID, tableName.getNameAsString() };
+ runVerifyReplication(argsWithRawAsTrue, 1, 0);
+ } finally {
+ if (lHtable1 != null) {
+ lHtable1.close();
+ }
+ if (lHtable2 != null) {
+ lHtable2.close();
+ }
+ }
+ }
+
+ // VerifyReplication should honor versions option
+ @Test
+ public void testHBase14905() throws Exception {
+ // normal Batch tests
+ byte[] qualifierName = Bytes.toBytes("f1");
+ Put put = new Put(Bytes.toBytes("r1"));
+ put.addColumn(famName, qualifierName, Bytes.toBytes("v1002"));
+ htable1.put(put);
+ put.addColumn(famName, qualifierName, Bytes.toBytes("v1001"));
+ htable1.put(put);
+ put.addColumn(famName, qualifierName, Bytes.toBytes("v1112"));
+ htable1.put(put);
+
+ Scan scan = new Scan();
+ scan.readVersions(100);
+ ResultScanner scanner1 = htable1.getScanner(scan);
+ Result[] res1 = scanner1.next(1);
+ scanner1.close();
+
+ assertEquals(1, res1.length);
+ assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
+
+ for (int i = 0; i < NB_RETRIES; i++) {
+ scan = new Scan();
+ scan.readVersions(100);
+ scanner1 = htable2.getScanner(scan);
+ res1 = scanner1.next(1);
+ scanner1.close();
+ if (res1.length != 1) {
+ LOG.info("Only got " + res1.length + " rows");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size();
+ if (cellNumber != 3) {
+ LOG.info("Only got " + cellNumber + " cells");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ break;
+ }
+ }
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for normal batch replication");
+ }
+ }
+
+ put.addColumn(famName, qualifierName, Bytes.toBytes("v1111"));
+ htable2.put(put);
+ put.addColumn(famName, qualifierName, Bytes.toBytes("v1112"));
+ htable2.put(put);
+
+ scan = new Scan();
+ scan.readVersions(100);
+ scanner1 = htable2.getScanner(scan);
+ res1 = scanner1.next(NB_ROWS_IN_BATCH);
+ scanner1.close();
+
+ assertEquals(1, res1.length);
+ assertEquals(5, res1[0].getColumnCells(famName, qualifierName).size());
+
+ String[] args = new String[] { "--versions=100", PEER_ID, tableName.getNameAsString() };
+ runVerifyReplication(args, 0, 1);
+ }
+
+ // VerifyReplication should honor versions option
+ @Test
+ public void testVersionMismatchHBase14905() throws Exception {
+ // normal Batch tests
+ byte[] qualifierName = Bytes.toBytes("f1");
+ Put put = new Put(Bytes.toBytes("r1"));
+ long ts = System.currentTimeMillis();
+ put.addColumn(famName, qualifierName, ts + 1, Bytes.toBytes("v1"));
+ htable1.put(put);
+ put.addColumn(famName, qualifierName, ts + 2, Bytes.toBytes("v2"));
+ htable1.put(put);
+ put.addColumn(famName, qualifierName, ts + 3, Bytes.toBytes("v3"));
+ htable1.put(put);
+
+ Scan scan = new Scan();
+ scan.readVersions(100);
+ ResultScanner scanner1 = htable1.getScanner(scan);
+ Result[] res1 = scanner1.next(1);
+ scanner1.close();
+
+ assertEquals(1, res1.length);
+ assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
+
+ for (int i = 0; i < NB_RETRIES; i++) {
+ scan = new Scan();
+ scan.readVersions(100);
+ scanner1 = htable2.getScanner(scan);
+ res1 = scanner1.next(1);
+ scanner1.close();
+ if (res1.length != 1) {
+ LOG.info("Only got " + res1.length + " rows");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ int cellNumber = res1[0].getColumnCells(famName, Bytes.toBytes("f1")).size();
+ if (cellNumber != 3) {
+ LOG.info("Only got " + cellNumber + " cells");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ break;
+ }
+ }
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for normal batch replication");
+ }
+ }
+
+ try {
+ // Disabling replication and modifying the particular version of the cell to validate the
+ // feature.
+ hbaseAdmin.disableReplicationPeer(PEER_ID);
+ Put put2 = new Put(Bytes.toBytes("r1"));
+ put2.addColumn(famName, qualifierName, ts + 2, Bytes.toBytes("v99"));
+ htable2.put(put2);
+
+ scan = new Scan();
+ scan.readVersions(100);
+ scanner1 = htable2.getScanner(scan);
+ res1 = scanner1.next(NB_ROWS_IN_BATCH);
+ scanner1.close();
+ assertEquals(1, res1.length);
+ assertEquals(3, res1[0].getColumnCells(famName, qualifierName).size());
+
+ String[] args = new String[] { "--versions=100", PEER_ID, tableName.getNameAsString() };
+ runVerifyReplication(args, 0, 1);
+ } finally {
+ hbaseAdmin.enableReplicationPeer(PEER_ID);
+ }
+ }
+
+ @Test
+ public void testVerifyReplicationPrefixFiltering() throws Exception {
+ final byte[] prefixRow = Bytes.toBytes("prefixrow");
+ final byte[] prefixRow2 = Bytes.toBytes("secondrow");
+ loadData("prefixrow", prefixRow);
+ loadData("secondrow", prefixRow2);
+ loadData("aaa", row);
+ loadData("zzz", row);
+ waitForReplication(NB_ROWS_IN_BATCH * 4, NB_RETRIES * 4);
+ String[] args =
+ new String[] { "--row-prefixes=prefixrow,secondrow", PEER_ID, tableName.getNameAsString() };
+ runVerifyReplication(args, NB_ROWS_IN_BATCH * 2, 0);
+ }
+
+ @Test
+ public void testVerifyReplicationSnapshotArguments() {
+ String[] args =
+ new String[] { "--sourceSnapshotName=snapshot1", "2", tableName.getNameAsString() };
+ assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
+
+ args = new String[] { "--sourceSnapshotTmpDir=tmp", "2", tableName.getNameAsString() };
+ assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
+
+ args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=tmp", "2",
+ tableName.getNameAsString() };
+ assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
+
+ args = new String[] { "--peerSnapshotName=snapshot1", "2", tableName.getNameAsString() };
+ assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
+
+ args = new String[] { "--peerSnapshotTmpDir=/tmp/", "2", tableName.getNameAsString() };
+ assertFalse(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
+
+ args = new String[] { "--peerSnapshotName=snapshot1", "--peerSnapshotTmpDir=/tmp/",
+ "--peerFSAddress=tempfs", "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2",
+ tableName.getNameAsString() };
+ assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
+
+ args = new String[] { "--sourceSnapshotName=snapshot1", "--sourceSnapshotTmpDir=/tmp/",
+ "--peerSnapshotName=snapshot2", "--peerSnapshotTmpDir=/tmp/", "--peerFSAddress=tempfs",
+ "--peerHBaseRootAddress=hdfs://tempfs:50070/hbase/", "2", tableName.getNameAsString() };
+
+ assertTrue(Lists.newArrayList(args).toString(), new VerifyReplication().doCommandLine(args));
+ }
+
+ private void checkRestoreTmpDir(Configuration conf, String restoreTmpDir, int expectedCount)
+ throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ FileStatus[] subDirectories = fs.listStatus(new Path(restoreTmpDir));
+ assertNotNull(subDirectories);
+ assertEquals(subDirectories.length, expectedCount);
+ for (int i = 0; i < expectedCount; i++) {
+ assertTrue(subDirectories[i].isDirectory());
+ }
+ }
+
+ @Test
+ public void testVerifyReplicationWithSnapshotSupport() throws Exception {
+ // Populate the tables, at the same time it guarantees that the tables are
+ // identical since it does the check
+ runSmallBatchTest();
+
+ // Take source and target tables snapshot
+ Path rootDir = FSUtils.getRootDir(conf1);
+ FileSystem fs = rootDir.getFileSystem(conf1);
+ String sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis();
+ SnapshotTestingUtils.createSnapshotAndValidate(utility1.getAdmin(), tableName,
+ new String(famName), sourceSnapshotName, rootDir, fs, true);
+
+ // Take target snapshot
+ Path peerRootDir = FSUtils.getRootDir(conf2);
+ FileSystem peerFs = peerRootDir.getFileSystem(conf2);
+ String peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis();
+ SnapshotTestingUtils.createSnapshotAndValidate(utility2.getAdmin(), tableName,
+ new String(famName), peerSnapshotName, peerRootDir, peerFs, true);
+
+ String peerFSAddress = peerFs.getUri().toString();
+ String temPath1 = utility1.getRandomDir().toString();
+ String temPath2 = "/tmp2";
+
+ String[] args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName,
+ "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName,
+ "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress,
+ "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() };
+
+ Job job = new VerifyReplication().createSubmittableJob(conf1, args);
+ if (job == null) {
+ fail("Job wasn't created, see the log");
+ }
+ if (!job.waitForCompletion(true)) {
+ fail("Job failed, see the log");
+ }
+ assertEquals(NB_ROWS_IN_BATCH,
+ job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
+ assertEquals(0,
+ job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
+
+ checkRestoreTmpDir(conf1, temPath1, 1);
+ checkRestoreTmpDir(conf2, temPath2, 1);
+
+ Scan scan = new Scan();
+ ResultScanner rs = htable2.getScanner(scan);
+ Put put = null;
+ for (Result result : rs) {
+ put = new Put(result.getRow());
+ Cell firstVal = result.rawCells()[0];
+ put.addColumn(CellUtil.cloneFamily(firstVal), CellUtil.cloneQualifier(firstVal),
+ Bytes.toBytes("diff data"));
+ htable2.put(put);
+ }
+ Delete delete = new Delete(put.getRow());
+ htable2.delete(delete);
+
+ sourceSnapshotName = "sourceSnapshot-" + System.currentTimeMillis();
+ SnapshotTestingUtils.createSnapshotAndValidate(utility1.getAdmin(), tableName,
+ new String(famName), sourceSnapshotName, rootDir, fs, true);
+
+ peerSnapshotName = "peerSnapshot-" + System.currentTimeMillis();
+ SnapshotTestingUtils.createSnapshotAndValidate(utility2.getAdmin(), tableName,
+ new String(famName), peerSnapshotName, peerRootDir, peerFs, true);
+
+ args = new String[] { "--sourceSnapshotName=" + sourceSnapshotName,
+ "--sourceSnapshotTmpDir=" + temPath1, "--peerSnapshotName=" + peerSnapshotName,
+ "--peerSnapshotTmpDir=" + temPath2, "--peerFSAddress=" + peerFSAddress,
+ "--peerHBaseRootAddress=" + FSUtils.getRootDir(conf2), "2", tableName.getNameAsString() };
+
+ job = new VerifyReplication().createSubmittableJob(conf1, args);
+ if (job == null) {
+ fail("Job wasn't created, see the log");
+ }
+ if (!job.waitForCompletion(true)) {
+ fail("Job failed, see the log");
+ }
+ assertEquals(0,
+ job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue());
+ assertEquals(NB_ROWS_IN_BATCH,
+ job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
+
+ checkRestoreTmpDir(conf1, temPath1, 2);
+ checkRestoreTmpDir(conf2, temPath2, 2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8d6ecda/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
index 0592e2d..afb975d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -18,24 +18,38 @@
*/
package org.apache.hadoop.hbase.replication;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.NavigableMap;
import java.util.TreeMap;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -95,9 +109,72 @@ public class TestReplicationBase {
return Arrays.asList(false, true);
}
- /**
- * @throws java.lang.Exception
- */
+ protected final void cleanUp() throws IOException, InterruptedException {
+ // Starting and stopping replication can make us miss new logs,
+ // rolling like this makes sure the most recent one gets added to the queue
+ for (JVMClusterUtil.RegionServerThread r : utility1.getHBaseCluster()
+ .getRegionServerThreads()) {
+ utility1.getAdmin().rollWALWriter(r.getRegionServer().getServerName());
+ }
+ int rowCount = utility1.countRows(tableName);
+ utility1.deleteTableData(tableName);
+ // truncating the table will send one Delete per row to the slave cluster
+ // in an async fashion, which is why we cannot just call deleteTableData on
+ // utility2 since late writes could make it to the slave in some way.
+ // Instead, we truncate the first table and wait for all the Deletes to
+ // make it to the slave.
+ Scan scan = new Scan();
+ int lastCount = 0;
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for truncate");
+ }
+ ResultScanner scanner = htable2.getScanner(scan);
+ Result[] res = scanner.next(rowCount);
+ scanner.close();
+ if (res.length != 0) {
+ if (res.length < lastCount) {
+ i--; // Don't increment timeout if we make progress
+ }
+ lastCount = res.length;
+ LOG.info("Still got " + res.length + " rows");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ break;
+ }
+ }
+ }
+
+ protected static void waitForReplication(int expectedRows, int retries)
+ throws IOException, InterruptedException {
+ Scan scan;
+ for (int i = 0; i < retries; i++) {
+ scan = new Scan();
+ if (i== retries -1) {
+ fail("Waited too much time for normal batch replication");
+ }
+ ResultScanner scanner = htable2.getScanner(scan);
+ Result[] res = scanner.next(expectedRows);
+ scanner.close();
+ if (res.length != expectedRows) {
+ LOG.info("Only got " + res.length + " rows");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ break;
+ }
+ }
+ }
+
+ protected static void loadData(String prefix, byte[] row) throws IOException {
+ List<Put> puts = new ArrayList<>(NB_ROWS_IN_BATCH);
+ for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
+ Put put = new Put(Bytes.toBytes(prefix + Integer.toString(i)));
+ put.addColumn(famName, row, row);
+ puts.add(put);
+ }
+ htable1.put(puts);
+ }
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
@@ -149,20 +226,17 @@ public class TestReplicationBase {
// as a component in deciding maximum number of parallel batches to send to the peer cluster.
utility2.startMiniCluster(4);
- ReplicationPeerConfig rpc = new ReplicationPeerConfig();
- rpc.setClusterKey(utility2.getClusterKey());
+ ReplicationPeerConfig rpc =
+ ReplicationPeerConfig.newBuilder().setClusterKey(utility2.getClusterKey()).build();
hbaseAdmin = ConnectionFactory.createConnection(conf1).getAdmin();
hbaseAdmin.addReplicationPeer("2", rpc);
- HTableDescriptor table = new HTableDescriptor(tableName);
- HColumnDescriptor fam = new HColumnDescriptor(famName);
- fam.setMaxVersions(100);
- fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
- table.addFamily(fam);
- fam = new HColumnDescriptor(noRepfamName);
- table.addFamily(fam);
+ TableDescriptor table = TableDescriptorBuilder.newBuilder(tableName)
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName).setMaxVersions(100)
+ .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build();
scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for(HColumnDescriptor f : table.getColumnFamilies()) {
+ for (ColumnFamilyDescriptor f : table.getColumnFamilies()) {
scopes.put(f.getName(), f.getScope());
}
Connection connection1 = ConnectionFactory.createConnection(conf1);
@@ -179,9 +253,60 @@ public class TestReplicationBase {
htable2 = connection2.getTable(tableName);
}
- /**
- * @throws java.lang.Exception
- */
+ protected static void runSimplePutDeleteTest() throws IOException, InterruptedException {
+ Put put = new Put(row);
+ put.addColumn(famName, row, row);
+
+ htable1 = utility1.getConnection().getTable(tableName);
+ htable1.put(put);
+
+ Get get = new Get(row);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for put replication");
+ }
+ Result res = htable2.get(get);
+ if (res.isEmpty()) {
+ LOG.info("Row not available");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ assertArrayEquals(res.value(), row);
+ break;
+ }
+ }
+
+ Delete del = new Delete(row);
+ htable1.delete(del);
+
+ get = new Get(row);
+ for (int i = 0; i < NB_RETRIES; i++) {
+ if (i == NB_RETRIES - 1) {
+ fail("Waited too much time for del replication");
+ }
+ Result res = htable2.get(get);
+ if (res.size() >= 1) {
+ LOG.info("Row not deleted");
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ break;
+ }
+ }
+ }
+
+ protected static void runSmallBatchTest() throws IOException, InterruptedException {
+ // normal Batch tests
+ loadData("", row);
+
+ Scan scan = new Scan();
+
+ ResultScanner scanner1 = htable1.getScanner(scan);
+ Result[] res1 = scanner1.next(NB_ROWS_IN_BATCH);
+ scanner1.close();
+ assertEquals(NB_ROWS_IN_BATCH, res1.length);
+
+ waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
+ }
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
htable2.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/d8d6ecda/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java
new file mode 100644
index 0000000..4a43aa7
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
+import org.apache.hadoop.hbase.replication.regionserver.Replication;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSource;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ ReplicationTests.class, LargeTests.class })
+public class TestReplicationEmptyWALRecovery extends TestReplicationBase {
+
+ @Before
+ public void setUp() throws IOException, InterruptedException {
+ cleanUp();
+ }
+
+ /**
+ * Waits until there is only one log(the current writing one) in the replication queue
+ * @param numRs number of regionservers
+ */
+ private void waitForLogAdvance(int numRs) throws Exception {
+ Waiter.waitFor(conf1, 10000, new Waiter.Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ for (int i = 0; i < numRs; i++) {
+ HRegionServer hrs = utility1.getHBaseCluster().getRegionServer(i);
+ RegionInfo regionInfo =
+ utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
+ WAL wal = hrs.getWAL(regionInfo);
+ Path currentFile = ((AbstractFSWAL<?>) wal).getCurrentFileName();
+ Replication replicationService = (Replication) utility1.getHBaseCluster()
+ .getRegionServer(i).getReplicationSourceService();
+ for (ReplicationSourceInterface rsi : replicationService.getReplicationManager()
+ .getSources()) {
+ ReplicationSource source = (ReplicationSource) rsi;
+ if (!currentFile.equals(source.getCurrentPath())) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ });
+ }
+
+ @Test
+ public void testEmptyWALRecovery() throws Exception {
+ final int numRs = utility1.getHBaseCluster().getRegionServerThreads().size();
+
+ // for each RS, create an empty wal with same walGroupId
+ final List<Path> emptyWalPaths = new ArrayList<>();
+ long ts = System.currentTimeMillis();
+ for (int i = 0; i < numRs; i++) {
+ RegionInfo regionInfo =
+ utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
+ WAL wal = utility1.getHBaseCluster().getRegionServer(i).getWAL(regionInfo);
+ Path currentWalPath = AbstractFSWALProvider.getCurrentFileName(wal);
+ String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(currentWalPath.getName());
+ Path emptyWalPath = new Path(utility1.getDataTestDir(), walGroupId + "." + ts);
+ utility1.getTestFileSystem().create(emptyWalPath).close();
+ emptyWalPaths.add(emptyWalPath);
+ }
+
+ // inject our empty wal into the replication queue, and then roll the original wal, which
+ // enqueues a new wal behind our empty wal. We must roll the wal here as now we use the WAL to
+ // determine if the file being replicated currently is still opened for write, so just inject a
+ // new wal to the replication queue does not mean the previous file is closed.
+ for (int i = 0; i < numRs; i++) {
+ HRegionServer hrs = utility1.getHBaseCluster().getRegionServer(i);
+ Replication replicationService = (Replication) hrs.getReplicationSourceService();
+ replicationService.preLogRoll(null, emptyWalPaths.get(i));
+ replicationService.postLogRoll(null, emptyWalPaths.get(i));
+ RegionInfo regionInfo =
+ utility1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
+ WAL wal = hrs.getWAL(regionInfo);
+ wal.rollWriter(true);
+ }
+
+ // ReplicationSource should advance past the empty wal, or else the test will fail
+ waitForLogAdvance(numRs);
+
+ // we're now writing to the new wal
+ // if everything works, the source should've stopped reading from the empty wal, and start
+ // replicating from the new wal
+ runSimplePutDeleteTest();
+ }
+}
[26/31] hbase git commit: HBASE-19757 System table gets stuck after
enabling region server group feature in secure cluster
Posted by zh...@apache.org.
HBASE-19757 System table gets stuck after enabling region server group feature in secure cluster
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fd435070
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fd435070
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fd435070
Branch: refs/heads/HBASE-19064
Commit: fd43507069e3ae8d9cb689e69e3b5e211cf36e99
Parents: 8b520ce
Author: tedyu <yu...@gmail.com>
Authored: Thu Jan 18 17:56:07 2018 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jan 18 17:56:07 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fd435070/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
index 9ba1218..956f8ba 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
@@ -400,14 +400,8 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
orphanTables.add(TableName.valueOf(entry));
}
- final List<TableName> specialTables;
- if(!masterServices.isInitialized()) {
- specialTables = Arrays.asList(AccessControlLists.ACL_TABLE_NAME, TableName.META_TABLE_NAME,
- TableName.NAMESPACE_TABLE_NAME, RSGROUP_TABLE_NAME);
- } else {
- specialTables =
- masterServices.listTableNamesByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR);
- }
+ final List<TableName> specialTables = Arrays.asList(AccessControlLists.ACL_TABLE_NAME,
+ TableName.META_TABLE_NAME, TableName.NAMESPACE_TABLE_NAME, RSGROUP_TABLE_NAME);
for (TableName table : specialTables) {
orphanTables.add(table);
[28/31] hbase git commit: HBASE-19825 Fix hadoop3 compat test
failures, need commons-beanutils from hadoop 3
Posted by zh...@apache.org.
HBASE-19825 Fix hadoop3 compat test failures, need commons-beanutils from hadoop 3
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9ba961a5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9ba961a5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9ba961a5
Branch: refs/heads/HBASE-19064
Commit: 9ba961a5eff2561524eea51eed7c2440109baf89
Parents: 2018ffc
Author: Umesh Agashe <ua...@cloudera.com>
Authored: Thu Jan 18 16:09:32 2018 -0800
Committer: Apekshit Sharma <ap...@cloudera.com>
Committed: Thu Jan 18 18:28:54 2018 -0800
----------------------------------------------------------------------
pom.xml | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9ba961a5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bbfb122..cb1d0bf 100755
--- a/pom.xml
+++ b/pom.xml
@@ -2889,10 +2889,6 @@
<artifactId>htrace-core</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- </exclusion>
- <exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
[29/31] hbase git commit: HBASE-19083 Introduce a new log writer
which can write to two HDFSes
Posted by zh...@apache.org.
HBASE-19083 Introduce a new log writer which can write to two HDFSes
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/52d0ed18
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/52d0ed18
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/52d0ed18
Branch: refs/heads/HBASE-19064
Commit: 52d0ed187a23be01f1d8a75ab8f7dd7bf7a66de2
Parents: 9ba961a
Author: zhangduo <zh...@apache.org>
Authored: Thu Jan 11 21:08:02 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Fri Jan 19 21:20:34 2018 +0800
----------------------------------------------------------------------
.../hbase/regionserver/wal/AsyncFSWAL.java | 26 ++--
.../regionserver/wal/CombinedAsyncWriter.java | 134 ++++++++++++++++++
.../hbase/regionserver/wal/DualAsyncFSWAL.java | 67 +++++++++
.../wal/AbstractTestProtobufLog.java | 111 +++------------
.../regionserver/wal/ProtobufLogTestHelper.java | 99 ++++++++++++++
.../regionserver/wal/TestAsyncProtobufLog.java | 44 ++----
.../wal/TestCombinedAsyncWriter.java | 136 +++++++++++++++++++
.../hbase/regionserver/wal/TestProtobufLog.java | 15 +-
.../regionserver/wal/WriterOverAsyncWriter.java | 63 +++++++++
9 files changed, 541 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/52d0ed18/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index 0ace782..6ed3eea 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -606,8 +606,7 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
}
}
- @Override
- protected AsyncWriter createWriterInstance(Path path) throws IOException {
+ protected final AsyncWriter createAsyncWriter(FileSystem fs, Path path) throws IOException {
try {
return AsyncFSWALProvider.createAsyncWriter(conf, fs, path, false, eventLoopGroup,
channelClass);
@@ -623,6 +622,11 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
}
}
+ @Override
+ protected AsyncWriter createWriterInstance(Path path) throws IOException {
+ return createAsyncWriter(fs, path);
+ }
+
private void waitForSafePoint() {
consumeLock.lock();
try {
@@ -665,21 +669,21 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
} finally {
consumeLock.unlock();
}
- return executeClose(closeExecutor, oldWriter);
+ return executeClose(oldWriter);
}
@Override
protected void doShutdown() throws IOException {
waitForSafePoint();
- executeClose(closeExecutor, writer);
+ executeClose(writer);
closeExecutor.shutdown();
try {
if (!closeExecutor.awaitTermination(waitOnShutdownInSeconds, TimeUnit.SECONDS)) {
- LOG.error("We have waited " + waitOnShutdownInSeconds + " seconds but"
- + " the close of async writer doesn't complete."
- + "Please check the status of underlying filesystem"
- + " or increase the wait time by the config \""
- + ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS + "\"");
+ LOG.error("We have waited " + waitOnShutdownInSeconds + " seconds but" +
+ " the close of async writer doesn't complete." +
+ "Please check the status of underlying filesystem" +
+ " or increase the wait time by the config \"" + ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS +
+ "\"");
}
} catch (InterruptedException e) {
LOG.error("The wait for close of async writer is interrupted");
@@ -692,7 +696,7 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
}
}
- private static long executeClose(ExecutorService closeExecutor, AsyncWriter writer) {
+ protected final long executeClose(AsyncWriter writer) {
long fileLength;
if (writer != null) {
fileLength = writer.getLength();
@@ -700,7 +704,7 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
try {
writer.close();
} catch (IOException e) {
- LOG.warn("close old writer failed", e);
+ LOG.warn("close writer failed", e);
}
});
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/52d0ed18/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
new file mode 100644
index 0000000..8ecfede
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.hbase.wal.WAL.Entry;
+import org.apache.hadoop.hbase.wal.WALProvider.AsyncWriter;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
+
+/**
+ * An {@link AsyncWriter} wrapper which writes data to a set of {@link AsyncWriter} instances.
+ */
+@InterfaceAudience.Private
+public abstract class CombinedAsyncWriter implements AsyncWriter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CombinedAsyncWriter.class);
+
+ protected final ImmutableList<AsyncWriter> writers;
+
+ protected CombinedAsyncWriter(ImmutableList<AsyncWriter> writers) {
+ this.writers = writers;
+ }
+
+ @Override
+ public long getLength() {
+ return writers.get(0).getLength();
+ }
+
+ @Override
+ public void close() throws IOException {
+ Exception error = null;
+ for (AsyncWriter writer : writers) {
+ try {
+ writer.close();
+ } catch (Exception e) {
+ LOG.warn("close writer failed", e);
+ if (error == null) {
+ error = e;
+ }
+ }
+ }
+ if (error != null) {
+ throw new IOException("Failed to close at least one writer, please see the warn log above. " +
+ "The cause is the first exception occured", error);
+ }
+ }
+
+ protected abstract void doSync(CompletableFuture<Long> future);
+
+ @Override
+ public CompletableFuture<Long> sync() {
+ CompletableFuture<Long> future = new CompletableFuture<>();
+ doSync(future);
+ return future;
+ }
+
+ @Override
+ public void append(Entry entry) {
+ writers.forEach(w -> w.append(entry));
+ }
+
+ public enum Mode {
+ SEQUENTIAL, PARALLEL
+ }
+
+ public static CombinedAsyncWriter create(Mode mode, AsyncWriter writer, AsyncWriter... writers) {
+ ImmutableList<AsyncWriter> ws =
+ ImmutableList.<AsyncWriter> builder().add(writer).add(writers).build();
+ switch (mode) {
+ case SEQUENTIAL:
+ return new CombinedAsyncWriter(ws) {
+
+ private void doSync(CompletableFuture<Long> future, Long length, int index) {
+ if (index == writers.size()) {
+ future.complete(length);
+ return;
+ }
+ writers.get(index).sync().whenComplete((len, error) -> {
+ if (error != null) {
+ future.completeExceptionally(error);
+ return;
+ }
+ doSync(future, len, index + 1);
+ });
+ }
+
+ @Override
+ protected void doSync(CompletableFuture<Long> future) {
+ doSync(future, null, 0);
+ }
+ };
+ case PARALLEL:
+ return new CombinedAsyncWriter(ws) {
+
+ @Override
+ protected void doSync(CompletableFuture<Long> future) {
+ AtomicInteger remaining = new AtomicInteger(writers.size());
+ writers.forEach(w -> w.sync().whenComplete((length, error) -> {
+ if (error != null) {
+ future.completeExceptionally(error);
+ return;
+ }
+ if (remaining.decrementAndGet() == 0) {
+ future.complete(length);
+ }
+ }));
+ }
+ };
+ default:
+ throw new IllegalArgumentException("Unknown mode: " + mode);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52d0ed18/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
new file mode 100644
index 0000000..6bf9e02
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.wal.WALProvider.AsyncWriter;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+
+/**
+ * An AsyncFSWAL which writes data to two filesystems.
+ */
+@InterfaceAudience.Private
+public class DualAsyncFSWAL extends AsyncFSWAL {
+
+ private final FileSystem remoteFs;
+
+ private final Path remoteWalDir;
+
+ public DualAsyncFSWAL(FileSystem fs, FileSystem remoteFs, Path rootDir, Path remoteRootDir,
+ String logDir, String archiveDir, Configuration conf, List<WALActionsListener> listeners,
+ boolean failIfWALExists, String prefix, String suffix, EventLoopGroup eventLoopGroup,
+ Class<? extends Channel> channelClass) throws FailedLogCloseException, IOException {
+ super(fs, rootDir, logDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix,
+ eventLoopGroup, channelClass);
+ this.remoteFs = remoteFs;
+ this.remoteWalDir = new Path(remoteRootDir, logDir);
+ }
+
+ @Override
+ protected AsyncWriter createWriterInstance(Path path) throws IOException {
+ AsyncWriter localWriter = super.createWriterInstance(path);
+ AsyncWriter remoteWriter;
+ boolean succ = false;
+ try {
+ remoteWriter = createAsyncWriter(remoteFs, new Path(remoteWalDir, path.getName()));
+ succ = true;
+ } finally {
+ if (!succ) {
+ executeClose(localWriter);
+ }
+ }
+ return CombinedAsyncWriter.create(CombinedAsyncWriter.Mode.SEQUENTIAL, remoteWriter,
+ localWriter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52d0ed18/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestProtobufLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestProtobufLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestProtobufLog.java
index c0510d3..ebede2e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestProtobufLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestProtobufLog.java
@@ -18,34 +18,15 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.Closeable;
import java.io.IOException;
-
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.coprocessor.SampleRegionWALCoprocessor;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.wal.WALFactory;
-import org.apache.hadoop.hbase.wal.WALKeyImpl;
+import org.apache.hadoop.hbase.wal.WALProvider;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -57,8 +38,8 @@ import org.junit.rules.TestName;
/**
* WAL tests that can be reused across providers.
*/
-public abstract class AbstractTestProtobufLog<W extends Closeable> {
- protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+public abstract class AbstractTestProtobufLog {
+ protected static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
protected FileSystem fs;
protected Path dir;
@@ -94,14 +75,7 @@ public abstract class AbstractTestProtobufLog<W extends Closeable> {
TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);
// faster failover with cluster.shutdown();fs.close() idiom
- TEST_UTIL.getConfiguration()
- .setInt("hbase.ipc.client.connect.max.retries", 1);
- TEST_UTIL.getConfiguration().setInt(
- "dfs.client.block.recovery.retries", 1);
- TEST_UTIL.getConfiguration().setInt(
- "hbase.ipc.client.connection.maxidletime", 500);
- TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
- SampleRegionWALCoprocessor.class.getName());
+ TEST_UTIL.getConfiguration().setInt("dfs.client.block.recovery.retries", 1);
TEST_UTIL.startMiniDFSCluster(3);
}
@@ -132,77 +106,24 @@ public abstract class AbstractTestProtobufLog<W extends Closeable> {
* @throws IOException
*/
private void doRead(boolean withTrailer) throws IOException {
- final int columnCount = 5;
- final int recordCount = 5;
- final TableName tableName =
- TableName.valueOf("tablename");
- final byte[] row = Bytes.toBytes("row");
+ int columnCount = 5;
+ int recordCount = 5;
+ TableName tableName = TableName.valueOf("tablename");
+ byte[] row = Bytes.toBytes("row");
long timestamp = System.currentTimeMillis();
Path path = new Path(dir, "tempwal");
// delete the log if already exists, for test only
fs.delete(path, true);
- W writer = null;
- ProtobufLogReader reader = null;
- try {
- HRegionInfo hri = new HRegionInfo(tableName,
- HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
- HTableDescriptor htd = new HTableDescriptor(tableName);
- fs.mkdirs(dir);
- // Write log in pb format.
- writer = createWriter(path);
- for (int i = 0; i < recordCount; ++i) {
- WALKeyImpl key = new WALKeyImpl(
- hri.getEncodedNameAsBytes(), tableName, i, timestamp, HConstants.DEFAULT_CLUSTER_ID);
- WALEdit edit = new WALEdit();
- for (int j = 0; j < columnCount; ++j) {
- if (i == 0) {
- htd.addFamily(new HColumnDescriptor("column" + j));
- }
- String value = i + "" + j;
- edit.add(new KeyValue(row, row, row, timestamp, Bytes.toBytes(value)));
- }
- append(writer, new WAL.Entry(key, edit));
- }
- sync(writer);
- if (withTrailer) writer.close();
-
- // Now read the log using standard means.
- reader = (ProtobufLogReader) wals.createReader(fs, path);
- if (withTrailer) {
- assertNotNull(reader.trailer);
- } else {
- assertNull(reader.trailer);
- }
- for (int i = 0; i < recordCount; ++i) {
- WAL.Entry entry = reader.next();
- assertNotNull(entry);
- assertEquals(columnCount, entry.getEdit().size());
- assertArrayEquals(hri.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName());
- assertEquals(tableName, entry.getKey().getTablename());
- int idx = 0;
- for (Cell val : entry.getEdit().getCells()) {
- assertTrue(Bytes.equals(row, 0, row.length, val.getRowArray(), val.getRowOffset(),
- val.getRowLength()));
- String value = i + "" + idx;
- assertArrayEquals(Bytes.toBytes(value), CellUtil.cloneValue(val));
- idx++;
- }
- }
- WAL.Entry entry = reader.next();
- assertNull(entry);
- } finally {
- if (writer != null) {
- writer.close();
- }
- if (reader != null) {
- reader.close();
+ fs.mkdirs(dir);
+ try (WALProvider.Writer writer = createWriter(path)) {
+ ProtobufLogTestHelper.doWrite(writer, withTrailer, tableName, columnCount, recordCount, row,
+ timestamp);
+ try (ProtobufLogReader reader = (ProtobufLogReader) wals.createReader(fs, path)) {
+ ProtobufLogTestHelper.doRead(reader, withTrailer, tableName, columnCount, recordCount, row,
+ timestamp);
}
}
}
- protected abstract W createWriter(Path path) throws IOException;
-
- protected abstract void append(W writer, WAL.Entry entry) throws IOException;
-
- protected abstract void sync(W writer) throws IOException;
+ protected abstract WALProvider.Writer createWriter(Path path) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52d0ed18/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java
new file mode 100644
index 0000000..ba2f8e7
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.stream.IntStream;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALEdit;
+import org.apache.hadoop.hbase.wal.WALKeyImpl;
+import org.apache.hadoop.hbase.wal.WALProvider;
+
+/**
+ * Helper class for testing protobuf log.
+ */
+final class ProtobufLogTestHelper {
+
+ private ProtobufLogTestHelper() {
+ }
+
+ private static byte[] toValue(int prefix, int suffix) {
+ return Bytes.toBytes(prefix + "-" + suffix);
+ }
+
+ private static RegionInfo toRegionInfo(TableName tableName) {
+ return RegionInfoBuilder.newBuilder(tableName).setRegionId(1024).build();
+ }
+
+ public static void doWrite(WALProvider.Writer writer, boolean withTrailer, TableName tableName,
+ int columnCount, int recordCount, byte[] row, long timestamp) throws IOException {
+ RegionInfo hri = toRegionInfo(tableName);
+ for (int i = 0; i < recordCount; i++) {
+ WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, i, timestamp,
+ HConstants.DEFAULT_CLUSTER_ID);
+ WALEdit edit = new WALEdit();
+ int prefix = i;
+ IntStream.range(0, columnCount).mapToObj(j -> toValue(prefix, j))
+ .map(value -> new KeyValue(row, row, row, timestamp, value)).forEachOrdered(edit::add);
+ writer.append(new WAL.Entry(key, edit));
+ }
+ writer.sync();
+ if (withTrailer) {
+ writer.close();
+ }
+ }
+
+ public static void doRead(ProtobufLogReader reader, boolean withTrailer, TableName tableName,
+ int columnCount, int recordCount, byte[] row, long timestamp) throws IOException {
+ if (withTrailer) {
+ assertNotNull(reader.trailer);
+ } else {
+ assertNull(reader.trailer);
+ }
+ RegionInfo hri = toRegionInfo(tableName);
+ for (int i = 0; i < recordCount; ++i) {
+ WAL.Entry entry = reader.next();
+ assertNotNull(entry);
+ assertEquals(columnCount, entry.getEdit().size());
+ assertArrayEquals(hri.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName());
+ assertEquals(tableName, entry.getKey().getTablename());
+ int idx = 0;
+ for (Cell val : entry.getEdit().getCells()) {
+ assertTrue(Bytes.equals(row, 0, row.length, val.getRowArray(), val.getRowOffset(),
+ val.getRowLength()));
+ assertArrayEquals(toValue(i, idx), CellUtil.cloneValue(val));
+ idx++;
+ }
+ }
+ assertNull(reader.next());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52d0ed18/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncProtobufLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncProtobufLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncProtobufLog.java
index 86b4c89..c4c9792 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncProtobufLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncProtobufLog.java
@@ -17,30 +17,23 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
-import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
-
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
-
import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.concurrent.ExecutionException;
-
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.wal.AsyncFSWALProvider;
-import org.apache.hadoop.hbase.wal.WAL.Entry;
-import org.apache.hadoop.hbase.wal.WALProvider;
-import org.apache.hadoop.hbase.wal.WALProvider.AsyncWriter;
+import org.apache.hadoop.hbase.wal.WALProvider.Writer;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.experimental.categories.Category;
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
+
@Category({ RegionServerTests.class, MediumTests.class })
-public class TestAsyncProtobufLog extends AbstractTestProtobufLog<WALProvider.AsyncWriter> {
+public class TestAsyncProtobufLog extends AbstractTestProtobufLog {
private static EventLoopGroup EVENT_LOOP_GROUP;
@@ -60,25 +53,8 @@ public class TestAsyncProtobufLog extends AbstractTestProtobufLog<WALProvider.As
}
@Override
- protected AsyncWriter createWriter(Path path) throws IOException {
- return AsyncFSWALProvider.createAsyncWriter(TEST_UTIL.getConfiguration(), fs, path, false,
- EVENT_LOOP_GROUP.next(), CHANNEL_CLASS);
- }
-
- @Override
- protected void append(AsyncWriter writer, Entry entry) throws IOException {
- writer.append(entry);
- }
-
- @Override
- protected void sync(AsyncWriter writer) throws IOException {
- try {
- writer.sync().get();
- } catch (InterruptedException e) {
- throw new InterruptedIOException();
- } catch (ExecutionException e) {
- Throwables.propagateIfPossible(e.getCause());
- throw new IOException(e.getCause());
- }
+ protected Writer createWriter(Path path) throws IOException {
+ return new WriterOverAsyncWriter(AsyncFSWALProvider.createAsyncWriter(
+ TEST_UTIL.getConfiguration(), fs, path, false, EVENT_LOOP_GROUP.next(), CHANNEL_CLASS));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52d0ed18/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCombinedAsyncWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCombinedAsyncWriter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCombinedAsyncWriter.java
new file mode 100644
index 0000000..d74f9d8
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestCombinedAsyncWriter.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.AsyncFSWALProvider;
+import org.apache.hadoop.hbase.wal.AsyncFSWALProvider.AsyncWriter;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
+
+@RunWith(Parameterized.class)
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestCombinedAsyncWriter {
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static EventLoopGroup EVENT_LOOP_GROUP;
+
+ private static Class<? extends Channel> CHANNEL_CLASS;
+
+ private static WALFactory WALS;
+
+ @Rule
+ public final TestName name = new TestName();
+
+ @Parameter
+ public CombinedAsyncWriter.Mode mode;
+
+ @Parameters(name = "{index}: mode={0}")
+ public static List<Object[]> params() {
+ return Arrays.asList(new Object[] { CombinedAsyncWriter.Mode.SEQUENTIAL },
+ new Object[] { CombinedAsyncWriter.Mode.PARALLEL });
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ EVENT_LOOP_GROUP = new NioEventLoopGroup();
+ CHANNEL_CLASS = NioSocketChannel.class;
+ UTIL.startMiniDFSCluster(3);
+ UTIL.getTestFileSystem().mkdirs(UTIL.getDataTestDirOnTestFS());
+ WALS =
+ new WALFactory(UTIL.getConfiguration(), null, TestCombinedAsyncWriter.class.getSimpleName());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ if (WALS != null) {
+ WALS.close();
+ }
+ EVENT_LOOP_GROUP.shutdownGracefully().syncUninterruptibly();
+ UTIL.shutdownMiniDFSCluster();
+ }
+
+ @Test
+ public void testWithTrailer() throws IOException {
+ doTest(true);
+ }
+
+ @Test
+ public void testWithoutTrailer() throws IOException {
+ doTest(false);
+ }
+
+ private Path getPath(int index) throws IOException {
+ String methodName = name.getMethodName().replaceAll("[^A-Za-z0-9_-]", "_");
+ return new Path(UTIL.getDataTestDirOnTestFS(), methodName + "-" + index);
+ }
+
+ private void doTest(boolean withTrailer) throws IOException {
+ int columnCount = 5;
+ int recordCount = 5;
+ TableName tableName = TableName.valueOf("tablename");
+ byte[] row = Bytes.toBytes("row");
+ long timestamp = System.currentTimeMillis();
+ Path path1 = getPath(1);
+ Path path2 = getPath(2);
+ FileSystem fs = UTIL.getTestFileSystem();
+ Configuration conf = UTIL.getConfiguration();
+ try (
+ AsyncWriter writer1 = AsyncFSWALProvider.createAsyncWriter(conf, fs, path1, false,
+ EVENT_LOOP_GROUP.next(), CHANNEL_CLASS);
+ AsyncWriter writer2 = AsyncFSWALProvider.createAsyncWriter(conf, fs, path2, false,
+ EVENT_LOOP_GROUP.next(), CHANNEL_CLASS);
+ CombinedAsyncWriter writer = CombinedAsyncWriter.create(mode, writer1, writer2)) {
+ ProtobufLogTestHelper.doWrite(new WriterOverAsyncWriter(writer), withTrailer, tableName,
+ columnCount, recordCount, row, timestamp);
+ try (ProtobufLogReader reader = (ProtobufLogReader) WALS.createReader(fs, path1)) {
+ ProtobufLogTestHelper.doRead(reader, withTrailer, tableName, columnCount, recordCount, row,
+ timestamp);
+ }
+ try (ProtobufLogReader reader = (ProtobufLogReader) WALS.createReader(fs, path2)) {
+ ProtobufLogTestHelper.doRead(reader, withTrailer, tableName, columnCount, recordCount, row,
+ timestamp);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52d0ed18/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestProtobufLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestProtobufLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestProtobufLog.java
index 0add852..ae28760 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestProtobufLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestProtobufLog.java
@@ -18,31 +18,18 @@
package org.apache.hadoop.hbase.regionserver.wal;
import java.io.IOException;
-
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
-import org.apache.hadoop.hbase.wal.WAL.Entry;
import org.apache.hadoop.hbase.wal.FSHLogProvider;
-import org.apache.hadoop.hbase.wal.WALProvider;
import org.apache.hadoop.hbase.wal.WALProvider.Writer;
import org.junit.experimental.categories.Category;
@Category({ RegionServerTests.class, MediumTests.class })
-public class TestProtobufLog extends AbstractTestProtobufLog<WALProvider.Writer> {
+public class TestProtobufLog extends AbstractTestProtobufLog {
@Override
protected Writer createWriter(Path path) throws IOException {
return FSHLogProvider.createWriter(TEST_UTIL.getConfiguration(), fs, path, false);
}
-
- @Override
- protected void append(Writer writer, Entry entry) throws IOException {
- writer.append(entry);
- }
-
- @Override
- protected void sync(Writer writer) throws IOException {
- writer.sync();
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52d0ed18/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WriterOverAsyncWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WriterOverAsyncWriter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WriterOverAsyncWriter.java
new file mode 100644
index 0000000..da70ee5
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/WriterOverAsyncWriter.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.concurrent.ExecutionException;
+import org.apache.hadoop.hbase.wal.WAL.Entry;
+import org.apache.hadoop.hbase.wal.WALProvider;
+import org.apache.hadoop.hbase.wal.WALProvider.AsyncWriter;
+
+import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
+
+class WriterOverAsyncWriter implements WALProvider.Writer {
+
+ private final WALProvider.AsyncWriter asyncWriter;
+
+ public WriterOverAsyncWriter(AsyncWriter asyncWriter) {
+ this.asyncWriter = asyncWriter;
+ }
+
+ @Override
+ public void close() throws IOException {
+ asyncWriter.close();
+ }
+
+ @Override
+ public long getLength() {
+ return asyncWriter.getLength();
+ }
+
+ @Override
+ public void sync() throws IOException {
+ try {
+ asyncWriter.sync().get();
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException();
+ } catch (ExecutionException e) {
+ Throwables.propagateIfPossible(e.getCause(), IOException.class);
+ throw new IOException(e.getCause());
+ }
+ }
+
+ @Override
+ public void append(Entry entry) throws IOException {
+ asyncWriter.append(entry);
+ }
+}
[18/31] hbase git commit: HBASE-19812 TestFlushSnapshotFromClient
fails because of failing region.flush
Posted by zh...@apache.org.
HBASE-19812 TestFlushSnapshotFromClient fails because of failing region.flush
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/678b98a7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/678b98a7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/678b98a7
Branch: refs/heads/HBASE-19064
Commit: 678b98a72668bccbdb5bf7621c7e71afc05d6ffa
Parents: e3d1583
Author: zhangduo <zh...@apache.org>
Authored: Thu Jan 18 12:50:30 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Thu Jan 18 16:23:07 2018 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/MiniHBaseCluster.java | 34 +++++++++++---------
1 file changed, 19 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/678b98a7/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index 92715ee..e7f8496 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegion.FlushResult;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.security.User;
@@ -386,7 +387,6 @@ public class MiniHBaseCluster extends HBaseCluster {
* within the timeout.
*
* @return New RegionServerThread
- * @throws IOException
*/
public JVMClusterUtil.RegionServerThread startRegionServerAndWait(long timeout)
throws IOException {
@@ -463,7 +463,6 @@ public class MiniHBaseCluster extends HBaseCluster {
/**
* Starts a master thread running
*
- * @throws IOException
* @return New RegionServerThread
*/
public JVMClusterUtil.MasterThread startMaster() throws IOException {
@@ -615,9 +614,7 @@ public class MiniHBaseCluster extends HBaseCluster {
/**
* Shut down the mini HBase cluster
- * @throws IOException
*/
- @SuppressWarnings("deprecation")
public void shutdown() throws IOException {
if (this.hbaseCluster != null) {
this.hbaseCluster.shutdown();
@@ -644,29 +641,36 @@ public class MiniHBaseCluster extends HBaseCluster {
return master == null ? null : master.getClusterMetrics();
}
+ private void executeFlush(HRegion region) throws IOException {
+ // retry 5 times if we can not flush
+ for (int i = 0; i < 5; i++) {
+ FlushResult result = region.flush(true);
+ if (result.getResult() != FlushResult.Result.CANNOT_FLUSH) {
+ return;
+ }
+ Threads.sleep(1000);
+ }
+ }
+
/**
* Call flushCache on all regions on all participating regionservers.
- * @throws IOException
*/
public void flushcache() throws IOException {
- for (JVMClusterUtil.RegionServerThread t:
- this.hbaseCluster.getRegionServers()) {
- for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) {
- r.flush(true);
+ for (JVMClusterUtil.RegionServerThread t : this.hbaseCluster.getRegionServers()) {
+ for (HRegion r : t.getRegionServer().getOnlineRegionsLocalContext()) {
+ executeFlush(r);
}
}
}
/**
* Call flushCache on all regions of the specified table.
- * @throws IOException
*/
public void flushcache(TableName tableName) throws IOException {
- for (JVMClusterUtil.RegionServerThread t:
- this.hbaseCluster.getRegionServers()) {
- for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) {
- if(r.getTableDescriptor().getTableName().equals(tableName)) {
- r.flush(true);
+ for (JVMClusterUtil.RegionServerThread t : this.hbaseCluster.getRegionServers()) {
+ for (HRegion r : t.getRegionServer().getOnlineRegionsLocalContext()) {
+ if (r.getTableDescriptor().getTableName().equals(tableName)) {
+ executeFlush(r);
}
}
}
[15/31] hbase git commit: Revert "HBASE-19791 TestZKAsyncRegistry
hangs" Premature push
Posted by zh...@apache.org.
Revert "HBASE-19791 TestZKAsyncRegistry hangs"
Premature push
This reverts commit d3a306d81d3f087696fc6d45dd8d6bda939378b2.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eeb40ff6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eeb40ff6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eeb40ff6
Branch: refs/heads/HBASE-19064
Commit: eeb40ff66c7d5b148fd693780be64358e5f7385e
Parents: 53d0c23
Author: Michael Stack <st...@apache.org>
Authored: Wed Jan 17 15:19:48 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Wed Jan 17 15:19:48 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeb40ff6/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
index a0c4c25..3478549 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestZKAsyncRegistry.java
@@ -47,9 +47,6 @@ import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * Test the ZooKeeper Async Registry.
- */
@Category({ MediumTests.class, ClientTests.class })
public class TestZKAsyncRegistry {
private static final Logger LOG = LoggerFactory.getLogger(TestZKAsyncRegistry.class);
[19/31] hbase git commit: HBASE-19806 Lower max versions for table
column family of hbase:meta
Posted by zh...@apache.org.
HBASE-19806 Lower max versions for table column family of hbase:meta
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7573e07c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7573e07c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7573e07c
Branch: refs/heads/HBASE-19064
Commit: 7573e07cc69d6da672512d6fb01926275dc27971
Parents: 678b98a
Author: tedyu <yu...@gmail.com>
Authored: Thu Jan 18 07:57:22 2018 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jan 18 07:57:22 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/util/FSTableDescriptors.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7573e07c/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index 9ec5704..c76cd90 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -190,8 +190,8 @@ public class FSTableDescriptors implements TableDescriptors {
.setBloomFilterType(BloomType.NONE)
.build())
.addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.TABLE_FAMILY)
- // Ten is arbitrary number. Keep versions to help debugging.
- .setMaxVersions(10)
+ .setMaxVersions(conf.getInt(HConstants.HBASE_META_VERSIONS,
+ HConstants.DEFAULT_HBASE_META_VERSIONS))
.setInMemory(true)
.setBlocksize(8 * 1024)
.setScope(HConstants.REPLICATION_SCOPE_LOCAL)
[23/31] hbase git commit: HBASE-19815 Flakey
TestAssignmentManager.testAssignWithRandExec (Part Two).
Posted by zh...@apache.org.
HBASE-19815 Flakey TestAssignmentManager.testAssignWithRandExec (Part Two).
Part One cleaned up a ClassCastException.
Part Two adds the ServerCrashProcedure#handleRIT behavior to RecoverMetaProcedure.
Adds debug in the test.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/581fabe7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/581fabe7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/581fabe7
Branch: refs/heads/HBASE-19064
Commit: 581fabe7b2177a090af33517f2f7cb1cdab2c64b
Parents: 646770d
Author: Michael Stack <st...@apache.org>
Authored: Wed Jan 17 22:35:35 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Thu Jan 18 11:32:21 2018 -0800
----------------------------------------------------------------------
.../master/procedure/RecoverMetaProcedure.java | 38 +++++++++++++++++---
.../assignment/TestAssignmentManager.java | 9 ++++-
2 files changed, 41 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/581fabe7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java
index 50ef3e0..70d0d55 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RecoverMetaProcedure.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.assignment.AssignProcedure;
+import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
+import org.apache.hadoop.hbase.master.assignment.RegionTransitionProcedure;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
@@ -126,17 +128,17 @@ public class RecoverMetaProcedure
RegionInfoBuilder.FIRST_META_REGIONINFO, this.replicaId);
AssignProcedure metaAssignProcedure;
+ AssignmentManager am = master.getAssignmentManager();
if (failedMetaServer != null) {
- LOG.info(this + "; Assigning meta with new plan. previous meta server=" +
- failedMetaServer);
- metaAssignProcedure = master.getAssignmentManager().createAssignProcedure(hri);
+ handleRIT(env, hri, this.failedMetaServer);
+ LOG.info(this + "; Assigning meta with new plan; previous server=" + failedMetaServer);
+ metaAssignProcedure = am.createAssignProcedure(hri);
} else {
// get server carrying meta from zk
ServerName metaServer =
MetaTableLocator.getMetaRegionState(master.getZooKeeper()).getServerName();
LOG.info(this + "; Retaining meta assignment to server=" + metaServer);
- metaAssignProcedure =
- master.getAssignmentManager().createAssignProcedure(hri, metaServer);
+ metaAssignProcedure = am.createAssignProcedure(hri, metaServer);
}
addChildProcedure(metaAssignProcedure);
@@ -152,6 +154,32 @@ public class RecoverMetaProcedure
return Flow.HAS_MORE_STATE;
}
+ /**
+ * Is the region stuck assigning to this failedMetaServer? If so, cancel the call
+ * just as we do over in ServerCrashProcedure#handleRIT except less to do here; less context
+ * to carry.
+ */
+ private void handleRIT(MasterProcedureEnv env, RegionInfo ri, ServerName crashedServerName) {
+ AssignmentManager am = env.getAssignmentManager();
+ RegionTransitionProcedure rtp = am.getRegionStates().getRegionTransitionProcedure(ri);
+ if (rtp == null) {
+ return; // Nothing to do. Not in RIT.
+ }
+ // Make sure the RIT is against this crashed server. In the case where there are many
+ // processings of a crashed server -- backed up for whatever reason (slow WAL split)
+ // -- then a previous SCP may have already failed an assign, etc., and it may have a
+ // new location target; DO NOT fail these else we make for assign flux.
+ ServerName rtpServerName = rtp.getServer(env);
+ if (rtpServerName == null) {
+ LOG.warn("RIT with ServerName null! " + rtp);
+ } else if (rtpServerName.equals(crashedServerName)) {
+ LOG.info("pid=" + getProcId() + " found RIT " + rtp + "; " +
+ rtp.getRegionState(env).toShortString());
+ rtp.remoteCallFailed(env, crashedServerName,
+ new ServerCrashException(getProcId(), crashedServerName));
+ }
+ }
+
@Override
protected void rollbackState(MasterProcedureEnv env,
MasterProcedureProtos.RecoverMetaState recoverMetaState)
http://git-wip-us.apache.org/repos/asf/hbase/blob/581fabe7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
index 3ab915b..9b9f624 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.net.SocketTimeoutException;
import java.util.NavigableMap;
import java.util.Random;
@@ -206,7 +207,7 @@ public class TestAssignmentManager {
rsDispatcher.setMockRsExecutor(new RandRsExecutor());
// Loop a bunch of times so we hit various combos of exceptions.
for (int i = 0; i < 10; i++) {
- LOG.info("" + i);
+ LOG.info("ROUND=" + i);
AssignProcedure proc = am.createAssignProcedure(hri);
waitOnFuture(submitProcedure(proc));
}
@@ -445,6 +446,12 @@ public class TestAssignmentManager {
return future.get(5, TimeUnit.SECONDS);
} catch (ExecutionException e) {
LOG.info("ExecutionException", e);
+ Exception ee = (Exception)e.getCause();
+ if (ee instanceof InterruptedIOException) {
+ for (Procedure p: this.master.getMasterProcedureExecutor().getProcedures()) {
+ LOG.info(p.toStringDetails());
+ }
+ }
throw (Exception)e.getCause();
}
}
[30/31] hbase git commit: HBASE-19078 Add a remote peer cluster wal
directory config for synchronous replication
Posted by zh...@apache.org.
HBASE-19078 Add a remote peer cluster wal directory config for synchronous replication
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/373d2c0c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/373d2c0c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/373d2c0c
Branch: refs/heads/HBASE-19064
Commit: 373d2c0c4661bba0c9c17b8b08d4440c864bb4e6
Parents: 52d0ed1
Author: Guanghao Zhang <zg...@apache.org>
Authored: Sat Jan 13 18:55:28 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Fri Jan 19 21:20:34 2018 +0800
----------------------------------------------------------------------
.../replication/ReplicationPeerConfigUtil.java | 6 ++
.../replication/ReplicationPeerConfig.java | 21 +++++-
.../ReplicationPeerConfigBuilder.java | 7 ++
.../src/main/protobuf/Replication.proto | 1 +
.../replication/ReplicationPeerManager.java | 15 ++++
.../replication/TestReplicationAdmin.java | 77 ++++++++++++++++++++
.../src/main/ruby/hbase/replication_admin.rb | 17 +++--
hbase-shell/src/main/ruby/hbase_constants.rb | 1 +
.../src/main/ruby/shell/commands/add_peer.rb | 21 +++++-
.../src/main/ruby/shell/commands/list_peers.rb | 19 ++++-
.../test/ruby/hbase/replication_admin_test.rb | 16 ++++
11 files changed, 188 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
index a234a9b..642149b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
@@ -315,6 +315,9 @@ public final class ReplicationPeerConfigUtil {
excludeNamespacesList.stream().map(ByteString::toStringUtf8).collect(Collectors.toSet()));
}
+ if (peer.hasRemoteWALDir()) {
+ builder.setRemoteWALDir(peer.getRemoteWALDir());
+ }
return builder.build();
}
@@ -371,6 +374,9 @@ public final class ReplicationPeerConfigUtil {
}
}
+ if (peerConfig.getRemoteWALDir() != null) {
+ builder.setRemoteWALDir(peerConfig.getRemoteWALDir());
+ }
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index bf8d030..4c10c46 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -46,6 +46,8 @@ public class ReplicationPeerConfig {
private Map<TableName, ? extends Collection<String>> excludeTableCFsMap = null;
private Set<String> excludeNamespaces = null;
private long bandwidth = 0;
+ // Used by synchronous replication
+ private String remoteWALDir;
private ReplicationPeerConfig(ReplicationPeerConfigBuilderImpl builder) {
this.clusterKey = builder.clusterKey;
@@ -64,6 +66,7 @@ public class ReplicationPeerConfig {
builder.excludeNamespaces != null ? Collections.unmodifiableSet(builder.excludeNamespaces)
: null;
this.bandwidth = builder.bandwidth;
+ this.remoteWALDir = builder.remoteWALDir;
}
private Map<TableName, List<String>>
@@ -210,6 +213,10 @@ public class ReplicationPeerConfig {
return this;
}
+ public String getRemoteWALDir() {
+ return this.remoteWALDir;
+ }
+
public static ReplicationPeerConfigBuilder newBuilder() {
return new ReplicationPeerConfigBuilderImpl();
}
@@ -223,7 +230,8 @@ public class ReplicationPeerConfig {
.setReplicateAllUserTables(peerConfig.replicateAllUserTables())
.setExcludeTableCFsMap(peerConfig.getExcludeTableCFsMap())
.setExcludeNamespaces(peerConfig.getExcludeNamespaces())
- .setBandwidth(peerConfig.getBandwidth());
+ .setBandwidth(peerConfig.getBandwidth())
+ .setRemoteWALDir(peerConfig.getRemoteWALDir());
return builder;
}
@@ -250,6 +258,8 @@ public class ReplicationPeerConfig {
private long bandwidth = 0;
+ private String remoteWALDir = null;
+
@Override
public ReplicationPeerConfigBuilder setClusterKey(String clusterKey) {
this.clusterKey = clusterKey;
@@ -313,6 +323,12 @@ public class ReplicationPeerConfig {
}
@Override
+ public ReplicationPeerConfigBuilder setRemoteWALDir(String dir) {
+ this.remoteWALDir = dir;
+ return this;
+ }
+
+ @Override
public ReplicationPeerConfig build() {
// It would be nice to validate the configuration, but we have to work with "old" data
// from ZK which makes it much more difficult.
@@ -341,6 +357,9 @@ public class ReplicationPeerConfig {
}
}
builder.append("bandwidth=").append(bandwidth);
+ if (this.remoteWALDir != null) {
+ builder.append(",remoteWALDir=").append(remoteWALDir);
+ }
return builder.toString();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java
index 0b2f2e2..eac98c6 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java
@@ -138,6 +138,13 @@ public interface ReplicationPeerConfigBuilder {
ReplicationPeerConfigBuilder setExcludeNamespaces(Set<String> namespaces);
/**
+ * Set the remote peer cluster's wal directory. Used by synchronous replication.
+ * @param dir the remote peer cluster's wal directory
+ * @return {@code this}
+ */
+ ReplicationPeerConfigBuilder setRemoteWALDir(String dir);
+
+ /**
* Builds the configuration object from the current state of {@code this}.
* @return A {@link ReplicationPeerConfig} instance.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-protocol-shaded/src/main/protobuf/Replication.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Replication.proto b/hbase-protocol-shaded/src/main/protobuf/Replication.proto
index 9f7b4c2..44295d8 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Replication.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Replication.proto
@@ -48,6 +48,7 @@ message ReplicationPeer {
optional bool replicate_all = 8;
repeated TableCF exclude_table_cfs = 9;
repeated bytes exclude_namespaces = 10;
+ optional string remoteWALDir = 11;
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 19fc7f4..d715e2e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -146,6 +146,21 @@ public class ReplicationPeerManager {
oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
" does not match new class '" + peerConfig.getReplicationEndpointImpl() + "'");
}
+
+ if (!isStringEquals(peerConfig.getRemoteWALDir(), oldPeerConfig.getRemoteWALDir())) {
+ throw new DoNotRetryIOException(
+ "Changing the remote wal dir on an existing peer is not allowed. Existing remote wal " +
+ "dir '" + oldPeerConfig.getRemoteWALDir() + "' for peer " + peerId +
+ " does not match new remote wal dir '" + peerConfig.getRemoteWALDir() + "'");
+ }
+
+ if (oldPeerConfig.getRemoteWALDir() != null) {
+ if (!ReplicationUtils.isKeyConfigEqual(oldPeerConfig, peerConfig)) {
+ throw new DoNotRetryIOException(
+ "Changing the replicated namespace/table config on a synchronous replication "
+ + "peer(peerId: " + peerId + ") is not allowed.");
+ }
+ }
}
public void addPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index af4e362..cb06590 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -900,4 +900,81 @@ public class TestReplicationAdmin {
// OK
}
}
+
+ @Test
+ public void testPeerRemoteWALDir() throws Exception {
+ String rootDir = "hdfs://srv1:9999/hbase";
+ ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder();
+ builder.setClusterKey(KEY_ONE);
+ hbaseAdmin.addReplicationPeer(ID_ONE, builder.build());
+
+ ReplicationPeerConfig rpc = hbaseAdmin.getReplicationPeerConfig(ID_ONE);
+ assertNull(rpc.getRemoteWALDir());
+
+ try {
+ builder.setRemoteWALDir("hdfs://srv2:8888/hbase");
+ hbaseAdmin.updateReplicationPeerConfig(ID_ONE, builder.build());
+ fail("Change remote wal dir is not allowed");
+ } catch (Exception e) {
+ // OK
+ }
+
+ builder = ReplicationPeerConfig.newBuilder();
+ builder.setClusterKey(KEY_SECOND);
+ builder.setRemoteWALDir(rootDir);
+ hbaseAdmin.addReplicationPeer(ID_SECOND, builder.build());
+
+ rpc = hbaseAdmin.getReplicationPeerConfig(ID_SECOND);
+ assertEquals(rootDir, rpc.getRemoteWALDir());
+
+ try {
+ builder.setRemoteWALDir("hdfs://srv2:8888/hbase");
+ hbaseAdmin.updateReplicationPeerConfig(ID_SECOND, builder.build());
+ fail("Change remote wal dir is not allowed");
+ } catch (Exception e) {
+ // OK
+ }
+
+ try {
+ builder.setRemoteWALDir(null);
+ hbaseAdmin.updateReplicationPeerConfig(ID_SECOND, builder.build());
+ fail("Change remote wal dir is not allowed");
+ } catch (Exception e) {
+ // OK
+ }
+
+ try {
+ builder = ReplicationPeerConfig.newBuilder(rpc);
+ builder.setReplicateAllUserTables(false);
+ hbaseAdmin.updateReplicationPeerConfig(ID_SECOND, builder.build());
+ fail(
+ "Change replicated namespace/table config on an existing synchronous peer is not allowed");
+ } catch (Exception e) {
+ // OK
+ }
+
+ try {
+ builder = ReplicationPeerConfig.newBuilder(rpc);
+ Set<String> namespaces = new HashSet<>();
+ namespaces.add("ns1");
+ builder.setExcludeNamespaces(namespaces);
+ hbaseAdmin.updateReplicationPeerConfig(ID_SECOND, builder.build());
+ fail(
+ "Change replicated namespace/table config on an existing synchronous peer is not allowed");
+ } catch (Exception e) {
+ // OK
+ }
+
+ try {
+ builder = ReplicationPeerConfig.newBuilder(rpc);
+ Map<TableName, List<String>> tableCfs = new HashMap<>();
+ tableCfs.put(TableName.valueOf(name.getMethodName()), new ArrayList<>());
+ builder.setExcludeTableCFsMap(tableCfs);
+ hbaseAdmin.updateReplicationPeerConfig(ID_SECOND, builder.build());
+ fail(
+ "Change replicated namespace/table config on an existing synchronous peer is not allowed");
+ } catch (Exception e) {
+ // OK
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-shell/src/main/ruby/hbase/replication_admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/replication_admin.rb b/hbase-shell/src/main/ruby/hbase/replication_admin.rb
index b9d4a0c..ba7d191 100644
--- a/hbase-shell/src/main/ruby/hbase/replication_admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/replication_admin.rb
@@ -64,16 +64,20 @@ module Hbase
table_cfs = args.fetch(TABLE_CFS, nil)
namespaces = args.fetch(NAMESPACES, nil)
peer_state = args.fetch(STATE, nil)
+ remote_wal_dir = args.fetch(REMOTE_WAL_DIR, nil)
# Create and populate a ReplicationPeerConfig
- builder = org.apache.hadoop.hbase.replication.ReplicationPeerConfig
- .newBuilder()
+ builder = ReplicationPeerConfig.newBuilder()
builder.set_cluster_key(cluster_key)
unless endpoint_classname.nil?
builder.set_replication_endpoint_impl(endpoint_classname)
end
+ unless remote_wal_dir.nil?
+ builder.setRemoteWALDir(remote_wal_dir)
+ end
+
unless config.nil?
builder.putAllConfiguration(config)
end
@@ -228,8 +232,7 @@ module Hbase
namespaces.each do |n|
ns_set.add(n)
end
- builder = org.apache.hadoop.hbase.replication.ReplicationPeerConfig
- .newBuilder(rpc)
+ builder = ReplicationPeerConfig.newBuilder(rpc)
builder.setNamespaces(ns_set)
@admin.updateReplicationPeerConfig(id, builder.build)
end
@@ -248,8 +251,7 @@ module Hbase
ns_set.remove(n)
end
end
- builder = org.apache.hadoop.hbase.replication.ReplicationPeerConfig
- .newBuilder(rpc)
+ builder = ReplicationPeerConfig.newBuilder(rpc)
builder.setNamespaces(ns_set)
@admin.updateReplicationPeerConfig(id, builder.build)
end
@@ -361,8 +363,7 @@ module Hbase
# Create and populate a ReplicationPeerConfig
replication_peer_config = get_peer_config(id)
- builder = org.apache.hadoop.hbase.replication.ReplicationPeerConfig
- .newBuilder(replication_peer_config)
+ builder = ReplicationPeerConfig.newBuilder(replication_peer_config)
unless config.nil?
builder.putAllConfiguration(config)
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-shell/src/main/ruby/hbase_constants.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb
index 28484cb..2870dfb 100644
--- a/hbase-shell/src/main/ruby/hbase_constants.rb
+++ b/hbase-shell/src/main/ruby/hbase_constants.rb
@@ -77,6 +77,7 @@ module HBaseConstants
VALUE = 'VALUE'.freeze
ENDPOINT_CLASSNAME = 'ENDPOINT_CLASSNAME'.freeze
CLUSTER_KEY = 'CLUSTER_KEY'.freeze
+ REMOTE_WAL_DIR = 'REMOTE_WAL_DIR'.freeze
TABLE_CFS = 'TABLE_CFS'.freeze
NAMESPACES = 'NAMESPACES'.freeze
STATE = 'STATE'.freeze
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-shell/src/main/ruby/shell/commands/add_peer.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/add_peer.rb b/hbase-shell/src/main/ruby/shell/commands/add_peer.rb
index eb2da83..4b6f294 100644
--- a/hbase-shell/src/main/ruby/shell/commands/add_peer.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/add_peer.rb
@@ -35,7 +35,7 @@ to the peer cluster.
An optional parameter for table column families identifies which tables and/or column families
will be replicated to the peer cluster.
-Notice: Set a namespace in the peer config means that all tables in this namespace
+Note: Set a namespace in the peer config means that all tables in this namespace
will be replicated to the peer cluster. So if you already have set a namespace in peer config,
then you can't set this namespace's tables in the peer config again.
@@ -74,6 +74,25 @@ the key TABLE_CFS.
Note: Either CLUSTER_KEY or ENDPOINT_CLASSNAME must be specified. If ENDPOINT_CLASSNAME is specified, CLUSTER_KEY is
optional and should only be specified if a particular custom endpoint requires it.
+The default replication peer is asynchronous. You can also add a synchronous replication peer
+with REMOTE_WAL_DIR parameter. Meanwhile, synchronous replication peer also support other optional
+config for asynchronous replication peer.
+
+Examples:
+
+ hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase",
+ REMOTE_WAL_DIR => "hdfs://srv1:9999/hbase"
+ hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase",
+ STATE => "ENABLED", REMOTE_WAL_DIR => "hdfs://srv1:9999/hbase"
+ hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase",
+ STATE => "DISABLED", REMOTE_WAL_DIR => "hdfs://srv1:9999/hbase"
+ hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase",
+ REMOTE_WAL_DIR => "hdfs://srv1:9999/hbase", NAMESPACES => ["ns1", "ns2"]
+ hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase",
+ REMOTE_WAL_DIR => "hdfs://srv1:9999/hbase", TABLE_CFS => { "table1" => [] }
+
+Note: The REMOTE_WAL_DIR is not allowed to change.
+
EOF
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/list_peers.rb b/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
index 522d23d..caeab86 100644
--- a/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/list_peers.rb
@@ -39,7 +39,8 @@ EOF
peers = replication_admin.list_peers
formatter.header(%w[PEER_ID CLUSTER_KEY ENDPOINT_CLASSNAME
- STATE REPLICATE_ALL NAMESPACES TABLE_CFS BANDWIDTH])
+ REMOTE_ROOT_DIR STATE REPLICATE_ALL
+ NAMESPACES TABLE_CFS BANDWIDTH])
peers.each do |peer|
id = peer.getPeerId
@@ -52,8 +53,20 @@ EOF
namespaces = replication_admin.show_peer_namespaces(config)
tableCFs = replication_admin.show_peer_tableCFs_by_config(config)
end
- formatter.row([id, config.getClusterKey,
- config.getReplicationEndpointImpl, state,
+ cluster_key = 'nil'
+ unless config.getClusterKey.nil?
+ cluster_key = config.getClusterKey
+ end
+ endpoint_classname = 'nil'
+ unless config.getReplicationEndpointImpl.nil?
+ endpoint_classname = config.getReplicationEndpointImpl
+ end
+ remote_root_dir = 'nil'
+ unless config.getRemoteWALDir.nil?
+ remote_root_dir = config.getRemoteWALDir
+ end
+ formatter.row([id, cluster_key, endpoint_classname,
+ remote_root_dir, state,
config.replicateAllUserTables, namespaces, tableCFs,
config.getBandwidth])
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/373d2c0c/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb b/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb
index 0f84396..7f2b6ae 100644
--- a/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb
+++ b/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb
@@ -97,6 +97,22 @@ module Hbase
command(:remove_peer, @peer_id)
end
+ define_test "add_peer: remote wal dir" do
+ cluster_key = "server1.cie.com:2181:/hbase"
+ remote_wal_dir = "hdfs://srv1:9999/hbase"
+ args = { CLUSTER_KEY => cluster_key, REMOTE_WAL_DIR => remote_wal_dir }
+ command(:add_peer, @peer_id, args)
+
+ assert_equal(1, command(:list_peers).length)
+ peer = command(:list_peers).get(0)
+ assert_equal(@peer_id, peer.getPeerId)
+ assert_equal(cluster_key, peer.getPeerConfig.getClusterKey)
+ assert_equal(remote_wal_dir, peer.getPeerConfig.getRemoteWALDir)
+
+ # cleanup for future tests
+ command(:remove_peer, @peer_id)
+ end
+
define_test "add_peer: single zk cluster key with enabled/disabled state" do
cluster_key = "server1.cie.com:2181:/hbase"
[16/31] hbase git commit: HBASE-19784 bin/stop-hbase more friendly
when hbase already stopped
Posted by zh...@apache.org.
HBASE-19784 bin/stop-hbase more friendly when hbase already stopped
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b1729a29
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b1729a29
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b1729a29
Branch: refs/heads/HBASE-19064
Commit: b1729a298bbcfeb54ba95821096408612e8ccef5
Parents: eeb40ff
Author: Mike Drob <md...@apache.org>
Authored: Fri Jan 12 08:34:49 2018 -0600
Committer: Michael Stack <st...@apache.org>
Committed: Wed Jan 17 15:48:53 2018 -0800
----------------------------------------------------------------------
bin/stop-hbase.sh | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1729a29/bin/stop-hbase.sh
----------------------------------------------------------------------
diff --git a/bin/stop-hbase.sh b/bin/stop-hbase.sh
index 8b81dee..4a19681 100755
--- a/bin/stop-hbase.sh
+++ b/bin/stop-hbase.sh
@@ -44,17 +44,20 @@ logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out
loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}"
pid=${HBASE_PID_DIR:-/tmp}/hbase-$HBASE_IDENT_STRING-master.pid
-echo -n stopping hbase
-echo "`date` Stopping hbase (via master)" >> $loglog
+if [[ -e $pid ]]; then
+ echo -n stopping hbase
+ echo "`date` Stopping hbase (via master)" >> $loglog
-nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \
- --config "${HBASE_CONF_DIR}" \
- master stop "$@" > "$logout" 2>&1 < /dev/null &
+ nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \
+ --config "${HBASE_CONF_DIR}" \
+ master stop "$@" > "$logout" 2>&1 < /dev/null &
-waitForProcessEnd `cat $pid` 'stop-master-command'
-
-rm -f $pid
+ waitForProcessEnd `cat $pid` 'stop-master-command'
+ rm -f $pid
+else
+ echo no hbase master found
+fi
# distributed == false means that the HMaster will kill ZK when it exits
# HBASE-6504 - only take the first line of the output in case verbose gc is on
[25/31] hbase git commit: HBASE-19822 HFileCleaner threads stick
around after shutdown stuck on queue#take Interrupt wait on queue when cancel
is called.
Posted by zh...@apache.org.
HBASE-19822 HFileCleaner threads stick around after shutdown stuck on queue#take Interrupt wait on queue when cancel is called.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8b520ce5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8b520ce5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8b520ce5
Branch: refs/heads/HBASE-19064
Commit: 8b520ce50d0906ce1821f2101da395d222bf6b76
Parents: cde1f82
Author: Michael Stack <st...@apache.org>
Authored: Thu Jan 18 11:12:31 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Thu Jan 18 13:52:51 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/master/cleaner/HFileCleaner.java | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b520ce5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index f20dcc4..640c8f7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -231,7 +231,7 @@ public class HFileCleaner extends CleanerChore<BaseHFileCleanerDelegate> {
try {
task = queue.take();
} catch (InterruptedException e) {
- LOG.debug("Interrupted while trying to take a task from queue", e);
+ LOG.trace("Interrupted while trying to take a task from queue", e);
break;
}
if (task != null) {
@@ -445,4 +445,12 @@ public class HFileCleaner extends CleanerChore<BaseHFileCleanerDelegate> {
}
return updated;
}
+
+ @Override
+ public synchronized void cancel(boolean mayInterruptIfRunning) {
+ super.cancel(mayInterruptIfRunning);
+ for (Thread t: this.threads) {
+ t.interrupt();
+ }
+ }
}
[24/31] hbase git commit: HBASE-19799 Add web UI to rsgroup
Posted by zh...@apache.org.
HBASE-19799 Add web UI to rsgroup
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cde1f821
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cde1f821
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cde1f821
Branch: refs/heads/HBASE-19064
Commit: cde1f821ebec14a30214a15aebb02c936f27fd87
Parents: 581fabe
Author: Guangxu Cheng <gu...@gmail.com>
Authored: Thu Jan 18 15:16:29 2018 +0800
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jan 18 12:49:43 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/RSGroupTableAccessor.java | 82 ++++
.../hadoop/hbase/protobuf/ProtobufUtil.java | 14 +
hbase-protocol/src/main/protobuf/RSGroup.proto | 34 ++
hbase-rsgroup/src/main/protobuf/RSGroup.proto | 34 --
.../hbase/tmpl/master/MasterStatusTmpl.jamon | 7 +-
.../hbase/tmpl/master/RSGroupListTmpl.jamon | 352 +++++++++++++++
.../resources/hbase-webapps/master/rsgroup.jsp | 441 +++++++++++++++++++
7 files changed, 929 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-client/src/main/java/org/apache/hadoop/hbase/RSGroupTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RSGroupTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RSGroupTableAccessor.java
new file mode 100644
index 0000000..06a7604
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RSGroupTableAccessor.java
@@ -0,0 +1,82 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
+import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Read rs group information from <code>hbase:rsgroup</code>.
+ */
+@InterfaceAudience.Private
+public final class RSGroupTableAccessor {
+
+ //Assigned before user tables
+ private static final TableName RSGROUP_TABLE_NAME =
+ TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "rsgroup");
+ private static final byte[] META_FAMILY_BYTES = Bytes.toBytes("m");
+ private static final byte[] META_QUALIFIER_BYTES = Bytes.toBytes("i");
+
+ public static List<RSGroupInfo> getAllRSGroupInfo(Connection connection)
+ throws IOException {
+ try (Table rsGroupTable = connection.getTable(RSGROUP_TABLE_NAME)) {
+ List<RSGroupInfo> rsGroupInfos = new ArrayList<>();
+ for (Result result : rsGroupTable.getScanner(new Scan())) {
+ RSGroupInfo rsGroupInfo = getRSGroupInfo(result);
+ if (rsGroupInfo != null) {
+ rsGroupInfos.add(rsGroupInfo);
+ }
+ }
+ return rsGroupInfos;
+ }
+ }
+
+ private static RSGroupInfo getRSGroupInfo(Result result) throws IOException {
+ byte[] rsGroupInfo = result.getValue(META_FAMILY_BYTES, META_QUALIFIER_BYTES);
+ if (rsGroupInfo == null) {
+ return null;
+ }
+ RSGroupProtos.RSGroupInfo proto =
+ RSGroupProtos.RSGroupInfo.parseFrom(rsGroupInfo);
+ return ProtobufUtil.toGroupInfo(proto);
+ }
+
+ public static RSGroupInfo getRSGroupInfo(Connection connection, byte[] rsGroupName)
+ throws IOException {
+ try (Table rsGroupTable = connection.getTable(RSGROUP_TABLE_NAME)){
+ Result result = rsGroupTable.get(new Get(rsGroupName));
+ return getRSGroupInfo(result);
+ }
+ }
+
+ private RSGroupTableAccessor() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 2f2dc86..fd263b0 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.net.Address;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
@@ -92,7 +93,9 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
+import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
+import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.Bytes;
@@ -1819,4 +1822,15 @@ public final class ProtobufUtil {
int port = Addressing.parsePort(str);
return ServerName.valueOf(hostname, port, -1L);
}
+
+ public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) {
+ RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName());
+ for(HBaseProtos.ServerName el: proto.getServersList()) {
+ RSGroupInfo.addServer(Address.fromParts(el.getHostName(), el.getPort()));
+ }
+ for(HBaseProtos.TableName pTableName: proto.getTablesList()) {
+ RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName));
+ }
+ return RSGroupInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-protocol/src/main/protobuf/RSGroup.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/protobuf/RSGroup.proto b/hbase-protocol/src/main/protobuf/RSGroup.proto
new file mode 100644
index 0000000..7358941
--- /dev/null
+++ b/hbase-protocol/src/main/protobuf/RSGroup.proto
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RSGroupProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+message RSGroupInfo {
+ required string name = 1;
+ repeated ServerName servers = 4;
+ repeated TableName tables = 3;
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-rsgroup/src/main/protobuf/RSGroup.proto
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/src/main/protobuf/RSGroup.proto b/hbase-rsgroup/src/main/protobuf/RSGroup.proto
deleted file mode 100644
index 7358941..0000000
--- a/hbase-rsgroup/src/main/protobuf/RSGroup.proto
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "RSGroupProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-
-message RSGroupInfo {
- required string name = 1;
- repeated ServerName servers = 4;
- repeated TableName tables = 3;
-}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 292a668..d0342db 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -184,7 +184,12 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
or re-run HBCK in repair mode.
</div>
</%if>
-
+ <%if master.getMasterCoprocessorHost().findCoprocessor("RSGroupAdminEndpoint") != null %>
+ <section>
+ <h2>RSGroup</h2>
+ <& RSGroupListTmpl; master= master; serverManager= serverManager&>
+ </section>
+ </%if>
<section>
<h2>Region Servers</h2>
<& RegionServerListTmpl; master= master; servers = servers &>
http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
new file mode 100644
index 0000000..9f9831f
--- /dev/null
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
@@ -0,0 +1,352 @@
+<%doc>
+Copyright The Apache Software Foundation
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+ </%doc>
+
+<%args>
+HMaster master;
+ServerManager serverManager;
+</%args>
+
+<%import>
+ java.util.Collections;
+ java.util.List;
+ java.util.Map;
+ java.util.Set;
+ java.util.stream.Collectors;
+ org.apache.hadoop.hbase.master.HMaster;
+ org.apache.hadoop.hbase.ServerLoad;
+ org.apache.hadoop.hbase.RSGroupTableAccessor;
+ org.apache.hadoop.hbase.master.ServerManager;
+ org.apache.hadoop.hbase.net.Address;
+ org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
+ org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
+</%import>
+<%java>
+List<RSGroupInfo> groups = RSGroupTableAccessor.getAllRSGroupInfo(master.getConnection());
+</%java>
+
+<%if (groups != null && groups.size() > 0)%>
+
+<%java>
+RSGroupInfo [] rsGroupInfos = groups.toArray(new RSGroupInfo[groups.size()]);
+Map<Address, ServerLoad> collectServers = Collections.emptyMap();
+if (master.getServerManager() != null) {
+ collectServers =
+ master.getServerManager().getOnlineServers().entrySet().stream()
+ .collect(Collectors.toMap(p -> p.getKey().getAddress(), Map.Entry::getValue));
+}
+</%java>
+
+<div class="tabbable">
+ <ul class="nav nav-pills">
+ <li class="active"><a href="#tab_rsgroup_baseStats" data-toggle="tab">Base Stats</a></li>
+ <li class=""><a href="#tab_rsgroup_memoryStats" data-toggle="tab">Memory</a></li>
+ <li class=""><a href="#tab_rsgroup_requestStats" data-toggle="tab">Requests</a></li>
+ <li class=""><a href="#tab_rsgroup_storeStats" data-toggle="tab">Storefiles</a></li>
+ <li class=""><a href="#tab_rsgroup_compactStats" data-toggle="tab">Compactions</a></li>
+ </ul>
+ <div class="tab-content" style="padding-bottom: 9px; border-bottom: 1px solid #ddd;">
+ <div class="tab-pane active" id="tab_rsgroup_baseStats">
+ <& rsgroup_baseStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &>
+ </div>
+ <div class="tab-pane" id="tab_rsgroup_memoryStats">
+ <& rsgroup_memoryStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &>
+ </div>
+ <div class="tab-pane" id="tab_rsgroup_requestStats">
+ <& rsgroup_requestStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &>
+ </div>
+ <div class="tab-pane" id="tab_rsgroup_storeStats">
+ <& rsgroup_storeStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &>
+ </div>
+ <div class="tab-pane" id="tab_rsgroup_compactStats">
+ <& rsgroup_compactStats; rsGroupInfos = rsGroupInfos; collectServers= collectServers &>
+ </div>
+ </div>
+</div>
+
+</%if>
+
+<%def rsgroup_baseStats>
+<%args>
+ RSGroupInfo [] rsGroupInfos;
+ Map<Address, ServerLoad> collectServers;
+</%args>
+<table class="table table-striped">
+<tr>
+ <th>RSGroup Name</th>
+ <th>Num. Online Servers</th>
+ <th>Num. Dead Servers</th>
+ <th>Num. Tables</th>
+ <th>Requests Per Second</th>
+ <th>Num. Regions</th>
+ <th>Average Load</th>
+</tr>
+<%java>
+ int totalOnlineServers = 0;
+ int totalDeadServers = 0;
+ int totalTables = 0;
+ int totalRequests = 0;
+ int totalRegions = 0;
+ for (RSGroupInfo rsGroupInfo: rsGroupInfos) {
+ String rsGroupName = rsGroupInfo.getName();
+ int onlineServers = 0;
+ int deadServers = 0;
+ int tables = 0;
+ long requestsPerSecond = 0;
+ int numRegionsOnline = 0;
+ Set<Address> servers = rsGroupInfo.getServers();
+ for (Address server : servers) {
+ ServerLoad sl = collectServers.get(server);
+ if (sl != null) {
+ requestsPerSecond += sl.getNumberOfRequests();
+ numRegionsOnline += sl.getNumberOfRegions();
+ //rsgroup total
+ totalRegions += sl.getNumberOfRegions();
+ totalRequests += sl.getNumberOfRequests();
+ totalOnlineServers++;
+ onlineServers++;
+ } else {
+ totalDeadServers++;
+ deadServers++;
+ }
+ }
+ tables = rsGroupInfo.getTables().size();
+ totalTables += tables;
+ double avgLoad = onlineServers == 0 ? 0 :
+ (double)numRegionsOnline / (double)onlineServers;
+</%java>
+<tr>
+ <td><& rsGroupLink; rsGroupName=rsGroupName; &></td>
+ <td><% onlineServers %></td>
+ <td><% deadServers %></td>
+ <td><% tables %></td>
+ <td><% requestsPerSecond %></td>
+ <td><% numRegionsOnline %></td>
+ <td><% avgLoad %></td>
+</tr>
+<%java>
+}
+</%java>
+<tr><td>Total:<% rsGroupInfos.length %></td>
+<td><% totalOnlineServers %></td>
+<td><% totalDeadServers %></td>
+<td><% totalTables %></td>
+<td><% totalRequests %></td>
+<td><% totalRegions %></td>
+<td><% master.getServerManager().getAverageLoad() %></td>
+</tr>
+</table>
+</%def>
+
+<%def rsgroup_memoryStats>
+<%args>
+ RSGroupInfo [] rsGroupInfos;
+ Map<Address, ServerLoad> collectServers;
+</%args>
+<table class="table table-striped">
+<tr>
+ <th>RSGroup Name</th>
+ <th>Used Heap</th>
+ <th>Max Heap</th>
+ <th>Memstore Size</th>
+
+</tr>
+<%java>
+ for (RSGroupInfo rsGroupInfo: rsGroupInfos) {
+ String rsGroupName = rsGroupInfo.getName();
+ long usedHeap = 0;
+ long maxHeap = 0;
+ long memstoreSize = 0;
+ for (Address server : rsGroupInfo.getServers()) {
+ ServerLoad sl = collectServers.get(server);
+ if (sl != null) {
+ usedHeap += sl.getUsedHeapMB();
+ maxHeap += sl.getMaxHeapMB();
+ memstoreSize += sl.getMemstoreSizeInMB();
+ }
+ }
+</%java>
+<tr>
+ <td><& rsGroupLink; rsGroupName=rsGroupName; &></td>
+ <td><% TraditionalBinaryPrefix.long2String(usedHeap
+ * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+ <td><% TraditionalBinaryPrefix.long2String(maxHeap
+ * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+ <td><% TraditionalBinaryPrefix.long2String(memstoreSize
+ * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+
+</tr>
+<%java>
+}
+</%java>
+</table>
+</%def>
+
+<%def rsgroup_requestStats>
+<%args>
+ RSGroupInfo [] rsGroupInfos;
+ Map<Address, ServerLoad> collectServers;
+</%args>
+<table class="table table-striped">
+<tr>
+ <th>RSGroup Name</th>
+ <th>Request Per Second</th>
+ <th>Read Request Count</th>
+ <th>Write Request Count</th>
+</tr>
+<%java>
+ for (RSGroupInfo rsGroupInfo: rsGroupInfos) {
+ String rsGroupName = rsGroupInfo.getName();
+ long requestsPerSecond = 0;
+ long readRequests = 0;
+ long writeRequests = 0;
+ for (Address server : rsGroupInfo.getServers()) {
+ ServerLoad sl = collectServers.get(server);
+ if (sl != null) {
+ requestsPerSecond += sl.getNumberOfRequests();
+ readRequests += sl.getReadRequestsCount();
+ writeRequests += sl.getWriteRequestsCount();
+ }
+ }
+</%java>
+<tr>
+<td><& rsGroupLink; rsGroupName=rsGroupName; &></td>
+<td><% requestsPerSecond %></td>
+<td><% readRequests %></td>
+<td><% writeRequests %></td>
+</tr>
+<%java>
+}
+</%java>
+</table>
+</%def>
+
+
+<%def rsgroup_storeStats>
+<%args>
+ RSGroupInfo [] rsGroupInfos;
+ Map<Address, ServerLoad> collectServers;
+</%args>
+<table class="table table-striped">
+<tr>
+ <th>RSGroup Name</th>
+ <th>Num. Stores</th>
+ <th>Num. Storefiles</th>
+ <th>Storefile Size Uncompressed</th>
+ <th>Storefile Size</th>
+ <th>Index Size</th>
+ <th>Bloom Size</th>
+</tr>
+<%java>
+ for (RSGroupInfo rsGroupInfo: rsGroupInfos) {
+ String rsGroupName = rsGroupInfo.getName();
+ int numStores = 0;
+ long numStorefiles = 0;
+ long uncompressedStorefileSize = 0;
+ long storefileSize = 0;
+ long indexSize = 0;
+ long bloomSize = 0;
+ int count = 0;
+ for (Address server : rsGroupInfo.getServers()) {
+ ServerLoad sl = collectServers.get(server);
+ if (sl != null) {
+ numStores += sl.getStores();
+ numStorefiles += sl.getStorefiles();
+ uncompressedStorefileSize += sl.getStoreUncompressedSizeMB();
+ storefileSize += sl.getStorefileSizeInMB();
+ indexSize += sl.getTotalStaticIndexSizeKB();
+ bloomSize += sl.getTotalStaticBloomSizeKB();
+ count++;
+ }
+ }
+</%java>
+<tr>
+<td><& rsGroupLink; rsGroupName=rsGroupName; &></td>
+<td><% numStores %></td>
+<td><% numStorefiles %></td>
+<td><% TraditionalBinaryPrefix.long2String(
+ uncompressedStorefileSize * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+<td><% TraditionalBinaryPrefix.long2String(
+ storefileSize * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+<td><% TraditionalBinaryPrefix.long2String(
+ indexSize * TraditionalBinaryPrefix.KILO.value, "B", 1) %></td>
+<td><% TraditionalBinaryPrefix.long2String(
+ bloomSize * TraditionalBinaryPrefix.KILO.value, "B", 1) %></td>
+</tr>
+<%java>
+}
+</%java>
+</table>
+</%def>
+
+<%def rsgroup_compactStats>
+<%args>
+ RSGroupInfo [] rsGroupInfos;
+ Map<Address, ServerLoad> collectServers;
+</%args>
+<table class="table table-striped">
+<tr>
+ <th>RSGroup Name</th>
+ <th>Num. Compacting KVs</th>
+ <th>Num. Compacted KVs</th>
+ <th>Remaining KVs</th>
+ <th>Compaction Progress</th>
+</tr>
+<%java>
+ for (RSGroupInfo rsGroupInfo: rsGroupInfos) {
+ String rsGroupName = rsGroupInfo.getName();
+ int numStores = 0;
+ long totalCompactingKVs = 0;
+ long numCompactedKVs = 0;
+ long remainingKVs = 0;
+ long compactionProgress = 0;
+ for (Address server : rsGroupInfo.getServers()) {
+ ServerLoad sl = collectServers.get(server);
+ if (sl != null) {
+ totalCompactingKVs += sl.getTotalCompactingKVs();
+ numCompactedKVs += sl.getCurrentCompactedKVs();
+ }
+ }
+ remainingKVs = totalCompactingKVs - numCompactedKVs;
+ String percentDone = "";
+ if (totalCompactingKVs > 0) {
+ percentDone = String.format("%.2f", 100 *
+ ((float) numCompactedKVs / totalCompactingKVs)) + "%";
+ }
+</%java>
+<tr>
+<td><& rsGroupLink; rsGroupName=rsGroupName; &></td>
+<td><% totalCompactingKVs %></td>
+<td><% numCompactedKVs %></td>
+<td><% remainingKVs %></td>
+<td><% percentDone %></td>
+</tr>
+<%java>
+}
+</%java>
+</table>
+</%def>
+
+
+<%def rsGroupLink>
+ <%args>
+ String rsGroupName;
+ </%args>
+ <a href=rsgroup.jsp?name=<% rsGroupName %>><% rsGroupName %></a>
+</%def>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/cde1f821/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp
new file mode 100644
index 0000000..9f95b76
--- /dev/null
+++ b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp
@@ -0,0 +1,441 @@
+<%--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--%>
+<%@ page contentType="text/html;charset=UTF-8"
+ import="java.util.ArrayList"
+ import="java.util.Collections"
+ import="java.util.Date"
+ import="java.util.List"
+ import="java.util.Map"
+ import="java.util.regex.Pattern"
+ import="java.util.stream.Stream"
+ import="java.util.stream.Collectors"
+ import="org.apache.hadoop.hbase.HTableDescriptor"
+ import="org.apache.hadoop.hbase.RSGroupTableAccessor"
+ import="org.apache.hadoop.hbase.ServerLoad"
+ import="org.apache.hadoop.hbase.ServerName"
+ import="org.apache.hadoop.hbase.TableName"
+ import="org.apache.hadoop.hbase.client.Admin"
+ import="org.apache.hadoop.hbase.client.RegionInfo"
+ import="org.apache.hadoop.hbase.client.TableState"
+ import="org.apache.hadoop.hbase.client.TableDescriptor"
+ import="org.apache.hadoop.hbase.master.HMaster"
+ import="org.apache.hadoop.hbase.master.RegionState"
+ import="org.apache.hadoop.hbase.net.Address"
+ import="org.apache.hadoop.hbase.rsgroup.RSGroupInfo"
+ import="org.apache.hadoop.hbase.util.Bytes"
+ import="org.apache.hadoop.hbase.util.VersionInfo"
+ import="org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix"%>
+<%
+ HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
+ String rsGroupName = request.getParameter("name");
+ List<Address> rsGroupServers = new ArrayList<>();
+ List<TableName> rsGroupTables = new ArrayList<>();
+ RSGroupInfo rsGroupInfo = null;
+ if (rsGroupName != null && !rsGroupName.isEmpty()) {
+ rsGroupInfo = RSGroupTableAccessor.getRSGroupInfo(
+ master.getConnection(), Bytes.toBytes(rsGroupName));
+ if (rsGroupInfo != null) {
+ rsGroupServers.addAll(rsGroupInfo.getServers());
+ rsGroupTables.addAll(rsGroupInfo.getTables());
+ }
+ }
+ Collections.sort(rsGroupServers);
+ rsGroupTables.sort((o1, o2) -> {
+ int compare = Bytes.compareTo(o1.getNamespace(), o2.getNamespace());
+ if (compare != 0)
+ return compare;
+ compare = Bytes.compareTo(o1.getQualifier(), o2.getQualifier());
+ if (compare != 0)
+ return compare;
+ return 0;
+ });
+
+ Map<Address, ServerLoad> onlineServers = Collections.emptyMap();
+ Map<Address, ServerName> serverMaping = Collections.emptyMap();
+ if (master.getServerManager() != null) {
+ onlineServers = master.getServerManager().getOnlineServers().entrySet().stream()
+ .collect(Collectors.toMap(p -> p.getKey().getAddress(), Map.Entry::getValue));
+ serverMaping =
+ master.getServerManager().getOnlineServers().entrySet().stream()
+ .collect(Collectors.toMap(p -> p.getKey().getAddress(), Map.Entry::getKey));
+ }
+ pageContext.setAttribute("pageTitle", "RSGroup: " + rsGroupName);
+%>
+<jsp:include page="header.jsp">
+ <jsp:param name="pageTitle" value="${pageTitle}"/>
+</jsp:include>
+<div class="container-fluid content">
+ <% if (rsGroupName == null || rsGroupName.isEmpty() || rsGroupInfo == null) { %>
+ <div class="row inner_header">
+ <div class="page-header">
+ <h1>RSGroup: "<%= rsGroupName %>" does not exist</h1>
+ </div>
+ </div>
+ <p>Go <a href="javascript:history.back()">Back</a>, or wait for the redirect.
+ <% } else { %>
+ <div class="container-fluid content">
+ <div class="row">
+ <div class="page-header">
+ <h1>RSGroup: <%= rsGroupName %></h1>
+ </div>
+ </div>
+ </div>
+ <div class="container-fluid content">
+ <div class="row">
+ <div class="inner_header">
+ <h1>Region Servers</h1>
+ </div>
+ </div>
+ <div class="tabbable">
+ <% if (rsGroupServers != null && rsGroupServers.size() > 0) { %>
+ <ul class="nav nav-pills">
+ <li class="active">
+ <a href="#tab_baseStats" data-toggle="tab">Base Stats</a>
+ </li>
+ <li class="">
+ <a href="#tab_memoryStats" data-toggle="tab">Memory</a>
+ </li>
+ <li class="">
+ <a href="#tab_requestStats" data-toggle="tab">Requests</a>
+ </li>
+ <li class="">
+ <a href="#tab_storeStats" data-toggle="tab">Storefiles</a>
+ </li>
+ <li class="">
+ <a href="#tab_compactStats" data-toggle="tab">Compactions</a>
+ </li>
+ </ul>
+
+ <div class="tab-content" style="padding-bottom: 9px; border-bottom: 1px solid #ddd;">
+ <div class="tab-pane active" id="tab_baseStats">
+ <table class="table table-striped">
+ <tr>
+ <th>ServerName</th>
+ <th>Start time</th>
+ <th>Last contact</th>
+ <th>Version</th>
+ <th>Requests Per Second</th>
+ <th>Num. Regions</th>
+ </tr>
+ <% int totalRegions = 0;
+ int totalRequests = 0;
+ int inconsistentNodeNum = 0;
+ String masterVersion = VersionInfo.getVersion();
+ for (Address server: rsGroupServers) {
+ ServerName serverName = serverMaping.get(server);
+ if (serverName != null) {
+ ServerLoad sl = onlineServers.get(server);
+ String version = master.getRegionServerVersion(serverName);
+ if (!masterVersion.equals(version)) {
+ inconsistentNodeNum ++;
+ }
+ double requestsPerSecond = 0.0;
+ int numRegionsOnline = 0;
+ long lastContact = 0;
+ if (sl != null) {
+ requestsPerSecond = sl.getRequestsPerSecond();
+ numRegionsOnline = sl.getNumberOfRegions();
+ totalRegions += sl.getNumberOfRegions();
+ totalRequests += sl.getNumberOfRequests();
+ lastContact = (System.currentTimeMillis() - sl.getReportTime())/1000;
+ }
+ long startcode = serverName.getStartcode();
+ int infoPort = master.getRegionServerInfoPort(serverName);
+ String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";%>
+ <tr>
+ <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td>
+ <td><%= new Date(startcode) %></td>
+ <td><%= lastContact %></td>
+ <td><%= version %></td>
+ <td><%= String.format("%.0f", requestsPerSecond) %></td>
+ <td><%= numRegionsOnline %></td>
+ </tr>
+ <% } else { %>
+ <tr>
+ <td style="color:rgb(192,192,192);"><%= server %></td>
+ <td style="color:rgb(192,192,192);"><%= "Dead" %></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ </tr>
+ <% } %>
+ <% } %>
+ <tr><td>Total:<%= rsGroupServers.size() %></td>
+ <td></td>
+ <td></td>
+ <%if (inconsistentNodeNum > 0) { %>
+ <td style="color:red;"><%= inconsistentNodeNum %> nodes with inconsistent version</td>
+ <%} else { %>
+ <td></td>
+ <%} %>
+ <td><%= totalRequests %></td>
+ <td><%= totalRegions %></td>
+ </tr>
+ </table>
+ </div>
+ <div class="tab-pane" id="tab_memoryStats">
+ <table class="table table-striped">
+ <tr>
+ <th>ServerName</th>
+ <th>Used Heap</th>
+ <th>Max Heap</th>
+ <th>Memstore Size</th>
+ </tr>
+ <% for (Address server: rsGroupServers) {
+ ServerName serverName = serverMaping.get(server);
+ ServerLoad sl = onlineServers.get(server);
+ if (sl != null && serverName != null) {
+ int infoPort = master.getRegionServerInfoPort(serverName);
+ String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";
+ %>
+ <tr>
+ <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td>
+ <td><%= TraditionalBinaryPrefix.long2String(sl.getUsedHeapMB()
+ * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+ <td><%= TraditionalBinaryPrefix.long2String(sl.getMaxHeapMB()
+ * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+ <td><%= TraditionalBinaryPrefix.long2String(sl.getMemstoreSizeInMB()
+ * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+ </tr>
+ <% } else { %>
+ <tr>
+ <td style="color:rgb(192,192,192);"><%= server %></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ </tr>
+ <% }
+ } %>
+ </table>
+ </div>
+ <div class="tab-pane" id="tab_requestStats">
+ <table class="table table-striped">
+ <tr>
+ <th>ServerName</th>
+ <th>Request Per Second</th>
+ <th>Read Request Count</th>
+ <th>Write Request Count</th>
+ </tr>
+ <% for (Address server: rsGroupServers) {
+ ServerName serverName = serverMaping.get(server);
+ ServerLoad sl = onlineServers.get(server);
+ if (sl != null && serverName != null) {
+ int infoPort = master.getRegionServerInfoPort(serverName);
+ String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";
+ %>
+ <tr>
+ <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td>
+ <td><%= String.format("%.0f", sl.getRequestsPerSecond()) %></td>
+ <td><%= sl.getReadRequestsCount() %></td>
+ <td><%= sl.getWriteRequestsCount() %></td>
+ </tr>
+ <% } else { %>
+ <tr>
+ <td style="color:rgb(192,192,192);"><%= server %></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ </tr>
+ <% }
+ } %>
+ </table>
+ </div>
+ <div class="tab-pane" id="tab_storeStats">
+ <table class="table table-striped">
+ <tr>
+ <th>ServerName</th>
+ <th>Num. Stores</th>
+ <th>Num. Storefiles</th>
+ <th>Storefile Size Uncompressed</th>
+ <th>Storefile Size</th>
+ <th>Index Size</th>
+ <th>Bloom Size</th>
+ </tr>
+ <% for (Address server: rsGroupServers) {
+ ServerName serverName = serverMaping.get(server);
+ ServerLoad sl = onlineServers.get(server);
+ if (sl != null && serverName != null) {
+ int infoPort = master.getRegionServerInfoPort(serverName);
+ String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";
+ %>
+ <tr>
+ <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td>
+ <td><%= sl.getStores() %></td>
+ <td><%= sl.getStorefiles() %></td>
+ <td><%= TraditionalBinaryPrefix.long2String(
+ sl.getStoreUncompressedSizeMB() * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+ <td><%= TraditionalBinaryPrefix.long2String(sl.getStorefileSizeInMB()
+ * TraditionalBinaryPrefix.MEGA.value, "B", 1) %></td>
+ <td><%= TraditionalBinaryPrefix.long2String(sl.getTotalStaticIndexSizeKB()
+ * TraditionalBinaryPrefix.KILO.value, "B", 1) %></td>
+ <td><%= TraditionalBinaryPrefix.long2String(sl.getTotalStaticBloomSizeKB()
+ * TraditionalBinaryPrefix.KILO.value, "B", 1) %></td>
+ </tr>
+ <% } else { %>
+ <tr>
+ <td style="color:rgb(192,192,192);"><%= server %></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ </tr>
+ <% }
+ } %>
+ </table>
+ </div>
+ <div class="tab-pane" id="tab_compactStats">
+ <table class="table table-striped">
+ <tr>
+ <th>ServerName</th>
+ <th>Num. Compacting KVs</th>
+ <th>Num. Compacted KVs</th>
+ <th>Remaining KVs</th>
+ <th>Compaction Progress</th>
+ </tr>
+ <% for (Address server: rsGroupServers) {
+ ServerName serverName = serverMaping.get(server);
+ ServerLoad sl = onlineServers.get(server);
+ if (sl != null && serverName != null) {
+ String percentDone = "";
+ if (sl.getTotalCompactingKVs() > 0) {
+ percentDone = String.format("%.2f", 100 *
+ ((float) sl.getCurrentCompactedKVs() / sl.getTotalCompactingKVs())) + "%";
+ }
+ int infoPort = master.getRegionServerInfoPort(serverName);
+ String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";
+ %>
+ <tr>
+ <td><a href="<%= url %>"><%= serverName.getServerName() %></a></td>
+ <td><%= sl.getTotalCompactingKVs() %></td>
+ <td><%= sl.getCurrentCompactedKVs() %></td>
+ <td><%= sl.getTotalCompactingKVs() - sl.getCurrentCompactedKVs() %></td>
+ <td><%= percentDone %></td>
+ </tr>
+ <% } else { %>
+ <tr>
+ <td style="color:rgb(192,192,192);"><%= server %></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ </tr>
+ <% }
+ } %>
+ </table>
+ </div>
+ </div>
+ <% } else { %>
+ <p> No Region Servers</p>
+ <% } %>
+ </div>
+ </div>
+ <br />
+
+ <div class="container-fluid content">
+ <div class="row inner_header">
+ <div class="page-header">
+ <h1>Tables</h1>
+ </div>
+ </div>
+
+ <% if (rsGroupTables != null && rsGroupTables.size() > 0) {
+ HTableDescriptor[] tables = null;
+ try (Admin admin = master.getConnection().getAdmin()) {
+ tables = master.isInitialized() ? admin.listTables((Pattern)null, true) : null;
+ }
+ Map<TableName, HTableDescriptor> tableDescriptors
+ = Stream.of(tables).collect(Collectors.toMap(TableDescriptor::getTableName, p -> p));
+ %>
+ <table class="table table-striped">
+ <tr>
+ <th>Namespace</th>
+ <th>Table</th>
+ <th>Stats</th>
+ <th>Online Regions</th>
+ <th>Offline Regions</th>
+ <th>Failed Regions</th>
+ <th>Split Regions</th>
+ <th>Other Regions</th>
+ <th>Description</th>
+ </tr>
+ <% for(TableName tableName : rsGroupTables) {
+ HTableDescriptor htDesc = tableDescriptors.get(tableName);
+ if(htDesc == null) {
+ %>
+ <tr>
+ <td><%= tableName.getNamespaceAsString() %></td>
+ <td><%= tableName.getQualifierAsString() %></td>
+ <td style="color:rgb(0,0,255);"><%= "DELETED" %></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ <td></td>
+ </tr>
+ <% } else { %>
+ <tr>
+ <td><%= tableName.getNamespaceAsString() %></td>
+ <td><a href="/table.jsp?name=<%= tableName.getNameAsString() %>"><%= tableName.getQualifierAsString() %></a></td>
+ <% TableState.State tableState = master.getTableStateManager().getTableState(tableName);
+ if(TableState.isInStates(tableState,
+ TableState.State.DISABLED, TableState.State.DISABLING)) {
+ %>
+ <td style="color:red;"><%= tableState.name() %></td>
+ <% } else { %>
+ <td><%= tableState.name() %></td>
+ <% } %>
+ <% Map<RegionState.State, List<RegionInfo>> tableRegions =
+ master.getAssignmentManager().getRegionStates().getRegionByStateOfTable(tableName);
+ int openRegionsCount = tableRegions.get(RegionState.State.OPEN).size();
+ int offlineRegionsCount = tableRegions.get(RegionState.State.OFFLINE).size();
+ int splitRegionsCount = tableRegions.get(RegionState.State.SPLIT).size();
+ int failedRegionsCount = tableRegions.get(RegionState.State.FAILED_OPEN).size()
+ + tableRegions.get(RegionState.State.FAILED_CLOSE).size();
+ int otherRegionsCount = 0;
+ for (List<RegionInfo> list: tableRegions.values()) {
+ otherRegionsCount += list.size();
+ }
+ // now subtract known states
+ otherRegionsCount = otherRegionsCount - openRegionsCount
+ - failedRegionsCount - offlineRegionsCount
+ - splitRegionsCount;
+ %>
+ <td><%= openRegionsCount %></td>
+ <td><%= offlineRegionsCount %></td>
+ <td><%= failedRegionsCount %></td>
+ <td><%= splitRegionsCount %></td>
+ <td><%= otherRegionsCount %></td>
+ <td><%= htDesc.toStringCustomizedValues() %></td>
+ </tr>
+ <% }
+ } %>
+ <p> <%= rsGroupTables.size() %> table(s) in set.</p>
+ </table>
+ <% } else { %>
+ <p> No Tables</p>
+ <% } %>
+ </div>
+<% } %>
+</div>
+<jsp:include page="footer.jsp" />
[13/31] hbase git commit: HBASE-19810 Fix findbugs and error-prone
warnings in hbase-metrics (branch-2)
Posted by zh...@apache.org.
HBASE-19810 Fix findbugs and error-prone warnings in hbase-metrics (branch-2)
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c1a8dc09
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c1a8dc09
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c1a8dc09
Branch: refs/heads/HBASE-19064
Commit: c1a8dc09d64ef3f6062aced595c3bb918724025d
Parents: c269e63
Author: Peter Somogyi <ps...@cloudera.com>
Authored: Wed Jan 17 10:05:03 2018 +0100
Committer: Michael Stack <st...@apache.org>
Committed: Wed Jan 17 11:25:48 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java | 2 ++
.../apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java | 5 +++++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1a8dc09/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java
----------------------------------------------------------------------
diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java
index 6871f7e..7b79fba 100644
--- a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java
+++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java
@@ -65,6 +65,7 @@ public class HistogramImpl implements Histogram {
histogram.add(value, 1);
}
+ @Override
public long getCount() {
return counter.getCount();
}
@@ -73,6 +74,7 @@ public class HistogramImpl implements Histogram {
return this.histogram.getMax();
}
+ @Override
public Snapshot snapshot() {
return histogram.snapshotAndReset();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1a8dc09/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java
----------------------------------------------------------------------
diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java
index d72c68f..3826e66 100644
--- a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java
+++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java
@@ -51,22 +51,27 @@ public class MetricRegistriesImpl extends MetricRegistries {
return registries.put(info, () -> factory.create(info));
}
+ @Override
public boolean remove(MetricRegistryInfo key) {
return registries.remove(key) == null;
}
+ @Override
public Optional<MetricRegistry> get(MetricRegistryInfo info) {
return Optional.ofNullable(registries.get(info));
}
+ @Override
public Collection<MetricRegistry> getMetricRegistries() {
return Collections.unmodifiableCollection(registries.values());
}
+ @Override
public void clear() {
registries.clear();
}
+ @Override
public Set<MetricRegistryInfo> getMetricRegistryInfos() {
return Collections.unmodifiableSet(registries.keySet());
}
[04/31] hbase git commit: HBASE-19789 Not exclude flaky tests from
nightly builds.
Posted by zh...@apache.org.
HBASE-19789 Not exclude flaky tests from nightly builds.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9917244b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9917244b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9917244b
Branch: refs/heads/HBASE-19064
Commit: 9917244be2c7de7dccf870240f08a90baf341bbb
Parents: d155a4e
Author: Apekshit Sharma <ap...@apache.org>
Authored: Fri Jan 12 16:22:06 2018 -0800
Committer: Apekshit Sharma <ap...@apache.org>
Committed: Tue Jan 16 10:07:48 2018 -0800
----------------------------------------------------------------------
dev-support/Jenkinsfile | 7 +++++--
dev-support/hbase_nightly_yetus.sh | 7 +++++--
2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9917244b/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index dcef649..14092d5 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -44,8 +44,11 @@ pipeline {
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
- // Only used for master branch.
- EXCLUDE_TESTS_URL = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
+ // Flaky urls for different branches. Replace '-' and '.' in branch name by '_' because those
+ // characters are not allowed in bash variable name.
+ // Not excluding flakies from the nightly build for now.
+ // EXCLUDE_TESTS_URL_master = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests/lastSuccessfulBuild/artifact/excludes/'
+ // EXCLUDE_TESTS_URL_branch_2 = 'https://builds.apache.org/job/HBase-Find-Flaky-Tests-branch2.0/lastSuccessfulBuild/artifact/excludes/'
}
parameters {
booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.
http://git-wip-us.apache.org/repos/asf/hbase/blob/9917244b/dev-support/hbase_nightly_yetus.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index e1175d2..4e67354 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -69,10 +69,13 @@ YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
# Currently, flaky list is calculated only for master branch.
-if [[ -n "${EXCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then
+UNDERSCORED_BRANCH_NAME=$(echo ${BRANCH_NAME} | tr '.-' '_')
+EXCLUDE_TESTS_URL=$(eval echo "\$EXCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}")
+INCLUDE_TESTS_URL=$(eval echo "\$INCLUDE_TESTS_URL_${UNDERSCORED_BRANCH_NAME}")
+if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
YETUS_ARGS=("--exclude-tests-url=${EXCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
fi
-if [[ -n "${INCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then
+if [[ -n "${INCLUDE_TESTS_URL}" ]]; then
YETUS_ARGS=("--include-tests-url=${INCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
fi
[09/31] hbase git commit: HBASE-19736 Remove BaseLogCleanerDelegate
deprecated #isLogDeletable(FileStatus) and use #isFileDeletable(FileStatus)
instead
Posted by zh...@apache.org.
HBASE-19736 Remove BaseLogCleanerDelegate deprecated #isLogDeletable(FileStatus) and use #isFileDeletable(FileStatus) instead
Signed-off-by: Chia-Ping Tsai <ch...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8a8be33c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8a8be33c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8a8be33c
Branch: refs/heads/HBASE-19064
Commit: 8a8be33c71c851fbe0dc57543f49d549e19385a3
Parents: d8d6ecd
Author: Reid Chan <re...@outlook.com>
Authored: Tue Jan 9 17:51:05 2018 +0800
Committer: Chia-Ping Tsai <ch...@gmail.com>
Committed: Wed Jan 17 21:16:23 2018 +0800
----------------------------------------------------------------------
.../master/cleaner/BaseHFileCleanerDelegate.java | 2 +-
.../master/cleaner/BaseLogCleanerDelegate.java | 17 +++--------------
.../hbase/master/cleaner/TimeToLiveLogCleaner.java | 2 +-
3 files changed, 5 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a8be33c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
index 8de59bd..d8a3f92 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseHFileCleanerDelegate.java
@@ -23,7 +23,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* Base class for the hfile cleaning function inside the master. By default, only the
* {@link TimeToLiveHFileCleaner} is called.
* <p>
- * If other effects are needed, implement your own LogCleanerDelegate and add it to the
+ * If other effects are needed, implement your own HFileCleanerDelegate and add it to the
* configuration "hbase.master.hfilecleaner.plugins", which is a comma-separated list of fully
* qualified class names. The <code>HFileCleaner</code> will build the cleaner chain in
* order the order specified by the configuration.
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a8be33c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
index b1f9d0b..fcfeb79 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseLogCleanerDelegate.java
@@ -21,8 +21,9 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.fs.FileStatus;
/**
- * Base class for the log cleaning function inside the master. By default, two
- * cleaners: <code>TimeToLiveLogCleaner</code> and
+ * Base class for the log cleaning function inside the master. By default, three
+ * cleaners: <code>TimeToLiveLogCleaner</code>,
+ * <code>TimeToLiveProcedureWALCleaner</code> and
* <code>ReplicationLogCleaner</code> are called in order. So if other effects
* are needed, implement your own LogCleanerDelegate and add it to the
* configuration "hbase.master.logcleaner.plugins", which is a comma-separated
@@ -40,18 +41,6 @@ public abstract class BaseLogCleanerDelegate extends BaseFileCleanerDelegate {
@Override
public boolean isFileDeletable(FileStatus fStat) {
- return isLogDeletable(fStat);
- }
-
- /**
- * Should the master delete the log or keep it?
- * <p>
- * Implementing classes should override {@link #isFileDeletable(FileStatus)} instead.
- * @param fStat file status of the file
- * @return true if the log is deletable, false (default) if not
- */
- @Deprecated
- public boolean isLogDeletable(FileStatus fStat) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8a8be33c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
index 7385273..670bd88 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveLogCleaner.java
@@ -41,7 +41,7 @@ public class TimeToLiveLogCleaner extends BaseLogCleanerDelegate {
private boolean stopped = false;
@Override
- public boolean isLogDeletable(FileStatus fStat) {
+ public boolean isFileDeletable(FileStatus fStat) {
// Files are validated for the second time here,
// if it causes a bottleneck this logic needs refactored
if (!AbstractFSWALProvider.validateWALFilename(fStat.getPath().getName())) {
[31/31] hbase git commit: HBASE-19747 Introduce a special WALProvider
for synchronous replication
Posted by zh...@apache.org.
HBASE-19747 Introduce a special WALProvider for synchronous replication
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b56bf0db
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b56bf0db
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b56bf0db
Branch: refs/heads/HBASE-19064
Commit: b56bf0dbb286e60f1a2519f25e86ff1ebdffd546
Parents: 373d2c0
Author: zhangduo <zh...@apache.org>
Authored: Fri Jan 19 18:38:39 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Fri Jan 19 21:20:34 2018 +0800
----------------------------------------------------------------------
.../hbase/regionserver/wal/AbstractFSWAL.java | 7 +
.../hbase/regionserver/wal/AsyncFSWAL.java | 1 -
.../hbase/regionserver/wal/DualAsyncFSWAL.java | 4 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 3 -
.../regionserver/PeerActionListener.java | 33 +++
.../SynchronousReplicationPeerProvider.java | 35 +++
.../hadoop/hbase/wal/AbstractFSWALProvider.java | 1 +
.../hadoop/hbase/wal/AsyncFSWALProvider.java | 18 +-
.../hbase/wal/NettyAsyncFSWALConfigHelper.java | 8 +-
.../hbase/wal/RegionGroupingProvider.java | 23 +-
.../wal/SynchronousReplicationWALProvider.java | 220 +++++++++++++++++++
.../org/apache/hadoop/hbase/wal/WALFactory.java | 40 +++-
.../org/apache/hadoop/hbase/wal/WALKeyImpl.java | 16 +-
.../regionserver/TestCompactionPolicy.java | 1 +
.../regionserver/TestFailedAppendAndSync.java | 122 +++++-----
.../hadoop/hbase/regionserver/TestHRegion.java | 27 ++-
.../TestHRegionWithInMemoryFlush.java | 8 +-
.../hbase/regionserver/TestRegionIncrement.java | 21 +-
.../hbase/regionserver/TestWALLockup.java | 17 +-
.../regionserver/wal/AbstractTestWALReplay.java | 1 +
.../regionserver/wal/ProtobufLogTestHelper.java | 44 +++-
.../hbase/regionserver/wal/TestAsyncFSWAL.java | 24 +-
.../regionserver/wal/TestAsyncWALReplay.java | 4 +-
.../hbase/regionserver/wal/TestFSHLog.java | 21 +-
.../hbase/regionserver/wal/TestWALReplay.java | 8 +-
.../apache/hadoop/hbase/wal/IOTestProvider.java | 1 +
.../TestSynchronousReplicationWALProvider.java | 153 +++++++++++++
27 files changed, 678 insertions(+), 183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index 54a5cd3..a37bda3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -435,6 +435,13 @@ public abstract class AbstractFSWAL<W extends WriterBase> implements WAL {
this.implClassName = getClass().getSimpleName();
}
+ /**
+ * Used to initialize the WAL. Usually just call rollWriter to create the first log writer.
+ */
+ public void init() throws IOException {
+ rollWriter();
+ }
+
@Override
public void registerWALActionsListener(WALActionsListener listener) {
this.listeners.add(listener);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index 6ed3eea..15b812d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -247,7 +247,6 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
batchSize = conf.getLong(WAL_BATCH_SIZE, DEFAULT_WAL_BATCH_SIZE);
waitOnShutdownInSeconds = conf.getInt(ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS,
DEFAULT_ASYNC_WAL_WAIT_ON_SHUTDOWN_IN_SECONDS);
- rollWriter();
}
private static boolean waitingRoll(int epochAndState) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
index 6bf9e02..f92ce93 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/DualAsyncFSWAL.java
@@ -38,14 +38,14 @@ public class DualAsyncFSWAL extends AsyncFSWAL {
private final Path remoteWalDir;
- public DualAsyncFSWAL(FileSystem fs, FileSystem remoteFs, Path rootDir, Path remoteRootDir,
+ public DualAsyncFSWAL(FileSystem fs, FileSystem remoteFs, Path rootDir, Path remoteWalDir,
String logDir, String archiveDir, Configuration conf, List<WALActionsListener> listeners,
boolean failIfWALExists, String prefix, String suffix, EventLoopGroup eventLoopGroup,
Class<? extends Channel> channelClass) throws FailedLogCloseException, IOException {
super(fs, rootDir, logDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix,
eventLoopGroup, channelClass);
this.remoteFs = remoteFs;
- this.remoteWalDir = new Path(remoteRootDir, logDir);
+ this.remoteWalDir = remoteWalDir;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index a1dd2d0..350a54e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -212,9 +212,6 @@ public class FSHLog extends AbstractFSWAL<Writer> {
5);
this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 2);
- // rollWriter sets this.hdfs_out if it can.
- rollWriter();
-
// This is the 'writer' -- a single threaded executor. This single thread 'consumes' what is
// put on the ring buffer.
String hostingThreadName = Thread.currentThread().getName();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerActionListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerActionListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerActionListener.java
new file mode 100644
index 0000000..74ad626
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerActionListener.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Get notification for replication peer events. Mainly used for telling the
+ * {@link org.apache.hadoop.hbase.wal.SynchronousReplicationWALProvider} to close some WAL if not
+ * used any more.
+ * <p>
+ * TODO: Also need a synchronous peer state change notification.
+ */
+@InterfaceAudience.Private
+public interface PeerActionListener {
+
+ default void peerRemoved(String peerId) {}
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SynchronousReplicationPeerProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SynchronousReplicationPeerProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SynchronousReplicationPeerProvider.java
new file mode 100644
index 0000000..b4e04fb
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/SynchronousReplicationPeerProvider.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import java.util.Optional;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Get the peer id and remote root dir if the region is synchronously replicated.
+ */
+@InterfaceAudience.Private
+public interface SynchronousReplicationPeerProvider {
+
+ /**
+ * Return the peer id and remote WAL directory if the region is synchronously replicated.
+ */
+ Optional<Pair<String, String>> getPeerIdAndRemoteWALDir(RegionInfo info);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
index 74d502e..a01bf51 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
@@ -139,6 +139,7 @@ public abstract class AbstractFSWALProvider<T extends AbstractFSWAL<?>> implemen
if (walCopy == null) {
walCopy = createWAL();
wal = walCopy;
+ walCopy.init();
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java
index 9c62bed..84e859d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AsyncFSWALProvider.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.wal;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -31,12 +30,10 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
import org.apache.hbase.thirdparty.io.netty.channel.Channel;
import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
-import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
/**
* A WAL provider that use {@link AsyncFSWAL}.
@@ -61,6 +58,7 @@ public class AsyncFSWALProvider extends AbstractFSWALProvider<AsyncFSWAL> {
private EventLoopGroup eventLoopGroup;
private Class<? extends Channel> channelClass;
+
@Override
protected AsyncFSWAL createWAL() throws IOException {
return new AsyncFSWAL(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf),
@@ -73,15 +71,9 @@ public class AsyncFSWALProvider extends AbstractFSWALProvider<AsyncFSWAL> {
@Override
protected void doInit(Configuration conf) throws IOException {
Pair<EventLoopGroup, Class<? extends Channel>> eventLoopGroupAndChannelClass =
- NettyAsyncFSWALConfigHelper.getEventLoopConfig(conf);
- if (eventLoopGroupAndChannelClass != null) {
- eventLoopGroup = eventLoopGroupAndChannelClass.getFirst();
- channelClass = eventLoopGroupAndChannelClass.getSecond();
- } else {
- eventLoopGroup = new NioEventLoopGroup(1,
- new DefaultThreadFactory("AsyncFSWAL", true, Thread.MAX_PRIORITY));
- channelClass = NioSocketChannel.class;
- }
+ NettyAsyncFSWALConfigHelper.getEventLoopConfig(conf);
+ eventLoopGroup = eventLoopGroupAndChannelClass.getFirst();
+ channelClass = eventLoopGroupAndChannelClass.getSecond();
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NettyAsyncFSWALConfigHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NettyAsyncFSWALConfigHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NettyAsyncFSWALConfigHelper.java
index 0836b5d..00ccb71 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NettyAsyncFSWALConfigHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/NettyAsyncFSWALConfigHelper.java
@@ -26,6 +26,9 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.io.netty.channel.Channel;
import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
+import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
/**
* Helper class for passing netty event loop config to {@link AsyncFSWALProvider}.
@@ -55,7 +58,10 @@ public class NettyAsyncFSWALConfigHelper {
static Pair<EventLoopGroup, Class<? extends Channel>> getEventLoopConfig(Configuration conf) {
String name = conf.get(EVENT_LOOP_CONFIG);
if (StringUtils.isBlank(name)) {
- return null;
+ // create new event loop group if config is empty
+ return Pair.<EventLoopGroup, Class<? extends Channel>> newPair(
+ new NioEventLoopGroup(0, new DefaultThreadFactory("AsyncFSWAL", true, Thread.MAX_PRIORITY)),
+ NioSocketChannel.class);
}
return EVENT_LOOP_CONFIG_MAP.get(name);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
index a3e54a5..05eb684 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
@@ -135,18 +135,20 @@ public class RegionGroupingProvider implements WALProvider {
private final KeyLocker<String> createLock = new KeyLocker<>();
- private RegionGroupingStrategy strategy = null;
- private WALFactory factory = null;
- private List<WALActionsListener> listeners = null;
- private String providerId = null;
+ private RegionGroupingStrategy strategy;
+ private WALFactory factory;
+ private Configuration conf;
+ private List<WALActionsListener> listeners;
+ private String providerId;
private Class<? extends WALProvider> providerClass;
@Override
- public void init(final WALFactory factory, final Configuration conf,
- final List<WALActionsListener> listeners, final String providerId) throws IOException {
+ public void init(WALFactory factory, Configuration conf, List<WALActionsListener> listeners,
+ String providerId) throws IOException {
if (null != strategy) {
throw new IllegalStateException("WALProvider.init should only be called once.");
}
+ this.conf = conf;
this.factory = factory;
this.listeners = null == listeners ? null : Collections.unmodifiableList(listeners);
StringBuilder sb = new StringBuilder().append(factory.factoryId);
@@ -163,11 +165,10 @@ public class RegionGroupingProvider implements WALProvider {
}
private WALProvider createProvider(String group) throws IOException {
- if (META_WAL_PROVIDER_ID.equals(providerId)) {
- return factory.createProvider(providerClass, listeners, META_WAL_PROVIDER_ID);
- } else {
- return factory.createProvider(providerClass, listeners, group);
- }
+ WALProvider provider = WALFactory.createProvider(providerClass);
+ provider.init(factory, conf, listeners,
+ META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : group);
+ return provider;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SynchronousReplicationWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SynchronousReplicationWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SynchronousReplicationWALProvider.java
new file mode 100644
index 0000000..de1b56d
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/SynchronousReplicationWALProvider.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.apache.hadoop.hbase.wal.AbstractFSWALProvider.WAL_FILE_NAME_DELIMITER;
+import static org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWALArchiveDirectoryName;
+import static org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWALDirectoryName;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.regionserver.wal.DualAsyncFSWAL;
+import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+import org.apache.hadoop.hbase.replication.regionserver.PeerActionListener;
+import org.apache.hadoop.hbase.replication.regionserver.SynchronousReplicationPeerProvider;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.KeyLocker;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.Streams;
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+
+/**
+ * The special {@link WALProvider} for synchronous replication.
+ * <p>
+ * It works like an interceptor, when getting WAL, first it will check if the given region should be
+ * replicated synchronously, if so it will return a special WAL for it, otherwise it will delegate
+ * the request to the normal {@link WALProvider}.
+ */
+@InterfaceAudience.Private
+public class SynchronousReplicationWALProvider implements WALProvider, PeerActionListener {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SynchronousReplicationWALProvider.class);
+
+ private static final String LOG_SUFFIX = ".syncrep";
+
+ private final WALProvider provider;
+
+ private final SynchronousReplicationPeerProvider peerProvider;
+
+ private WALFactory factory;
+
+ private Configuration conf;
+
+ private List<WALActionsListener> listeners;
+
+ private EventLoopGroup eventLoopGroup;
+
+ private Class<? extends Channel> channelClass;
+
+ private AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private final ConcurrentMap<String, DualAsyncFSWAL> peerId2WAL = new ConcurrentHashMap<>();
+
+ private final KeyLocker<String> createLock = new KeyLocker<>();
+
+ SynchronousReplicationWALProvider(WALProvider provider,
+ SynchronousReplicationPeerProvider peerProvider) {
+ this.provider = provider;
+ this.peerProvider = peerProvider;
+ }
+
+ @Override
+ public void init(WALFactory factory, Configuration conf, List<WALActionsListener> listeners,
+ String providerId) throws IOException {
+ if (!initialized.compareAndSet(false, true)) {
+ throw new IllegalStateException("WALProvider.init should only be called once.");
+ }
+ provider.init(factory, conf, listeners, providerId);
+ this.conf = conf;
+ this.factory = factory;
+ this.listeners = listeners;
+ Pair<EventLoopGroup, Class<? extends Channel>> eventLoopGroupAndChannelClass =
+ NettyAsyncFSWALConfigHelper.getEventLoopConfig(conf);
+ eventLoopGroup = eventLoopGroupAndChannelClass.getFirst();
+ channelClass = eventLoopGroupAndChannelClass.getSecond();
+ }
+
+ private String getLogPrefix(String peerId) {
+ return factory.factoryId + WAL_FILE_NAME_DELIMITER + peerId;
+ }
+
+ private DualAsyncFSWAL createWAL(String peerId, String remoteWALDir) throws IOException {
+ Path remoteWALDirPath = new Path(remoteWALDir);
+ FileSystem remoteFs = remoteWALDirPath.getFileSystem(conf);
+ return new DualAsyncFSWAL(CommonFSUtils.getWALFileSystem(conf), remoteFs,
+ CommonFSUtils.getWALRootDir(conf), new Path(remoteWALDirPath, peerId),
+ getWALDirectoryName(factory.factoryId), getWALArchiveDirectoryName(conf, factory.factoryId),
+ conf, listeners, true, getLogPrefix(peerId), LOG_SUFFIX, eventLoopGroup, channelClass);
+ }
+
+ private DualAsyncFSWAL getWAL(String peerId, String remoteWALDir) throws IOException {
+ DualAsyncFSWAL wal = peerId2WAL.get(peerId);
+ if (wal != null) {
+ return wal;
+ }
+ Lock lock = createLock.acquireLock(peerId);
+ try {
+ wal = peerId2WAL.get(peerId);
+ if (wal == null) {
+ wal = createWAL(peerId, remoteWALDir);
+ peerId2WAL.put(peerId, wal);
+ wal.init();
+ }
+ return wal;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public WAL getWAL(RegionInfo region) throws IOException {
+ Optional<Pair<String, String>> peerIdAndRemoteWALDir =
+ peerProvider.getPeerIdAndRemoteWALDir(region);
+ if (peerIdAndRemoteWALDir.isPresent()) {
+ Pair<String, String> pair = peerIdAndRemoteWALDir.get();
+ return getWAL(pair.getFirst(), pair.getSecond());
+ } else {
+ return provider.getWAL(region);
+ }
+ }
+
+ private Stream<WAL> getWALStream() {
+ return Streams.concat(peerId2WAL.values().stream(), provider.getWALs().stream());
+ }
+
+ @Override
+ public List<WAL> getWALs() {
+ return getWALStream().collect(Collectors.toList());
+ }
+
+ @Override
+ public void shutdown() throws IOException {
+ // save the last exception and rethrow
+ IOException failure = null;
+ for (DualAsyncFSWAL wal : peerId2WAL.values()) {
+ try {
+ wal.shutdown();
+ } catch (IOException e) {
+ LOG.error("Shutdown WAL failed", e);
+ failure = e;
+ }
+ }
+ provider.shutdown();
+ if (failure != null) {
+ throw failure;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ // save the last exception and rethrow
+ IOException failure = null;
+ for (DualAsyncFSWAL wal : peerId2WAL.values()) {
+ try {
+ wal.close();
+ } catch (IOException e) {
+ LOG.error("Close WAL failed", e);
+ failure = e;
+ }
+ }
+ provider.close();
+ if (failure != null) {
+ throw failure;
+ }
+ }
+
+ @Override
+ public long getNumLogFiles() {
+ return peerId2WAL.size() + provider.getNumLogFiles();
+ }
+
+ @Override
+ public long getLogFileSize() {
+ return peerId2WAL.values().stream().mapToLong(DualAsyncFSWAL::getLogFileSize).sum() +
+ provider.getLogFileSize();
+ }
+
+ @Override
+ public void peerRemoved(String peerId) {
+ WAL wal = peerId2WAL.remove(peerId);
+ if (wal != null) {
+ try {
+ wal.close();
+ } catch (IOException e) {
+ LOG.error("Close WAL failed", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
index d70b8cd..adc76ac 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+import org.apache.hadoop.hbase.replication.regionserver.SynchronousReplicationPeerProvider;
import org.apache.hadoop.hbase.replication.regionserver.WALFileLengthProvider;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -135,13 +136,10 @@ public class WALFactory implements WALFileLengthProvider {
}
}
- WALProvider createProvider(Class<? extends WALProvider> clazz,
- List<WALActionsListener> listeners, String providerId) throws IOException {
- LOG.info("Instantiating WALProvider of type " + clazz);
+ static WALProvider createProvider(Class<? extends WALProvider> clazz) throws IOException {
+ LOG.info("Instantiating WALProvider of type {}", clazz);
try {
- final WALProvider result = clazz.newInstance();
- result.init(this, conf, listeners, providerId);
- return result;
+ return clazz.newInstance();
} catch (InstantiationException exception) {
LOG.error("couldn't set up WALProvider, the configured class is " + clazz);
LOG.debug("Exception details for failure to load WALProvider.", exception);
@@ -154,13 +152,14 @@ public class WALFactory implements WALFileLengthProvider {
}
/**
- * instantiate a provider from a config property.
- * requires conf to have already been set (as well as anything the provider might need to read).
+ * instantiate a provider from a config property. requires conf to have already been set (as well
+ * as anything the provider might need to read).
*/
- WALProvider getProvider(final String key, final String defaultValue,
+ private WALProvider getProvider(final String key, final String defaultValue,
final List<WALActionsListener> listeners, final String providerId) throws IOException {
- Class<? extends WALProvider> clazz = getProviderClass(key, defaultValue);
- return createProvider(clazz, listeners, providerId);
+ WALProvider provider = createProvider(getProviderClass(key, defaultValue));
+ provider.init(this, conf, listeners, providerId);
+ return provider;
}
/**
@@ -192,6 +191,25 @@ public class WALFactory implements WALFileLengthProvider {
}
/**
+ * A temporary constructor for testing synchronous replication.
+ * <p>
+ * Remove it once we can integrate the synchronous replication logic in RS.
+ */
+ @VisibleForTesting
+ WALFactory(Configuration conf, List<WALActionsListener> listeners, String factoryId,
+ SynchronousReplicationPeerProvider peerProvider) throws IOException {
+ timeoutMillis = conf.getInt("hbase.hlog.open.timeout", 300000);
+ /* TODO Both of these are probably specific to the fs wal provider */
+ logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
+ AbstractFSWALProvider.Reader.class);
+ this.conf = conf;
+ this.factoryId = factoryId;
+ WALProvider provider = createProvider(getProviderClass(WAL_PROVIDER, DEFAULT_WAL_PROVIDER));
+ this.provider = new SynchronousReplicationWALProvider(provider, peerProvider);
+ this.provider.init(this, conf, listeners, null);
+ }
+
+ /**
* Shutdown all WALs and clean up any underlying storage.
* Use only when you will not need to replay and edits that have gone to any wals from this
* factory.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
index 983fae9..72e2598 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
@@ -135,13 +135,21 @@ public class WALKeyImpl implements WALKey {
}
@VisibleForTesting
- public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename,
- long logSeqNum,
+ public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, long logSeqNum,
final long now, UUID clusterId) {
List<UUID> clusterIds = new ArrayList<>(1);
clusterIds.add(clusterId);
- init(encodedRegionName, tablename, logSeqNum, now, clusterIds,
- HConstants.NO_NONCE, HConstants.NO_NONCE, null, null);
+ init(encodedRegionName, tablename, logSeqNum, now, clusterIds, HConstants.NO_NONCE,
+ HConstants.NO_NONCE, null, null);
+ }
+
+ @VisibleForTesting
+ public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, long logSeqNum,
+ final long now, UUID clusterId, MultiVersionConcurrencyControl mvcc) {
+ List<UUID> clusterIds = new ArrayList<>(1);
+ clusterIds.add(clusterId);
+ init(encodedRegionName, tablename, logSeqNum, now, clusterIds, HConstants.NO_NONCE,
+ HConstants.NO_NONCE, mvcc, null);
}
// TODO: Fix being able to pass in sequenceid.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
index ca4b227..939f35c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionPolicy.java
@@ -100,6 +100,7 @@ public class TestCompactionPolicy {
HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
hlog = new FSHLog(fs, basedir, logName, conf);
+ hlog.init();
ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
region = HRegion.createHRegion(info, basedir, conf, htd, hlog);
region.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
index 84e424e..55cd867 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFailedAppendAndSync.java
@@ -98,65 +98,64 @@ public class TestFailedAppendAndSync {
return name.getMethodName();
}
- /**
- * Reproduce locking up that happens when we get an exceptions appending and syncing.
- * See HBASE-14317.
- * First I need to set up some mocks for Server and RegionServerServices. I also need to
- * set up a dodgy WAL that will throw an exception when we go to append to it.
- */
- @Test (timeout=300000)
- public void testLockupAroundBadAssignSync() throws IOException {
+ // Dodgy WAL. Will throw exceptions when flags set.
+ class DodgyFSLog extends FSHLog {
+ volatile boolean throwSyncException = false;
+ volatile boolean throwAppendException = false;
final AtomicLong rolls = new AtomicLong(0);
- // Dodgy WAL. Will throw exceptions when flags set.
- class DodgyFSLog extends FSHLog {
- volatile boolean throwSyncException = false;
- volatile boolean throwAppendException = false;
- public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration conf)
- throws IOException {
- super(fs, root, logDir, conf);
- }
-
- @Override
- public byte[][] rollWriter(boolean force) throws FailedLogCloseException, IOException {
- byte [][] regions = super.rollWriter(force);
- rolls.getAndIncrement();
- return regions;
- }
+ public DodgyFSLog(FileSystem fs, Path root, String logDir, Configuration conf)
+ throws IOException {
+ super(fs, root, logDir, conf);
+ }
- @Override
- protected Writer createWriterInstance(Path path) throws IOException {
- final Writer w = super.createWriterInstance(path);
- return new Writer() {
- @Override
- public void close() throws IOException {
- w.close();
- }
+ @Override
+ public byte[][] rollWriter(boolean force) throws FailedLogCloseException, IOException {
+ byte[][] regions = super.rollWriter(force);
+ rolls.getAndIncrement();
+ return regions;
+ }
- @Override
- public void sync() throws IOException {
- if (throwSyncException) {
- throw new IOException("FAKE! Failed to replace a bad datanode...");
- }
- w.sync();
- }
+ @Override
+ protected Writer createWriterInstance(Path path) throws IOException {
+ final Writer w = super.createWriterInstance(path);
+ return new Writer() {
+ @Override
+ public void close() throws IOException {
+ w.close();
+ }
- @Override
- public void append(Entry entry) throws IOException {
- if (throwAppendException) {
- throw new IOException("FAKE! Failed to replace a bad datanode...");
- }
- w.append(entry);
- }
+ @Override
+ public void sync() throws IOException {
+ if (throwSyncException) {
+ throw new IOException("FAKE! Failed to replace a bad datanode...");
+ }
+ w.sync();
+ }
- @Override
- public long getLength() {
- return w.getLength();
- }
- };
+ @Override
+ public void append(Entry entry) throws IOException {
+ if (throwAppendException) {
+ throw new IOException("FAKE! Failed to replace a bad datanode...");
}
- }
+ w.append(entry);
+ }
+ @Override
+ public long getLength() {
+ return w.getLength();
+ }
+ };
+ }
+ }
+ /**
+ * Reproduce locking up that happens when we get an exceptions appending and syncing.
+ * See HBASE-14317.
+ * First I need to set up some mocks for Server and RegionServerServices. I also need to
+ * set up a dodgy WAL that will throw an exception when we go to append to it.
+ */
+ @Test (timeout=300000)
+ public void testLockupAroundBadAssignSync() throws IOException {
// Make up mocked server and services.
Server server = mock(Server.class);
when(server.getConfiguration()).thenReturn(CONF);
@@ -168,6 +167,7 @@ public class TestFailedAppendAndSync {
FileSystem fs = FileSystem.get(CONF);
Path rootDir = new Path(dir + getName());
DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, getName(), CONF);
+ dodgyWAL.init();
LogRoller logRoller = new LogRoller(server, services);
logRoller.addWAL(dodgyWAL);
logRoller.start();
@@ -188,7 +188,7 @@ public class TestFailedAppendAndSync {
} catch (IOException ioe) {
fail();
}
- long rollsCount = rolls.get();
+ long rollsCount = dodgyWAL.rolls.get();
try {
dodgyWAL.throwAppendException = true;
dodgyWAL.throwSyncException = false;
@@ -198,8 +198,10 @@ public class TestFailedAppendAndSync {
} catch (IOException ioe) {
threwOnAppend = true;
}
- while (rollsCount == rolls.get()) Threads.sleep(100);
- rollsCount = rolls.get();
+ while (rollsCount == dodgyWAL.rolls.get()) {
+ Threads.sleep(100);
+ }
+ rollsCount = dodgyWAL.rolls.get();
// When we get to here.. we should be ok. A new WAL has been put in place. There were no
// appends to sync. We should be able to continue.
@@ -213,14 +215,16 @@ public class TestFailedAppendAndSync {
} catch (IOException ioe) {
threwOnBoth = true;
}
- while (rollsCount == rolls.get()) Threads.sleep(100);
+ while (rollsCount == dodgyWAL.rolls.get()) {
+ Threads.sleep(100);
+ }
// Again, all should be good. New WAL and no outstanding unsync'd edits so we should be able
// to just continue.
// So, should be no abort at this stage. Verify.
- Mockito.verify(server, Mockito.atLeast(0)).
- abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
+ Mockito.verify(server, Mockito.atLeast(0)).abort(Mockito.anyString(),
+ Mockito.any(Throwable.class));
try {
dodgyWAL.throwAppendException = false;
dodgyWAL.throwSyncException = true;
@@ -235,8 +239,8 @@ public class TestFailedAppendAndSync {
// happens. If it don't we'll timeout the whole test. That is fine.
while (true) {
try {
- Mockito.verify(server, Mockito.atLeast(1)).
- abort(Mockito.anyString(), (Throwable)Mockito.anyObject());
+ Mockito.verify(server, Mockito.atLeast(1)).abort(Mockito.anyString(),
+ Mockito.any(Throwable.class));
break;
} catch (WantedButNotInvoked t) {
Threads.sleep(1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 3c11b31..f4b8616 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -219,7 +218,6 @@ public class TestHRegion {
protected static HBaseTestingUtility TEST_UTIL;
public static Configuration CONF ;
private String dir;
- private static FileSystem FILESYSTEM;
private final int MAX_VERSIONS = 2;
// Test names
@@ -241,7 +239,6 @@ public class TestHRegion {
@Before
public void setup() throws IOException {
TEST_UTIL = HBaseTestingUtility.createLocalHTU();
- FILESYSTEM = TEST_UTIL.getTestFileSystem();
CONF = TEST_UTIL.getConfiguration();
dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
method = name.getMethodName();
@@ -343,6 +340,7 @@ public class TestHRegion {
FileSystem fs = FileSystem.get(CONF);
Path rootDir = new Path(dir + "testMemstoreSnapshotSize");
MyFaultyFSLog faultyLog = new MyFaultyFSLog(fs, rootDir, "testMemstoreSnapshotSize", CONF);
+ faultyLog.init();
HRegion region = initHRegion(tableName, null, null, false, Durability.SYNC_WAL, faultyLog,
COLUMN_FAMILY_BYTES);
@@ -354,7 +352,6 @@ public class TestHRegion {
Put put = new Put(value);
put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
faultyLog.setFailureType(FaultyFSLog.FailureType.SYNC);
-
boolean threwIOE = false;
try {
region.put(put);
@@ -390,6 +387,7 @@ public class TestHRegion {
FileSystem fs = FileSystem.get(CONF);
Path rootDir = new Path(dir + testName);
FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF);
+ hLog.init();
HRegion region = initHRegion(tableName, null, null, false, Durability.SYNC_WAL, hLog,
COLUMN_FAMILY_BYTES);
HStore store = region.getStore(COLUMN_FAMILY_BYTES);
@@ -1164,6 +1162,7 @@ public class TestHRegion {
FailAppendFlushMarkerWAL wal =
new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf),
method, walConf);
+ wal.init();
this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW,
HConstants.EMPTY_END_ROW, false, Durability.USE_DEFAULT, wal, family);
try {
@@ -1195,7 +1194,7 @@ public class TestHRegion {
wal.flushActions = new FlushAction [] {FlushAction.COMMIT_FLUSH};
wal = new FailAppendFlushMarkerWAL(FileSystem.get(walConf), FSUtils.getRootDir(walConf),
method, walConf);
-
+ wal.init();
this.region = initHRegion(tableName, HConstants.EMPTY_START_ROW,
HConstants.EMPTY_END_ROW, false, Durability.USE_DEFAULT, wal, family);
region.put(put);
@@ -2447,6 +2446,7 @@ public class TestHRegion {
FileSystem fs = FileSystem.get(CONF);
Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL");
FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF);
+ hLog.init();
// This chunk creation is done throughout the code base. Do we want to move it into core?
// It is missing from this test. W/o it we NPE.
ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
@@ -2499,9 +2499,9 @@ public class TestHRegion {
RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
// Because the preBatchMutate returns void, we can't do usual Mockito when...then form. Must
// do below format (from Mockito doc).
- Mockito.doAnswer(new Answer() {
+ Mockito.doAnswer(new Answer<Void>() {
@Override
- public Object answer(InvocationOnMock invocation) throws Throwable {
+ public Void answer(InvocationOnMock invocation) throws Throwable {
MiniBatchOperationInProgress<Mutation> mb = invocation.getArgument(0);
mb.addOperationsFromCP(0, new Mutation[]{addPut});
return null;
@@ -3792,9 +3792,12 @@ public class TestHRegion {
boolean previousEmpty = res.isEmpty();
res.clear();
- InternalScanner scanner = region.getScanner(scan);
- while (scanner.next(res))
- ;
+ try (InternalScanner scanner = region.getScanner(scan)) {
+ boolean moreRows;
+ do {
+ moreRows = scanner.next(res);
+ } while (moreRows);
+ }
if (!res.isEmpty() || !previousEmpty || i > compactInterval) {
assertEquals("i=" + i, expectedCount, res.size());
long timestamp = res.get(0).getTimestamp();
@@ -3890,7 +3893,7 @@ public class TestHRegion {
region.put(put);
numPutsFinished++;
if (numPutsFinished > 0 && numPutsFinished % 47 == 0) {
- System.out.println("put iteration = " + numPutsFinished);
+ LOG.debug("put iteration = {}", numPutsFinished);
Delete delete = new Delete(row, (long) numPutsFinished - 30);
region.delete(delete);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
index ac10a35..cf47d6d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionWithInMemoryFlush.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
-
import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
@@ -30,19 +29,14 @@ import org.apache.hadoop.hbase.wal.WAL;
import org.junit.ClassRule;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestRule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* A test similar to TestHRegion, but with in-memory flush families.
* Also checks wal truncation after in-memory compaction.
*/
@Category({VerySlowRegionServerTests.class, LargeTests.class})
-@SuppressWarnings("deprecation")
public class TestHRegionWithInMemoryFlush extends TestHRegion{
- // Do not spin up clusters in here. If you need to spin up a cluster, do it
- // over in TestHRegionOnCluster.
- private static final Logger LOG = LoggerFactory.getLogger(TestHRegionWithInMemoryFlush.class);
+
@ClassRule
public static final TestRule timeout =
CategoryBasedTimeout.forClass(TestHRegionWithInMemoryFlush.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java
index d913c67..b9350c4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionIncrement.java
@@ -23,13 +23,13 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Scan;
@@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.client.TestIncrementsFromClientSide;
import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.wal.WAL;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@@ -81,12 +80,12 @@ public class TestRegionIncrement {
}
private HRegion getRegion(final Configuration conf, final String tableName) throws IOException {
- WAL wal = new FSHLog(FileSystem.get(conf), TEST_UTIL.getDataTestDir(),
- TEST_UTIL.getDataTestDir().toString(), conf);
+ FSHLog wal = new FSHLog(FileSystem.get(conf), TEST_UTIL.getDataTestDir(),
+ TEST_UTIL.getDataTestDir().toString(), conf);
+ wal.init();
ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
- return (HRegion)TEST_UTIL.createLocalHRegion(Bytes.toBytes(tableName),
- HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, tableName, conf,
- false, Durability.SKIP_WAL, wal, INCREMENT_BYTES);
+ return TEST_UTIL.createLocalHRegion(TableName.valueOf(tableName), HConstants.EMPTY_BYTE_ARRAY,
+ HConstants.EMPTY_BYTE_ARRAY, false, Durability.SKIP_WAL, wal, INCREMENT_BYTES);
}
private void closeRegion(final HRegion region) throws IOException {
@@ -170,8 +169,6 @@ public class TestRegionIncrement {
/**
* Have each thread update its own Cell. Avoid contention with another thread.
- * @throws IOException
- * @throws InterruptedException
*/
@Test
public void testUnContendedSingleCellIncrement()
@@ -209,13 +206,9 @@ public class TestRegionIncrement {
/**
* Have each thread update its own Cell. Avoid contention with another thread.
- * This is
- * @throws IOException
- * @throws InterruptedException
*/
@Test
- public void testContendedAcrossCellsIncrement()
- throws IOException, InterruptedException {
+ public void testContendedAcrossCellsIncrement() throws IOException, InterruptedException {
final HRegion region = getRegion(TEST_UTIL.getConfiguration(),
TestIncrementsFromClientSide.filterStringSoTableNameSafe(this.name.getMethodName()));
long startTime = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
index d69149c..12fae6b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -25,7 +24,6 @@ import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -34,7 +32,6 @@ import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
@@ -46,12 +43,12 @@ import org.apache.hadoop.hbase.regionserver.wal.DamagedWALException;
import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
-import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.wal.WALKey;
import org.apache.hadoop.hbase.wal.WALKeyImpl;
import org.apache.hadoop.hbase.wal.WALProvider.Writer;
@@ -210,6 +207,7 @@ public class TestWALLockup {
FileSystem fs = FileSystem.get(CONF);
Path rootDir = new Path(dir + getName());
DodgyFSLog dodgyWAL = new DodgyFSLog(fs, rootDir, getName(), CONF);
+ dodgyWAL.init();
Path originalWAL = dodgyWAL.getCurrentFileName();
// I need a log roller running.
LogRoller logRoller = new LogRoller(server, services);
@@ -217,7 +215,6 @@ public class TestWALLockup {
// There is no 'stop' once a logRoller is running.. it just dies.
logRoller.start();
// Now get a region and start adding in edits.
- HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME);
final HRegion region = initHRegion(tableName, null, null, dodgyWAL);
byte [] bytes = Bytes.toBytes(getName());
NavigableMap<byte[], Integer> scopes = new TreeMap<>(
@@ -231,7 +228,7 @@ public class TestWALLockup {
Put put = new Put(bytes);
put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
WALKeyImpl key = new WALKeyImpl(region.getRegionInfo().getEncodedNameAsBytes(),
- htd.getTableName(), System.currentTimeMillis(), mvcc, scopes);
+ TableName.META_TABLE_NAME, System.currentTimeMillis(), mvcc, scopes);
WALEdit edit = new WALEdit();
CellScanner CellScanner = put.cellScanner();
assertTrue(CellScanner.advance());
@@ -383,10 +380,11 @@ public class TestWALLockup {
FileSystem fs = FileSystem.get(CONF);
Path rootDir = new Path(dir + getName());
DodgyFSLog dodgyWAL1 = new DodgyFSLog(fs, rootDir, getName(), CONF);
-
+ dodgyWAL1.init();
Path rootDir2 = new Path(dir + getName() + "2");
final DodgyFSLog dodgyWAL2 = new DodgyFSLog(fs, rootDir2, getName() + "2",
CONF);
+ dodgyWAL2.init();
// Add a listener to force ringbuffer event handler sleep for a while
dodgyWAL2.registerWALActionsListener(new DummyWALActionsListener());
@@ -397,7 +395,6 @@ public class TestWALLockup {
// There is no 'stop' once a logRoller is running.. it just dies.
logRoller.start();
// Now get a region and start adding in edits.
- HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME);
final HRegion region = initHRegion(tableName, null, null, dodgyWAL1);
byte[] bytes = Bytes.toBytes(getName());
NavigableMap<byte[], Integer> scopes = new TreeMap<>(
@@ -408,7 +405,7 @@ public class TestWALLockup {
Put put = new Put(bytes);
put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("1"), bytes);
WALKeyImpl key = new WALKeyImpl(region.getRegionInfo().getEncodedNameAsBytes(),
- htd.getTableName(), System.currentTimeMillis(), mvcc, scopes);
+ TableName.META_TABLE_NAME, System.currentTimeMillis(), mvcc, scopes);
WALEdit edit = new WALEdit();
CellScanner CellScanner = put.cellScanner();
assertTrue(CellScanner.advance());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
index d90a5df..4e377bf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
@@ -1100,6 +1100,7 @@ public abstract class AbstractTestWALReplay {
private MockWAL createMockWAL() throws IOException {
MockWAL wal = new MockWAL(fs, hbaseRootDir, logName, conf);
+ wal.init();
// Set down maximum recovery so we dfsclient doesn't linger retrying something
// long gone.
HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java
index ba2f8e7..549add3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogTestHelper.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALEdit;
@@ -41,7 +42,7 @@ import org.apache.hadoop.hbase.wal.WALProvider;
/**
* Helper class for testing protobuf log.
*/
-final class ProtobufLogTestHelper {
+public final class ProtobufLogTestHelper {
private ProtobufLogTestHelper() {
}
@@ -54,17 +55,22 @@ final class ProtobufLogTestHelper {
return RegionInfoBuilder.newBuilder(tableName).setRegionId(1024).build();
}
+ private static WAL.Entry generateEdit(int i, RegionInfo hri, TableName tableName, byte[] row,
+ int columnCount, long timestamp, MultiVersionConcurrencyControl mvcc) {
+ WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, i, timestamp,
+ HConstants.DEFAULT_CLUSTER_ID, mvcc);
+ WALEdit edit = new WALEdit();
+ int prefix = i;
+ IntStream.range(0, columnCount).mapToObj(j -> toValue(prefix, j))
+ .map(value -> new KeyValue(row, row, row, timestamp, value)).forEachOrdered(edit::add);
+ return new WAL.Entry(key, edit);
+ }
+
public static void doWrite(WALProvider.Writer writer, boolean withTrailer, TableName tableName,
int columnCount, int recordCount, byte[] row, long timestamp) throws IOException {
RegionInfo hri = toRegionInfo(tableName);
for (int i = 0; i < recordCount; i++) {
- WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, i, timestamp,
- HConstants.DEFAULT_CLUSTER_ID);
- WALEdit edit = new WALEdit();
- int prefix = i;
- IntStream.range(0, columnCount).mapToObj(j -> toValue(prefix, j))
- .map(value -> new KeyValue(row, row, row, timestamp, value)).forEachOrdered(edit::add);
- writer.append(new WAL.Entry(key, edit));
+ writer.append(generateEdit(i, hri, tableName, row, columnCount, timestamp, null));
}
writer.sync();
if (withTrailer) {
@@ -72,14 +78,24 @@ final class ProtobufLogTestHelper {
}
}
- public static void doRead(ProtobufLogReader reader, boolean withTrailer, TableName tableName,
- int columnCount, int recordCount, byte[] row, long timestamp) throws IOException {
+ public static void doWrite(WAL wal, RegionInfo hri, TableName tableName, int columnCount,
+ int recordCount, byte[] row, long timestamp, MultiVersionConcurrencyControl mvcc)
+ throws IOException {
+ for (int i = 0; i < recordCount; i++) {
+ WAL.Entry entry = generateEdit(i, hri, tableName, row, columnCount, timestamp, mvcc);
+ wal.append(hri, entry.getKey(), entry.getEdit(), true);
+ }
+ wal.sync();
+ }
+
+ public static void doRead(ProtobufLogReader reader, boolean withTrailer, RegionInfo hri,
+ TableName tableName, int columnCount, int recordCount, byte[] row, long timestamp)
+ throws IOException {
if (withTrailer) {
assertNotNull(reader.trailer);
} else {
assertNull(reader.trailer);
}
- RegionInfo hri = toRegionInfo(tableName);
for (int i = 0; i < recordCount; ++i) {
WAL.Entry entry = reader.next();
assertNotNull(entry);
@@ -96,4 +112,10 @@ final class ProtobufLogTestHelper {
}
assertNull(reader.next());
}
+
+ public static void doRead(ProtobufLogReader reader, boolean withTrailer, TableName tableName,
+ int columnCount, int recordCount, byte[] row, long timestamp) throws IOException {
+ doRead(reader, withTrailer, toRegionInfo(tableName), tableName, columnCount, recordCount, row,
+ timestamp);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java
index 396b268..c073ba8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncFSWAL.java
@@ -17,14 +17,8 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
-import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
-
import java.io.IOException;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -35,6 +29,11 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.experimental.categories.Category;
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
+
/**
* Provides AsyncFSWAL test cases.
*/
@@ -62,8 +61,10 @@ public class TestAsyncFSWAL extends AbstractTestFSWAL {
protected AbstractFSWAL<?> newWAL(FileSystem fs, Path rootDir, String logDir, String archiveDir,
Configuration conf, List<WALActionsListener> listeners, boolean failIfWALExists,
String prefix, String suffix) throws IOException {
- return new AsyncFSWAL(fs, rootDir, logDir, archiveDir, conf, listeners, failIfWALExists, prefix,
- suffix, GROUP, CHANNEL_CLASS);
+ AsyncFSWAL wal = new AsyncFSWAL(fs, rootDir, logDir, archiveDir, conf, listeners,
+ failIfWALExists, prefix, suffix, GROUP, CHANNEL_CLASS);
+ wal.init();
+ return wal;
}
@Override
@@ -71,15 +72,16 @@ public class TestAsyncFSWAL extends AbstractTestFSWAL {
String archiveDir, Configuration conf, List<WALActionsListener> listeners,
boolean failIfWALExists, String prefix, String suffix, final Runnable action)
throws IOException {
- return new AsyncFSWAL(fs, rootDir, logDir, archiveDir, conf, listeners, failIfWALExists, prefix,
- suffix, GROUP, CHANNEL_CLASS) {
+ AsyncFSWAL wal = new AsyncFSWAL(fs, rootDir, logDir, archiveDir, conf, listeners,
+ failIfWALExists, prefix, suffix, GROUP, CHANNEL_CLASS) {
@Override
void atHeadOfRingBufferEventHandlerAppend() {
action.run();
super.atHeadOfRingBufferEventHandlerAppend();
}
-
};
+ wal.init();
+ return wal;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncWALReplay.java
index 1157a61..9473168 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncWALReplay.java
@@ -61,7 +61,9 @@ public class TestAsyncWALReplay extends AbstractTestWALReplay {
@Override
protected WAL createWAL(Configuration c, Path hbaseRootDir, String logName) throws IOException {
- return new AsyncFSWAL(FileSystem.get(c), hbaseRootDir, logName,
+ AsyncFSWAL wal = new AsyncFSWAL(FileSystem.get(c), hbaseRootDir, logName,
HConstants.HREGION_OLDLOGDIR_NAME, c, null, true, null, null, GROUP, CHANNEL_CLASS);
+ wal.init();
+ return wal;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index cb45f49..da7e8be 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
@@ -18,6 +18,8 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
+import static org.junit.Assert.assertEquals;
+
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.List;
@@ -27,7 +29,6 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -53,13 +54,12 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import static org.junit.Assert.assertEquals;
-
/**
* Provides FSHLog test cases.
*/
@Category({ RegionServerTests.class, MediumTests.class })
public class TestFSHLog extends AbstractTestFSWAL {
+
@Rule
public TestName name = new TestName();
@@ -67,8 +67,10 @@ public class TestFSHLog extends AbstractTestFSWAL {
protected AbstractFSWAL<?> newWAL(FileSystem fs, Path rootDir, String walDir, String archiveDir,
Configuration conf, List<WALActionsListener> listeners, boolean failIfWALExists,
String prefix, String suffix) throws IOException {
- return new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists, prefix,
- suffix);
+ FSHLog wal =
+ new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix);
+ wal.init();
+ return wal;
}
@Override
@@ -76,8 +78,8 @@ public class TestFSHLog extends AbstractTestFSWAL {
String archiveDir, Configuration conf, List<WALActionsListener> listeners,
boolean failIfWALExists, String prefix, String suffix, final Runnable action)
throws IOException {
- return new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists, prefix,
- suffix) {
+ FSHLog wal = new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists,
+ prefix, suffix) {
@Override
void atHeadOfRingBufferEventHandlerAppend() {
@@ -85,6 +87,8 @@ public class TestFSHLog extends AbstractTestFSWAL {
super.atHeadOfRingBufferEventHandlerAppend();
}
};
+ wal.init();
+ return wal;
}
@Test
@@ -93,6 +97,7 @@ public class TestFSHLog extends AbstractTestFSWAL {
final String name = this.name.getMethodName();
FSHLog log = new FSHLog(FS, FSUtils.getRootDir(CONF), name, HConstants.HREGION_OLDLOGDIR_NAME,
CONF, null, true, null, null);
+ log.init();
try {
Field ringBufferEventHandlerField = FSHLog.class.getDeclaredField("ringBufferEventHandler");
ringBufferEventHandlerField.setAccessible(true);
@@ -135,7 +140,7 @@ public class TestFSHLog extends AbstractTestFSWAL {
try (FSHLog log =
new FSHLog(FS, FSUtils.getRootDir(CONF), name, HConstants.HREGION_OLDLOGDIR_NAME, CONF,
null, true, null, null)) {
-
+ log.init();
log.registerWALActionsListener(new WALActionsListener() {
@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit)
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 2758d4d..9d2f6c0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.regionserver.wal;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -30,14 +29,14 @@ import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.junit.BeforeClass;
import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestRule;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALReplay extends AbstractTestWALReplay {
- @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()).
- withLookingForStuckThread(true).build();
+ @Rule
+ public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass())
+ .withLookingForStuckThread(true).build();
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -49,6 +48,7 @@ public class TestWALReplay extends AbstractTestWALReplay {
@Override
protected WAL createWAL(Configuration c, Path hbaseRootDir, String logName) throws IOException {
FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c);
+ wal.init();
// Set down maximum recovery so we dfsclient doesn't linger retrying something
// long gone.
HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java
index 485e5b8..201d8da 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/IOTestProvider.java
@@ -102,6 +102,7 @@ public class IOTestProvider implements WALProvider {
AbstractFSWALProvider.getWALDirectoryName(factory.factoryId),
HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners, true, logPrefix,
META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);
+ log.init();
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b56bf0db/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSynchronousReplicationWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSynchronousReplicationWALProvider.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSynchronousReplicationWALProvider.java
new file mode 100644
index 0000000..da686b1
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSynchronousReplicationWALProvider.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.util.Optional;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
+import org.apache.hadoop.hbase.regionserver.wal.DualAsyncFSWAL;
+import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
+import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogTestHelper;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestSynchronousReplicationWALProvider {
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static String PEER_ID = "1";
+
+ private static String REMOTE_WAL_DIR = "/RemoteWAL";
+
+ private static TableName TABLE = TableName.valueOf("table");
+
+ private static TableName TABLE_NO_REP = TableName.valueOf("table-no-rep");
+
+ private static RegionInfo REGION = RegionInfoBuilder.newBuilder(TABLE).build();
+
+ private static RegionInfo REGION_NO_REP = RegionInfoBuilder.newBuilder(TABLE_NO_REP).build();
+
+ private static WALFactory FACTORY;
+
+ private static Optional<Pair<String, String>> getPeerIdAndRemoteWALDir(RegionInfo info) {
+ if (info.getTable().equals(TABLE)) {
+ return Optional.of(Pair.newPair(PEER_ID, REMOTE_WAL_DIR));
+ } else {
+ return Optional.empty();
+ }
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ UTIL.startMiniDFSCluster(3);
+ FACTORY = new WALFactory(UTIL.getConfiguration(), null, "test",
+ TestSynchronousReplicationWALProvider::getPeerIdAndRemoteWALDir);
+ UTIL.getTestFileSystem().mkdirs(new Path(REMOTE_WAL_DIR, PEER_ID));
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws IOException {
+ FACTORY.close();
+ UTIL.shutdownMiniDFSCluster();
+ }
+
+ private void testReadWrite(DualAsyncFSWAL wal) throws Exception {
+ int recordCount = 100;
+ int columnCount = 10;
+ byte[] row = Bytes.toBytes("testRow");
+ long timestamp = System.currentTimeMillis();
+ MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
+ ProtobufLogTestHelper.doWrite(wal, REGION, TABLE, columnCount, recordCount, row, timestamp,
+ mvcc);
+ Path localFile = wal.getCurrentFileName();
+ Path remoteFile = new Path(REMOTE_WAL_DIR + "/" + PEER_ID, localFile.getName());
+ try (ProtobufLogReader reader =
+ (ProtobufLogReader) FACTORY.createReader(UTIL.getTestFileSystem(), localFile)) {
+ ProtobufLogTestHelper.doRead(reader, false, REGION, TABLE, columnCount, recordCount, row,
+ timestamp);
+ }
+ try (ProtobufLogReader reader =
+ (ProtobufLogReader) FACTORY.createReader(UTIL.getTestFileSystem(), remoteFile)) {
+ ProtobufLogTestHelper.doRead(reader, false, REGION, TABLE, columnCount, recordCount, row,
+ timestamp);
+ }
+ wal.rollWriter();
+ DistributedFileSystem dfs = (DistributedFileSystem) UTIL.getDFSCluster().getFileSystem();
+ UTIL.waitFor(5000, new ExplainingPredicate<Exception>() {
+
+ @Override
+ public boolean evaluate() throws Exception {
+ return dfs.isFileClosed(localFile) && dfs.isFileClosed(remoteFile);
+ }
+
+ @Override
+ public String explainFailure() throws Exception {
+ StringBuilder sb = new StringBuilder();
+ if (!dfs.isFileClosed(localFile)) {
+ sb.append(localFile + " has not been closed yet.");
+ }
+ if (!dfs.isFileClosed(remoteFile)) {
+ sb.append(remoteFile + " has not been closed yet.");
+ }
+ return sb.toString();
+ }
+ });
+ try (ProtobufLogReader reader =
+ (ProtobufLogReader) FACTORY.createReader(UTIL.getTestFileSystem(), localFile)) {
+ ProtobufLogTestHelper.doRead(reader, true, REGION, TABLE, columnCount, recordCount, row,
+ timestamp);
+ }
+ try (ProtobufLogReader reader =
+ (ProtobufLogReader) FACTORY.createReader(UTIL.getTestFileSystem(), remoteFile)) {
+ ProtobufLogTestHelper.doRead(reader, true, REGION, TABLE, columnCount, recordCount, row,
+ timestamp);
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ WAL walNoRep = FACTORY.getWAL(REGION_NO_REP);
+ assertThat(walNoRep, not(instanceOf(DualAsyncFSWAL.class)));
+ DualAsyncFSWAL wal = (DualAsyncFSWAL) FACTORY.getWAL(REGION);
+ assertEquals(2, FACTORY.getWALs().size());
+ testReadWrite(wal);
+ SynchronousReplicationWALProvider walProvider =
+ (SynchronousReplicationWALProvider) FACTORY.getWALProvider();
+ walProvider.peerRemoved(PEER_ID);
+ assertEquals(1, FACTORY.getWALs().size());
+ }
+}
[10/31] hbase git commit: HBASE-19808 Reenable TestMultiParallel
Posted by zh...@apache.org.
HBASE-19808 Reenable TestMultiParallel
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a3c98b2d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a3c98b2d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a3c98b2d
Branch: refs/heads/HBASE-19064
Commit: a3c98b2dd8835db3989699d780e61c8321361170
Parents: 8a8be33
Author: Michael Stack <st...@apache.org>
Authored: Tue Jan 16 22:12:43 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Wed Jan 17 08:58:37 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/client/TestMultiParallel.java | 6 +++---
.../hbase/regionserver/TestRegionServerReadRequestMetrics.java | 5 ++++-
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c98b2d/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
index a81447b..68659e1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java
@@ -58,13 +58,11 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-@Ignore // Depends on Master being able to host regions. Needs fixing.
@Category({MediumTests.class, FlakeyTests.class})
public class TestMultiParallel {
private static final Logger LOG = LoggerFactory.getLogger(TestMultiParallel.class);
@@ -90,7 +88,9 @@ public class TestMultiParallel {
UTIL.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY,
KeyValueCodec.class.getCanonicalName());
UTIL.getConfiguration().setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
- UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true);
+ // We used to ask for system tables on Master exclusively but not needed by test and doesn't
+ // work anyways -- so commented out.
+ // UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true);
UTIL.getConfiguration()
.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyMasterObserver.class.getName());
UTIL.startMiniCluster(slaves);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c98b2d/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java
index c5afefa..21f4fb6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java
@@ -105,7 +105,7 @@ public class TestRegionServerReadRequestMetrics {
public static void setUpOnce() throws Exception {
// Default starts one regionserver only.
TEST_UTIL.getConfiguration().setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
- TEST_UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true);
+ // TEST_UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true);
TEST_UTIL.startMiniCluster();
admin = TEST_UTIL.getAdmin();
serverNames = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
@@ -326,6 +326,7 @@ public class TestRegionServerReadRequestMetrics {
testReadRequests(resultCount, 1, 0);
}
+ @Ignore // HBASE-19785
@Test
public void testReadRequestsCountWithFilter() throws Exception {
int resultCount;
@@ -372,6 +373,7 @@ public class TestRegionServerReadRequestMetrics {
// testReadRequests(resultCount, 0, 1);
}
+ @Ignore // HBASE-19785
@Test
public void testReadRequestsCountWithDeletedRow() throws Exception {
try {
@@ -409,6 +411,7 @@ public class TestRegionServerReadRequestMetrics {
}
}
+ @Ignore // See HBASE-19785
@Test
public void testReadRequestsWithCoprocessor() throws Exception {
TableName tableName = TableName.valueOf("testReadRequestsWithCoprocessor");
[17/31] hbase git commit: HBASE-19816 Refresh repliation sinks on
UnknownHostException
Posted by zh...@apache.org.
HBASE-19816 Refresh repliation sinks on UnknownHostException
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e3d15834
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e3d15834
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e3d15834
Branch: refs/heads/HBASE-19064
Commit: e3d15834cb50b4d93373de7d4f7aa43173dfcc39
Parents: b1729a2
Author: Scott Wilson <sw...@siftscience.com>
Authored: Wed Jan 17 13:18:30 2018 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Wed Jan 17 16:24:48 2018 -0800
----------------------------------------------------------------------
.../regionserver/HBaseInterClusterReplicationEndpoint.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e3d15834/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
index 1ef1198..49ea1f7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -398,7 +399,7 @@ public class HBaseInterClusterReplicationEndpoint extends HBaseReplicationEndpoi
"call to the remote cluster timed out, which is usually " +
"caused by a machine failure or a massive slowdown",
this.socketTimeoutMultiplier);
- } else if (ioe instanceof ConnectException) {
+ } else if (ioe instanceof ConnectException || ioe instanceof UnknownHostException) {
LOG.warn("Peer is unavailable, rechecking all sinks: ", ioe);
replicationSinkMgr.chooseSinks();
} else {