You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/01/23 10:24:36 UTC
[01/50] [abbrv] hbase git commit: HBASE-19774 incorrect behavior of
locateRegionInMeta [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-19397-branch-2 46a9db935 -> 7aaad6f4c (forced update)
HBASE-19774 incorrect behavior of locateRegionInMeta
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6f8c3124
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6f8c3124
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6f8c3124
Branch: refs/heads/HBASE-19397-branch-2
Commit: 6f8c31242a211e09acabd8b2c0a5d264839f72af
Parents: 5f62041
Author: Sergey Soldatov <ss...@apache.org>
Authored: Thu Jan 11 13:40:38 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Mon Jan 22 11:49:19 2018 -0800
----------------------------------------------------------------------
.../hbase/client/ConnectionImplementation.java | 23 ++++-----
.../client/TestScannersFromClientSide.java | 50 +++++++++++++++-----
2 files changed, 49 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6f8c3124/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 562630f..8807884 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -786,11 +786,14 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
// build the key of the meta region we should be looking for.
// the extra 9's on the end are necessary to allow "exact" matches
// without knowing the precise region names.
- byte[] metaKey = RegionInfo.createRegionName(tableName, row, HConstants.NINES, false);
+ byte[] metaStartKey = RegionInfo.createRegionName(tableName, row, HConstants.NINES, false);
+ byte[] metaStopKey =
+ RegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW, "", false);
Scan s = new Scan();
s.setReversed(true);
- s.withStartRow(metaKey);
+ s.withStartRow(metaStartKey);
+ s.withStopRow(metaStopKey, true);
s.addFamily(HConstants.CATALOG_FAMILY);
if (this.useMetaReplicas) {
@@ -840,12 +843,11 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
// convert the row result into the HRegionLocation we need!
RegionLocations locations = MetaTableAccessor.getRegionLocations(regionInfoRow);
if (locations == null || locations.getRegionLocation(replicaId) == null) {
- throw new IOException("HRegionInfo was null in " +
- tableName + ", row=" + regionInfoRow);
+ throw new IOException("RegionInfo null in " + tableName + ", row=" + regionInfoRow);
}
RegionInfo regionInfo = locations.getRegionLocation(replicaId).getRegion();
if (regionInfo == null) {
- throw new IOException("HRegionInfo was null or empty in " +
+ throw new IOException("RegionInfo null or empty in " +
TableName.META_TABLE_NAME + ", row=" + regionInfoRow);
}
@@ -857,13 +859,12 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
"hbase:meta might be damaged.");
}
if (regionInfo.isSplit()) {
- throw new RegionOfflineException(
- "the only available region for the required row is a split parent,"
- + " the daughters should be online soon: " + regionInfo.getRegionNameAsString());
+ throw new RegionOfflineException ("Region for row is a split parent, daughters not online: " +
+ regionInfo.getRegionNameAsString());
}
if (regionInfo.isOffline()) {
- throw new RegionOfflineException("the region is offline, could"
- + " be caused by a disable table call: " + regionInfo.getRegionNameAsString());
+ throw new RegionOfflineException("Region offline; disable table call? " +
+ regionInfo.getRegionNameAsString());
}
ServerName serverName = locations.getRegionLocation(replicaId).getServerName();
@@ -908,7 +909,7 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
// Only relocate the parent region if necessary
if(!(e instanceof RegionOfflineException ||
e instanceof NoServerForRegionException)) {
- relocateRegion(TableName.META_TABLE_NAME, metaKey, replicaId);
+ relocateRegion(TableName.META_TABLE_NAME, metaStartKey, replicaId);
}
} finally {
userRegionLock.unlock();
http://git-wip-us.apache.org/repos/asf/hbase/blob/6f8c3124/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java
index 5441e2b..9173726 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannersFromClientSide.java
@@ -16,19 +16,6 @@
*/
package org.apache.hadoop.hbase.client;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
-import java.util.stream.IntStream;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CompareOperator;
@@ -41,6 +28,7 @@ import org.apache.hadoop.hbase.HTestConst;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
@@ -61,6 +49,21 @@ import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.stream.IntStream;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+
/**
* A client-side test, mostly testing scanners with various parameters.
*/
@@ -236,6 +239,27 @@ public class TestScannersFromClientSide {
clientScanner.getCacheSize() <= 1);
}
+ /**
+ * Scan on not existing table should throw the exception with correct message
+ */
+ @Test
+ public void testScannerForNotExistingTable() {
+ String[] tableNames = {"A", "Z", "A:A", "Z:Z"};
+ for(String tableName : tableNames) {
+ try {
+ Table table = TEST_UTIL.getConnection().getTable(TableName.valueOf(tableName));
+ testSmallScan(table, true, 1, 5);
+ fail("TableNotFoundException was not thrown");
+ } catch (TableNotFoundException e) {
+ // We expect that the message for TableNotFoundException would have only the table name only
+ // Otherwise that would mean that localeRegionInMeta doesn't work properly
+ assertEquals(e.getMessage(), tableName);
+ } catch (Exception e) {
+ fail("Unexpected exception " + e.getMessage());
+ }
+ }
+ }
+
@Test
public void testSmallScan() throws Exception {
final TableName tableName = TableName.valueOf(name.getMethodName());
[49/50] [abbrv] hbase git commit: HBASE-19783 Change replication peer
cluster key/endpoint from a not-null value to null is not allowed
Posted by zh...@apache.org.
HBASE-19783 Change replication peer cluster key/endpoint from a not-null value to null is not allowed
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7aaad6f4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7aaad6f4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7aaad6f4
Branch: refs/heads/HBASE-19397-branch-2
Commit: 7aaad6f4c024019c12e47d380b135b00feed596d
Parents: 2198683
Author: Guanghao Zhang <zg...@apache.org>
Authored: Fri Jan 12 22:04:38 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:20:38 2018 +0800
----------------------------------------------------------------------
.../replication/ReplicationPeerManager.java | 28 +++++++++++++-------
1 file changed, 19 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7aaad6f4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 696b2d7..19fc7f4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -132,20 +132,19 @@ public class ReplicationPeerManager {
checkPeerConfig(peerConfig);
ReplicationPeerDescription desc = checkPeerExists(peerId);
ReplicationPeerConfig oldPeerConfig = desc.getPeerConfig();
- if (!StringUtils.isBlank(peerConfig.getClusterKey()) &&
- !peerConfig.getClusterKey().equals(oldPeerConfig.getClusterKey())) {
+ if (!isStringEquals(peerConfig.getClusterKey(), oldPeerConfig.getClusterKey())) {
throw new DoNotRetryIOException(
"Changing the cluster key on an existing peer is not allowed. Existing key '" +
- oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does not match new key '" +
- peerConfig.getClusterKey() + "'");
+ oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does not match new key '" +
+ peerConfig.getClusterKey() + "'");
}
- if (!StringUtils.isBlank(peerConfig.getReplicationEndpointImpl()) &&
- !peerConfig.getReplicationEndpointImpl().equals(oldPeerConfig.getReplicationEndpointImpl())) {
+ if (!isStringEquals(peerConfig.getReplicationEndpointImpl(),
+ oldPeerConfig.getReplicationEndpointImpl())) {
throw new DoNotRetryIOException("Changing the replication endpoint implementation class " +
- "on an existing peer is not allowed. Existing class '" +
- oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
- " does not match new class '" + peerConfig.getReplicationEndpointImpl() + "'");
+ "on an existing peer is not allowed. Existing class '" +
+ oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
+ " does not match new class '" + peerConfig.getReplicationEndpointImpl() + "'");
}
}
@@ -341,4 +340,15 @@ public class ReplicationPeerManager {
return new ReplicationPeerManager(peerStorage,
ReplicationStorageFactory.getReplicationQueueStorage(zk, conf), peers);
}
+
+ /**
+ * For replication peer cluster key or endpoint class, null and empty string is same. So here
+ * don't use {@link StringUtils#equals(CharSequence, CharSequence)} directly.
+ */
+ private boolean isStringEquals(String s1, String s2) {
+ if (StringUtils.isBlank(s1)) {
+ return StringUtils.isBlank(s2);
+ }
+ return s1.equals(s2);
+ }
}
[36/50] [abbrv] hbase git commit: HBASE-19633 Clean up the
replication queues in the postPeerModification stage when removing a peer
Posted by zh...@apache.org.
HBASE-19633 Clean up the replication queues in the postPeerModification stage when removing a peer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/16d080c1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/16d080c1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/16d080c1
Branch: refs/heads/HBASE-19397-branch-2
Commit: 16d080c1ad044a4eb1043e78263630bb36f46bf9
Parents: a791c23
Author: zhangduo <zh...@apache.org>
Authored: Tue Jan 2 09:57:23 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../replication/ReplicationPeerConfig.java | 2 +-
.../replication/VerifyReplication.java | 34 ++++++++++-------
.../hbase/replication/ReplicationPeers.java | 32 ++++++----------
.../replication/ZKReplicationQueueStorage.java | 3 +-
.../replication/ZKReplicationStorageBase.java | 4 +-
.../replication/TestReplicationStateBasic.java | 10 +----
.../master/replication/AddPeerProcedure.java | 5 +--
.../replication/DisablePeerProcedure.java | 3 +-
.../master/replication/EnablePeerProcedure.java | 3 +-
.../master/replication/ModifyPeerProcedure.java | 34 +++++++++--------
.../replication/RefreshPeerProcedure.java | 17 ++++-----
.../master/replication/RemovePeerProcedure.java | 7 ++--
.../replication/ReplicationPeerManager.java | 31 +++++++++++++++-
.../replication/UpdatePeerConfigProcedure.java | 3 +-
.../RemoteProcedureResultReporter.java | 3 +-
.../regionserver/RefreshPeerCallable.java | 5 ++-
.../regionserver/ReplicationSourceManager.java | 39 +++++++-------------
.../TestReplicationAdminUsingProcedure.java | 7 ++--
18 files changed, 124 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index b80ee16..fdae288 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -27,8 +27,8 @@ import java.util.Set;
import java.util.TreeMap;
import org.apache.hadoop.hbase.TableName;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* A configuration for the replication peer cluster.
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index f0070f0..fe45762 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.mapreduce.replication;
import java.io.IOException;
import java.util.Arrays;
import java.util.UUID;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -45,13 +44,14 @@ import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
-import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
+import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -66,6 +66,7 @@ import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
@@ -333,19 +334,24 @@ public class VerifyReplication extends Configured implements Tool {
final Configuration conf, String peerId) throws IOException {
ZKWatcher localZKW = null;
try {
- localZKW = new ZKWatcher(conf, "VerifyReplication",
- new Abortable() {
- @Override public void abort(String why, Throwable e) {}
- @Override public boolean isAborted() {return false;}
- });
-
- ReplicationPeers rp = ReplicationFactory.getReplicationPeers(localZKW, conf);
- rp.init();
+ localZKW = new ZKWatcher(conf, "VerifyReplication", new Abortable() {
+ @Override
+ public void abort(String why, Throwable e) {
+ }
- return Pair.newPair(rp.getPeerConfig(peerId), rp.getPeerClusterConfiguration(peerId));
+ @Override
+ public boolean isAborted() {
+ return false;
+ }
+ });
+ ReplicationPeerStorage storage =
+ ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf);
+ ReplicationPeerConfig peerConfig = storage.getPeerConfig(peerId);
+ return Pair.newPair(peerConfig,
+ ReplicationPeers.getPeerClusterConfiguration(peerConfig, conf));
} catch (ReplicationException e) {
- throw new IOException(
- "An error occurred while trying to connect to the remove peer cluster", e);
+ throw new IOException("An error occurred while trying to connect to the remove peer cluster",
+ e);
} finally {
if (localZKW != null) {
localZKW.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index e58482e..422801b 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -27,8 +27,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
@@ -39,20 +37,22 @@ import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesti
@InterfaceAudience.Private
public class ReplicationPeers {
- private static final Logger LOG = LoggerFactory.getLogger(ReplicationPeers.class);
-
private final Configuration conf;
// Map of peer clusters keyed by their id
private final ConcurrentMap<String, ReplicationPeerImpl> peerCache;
private final ReplicationPeerStorage peerStorage;
- protected ReplicationPeers(ZKWatcher zookeeper, Configuration conf) {
+ ReplicationPeers(ZKWatcher zookeeper, Configuration conf) {
this.conf = conf;
this.peerCache = new ConcurrentHashMap<>();
this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zookeeper, conf);
}
+ public Configuration getConf() {
+ return conf;
+ }
+
public void init() throws ReplicationException {
// Loading all existing peerIds into peer cache.
for (String peerId : this.peerStorage.listPeerIds()) {
@@ -120,22 +120,13 @@ public class ReplicationPeers {
return peerCache.keySet();
}
- public ReplicationPeerConfig getPeerConfig(String peerId) {
- ReplicationPeer replicationPeer = this.peerCache.get(peerId);
- if (replicationPeer == null) {
- throw new IllegalArgumentException("Peer with id= " + peerId + " is not cached");
- }
- return replicationPeer.getPeerConfig();
- }
-
- public Configuration getPeerClusterConfiguration(String peerId) throws ReplicationException {
- ReplicationPeerConfig peerConfig = peerStorage.getPeerConfig(peerId);
-
+ public static Configuration getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
+ Configuration baseConf) throws ReplicationException {
Configuration otherConf;
try {
- otherConf = HBaseConfiguration.createClusterConf(this.conf, peerConfig.getClusterKey());
+ otherConf = HBaseConfiguration.createClusterConf(baseConf, peerConfig.getClusterKey());
} catch (IOException e) {
- throw new ReplicationException("Can't get peer configuration for peerId=" + peerId, e);
+ throw new ReplicationException("Can't get peer configuration for peer " + peerConfig, e);
}
if (!peerConfig.getConfiguration().isEmpty()) {
@@ -179,8 +170,9 @@ public class ReplicationPeers {
>>>>>>> HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
*/
private ReplicationPeerImpl createPeer(String peerId) throws ReplicationException {
- ReplicationPeerConfig peerConf = peerStorage.getPeerConfig(peerId);
+ ReplicationPeerConfig peerConfig = peerStorage.getPeerConfig(peerId);
boolean enabled = peerStorage.isPeerEnabled(peerId);
- return new ReplicationPeerImpl(getPeerClusterConfiguration(peerId), peerId, enabled, peerConf);
+ return new ReplicationPeerImpl(getPeerClusterConfiguration(peerConfig, conf), peerId, enabled,
+ peerConfig);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
index 41f50d8..ee237f2 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
@@ -50,7 +49,7 @@ import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* ZK based replication queue storage.
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
index d09a56b..2321e4f 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
@@ -19,13 +19,13 @@ package org.apache.hadoop.hbase.replication;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream;
+import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
index 2589199..07c6c15 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.replication;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -238,12 +237,6 @@ public abstract class TestReplicationStateBasic {
} catch (ReplicationException e) {
}
- try {
- assertNull(rp.getPeerClusterConfiguration("bogus"));
- fail("Should have thrown an ReplicationException when passed a bogus peerId");
- } catch (ReplicationException e) {
- }
-
assertNumberOfPeers(0);
// Add some peers
@@ -258,7 +251,8 @@ public abstract class TestReplicationStateBasic {
fail("There are no connected peers, should have thrown an IllegalArgumentException");
} catch (IllegalArgumentException e) {
}
- assertEquals(KEY_ONE, ZKConfig.getZooKeeperClusterKey(rp.getPeerClusterConfiguration(ID_ONE)));
+ assertEquals(KEY_ONE, ZKConfig.getZooKeeperClusterKey(ReplicationPeers
+ .getPeerClusterConfiguration(rp.getPeerStorage().getPeerConfig(ID_ONE), rp.getConf())));
rp.getPeerStorage().removePeer(ID_ONE);
rp.removePeer(ID_ONE);
assertNumberOfPeers(1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
index a4f9b32..f0f7704 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
@@ -74,8 +73,8 @@ public class AddPeerProcedure extends ModifyPeerProcedure {
@Override
protected void postPeerModification(MasterProcedureEnv env) throws IOException {
- LOG.info("Successfully added " + (enabled ? "ENABLED" : "DISABLED") + " peer " + peerId +
- ", config " + peerConfig);
+ LOG.info("Successfully added {} peer {}, config {}", enabled ? "ENABLED" : "DISABLED", peerId,
+ peerConfig);
MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
if (cpHost != null) {
env.getMasterCoprocessorHost().postAddReplicationPeer(peerId, peerConfig);
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
index 10e35a8..0871575 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -62,7 +61,7 @@ public class DisablePeerProcedure extends ModifyPeerProcedure {
@Override
protected void postPeerModification(MasterProcedureEnv env) throws IOException {
- LOG.info("Successfully disabled peer " + peerId);
+ LOG.info("Successfully disabled peer {}", peerId);
MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.postDisableReplicationPeer(peerId);
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
index f2a9f01..890462f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -62,7 +61,7 @@ public class EnablePeerProcedure extends ModifyPeerProcedure {
@Override
protected void postPeerModification(MasterProcedureEnv env) throws IOException {
- LOG.info("Successfully enabled peer " + peerId);
+ LOG.info("Successfully enabled peer {}", peerId);
MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.postEnableReplicationPeer(peerId);
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index a682606..c225619 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
@@ -84,10 +83,13 @@ public abstract class ModifyPeerProcedure
* Called before we finish the procedure. The implementation can do some logging work, and also
* call the coprocessor hook if any.
* <p>
- * Notice that, since we have already done the actual work, throwing exception here will not fail
- * this procedure, we will just ignore it and finish the procedure as suceeded.
+ * Notice that, since we have already done the actual work, throwing {@code IOException} here will
+ * not fail this procedure, we will just ignore it and finish the procedure as suceeded. If
+ * {@code ReplicationException} is thrown we will retry since this usually means we fails to
+ * update the peer storage.
*/
- protected abstract void postPeerModification(MasterProcedureEnv env) throws IOException;
+ protected abstract void postPeerModification(MasterProcedureEnv env)
+ throws IOException, ReplicationException;
private void releaseLatch() {
ProcedurePrepareLatch.releaseLatch(latch, this);
@@ -101,16 +103,14 @@ public abstract class ModifyPeerProcedure
try {
prePeerModification(env);
} catch (IOException e) {
- LOG.warn(
- getClass().getName() + " failed to call CP hook or the pre check is failed for peer " +
- peerId + ", mark the procedure as failure and give up",
- e);
+ LOG.warn("{} failed to call pre CP hook or the pre check is failed for peer {}, " +
+ "mark the procedure as failure and give up", getClass().getName(), peerId, e);
setFailure("master-" + getPeerOperationType().name().toLowerCase() + "-peer", e);
releaseLatch();
return Flow.NO_MORE_STATE;
} catch (ReplicationException e) {
- LOG.warn(getClass().getName() + " failed to call prePeerModification for peer " + peerId +
- ", retry", e);
+ LOG.warn("{} failed to call prePeerModification for peer {}, retry", getClass().getName(),
+ peerId, e);
throw new ProcedureYieldException();
}
setNextState(PeerModificationState.UPDATE_PEER_STORAGE);
@@ -119,8 +119,8 @@ public abstract class ModifyPeerProcedure
try {
updatePeerStorage(env);
} catch (ReplicationException e) {
- LOG.warn(
- getClass().getName() + " update peer storage for peer " + peerId + " failed, retry", e);
+ LOG.warn("{} update peer storage for peer {} failed, retry", getClass().getName(), peerId,
+ e);
throw new ProcedureYieldException();
}
setNextState(PeerModificationState.REFRESH_PEER_ON_RS);
@@ -134,9 +134,13 @@ public abstract class ModifyPeerProcedure
case POST_PEER_MODIFICATION:
try {
postPeerModification(env);
+ } catch (ReplicationException e) {
+ LOG.warn("{} failed to call postPeerModification for peer {}, retry",
+ getClass().getName(), peerId, e);
+ throw new ProcedureYieldException();
} catch (IOException e) {
- LOG.warn(getClass().getName() + " failed to call prePeerModification for peer " + peerId +
- ", ignore since the procedure has already done", e);
+ LOG.warn("{} failed to call post CP hook for peer {}, " +
+ "ignore since the procedure has already done", getClass().getName(), peerId, e);
}
releaseLatch();
return Flow.NO_MORE_STATE;
@@ -175,7 +179,7 @@ public abstract class ModifyPeerProcedure
throws IOException, InterruptedException {
if (state == PeerModificationState.PRE_PEER_MODIFICATION) {
// actually the peer related operations has no rollback, but if we haven't done any
- // modifications on the peer storage, we can just return.
+ // modifications on the peer storage yet, we can just return.
return;
}
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
index ba4285f..1253ef9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
@@ -122,17 +121,15 @@ public class RefreshPeerProcedure extends Procedure<MasterProcedureEnv>
private void complete(MasterProcedureEnv env, Throwable error) {
if (event == null) {
- LOG.warn("procedure event for " + getProcId() +
- " is null, maybe the procedure is created when recovery",
- new Exception());
+ LOG.warn("procedure event for {} is null, maybe the procedure is created when recovery",
+ getProcId());
return;
}
if (error != null) {
- LOG.warn("Refresh peer " + peerId + " for " + type + " on " + targetServer + " failed",
- error);
+ LOG.warn("Refresh peer {} for {} on {} failed", peerId, type, targetServer, error);
this.succ = false;
} else {
- LOG.info("Refresh peer " + peerId + " for " + type + " on " + targetServer + " suceeded");
+ LOG.info("Refresh peer {} for {} on {} suceeded", peerId, type, targetServer);
this.succ = true;
}
@@ -168,9 +165,9 @@ public class RefreshPeerProcedure extends Procedure<MasterProcedureEnv>
dispatched = false;
}
if (!env.getRemoteDispatcher().addOperationToNode(targetServer, this)) {
- LOG.info("Can not add remote operation for refreshing peer " + peerId + " for " + type +
- " to " + targetServer + ", this usually because the server is already dead," +
- " give up and mark the procedure as complete");
+ LOG.info("Can not add remote operation for refreshing peer {} for {} to {}, " +
+ "this usually because the server is already dead, " +
+ "give up and mark the procedure as complete", peerId, type, targetServer);
return null;
}
dispatched = true;
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
index 6e9c384..64faf2b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -61,8 +60,10 @@ public class RemovePeerProcedure extends ModifyPeerProcedure {
}
@Override
- protected void postPeerModification(MasterProcedureEnv env) throws IOException {
- LOG.info("Successfully removed peer " + peerId);
+ protected void postPeerModification(MasterProcedureEnv env)
+ throws IOException, ReplicationException {
+ env.getReplicationPeerManager().removeAllQueuesAndHFileRefs(peerId);
+ LOG.info("Successfully removed peer {}", peerId);
MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.postRemoveReplicationPeer(peerId);
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index b6732d7..1414d22 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -28,7 +28,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -217,6 +216,36 @@ public class ReplicationPeerManager {
return desc != null ? Optional.of(desc.getPeerConfig()) : Optional.empty();
}
+ private void removeAllQueues0(String peerId) throws ReplicationException {
+ for (ServerName replicator : queueStorage.getListOfReplicators()) {
+ List<String> queueIds = queueStorage.getAllQueues(replicator);
+ for (String queueId : queueIds) {
+ ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
+ if (queueInfo.getPeerId().equals(peerId)) {
+ queueStorage.removeQueue(replicator, queueId);
+ }
+ }
+ queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
+ }
+ }
+
+ public void removeAllQueuesAndHFileRefs(String peerId) throws ReplicationException {
+ // Here we need two passes to address the problem of claimQueue. Maybe a claimQueue is still
+ // on-going when the refresh peer config procedure is done, if a RS which has already been
+ // scanned claims the queue of a RS which has not been scanned yet, we will miss that queue in
+ // the scan here, and if the RS who has claimed the queue crashed before creating recovered
+ // source, then the queue will leave there until the another RS detects the crash and helps
+ // removing the queue.
+ // A two pass scan can solve the problem. Anyway, the queue will not disappear during the
+ // claiming, it will either under the old RS or under the new RS, and a queue can only be
+ // claimed once after the refresh peer procedure done(as the next claim queue will just delete
+ // it), so we can make sure that a two pass scan will finally find the queue and remove it,
+ // unless it has already been removed by others.
+ removeAllQueues0(peerId);
+ removeAllQueues0(peerId);
+ queueStorage.removePeerFromHFileRefs(peerId);
+ }
+
private void checkPeerConfig(ReplicationPeerConfig peerConfig) throws DoNotRetryIOException {
checkClusterKey(peerConfig.getClusterKey());
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
index a43532d..3497447 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
@@ -70,7 +69,7 @@ public class UpdatePeerConfigProcedure extends ModifyPeerProcedure {
@Override
protected void postPeerModification(MasterProcedureEnv env) throws IOException {
- LOG.info("Successfully updated peer config of " + peerId + " to " + peerConfig);
+ LOG.info("Successfully updated peer config of {} to {}", peerId, peerConfig);
MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.postUpdateReplicationPeerConfig(peerId, peerConfig);
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
index e4be422..ac3e95a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
@@ -28,7 +28,8 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.TextFormat;
+import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RemoteProcedureResult;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneRequest;
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
index c3f33aa..7ada24b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
@@ -20,11 +20,12 @@ package org.apache.hadoop.hbase.replication.regionserver;
import org.apache.hadoop.hbase.executor.EventType;
import org.apache.hadoop.hbase.procedure2.RSProcedureCallable;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationType;
import org.apache.log4j.Logger;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationType;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshPeerParameter;
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 1359575..c289c74 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -59,7 +59,6 @@ import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationListener;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
@@ -306,9 +305,8 @@ public class ReplicationSourceManager implements ReplicationListener {
*/
@VisibleForTesting
ReplicationSourceInterface addSource(String id) throws IOException, ReplicationException {
- ReplicationPeerConfig peerConfig = replicationPeers.getPeerConfig(id);
ReplicationPeer peer = replicationPeers.getPeer(id);
- ReplicationSourceInterface src = getReplicationSource(id, peerConfig, peer);
+ ReplicationSourceInterface src = getReplicationSource(id, peer);
synchronized (this.walsById) {
this.sources.add(src);
Map<String, SortedSet<String>> walsByGroup = new HashMap<>();
@@ -499,8 +497,8 @@ public class ReplicationSourceManager implements ReplicationListener {
* @param peerId the id of the peer cluster
* @return the created source
*/
- private ReplicationSourceInterface getReplicationSource(String peerId,
- ReplicationPeerConfig peerConfig, ReplicationPeer replicationPeer) throws IOException {
+ private ReplicationSourceInterface getReplicationSource(String peerId, ReplicationPeer peer)
+ throws IOException {
RegionServerCoprocessorHost rsServerHost = null;
TableDescriptors tableDescriptors = null;
if (server instanceof HRegionServer) {
@@ -512,24 +510,24 @@ public class ReplicationSourceManager implements ReplicationListener {
ReplicationEndpoint replicationEndpoint = null;
try {
- String replicationEndpointImpl = peerConfig.getReplicationEndpointImpl();
+ String replicationEndpointImpl = peer.getPeerConfig().getReplicationEndpointImpl();
if (replicationEndpointImpl == null) {
// Default to HBase inter-cluster replication endpoint
replicationEndpointImpl = HBaseInterClusterReplicationEndpoint.class.getName();
}
replicationEndpoint = Class.forName(replicationEndpointImpl)
.asSubclass(ReplicationEndpoint.class).newInstance();
- if(rsServerHost != null) {
- ReplicationEndpoint newReplicationEndPoint = rsServerHost
- .postCreateReplicationEndPoint(replicationEndpoint);
- if(newReplicationEndPoint != null) {
+ if (rsServerHost != null) {
+ ReplicationEndpoint newReplicationEndPoint =
+ rsServerHost.postCreateReplicationEndPoint(replicationEndpoint);
+ if (newReplicationEndPoint != null) {
// Override the newly created endpoint from the hook with configured end point
replicationEndpoint = newReplicationEndPoint;
}
}
} catch (Exception e) {
- LOG.warn("Passed replication endpoint implementation throws errors"
- + " while initializing ReplicationSource for peer: " + peerId, e);
+ LOG.warn("Passed replication endpoint implementation throws errors" +
+ " while initializing ReplicationSource for peer: " + peerId, e);
throw new IOException(e);
}
@@ -539,8 +537,8 @@ public class ReplicationSourceManager implements ReplicationListener {
replicationEndpoint, walFileLengthProvider, metrics);
// init replication endpoint
- replicationEndpoint.init(new ReplicationEndpoint.Context(conf, replicationPeer.getConfiguration(),
- fs, peerId, clusterId, replicationPeer, metrics, tableDescriptors, server));
+ replicationEndpoint.init(new ReplicationEndpoint.Context(conf, peer.getConfiguration(), fs,
+ peerId, clusterId, peer, metrics, tableDescriptors, server));
return src;
}
@@ -736,17 +734,6 @@ public class ReplicationSourceManager implements ReplicationListener {
abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), peerId));
continue;
}
-
- ReplicationPeerConfig peerConfig = null;
- try {
- peerConfig = replicationPeers.getPeerConfig(actualPeerId);
- } catch (Exception e) {
- LOG.warn("Skipping failover for peer:" + actualPeerId + " of node " + deadRS
- + ", failed to read peer config", e);
- abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), peerId));
- continue;
- }
-
// track sources in walsByIdRecoveredQueues
Map<String, SortedSet<String>> walsByGroup = new HashMap<>();
walsByIdRecoveredQueues.put(peerId, walsByGroup);
@@ -761,7 +748,7 @@ public class ReplicationSourceManager implements ReplicationListener {
}
// enqueue sources
- ReplicationSourceInterface src = getReplicationSource(peerId, peerConfig, peer);
+ ReplicationSourceInterface src = getReplicationSource(peerId, peer);
// synchronized on oldsources to avoid adding recovered source for the to-be-removed peer
// see removePeer
synchronized (oldsources) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/16d080c1/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
index b09a8a7..1300376 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
@@ -15,19 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.client.replication;
import java.io.IOException;
-
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.TestReplicationBase;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.ImmutableList;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -39,6 +35,9 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
@Category({ MediumTests.class, ClientTests.class })
public class TestReplicationAdminUsingProcedure extends TestReplicationBase {
[29/50] [abbrv] hbase git commit: HBASE-19642 Fix locking for peer
modification procedure
Posted by zh...@apache.org.
HBASE-19642 Fix locking for peer modification procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f90fcc9e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f90fcc9e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f90fcc9e
Branch: refs/heads/HBASE-19397-branch-2
Commit: f90fcc9e7ae5124dfaf42b416603a8baf1debd48
Parents: 2417c32
Author: zhangduo <zh...@apache.org>
Authored: Wed Dec 27 18:27:13 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../procedure/MasterProcedureScheduler.java | 14 +++++++++++++
.../master/replication/ModifyPeerProcedure.java | 21 +++++++++++++++++---
2 files changed, 32 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f90fcc9e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index a25217c..4ecb3b1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -610,6 +610,20 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
public boolean requireExclusiveLock(Procedure proc) {
return requirePeerExclusiveLock((PeerProcedureInterface) proc);
}
+
+ @Override
+ public boolean isAvailable() {
+ if (isEmpty()) {
+ return false;
+ }
+ if (getLockStatus().hasExclusiveLock()) {
+ // if we have an exclusive lock already taken
+ // only child of the lock owner can be executed
+ Procedure nextProc = peek();
+ return nextProc != null && getLockStatus().hasLockAccess(nextProc);
+ }
+ return true;
+ }
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/hbase/blob/f90fcc9e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 279fbc7..a682606 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -46,6 +46,8 @@ public abstract class ModifyPeerProcedure
protected String peerId;
+ private volatile boolean locked;
+
// used to keep compatible with old client where we can only returns after updateStorage.
protected ProcedurePrepareLatch latch;
@@ -145,17 +147,30 @@ public abstract class ModifyPeerProcedure
@Override
protected LockState acquireLock(MasterProcedureEnv env) {
- return env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)
- ? LockState.LOCK_EVENT_WAIT
- : LockState.LOCK_ACQUIRED;
+ if (env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)) {
+ return LockState.LOCK_EVENT_WAIT;
+ }
+ locked = true;
+ return LockState.LOCK_ACQUIRED;
}
@Override
protected void releaseLock(MasterProcedureEnv env) {
+ locked = false;
env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId);
}
@Override
+ protected boolean holdLock(MasterProcedureEnv env) {
+ return true;
+ }
+
+ @Override
+ protected boolean hasLock(MasterProcedureEnv env) {
+ return locked;
+ }
+
+ @Override
protected void rollbackState(MasterProcedureEnv env, PeerModificationState state)
throws IOException, InterruptedException {
if (state == PeerModificationState.PRE_PEER_MODIFICATION) {
[14/50] [abbrv] hbase git commit: HBASE-19838 Can not shutdown backup
master cleanly when it has already tried to become the active master
Posted by zh...@apache.org.
HBASE-19838 Can not shutdown backup master cleanly when it has already tried to become the active master
On Master@shutdown, close the shared Master connection to kill any
ongoing RPCs by hosted clients.
M hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
Call close ont the Master shared clusterconnection to kill any ongoing
rpcs.
M hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
Remove guts of close; we were closing the Masters connection....not
our responsibility.
Added unit test written by Duo Zhang which demonstrates the case where
Master will not go down.
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/739b9b4a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/739b9b4a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/739b9b4a
Branch: refs/heads/HBASE-19397-branch-2
Commit: 739b9b4a8e3f50145953ed24ac18ed8ccb5ff869
Parents: f952779
Author: Michael Stack <st...@apache.org>
Authored: Mon Jan 22 14:44:16 2018 -0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 11:09:42 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 21 +++-
.../hadoop/hbase/master/ServerManager.java | 16 +--
.../hbase/regionserver/HRegionServer.java | 3 +
.../hbase/master/TestShutdownBackupMaster.java | 108 +++++++++++++++++++
4 files changed, 132 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/739b9b4a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0806ad4..bd52bd1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2616,7 +2616,7 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
- public void abort(final String msg, final Throwable t) {
+ public void abort(String reason, Throwable cause) {
if (isAborted() || isStopped()) {
return;
}
@@ -2625,8 +2625,9 @@ public class HMaster extends HRegionServer implements MasterServices {
LOG.error(HBaseMarkers.FATAL, "Master server abort: loaded coprocessors are: " +
getLoadedCoprocessors());
}
- if (t != null) {
- LOG.error(HBaseMarkers.FATAL, msg, t);
+ String msg = "***** ABORTING master " + this + ": " + reason + " *****";
+ if (cause != null) {
+ LOG.error(HBaseMarkers.FATAL, msg, cause);
} else {
LOG.error(HBaseMarkers.FATAL, msg);
}
@@ -2677,14 +2678,19 @@ public class HMaster extends HRegionServer implements MasterServices {
return rsFatals;
}
+ /**
+ * Shutdown the cluster.
+ * Master runs a coordinated stop of all RegionServers and then itself.
+ */
public void shutdown() throws IOException {
if (cpHost != null) {
cpHost.preShutdown();
}
-
+ // Tell the servermanager cluster is down.
if (this.serverManager != null) {
this.serverManager.shutdownCluster();
}
+ // Set the cluster down flag; broadcast across the cluster.
if (this.clusterStatusTracker != null){
try {
this.clusterStatusTracker.setClusterDown();
@@ -2692,6 +2698,13 @@ public class HMaster extends HRegionServer implements MasterServices {
LOG.error("ZooKeeper exception trying to set cluster as down in ZK", e);
}
}
+ // Shutdown our cluster connection. This will kill any hosted RPCs that might be going on;
+ // this is what we want especially if the Master is in startup phase doing call outs to
+ // hbase:meta, etc. when cluster is down. Without ths connection close, we'd have to wait on
+ // the rpc to timeout.
+ if (this.clusterConnection != null) {
+ this.clusterConnection.close();
+ }
}
public void stopMaster() throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/739b9b4a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index 3db6033..3c84bfb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -200,10 +200,8 @@ public class ServerManager {
Configuration c = master.getConfiguration();
maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
- this.connection = connect ? master.getClusterConnection() : null;
- this.rpcControllerFactory = this.connection == null
- ? null
- : connection.getRpcControllerFactory();
+ this.connection = connect? master.getClusterConnection(): null;
+ this.rpcControllerFactory = this.connection == null? null: connection.getRpcControllerFactory();
}
/**
@@ -968,16 +966,10 @@ public class ServerManager {
}
/**
- * Stop the ServerManager. Currently closes the connection to the master.
+ * Stop the ServerManager.
*/
public void stop() {
- if (connection != null) {
- try {
- connection.close();
- } catch (IOException e) {
- LOG.error("Attempt to close connection to master failed", e);
- }
- }
+ // Nothing to do.
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/739b9b4a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index bd7b4a0..1c1ce82 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -248,6 +248,9 @@ public class HRegionServer extends HasThread implements
* Cluster connection to be shared by services.
* Initialized at server startup and closed when server shuts down.
* Clients must never close it explicitly.
+ * Clients hosted by this Server should make use of this clusterConnection rather than create
+ * their own; if they create their own, there is no way for the hosting server to shutdown
+ * ongoing client RPCs.
*/
protected ClusterConnection clusterConnection;
http://git-wip-us.apache.org/repos/asf/hbase/blob/739b9b4a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java
new file mode 100644
index 0000000..02d7f2f
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertNotNull;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CategoryBasedTimeout;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestRule;
+
+/**
+ * Test to confirm that we will not hang when stop a backup master which is trying to become the
+ * active master. See HBASE-19838
+ */
+@Category({ MasterTests.class, MediumTests.class })
+public class TestShutdownBackupMaster {
+ @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()).
+ withLookingForStuckThread(true).build();
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static volatile CountDownLatch ARRIVE;
+
+ private static volatile CountDownLatch CONTINUE;
+
+ public static final class MockHMaster extends HMaster {
+
+ public MockHMaster(Configuration conf) throws IOException, KeeperException {
+ super(conf);
+ }
+
+ @Override
+ void initClusterSchemaService() throws IOException, InterruptedException {
+ if (ARRIVE != null) {
+ ARRIVE.countDown();
+ CONTINUE.await();
+ }
+ super.initClusterSchemaService();
+ }
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, MockHMaster.class, HMaster.class);
+ UTIL.startMiniCluster(2, 2);
+ UTIL.waitUntilAllSystemRegionsAssigned();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ // make sure that we can stop the cluster cleanly
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testShutdownWhileBecomingActive() throws InterruptedException {
+ MiniHBaseCluster cluster = UTIL.getHBaseCluster();
+ HMaster activeMaster = null;
+ HMaster backupMaster = null;
+ for (MasterThread t : cluster.getMasterThreads()) {
+ if (t.getMaster().isActiveMaster()) {
+ activeMaster = t.getMaster();
+ } else {
+ backupMaster = t.getMaster();
+ }
+ }
+ assertNotNull(activeMaster);
+ assertNotNull(backupMaster);
+ ARRIVE = new CountDownLatch(1);
+ CONTINUE = new CountDownLatch(1);
+ activeMaster.abort("Aborting active master for test");
+ // wait until we arrive the initClusterSchemaService
+ ARRIVE.await();
+ // killall RSes
+ cluster.getRegionServerThreads().stream().map(t -> t.getRegionServer())
+ .forEachOrdered(rs -> rs.abort("Aborting RS for test"));
+ CONTINUE.countDown();
+ }
+}
[41/50] [abbrv] hbase git commit: HBASE-19544 Add UTs for testing
concurrent modifications on replication peer
Posted by zh...@apache.org.
HBASE-19544 Add UTs for testing concurrent modifications on replication peer
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4caceda1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4caceda1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4caceda1
Branch: refs/heads/HBASE-19397-branch-2
Commit: 4caceda1ff694db9bd5aa6c54bfa908a97cdf54a
Parents: 54659f2
Author: Guanghao Zhang <zg...@apache.org>
Authored: Tue Jan 2 17:07:41 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../replication/TestReplicationAdmin.java | 69 ++++++++++++++++++++
1 file changed, 69 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4caceda1/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index f9629bd..9edd4a2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -31,6 +31,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -56,6 +57,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Unit testing of ReplicationAdmin
@@ -63,6 +66,8 @@ import org.junit.rules.TestName;
@Category({MediumTests.class, ClientTests.class})
public class TestReplicationAdmin {
+ private static final Logger LOG = LoggerFactory.getLogger(TestReplicationAdmin.class);
+
private final static HBaseTestingUtility TEST_UTIL =
new HBaseTestingUtility();
@@ -112,6 +117,70 @@ public class TestReplicationAdmin {
}
@Test
+ public void testConcurrentPeerOperations() throws Exception {
+ int threadNum = 5;
+ AtomicLong successCount = new AtomicLong(0);
+
+ // Test concurrent add peer operation
+ Thread[] addPeers = new Thread[threadNum];
+ for (int i = 0; i < threadNum; i++) {
+ addPeers[i] = new Thread(() -> {
+ try {
+ hbaseAdmin.addReplicationPeer(ID_ONE,
+ ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build());
+ successCount.incrementAndGet();
+ } catch (Exception e) {
+ LOG.debug("Got exception when add replication peer", e);
+ }
+ });
+ addPeers[i].start();
+ }
+ for (Thread addPeer : addPeers) {
+ addPeer.join();
+ }
+ assertEquals(1, successCount.get());
+
+ // Test concurrent remove peer operation
+ successCount.set(0);
+ Thread[] removePeers = new Thread[threadNum];
+ for (int i = 0; i < threadNum; i++) {
+ removePeers[i] = new Thread(() -> {
+ try {
+ hbaseAdmin.removeReplicationPeer(ID_ONE);
+ successCount.incrementAndGet();
+ } catch (Exception e) {
+ LOG.debug("Got exception when remove replication peer", e);
+ }
+ });
+ removePeers[i].start();
+ }
+ for (Thread removePeer : removePeers) {
+ removePeer.join();
+ }
+ assertEquals(1, successCount.get());
+
+ // Test concurrent add peer operation again
+ successCount.set(0);
+ addPeers = new Thread[threadNum];
+ for (int i = 0; i < threadNum; i++) {
+ addPeers[i] = new Thread(() -> {
+ try {
+ hbaseAdmin.addReplicationPeer(ID_ONE,
+ ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build());
+ successCount.incrementAndGet();
+ } catch (Exception e) {
+ LOG.debug("Got exception when add replication peer", e);
+ }
+ });
+ addPeers[i].start();
+ }
+ for (Thread addPeer : addPeers) {
+ addPeer.join();
+ }
+ assertEquals(1, successCount.get());
+ }
+
+ @Test
public void testAddInvalidPeer() {
ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder();
builder.setClusterKey(KEY_ONE);
[10/50] [abbrv] hbase git commit: HBASE-19811 Fix findbugs and
error-prone warnings in hbase-server (branch-2)
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 0c880f5..101e64b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -625,7 +625,7 @@ public class FSHLog extends AbstractFSWAL<Writer> {
} finally {
rollWriterLock.unlock();
}
- if (lowReplication || writer != null && writer.getLength() > logrollsize) {
+ if (lowReplication || (writer != null && writer.getLength() > logrollsize)) {
requestLogRoll(lowReplication);
}
}
@@ -866,7 +866,7 @@ public class FSHLog extends AbstractFSWAL<Writer> {
private final SyncFuture[] syncFutures;
// Had 'interesting' issues when this was non-volatile. On occasion, we'd not pass all
// syncFutures to the next sync'ing thread.
- private volatile int syncFuturesCount = 0;
+ private AtomicInteger syncFuturesCount = new AtomicInteger();
private volatile SafePointZigZagLatch zigzagLatch;
/**
* Set if we get an exception appending or syncing so that all subsequence appends and syncs on
@@ -894,10 +894,10 @@ public class FSHLog extends AbstractFSWAL<Writer> {
private void cleanupOutstandingSyncsOnException(final long sequence, final Exception e) {
// There could be handler-count syncFutures outstanding.
- for (int i = 0; i < this.syncFuturesCount; i++) {
+ for (int i = 0; i < this.syncFuturesCount.get(); i++) {
this.syncFutures[i].done(sequence, e);
}
- this.syncFuturesCount = 0;
+ this.syncFuturesCount.set(0);
}
/**
@@ -905,7 +905,7 @@ public class FSHLog extends AbstractFSWAL<Writer> {
*/
private boolean isOutstandingSyncs() {
// Look at SyncFutures in the EventHandler
- for (int i = 0; i < this.syncFuturesCount; i++) {
+ for (int i = 0; i < this.syncFuturesCount.get(); i++) {
if (!this.syncFutures[i].isDone()) {
return true;
}
@@ -938,9 +938,9 @@ public class FSHLog extends AbstractFSWAL<Writer> {
try {
if (truck.type() == RingBufferTruck.Type.SYNC) {
- this.syncFutures[this.syncFuturesCount++] = truck.unloadSync();
+ this.syncFutures[this.syncFuturesCount.getAndIncrement()] = truck.unloadSync();
// Force flush of syncs if we are carrying a full complement of syncFutures.
- if (this.syncFuturesCount == this.syncFutures.length) {
+ if (this.syncFuturesCount.get() == this.syncFutures.length) {
endOfBatch = true;
}
} else if (truck.type() == RingBufferTruck.Type.APPEND) {
@@ -979,7 +979,7 @@ public class FSHLog extends AbstractFSWAL<Writer> {
if (this.exception == null) {
// If not a batch, return to consume more events from the ring buffer before proceeding;
// we want to get up a batch of syncs and appends before we go do a filesystem sync.
- if (!endOfBatch || this.syncFuturesCount <= 0) {
+ if (!endOfBatch || this.syncFuturesCount.get() <= 0) {
return;
}
// syncRunnerIndex is bound to the range [0, Integer.MAX_INT - 1] as follows:
@@ -997,7 +997,7 @@ public class FSHLog extends AbstractFSWAL<Writer> {
// Below expects that the offer 'transfers' responsibility for the outstanding syncs to
// the syncRunner. We should never get an exception in here.
this.syncRunners[this.syncRunnerIndex].offer(sequence, this.syncFutures,
- this.syncFuturesCount);
+ this.syncFuturesCount.get());
} catch (Exception e) {
// Should NEVER get here.
requestLogRoll();
@@ -1010,7 +1010,7 @@ public class FSHLog extends AbstractFSWAL<Writer> {
? this.exception : new DamagedWALException("On sync", this.exception));
}
attainSafePoint(sequence);
- this.syncFuturesCount = 0;
+ this.syncFuturesCount.set(0);
} catch (Throwable t) {
LOG.error("UNEXPECTED!!! syncFutures.length=" + this.syncFutures.length, t);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java
index 57eccbb..4e88df0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSWALEntry.java
@@ -85,6 +85,7 @@ class FSWALEntry extends Entry {
}
}
+ @Override
public String toString() {
return "sequence=" + this.txid + ", " + super.toString();
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
index ebb6079..5d8d8c0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.java
@@ -102,10 +102,12 @@ public class ProtobufLogReader extends ReaderBase {
public long trailerSize() {
if (trailerPresent) {
// sizeof PB_WAL_COMPLETE_MAGIC + sizof trailerSize + trailer
- final long calculatedSize = PB_WAL_COMPLETE_MAGIC.length + Bytes.SIZEOF_INT + trailer.getSerializedSize();
+ final long calculatedSize = (long) PB_WAL_COMPLETE_MAGIC.length + Bytes.SIZEOF_INT
+ + trailer.getSerializedSize();
final long expectedSize = fileLength - walEditsStopOffset;
if (expectedSize != calculatedSize) {
- LOG.warn("After parsing the trailer, we expect the total footer to be "+ expectedSize +" bytes, but we calculate it as being " + calculatedSize);
+ LOG.warn("After parsing the trailer, we expect the total footer to be {} bytes, but we "
+ + "calculate it as being {}", expectedSize, calculatedSize);
}
return expectedSize;
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
index 2093421..42d0299 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
@@ -84,7 +84,7 @@ public abstract class BaseReplicationEndpoint extends AbstractService
for (String filterName : filterNames) {
try {
Class<?> clazz = Class.forName(filterName);
- filters.add((WALEntryFilter) clazz.newInstance());
+ filters.add((WALEntryFilter) clazz.getDeclaredConstructor().newInstance());
} catch (Exception e) {
LOG.error("Unable to create WALEntryFilter " + filterName, e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DefaultSourceFSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DefaultSourceFSConfigurationProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DefaultSourceFSConfigurationProvider.java
index b28c58f..8a4d331 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DefaultSourceFSConfigurationProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DefaultSourceFSConfigurationProvider.java
@@ -35,7 +35,7 @@ public class DefaultSourceFSConfigurationProvider implements SourceFSConfigurati
LoggerFactory.getLogger(DefaultSourceFSConfigurationProvider.class);
// Map containing all the source clusters configurations against their replication cluster id
- private Map<String, Configuration> sourceClustersConfs = new HashMap<>();
+ private final Map<String, Configuration> sourceClustersConfs = new HashMap<>();
private static final String XML = ".xml";
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index dcd79a6..28494e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -105,6 +105,7 @@ public class Replication implements
public Replication() {
}
+ @Override
public void initialize(Server server, FileSystem fs, Path logDir, Path oldLogDir,
WALFileLengthProvider walFileLengthProvider) throws IOException {
this.server = server;
@@ -165,12 +166,14 @@ public class Replication implements
/*
* Returns an object to listen to new wal changes
**/
+ @Override
public WALActionsListener getWALActionsListener() {
return this;
}
/**
* Stops replication service.
*/
+ @Override
public void stopReplicationService() {
join();
}
@@ -199,6 +202,7 @@ public class Replication implements
* @param sourceHFileArchiveDirPath Path that point to the source cluster hfile archive directory
* @throws IOException
*/
+ @Override
public void replicateLogEntries(List<WALEntry> entries, CellScanner cells,
String replicationClusterId, String sourceBaseNamespaceDirPath,
String sourceHFileArchiveDirPath) throws IOException {
@@ -211,6 +215,7 @@ public class Replication implements
* it starts
* @throws IOException
*/
+ @Override
public void startReplicationService() throws IOException {
try {
this.replicationManager.init();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
index 57e185a..902971e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java
@@ -110,7 +110,7 @@ public class ReplicationSink {
try {
@SuppressWarnings("rawtypes")
Class c = Class.forName(className);
- this.provider = (SourceFSConfigurationProvider) c.newInstance();
+ this.provider = (SourceFSConfigurationProvider) c.getDeclaredConstructor().newInstance();
} catch (Exception e) {
throw new IllegalArgumentException("Configured source fs configuration provider class "
+ className + " throws error.", e);
@@ -123,7 +123,7 @@ public class ReplicationSink {
WALEntrySinkFilter filter = null;
try {
filter = walEntryFilterClass == null? null:
- (WALEntrySinkFilter)walEntryFilterClass.newInstance();
+ (WALEntrySinkFilter)walEntryFilterClass.getDeclaredConstructor().newInstance();
} catch (Exception e) {
LOG.warn("Failed to instantiate " + walEntryFilterClass);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 16fb4a7..9db3cc2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -40,11 +40,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.TableName;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ChainWALEntryFilter;
import org.apache.hadoop.hbase.replication.ClusterMarkingEntryFilter;
@@ -61,8 +57,11 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.wal.WAL.Entry;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
/**
* Class that handles the source of a replication stream.
@@ -225,7 +224,8 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
// A peerId will not have "-" in its name, see HBASE-11394
peerId = peerClusterZnode.split("-")[0];
}
- Map<TableName, List<String>> tableCFMap = replicationPeers.getConnectedPeer(peerId).getTableCFs();
+ Map<TableName, List<String>> tableCFMap =
+ replicationPeers.getConnectedPeer(peerId).getTableCFs();
if (tableCFMap != null) {
List<String> tableCfs = tableCFMap.get(tableName);
if (tableCFMap.containsKey(tableName)
@@ -470,7 +470,8 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
}
if (this.replicationEndpoint != null) {
try {
- this.replicationEndpoint.awaitTerminated(sleepForRetries * maxRetriesMultiplier, TimeUnit.MILLISECONDS);
+ this.replicationEndpoint
+ .awaitTerminated(sleepForRetries * maxRetriesMultiplier, TimeUnit.MILLISECONDS);
} catch (TimeoutException te) {
LOG.warn("Got exception while waiting for endpoint to shutdown for replication source :"
+ this.peerClusterZnode,
@@ -494,7 +495,9 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
public Path getCurrentPath() {
// only for testing
for (ReplicationSourceShipper worker : workerThreads.values()) {
- if (worker.getCurrentPath() != null) return worker.getCurrentPath();
+ if (worker.getCurrentPath() != null) {
+ return worker.getCurrentPath();
+ }
}
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 23ae704..55ebdc1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -110,7 +110,7 @@ public class ReplicationSourceManager implements ReplicationListener {
private final Configuration conf;
private final FileSystem fs;
// The paths to the latest log of each wal group, for new coming peers
- private Set<Path> latestPaths;
+ private final Set<Path> latestPaths;
// Path to the wals directories
private final Path logDir;
// Path to the wal archive
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
index 4643a22..e56fab2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
@@ -355,7 +355,8 @@ public class ReplicationSourceWALReader extends Thread {
List<StoreDescriptor> stores = bld.getStoresList();
int totalStores = stores.size();
for (int j = 0; j < totalStores; j++) {
- totalStoreFilesSize += stores.get(j).getStoreFileSizeBytes();
+ totalStoreFilesSize =
+ (int) (totalStoreFilesSize + stores.get(j).getStoreFileSizeBytes());
}
} catch (IOException e) {
LOG.error("Failed to deserialize bulk load entry from wal edit. "
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java
index bfc415c..7e444cb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlFilter.java
@@ -154,6 +154,7 @@ class AccessControlFilter extends FilterBase {
/**
* @return The filter serialized using pb
*/
+ @Override
public byte [] toByteArray() {
// no implementation, server-side use only
throw new UnsupportedOperationException(
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 5a3c883..f191c9d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -2186,8 +2186,8 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
// Also using acl as table name to be inline with the results of global admin and will
// help in avoiding any leakage of information about being superusers.
for (String user: Superusers.getSuperUsers()) {
- perms.add(new UserPermission(user.getBytes(), AccessControlLists.ACL_TABLE_NAME, null,
- Action.values()));
+ perms.add(new UserPermission(Bytes.toBytes(user), AccessControlLists.ACL_TABLE_NAME,
+ null, Action.values()));
}
}
response = AccessControlUtil.buildGetUserPermissionsResponse(perms);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AuthResult.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AuthResult.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AuthResult.java
index 1949b98..cecca41 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AuthResult.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AuthResult.java
@@ -207,6 +207,7 @@ public class AuthResult {
return sb.toString();
}
+ @Override
public String toString() {
return "AuthResult" + toContextString();
}
@@ -279,6 +280,7 @@ public class AuthResult {
return this;
}
+ @Override
public String toString() {
String familiesString = toFamiliesString(families, family, qualifier);
String[] params = new String[] {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java
index de8ea5d..59b91a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java
@@ -330,6 +330,7 @@ public class AuthenticationTokenSecretManager
interrupt();
}
+ @Override
public void run() {
zkLeader.start();
zkLeader.waitToBecomeLeader();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ExpressionParser.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ExpressionParser.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ExpressionParser.java
index 0bc74b1..c2dd046 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ExpressionParser.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ExpressionParser.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.security.visibility;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
@@ -103,7 +104,8 @@ public class ExpressionParser {
}
index++;
} while (index < endPos && !isEndOfLabel(exp[index]));
- leafExp = new String(exp, labelOffset, index - labelOffset).trim();
+ leafExp =
+ new String(exp, labelOffset, index - labelOffset, StandardCharsets.UTF_8).trim();
if (leafExp.isEmpty()) {
throw new ParseException("Error parsing expression " + expS + " at column : " + index);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
index 1ba6029..6e00f40 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
@@ -36,6 +36,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import org.apache.hadoop.conf.Configuration;
@@ -753,8 +754,9 @@ public class VisibilityController implements MasterCoprocessor, RegionCoprocesso
logResult(true, "addLabels", "Adding labels allowed", null, labels, null);
int i = 0;
for (OperationStatus status : opStatus) {
- while (response.getResult(i) != successResult)
+ while (!Objects.equals(response.getResult(i), successResult)) {
i++;
+ }
if (status.getOperationStatusCode() != SUCCESS) {
RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
failureResultBuilder.setException(buildException(new DoNotRetryIOException(
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityNewVersionBehaivorTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityNewVersionBehaivorTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityNewVersionBehaivorTracker.java
index f6ed72f..f3e4853 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityNewVersionBehaivorTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityNewVersionBehaivorTracker.java
@@ -149,7 +149,7 @@ public class VisibilityNewVersionBehaivorTracker extends NewVersionBehaviorTrack
List<Tag> putVisTags = new ArrayList<>();
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(put, putVisTags);
return putVisTags.isEmpty() == delInfo.tags.isEmpty() && (
- putVisTags.isEmpty() && delInfo.tags.isEmpty() || VisibilityLabelServiceManager
+ (putVisTags.isEmpty() && delInfo.tags.isEmpty()) || VisibilityLabelServiceManager
.getInstance().getVisibilityLabelService()
.matchVisibility(putVisTags, putCellVisTagsFormat, delInfo.tags, delInfo.format));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/LeafExpressionNode.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/LeafExpressionNode.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/LeafExpressionNode.java
index 78b5037..fd479b4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/LeafExpressionNode.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/LeafExpressionNode.java
@@ -58,6 +58,7 @@ public class LeafExpressionNode implements ExpressionNode {
return true;
}
+ @Override
public LeafExpressionNode deepClone() {
LeafExpressionNode clone = new LeafExpressionNode(this.identifier);
return clone;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/NonLeafExpressionNode.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/NonLeafExpressionNode.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/NonLeafExpressionNode.java
index 77b34e9..83610fa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/NonLeafExpressionNode.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/NonLeafExpressionNode.java
@@ -91,6 +91,7 @@ public class NonLeafExpressionNode implements ExpressionNode {
return this.op == Operator.NOT;
}
+ @Override
public NonLeafExpressionNode deepClone() {
NonLeafExpressionNode clone = new NonLeafExpressionNode(this.op);
for (ExpressionNode exp : this.childExps) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/Operator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/Operator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/Operator.java
index fdec5ac..f7ffe46 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/Operator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/expression/Operator.java
@@ -23,12 +23,13 @@ import org.apache.yetus.audience.InterfaceAudience;
public enum Operator {
AND('&'), OR('|'), NOT('!');
- private char rep;
+ private final char rep;
private Operator(char rep) {
this.rep = rep;
}
+ @Override
public String toString() {
return String.valueOf(this.rep);
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index 7d7e526..97a74ed 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -300,7 +300,8 @@ public final class SnapshotInfo extends AbstractHBaseTool {
boolean inArchive = false;
long size = -1;
try {
- if ((inArchive = fs.exists(link.getArchivePath()))) {
+ if (fs.exists(link.getArchivePath())) {
+ inArchive = true;
size = fs.getFileStatus(link.getArchivePath()).getLen();
hfilesArchiveSize.addAndGet(size);
hfilesArchiveCount.incrementAndGet();
@@ -311,7 +312,8 @@ public final class SnapshotInfo extends AbstractHBaseTool {
!isArchivedFileStillReferenced(link.getArchivePath(), filesMap)) {
nonSharedHfilesArchiveSize.addAndGet(size);
}
- } else if (inArchive = fs.exists(link.getMobPath())) {
+ } else if (fs.exists(link.getMobPath())) {
+ inArchive = true;
size = fs.getFileStatus(link.getMobPath()).getLen();
hfilesMobSize.addAndGet(size);
hfilesMobCount.incrementAndGet();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
index a5468ee..1d7f4f6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java
@@ -75,23 +75,28 @@ public final class SnapshotManifestV1 {
this.fs = fs;
}
+ @Override
public HRegionFileSystem regionOpen(final RegionInfo regionInfo) throws IOException {
HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
fs, snapshotDir, regionInfo);
return snapshotRegionFs;
}
+ @Override
public void regionClose(final HRegionFileSystem region) {
}
+ @Override
public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) {
Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
return familyDir;
}
+ @Override
public void familyClose(final HRegionFileSystem region, final Path family) {
}
+ @Override
public void storeFile(final HRegionFileSystem region, final Path familyDir,
final StoreFileInfo storeFile) throws IOException {
Path referenceFile = new Path(familyDir, storeFile.getPath().getName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
index 4d35f0b..4e60d67 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java
@@ -77,12 +77,14 @@ public final class SnapshotManifestV2 {
this.fs = fs;
}
+ @Override
public SnapshotRegionManifest.Builder regionOpen(final RegionInfo regionInfo) {
SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
manifest.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
return manifest;
}
+ @Override
public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
// we should ensure the snapshot dir exist, maybe it has been deleted by master
// see HBASE-16464
@@ -99,6 +101,7 @@ public final class SnapshotManifestV2 {
}
}
+ @Override
public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
final SnapshotRegionManifest.Builder region, final byte[] familyName) {
SnapshotRegionManifest.FamilyFiles.Builder family =
@@ -107,11 +110,13 @@ public final class SnapshotManifestV2 {
return family;
}
+ @Override
public void familyClose(final SnapshotRegionManifest.Builder region,
final SnapshotRegionManifest.FamilyFiles.Builder family) {
region.addFamilyFiles(family.build());
}
+ @Override
public void storeFile(final SnapshotRegionManifest.Builder region,
final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
index c14c944..3cff047 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/LoadIncrementalHFiles.java
@@ -971,7 +971,7 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
continue;
}
Path familyDir = familyStat.getPath();
- byte[] familyName = familyDir.getName().getBytes();
+ byte[] familyName = Bytes.toBytes(familyDir.getName());
// Skip invalid family
try {
ColumnFamilyDescriptorBuilder.isLegalColumnFamilyName(familyName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BoundedPriorityBlockingQueue.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BoundedPriorityBlockingQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BoundedPriorityBlockingQueue.java
index 9950570..efad97e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BoundedPriorityBlockingQueue.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BoundedPriorityBlockingQueue.java
@@ -155,6 +155,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
this.queue = new PriorityQueue<>(capacity, comparator);
}
+ @Override
public boolean offer(E e) {
if (e == null) throw new NullPointerException();
@@ -171,6 +172,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
return false;
}
+ @Override
public void put(E e) throws InterruptedException {
if (e == null) throw new NullPointerException();
@@ -186,6 +188,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
}
}
+ @Override
public boolean offer(E e, long timeout, TimeUnit unit)
throws InterruptedException {
if (e == null) throw new NullPointerException();
@@ -206,6 +209,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
return true;
}
+ @Override
public E take() throws InterruptedException {
E result = null;
lock.lockInterruptibly();
@@ -221,6 +225,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
return result;
}
+ @Override
public E poll() {
E result = null;
lock.lock();
@@ -235,6 +240,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
return result;
}
+ @Override
public E poll(long timeout, TimeUnit unit)
throws InterruptedException {
long nanos = unit.toNanos(timeout);
@@ -254,6 +260,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
return result;
}
+ @Override
public E peek() {
lock.lock();
try {
@@ -263,6 +270,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
}
}
+ @Override
public int size() {
lock.lock();
try {
@@ -272,6 +280,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
}
}
+ @Override
public Iterator<E> iterator() {
throw new UnsupportedOperationException();
}
@@ -280,6 +289,7 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
return queue.comparator();
}
+ @Override
public int remainingCapacity() {
lock.lock();
try {
@@ -289,10 +299,12 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
}
}
+ @Override
public boolean remove(Object o) {
throw new UnsupportedOperationException();
}
+ @Override
public boolean contains(Object o) {
lock.lock();
try {
@@ -302,10 +314,12 @@ public class BoundedPriorityBlockingQueue<E> extends AbstractQueue<E> implements
}
}
+ @Override
public int drainTo(Collection<? super E> c) {
return drainTo(c, Integer.MAX_VALUE);
}
+ @Override
public int drainTo(Collection<? super E> c, int maxElements) {
if (c == null)
throw new NullPointerException();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java
index 4207f39..2cf3bb9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSMapRUtils.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
public class FSMapRUtils extends FSUtils {
private static final Logger LOG = LoggerFactory.getLogger(FSMapRUtils.class);
+ @Override
public void recoverFileLease(final FileSystem fs, final Path p,
Configuration conf, CancelableProgressable reporter) throws IOException {
LOG.info("Recovering file " + p.toString() +
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
index f258e6c..04a3384 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
@@ -52,7 +52,7 @@ class FSRegionScanner implements Runnable {
/**
* Maps each region to the RS with highest locality for that region.
*/
- private Map<String,String> regionToBestLocalityRSMapping;
+ private final Map<String,String> regionToBestLocalityRSMapping;
/**
* Maps region encoded names to maps of hostnames to fractional locality of
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
index c76cd90..c3f3bd8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
@@ -479,7 +479,7 @@ public class FSTableDescriptors implements TableDescriptors {
// Clean away old versions
for (FileStatus file : status) {
Path path = file.getPath();
- if (file != mostCurrent) {
+ if (!file.equals(mostCurrent)) {
if (!fs.delete(file.getPath(), false)) {
LOG.warn("Failed cleanup of " + path);
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index d0276c0..3ee13c4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -3800,7 +3800,7 @@ public class HBaseFsck extends Configured implements Closeable {
@Override
public int hashCode() {
int hash = Arrays.hashCode(getRegionName());
- hash ^= getRegionId();
+ hash = (int) (hash ^ getRegionId());
hash ^= Arrays.hashCode(getStartKey());
hash ^= Arrays.hashCode(getEndKey());
hash ^= Boolean.valueOf(isOffline()).hashCode();
@@ -3808,7 +3808,7 @@ public class HBaseFsck extends Configured implements Closeable {
if (regionServer != null) {
hash ^= regionServer.hashCode();
}
- hash ^= modTime;
+ hash = (int) (hash ^ modTime);
return hash;
}
}
@@ -4054,7 +4054,7 @@ public class HBaseFsck extends Configured implements Closeable {
return -1;
}
// both l.hdfsEntry and r.hdfsEntry must not be null.
- return (int) (l.hdfsEntry.hri.getRegionId()- r.hdfsEntry.hri.getRegionId());
+ return Long.compare(l.hdfsEntry.hri.getRegionId(), r.hdfsEntry.hri.getRegionId());
}
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdLock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdLock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdLock.java
index e06805c..eba9acd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdLock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/IdLock.java
@@ -52,6 +52,7 @@ public class IdLock {
this.id = id;
}
+ @Override
public String toString() {
return "id=" + id + ", numWaiter=" + numWaiters + ", isLocked="
+ locked;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
index 7b9cbb6..75b8ccd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
@@ -685,7 +685,7 @@ public class RegionSplitter {
}
}
try {
- return splitClass.asSubclass(SplitAlgorithm.class).newInstance();
+ return splitClass.asSubclass(SplitAlgorithm.class).getDeclaredConstructor().newInstance();
} catch (Exception e) {
throw new IOException("Problem loading split algorithm: ", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RowBloomContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RowBloomContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RowBloomContext.java
index 2819b82..f1c9ad3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RowBloomContext.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RowBloomContext.java
@@ -38,6 +38,7 @@ public class RowBloomContext extends BloomContext {
super(bloomFilterWriter, comparator);
}
+ @Override
public void addLastBloomKey(Writer writer) throws IOException {
if (this.getLastCell() != null) {
byte[] key = CellUtil.copyRow(this.getLastCell());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java
index a897f40..d3da773 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ShutdownHookManager.java
@@ -52,10 +52,12 @@ abstract public class ShutdownHookManager {
private static class ShutdownHookManagerV1 extends ShutdownHookManager {
// priority is ignored in hadoop versions earlier than 2.0
- public void addShutdownHook(Thread shutdownHookThread, int priority) {
+ @Override
+ public void addShutdownHook(Thread shutdownHookThread, int priority) {
Runtime.getRuntime().addShutdownHook(shutdownHookThread);
}
+ @Override
public boolean removeShutdownHook(Runnable shutdownHook) {
Thread shutdownHookThread = null;
if (!(shutdownHook instanceof Thread)) {
@@ -67,6 +69,7 @@ abstract public class ShutdownHookManager {
};
private static class ShutdownHookManagerV2 extends ShutdownHookManager {
+ @Override
public void addShutdownHook(Thread shutdownHookThread, int priority) {
try {
Methods.call(shutdownHookManagerClass,
@@ -79,6 +82,7 @@ abstract public class ShutdownHookManager {
}
}
+ @Override
public boolean removeShutdownHook(Runnable shutdownHook) {
try {
return (Boolean)
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
index 74d502e..d9badfa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
@@ -455,7 +456,7 @@ public abstract class AbstractFSWALProvider<T extends AbstractFSWAL<?>> implemen
} catch (FileNotFoundException fnfe) {
// If the log was archived, continue reading from there
Path archivedLog = AbstractFSWALProvider.getArchivedLogPath(path, conf);
- if (path != archivedLog) {
+ if (!Objects.equals(path, archivedLog)) {
return openReader(archivedLog, conf);
} else {
throw fnfe;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
index 725f9ff..2105490 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
@@ -185,7 +185,7 @@ class DisabledWALProvider implements WALProvider {
public void sync() {
if (!this.listeners.isEmpty()) {
for (WALActionsListener listener : this.listeners) {
- listener.postSync(0l, 0);
+ listener.postSync(0L, 0);
}
}
}
@@ -195,6 +195,7 @@ class DisabledWALProvider implements WALProvider {
sync();
}
+ @Override
public Long startCacheFlush(final byte[] encodedRegionName, Map<byte[], Long>
flushedFamilyNamesToSeq) {
return startCacheFlush(encodedRegionName, flushedFamilyNamesToSeq.keySet());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
index 14505a8..f1662bc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/FSHLogProvider.java
@@ -63,7 +63,7 @@ public class FSHLogProvider extends AbstractFSWALProvider<FSHLog> {
ProtobufLogWriter.class, Writer.class);
Writer writer = null;
try {
- writer = logWriterClass.newInstance();
+ writer = logWriterClass.getDeclaredConstructor().newInstance();
writer.init(fs, path, conf, overwritable);
return writer;
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
index a3e54a5..a0ef817 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/RegionGroupingProvider.java
@@ -104,19 +104,14 @@ public class RegionGroupingProvider implements WALProvider {
}
LOG.info("Instantiating RegionGroupingStrategy of type " + clazz);
try {
- final RegionGroupingStrategy result = clazz.newInstance();
+ final RegionGroupingStrategy result = clazz.getDeclaredConstructor().newInstance();
result.init(conf, providerId);
return result;
- } catch (InstantiationException exception) {
+ } catch (Exception e) {
LOG.error("couldn't set up region grouping strategy, check config key " +
REGION_GROUPING_STRATEGY);
- LOG.debug("Exception details for failure to load region grouping strategy.", exception);
- throw new IOException("couldn't set up region grouping strategy", exception);
- } catch (IllegalAccessException exception) {
- LOG.error("couldn't set up region grouping strategy, check config key " +
- REGION_GROUPING_STRATEGY);
- LOG.debug("Exception details for failure to load region grouping strategy.", exception);
- throw new IOException("couldn't set up region grouping strategy", exception);
+ LOG.debug("Exception details for failure to load region grouping strategy.", e);
+ throw new IOException("couldn't set up region grouping strategy", e);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 4008bb0..d478e4f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -99,6 +99,7 @@ public interface WAL extends Closeable, WALFileLengthProvider {
* underlying resources after this call; i.e. filesystem based WALs can archive or
* delete files.
*/
+ @Override
void close() throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
index d70b8cd..d59c824 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
@@ -75,7 +75,7 @@ public class WALFactory implements WALFileLengthProvider {
multiwal(RegionGroupingProvider.class),
asyncfs(AsyncFSWALProvider.class);
- Class<? extends WALProvider> clazz;
+ final Class<? extends WALProvider> clazz;
Providers(Class<? extends WALProvider> clazz) {
this.clazz = clazz;
}
@@ -139,17 +139,13 @@ public class WALFactory implements WALFileLengthProvider {
List<WALActionsListener> listeners, String providerId) throws IOException {
LOG.info("Instantiating WALProvider of type " + clazz);
try {
- final WALProvider result = clazz.newInstance();
+ final WALProvider result = clazz.getDeclaredConstructor().newInstance();
result.init(this, conf, listeners, providerId);
return result;
- } catch (InstantiationException exception) {
- LOG.error("couldn't set up WALProvider, the configured class is " + clazz);
- LOG.debug("Exception details for failure to load WALProvider.", exception);
- throw new IOException("couldn't set up WALProvider", exception);
- } catch (IllegalAccessException exception) {
+ } catch (Exception e) {
LOG.error("couldn't set up WALProvider, the configured class is " + clazz);
- LOG.debug("Exception details for failure to load WALProvider.", exception);
- throw new IOException("couldn't set up WALProvider", exception);
+ LOG.debug("Exception details for failure to load WALProvider.", e);
+ throw new IOException("couldn't set up WALProvider", e);
}
}
@@ -294,7 +290,7 @@ public class WALFactory implements WALFileLengthProvider {
AbstractFSWALProvider.Reader reader = null;
while (true) {
try {
- reader = lrClass.newInstance();
+ reader = lrClass.getDeclaredConstructor().newInstance();
reader.init(fs, path, conf, null);
return reader;
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
index 983fae9..0a5acda 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java
@@ -470,8 +470,8 @@ public class WALKeyImpl implements WALKey {
@Override
public int hashCode() {
int result = Bytes.hashCode(this.encodedRegionName);
- result ^= getSequenceId();
- result ^= this.writeTime;
+ result = (int) (result ^ getSequenceId());
+ result = (int) (result ^ this.writeTime);
return result;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
index 22f7e1a..97f80e0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
@@ -1084,7 +1084,7 @@ public class WALSplitter {
protected EntryBuffers entryBuffers;
protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();
- protected ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =
+ protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =
new ConcurrentHashMap<>();
@@ -1645,8 +1645,10 @@ public class WALSplitter {
List<IOException> thrown, List<Path> paths)
throws InterruptedException, ExecutionException {
for (final Map.Entry<byte[], RegionEntryBuffer> buffer : entryBuffers.buffers.entrySet()) {
- LOG.info("Submitting writeThenClose of " + buffer.getValue().encodedRegionName);
+ LOG.info("Submitting writeThenClose of {}",
+ Arrays.toString(buffer.getValue().encodedRegionName));
completionService.submit(new Callable<Void>() {
+ @Override
public Void call() throws Exception {
Path dst = writeThenClose(buffer.getValue());
paths.add(dst);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java
index aec4bbd..44d3e87 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java
@@ -151,6 +151,7 @@ public class AcidGuaranteesTestTool extends AbstractHBaseTool {
table = connection.getTable(TABLE_NAME);
}
+ @Override
public void doAnAction() throws Exception {
// Pick a random row to write into
byte[] targetRow = targetRows[rand.nextInt(targetRows.length)];
@@ -197,6 +198,7 @@ public class AcidGuaranteesTestTool extends AbstractHBaseTool {
table = connection.getTable(TABLE_NAME);
}
+ @Override
public void doAnAction() throws Exception {
Get g = new Get(targetRow);
Result res = table.get(g);
@@ -264,6 +266,7 @@ public class AcidGuaranteesTestTool extends AbstractHBaseTool {
table = connection.getTable(TABLE_NAME);
}
+ @Override
public void doAnAction() throws Exception {
Scan s = new Scan();
for (byte[] family : targetFamilies) {
@@ -344,6 +347,7 @@ public class AcidGuaranteesTestTool extends AbstractHBaseTool {
}
// Add a flusher
ctx.addThread(new RepeatingTestThread(ctx) {
+ @Override
public void doAnAction() throws Exception {
try {
admin.flush(TABLE_NAME);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
index 92581b8..5f2ffb2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.NavigableMap;
import org.apache.hadoop.conf.Configuration;
@@ -383,7 +384,7 @@ public abstract class HBaseTestCase extends TestCase {
if (res_value != null) {
assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
" at timestamp " +
- timestamp, value, new String(res_value));
+ timestamp, value, new String(res_value, StandardCharsets.UTF_8));
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index a686e33..75abd5e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -33,6 +33,7 @@ import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
@@ -1596,7 +1597,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
if (status.getSecond() != 0) {
LOG.debug(status.getSecond() - status.getFirst() + "/" + status.getSecond()
+ " regions updated.");
- Thread.sleep(1 * 1000l);
+ Thread.sleep(1 * 1000L);
} else {
LOG.debug("All regions updated.");
break;
@@ -1983,7 +1984,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
expectedCount = 1;
}
if (count != expectedCount) {
- String row = new String(new byte[] {b1,b2,b3});
+ String row = new String(new byte[] {b1,b2,b3}, StandardCharsets.UTF_8);
throw new RuntimeException("Row:" + row + " has a seen count of " + count + " " +
"instead of " + expectedCount);
}
@@ -2079,7 +2080,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
get.setConsistency(Consistency.TIMELINE);
Result result = table.get(get);
assertTrue(failMsg, result.containsColumn(f, null));
- assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
+ assertEquals(failMsg, 1, result.getColumnCells(f, null).size());
Cell cell = result.getColumnLatestCell(f, null);
assertTrue(failMsg,
Bytes.equals(data, 0, data.length, cell.getValueArray(), cell.getValueOffset(),
@@ -2114,7 +2115,7 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
if (!present) continue;
assertTrue(failMsg, result.containsColumn(f, null));
- assertEquals(failMsg, result.getColumnCells(f, null).size(), 1);
+ assertEquals(failMsg, 1, result.getColumnCells(f, null).size());
Cell cell = result.getColumnLatestCell(f, null);
assertTrue(failMsg,
Bytes.equals(data, 0, data.length, cell.getValueArray(), cell.getValueOffset(),
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
index 378f6ec..9959e31 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
@@ -165,7 +165,8 @@ public class MiniHBaseCluster extends HBaseCluster {
@Override
public void run() {
try {
- this.user.runAs(new PrivilegedAction<Object>(){
+ this.user.runAs(new PrivilegedAction<Object>() {
+ @Override
public Object run() {
runRegionServer();
return null;
@@ -195,6 +196,7 @@ public class MiniHBaseCluster extends HBaseCluster {
@Override
public void abort(final String reason, final Throwable cause) {
this.user.runAs(new PrivilegedAction<Object>() {
+ @Override
public Object run() {
abortRegionServer(reason, cause);
return null;
@@ -497,6 +499,7 @@ public class MiniHBaseCluster extends HBaseCluster {
* Returns the current active master, if available.
* @return the active HMaster, null if none is active.
*/
+ @Override
public MasterService.BlockingInterface getMasterAdminService() {
return this.hbaseCluster.getActiveMaster().getMasterRpcServices();
}
@@ -588,6 +591,7 @@ public class MiniHBaseCluster extends HBaseCluster {
* masters left.
* @throws InterruptedException
*/
+ @Override
public boolean waitForActiveAndReadyMaster(long timeout) throws IOException {
List<JVMClusterUtil.MasterThread> mts;
long start = System.currentTimeMillis();
@@ -628,6 +632,7 @@ public class MiniHBaseCluster extends HBaseCluster {
/**
* Shut down the mini HBase cluster
*/
+ @Override
public void shutdown() throws IOException {
if (this.hbaseCluster != null) {
this.hbaseCluster.shutdown();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java
index 1d8de45..86ac2f8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MultithreadedTestUtil.java
@@ -119,6 +119,7 @@ public abstract class MultithreadedTestUtil {
this.ctx = ctx;
}
+ @Override
public void run() {
try {
doWork();
@@ -143,6 +144,7 @@ public abstract class MultithreadedTestUtil {
super(ctx);
}
+ @Override
public final void doWork() throws Exception {
try {
while (ctx.shouldRun() && !stopped) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java
index 06cfdcf..f21d79d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHDFSBlocksDistribution.java
@@ -49,6 +49,7 @@ public class TestHDFSBlocksDistribution {
}
public class MockHDFSBlocksDistribution extends HDFSBlocksDistribution {
+ @Override
public Map<String,HostAndWeight> getHostAndWeights() {
HashMap<String, HostAndWeight> map = new HashMap<>();
map.put("test", new HostAndWeight(null, 100));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
index c5cda27..1210361 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -93,7 +94,7 @@ public class TestIOFencing {
}
public abstract static class CompactionBlockerRegion extends HRegion {
- volatile int compactCount = 0;
+ AtomicInteger compactCount = new AtomicInteger();
volatile CountDownLatch compactionsBlocked = new CountDownLatch(0);
volatile CountDownLatch compactionsWaiting = new CountDownLatch(0);
@@ -129,7 +130,7 @@ public class TestIOFencing {
try {
return super.compact(compaction, store, throughputController);
} finally {
- compactCount++;
+ compactCount.getAndIncrement();
}
}
@@ -139,7 +140,7 @@ public class TestIOFencing {
try {
return super.compact(compaction, store, throughputController, user);
} finally {
- compactCount++;
+ compactCount.getAndIncrement();
}
}
@@ -336,7 +337,7 @@ public class TestIOFencing {
}
LOG.info("Allowing compaction to proceed");
compactingRegion.allowCompactions();
- while (compactingRegion.compactCount == 0) {
+ while (compactingRegion.compactCount.get() == 0) {
Thread.sleep(1000);
}
// The server we killed stays up until the compaction that was started before it was killed
@@ -349,7 +350,7 @@ public class TestIOFencing {
FIRST_BATCH_COUNT + SECOND_BATCH_COUNT);
admin.majorCompact(TABLE_NAME);
startWaitTime = System.currentTimeMillis();
- while (newRegion.compactCount == 0) {
+ while (newRegion.compactCount.get() == 0) {
Thread.sleep(1000);
assertTrue("New region never compacted",
System.currentTimeMillis() - startWaitTime < 180000);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessorNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessorNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessorNoCluster.java
index 7b6c5a5..620abef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessorNoCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessorNoCluster.java
@@ -163,6 +163,7 @@ public class TestMetaTableAccessorNoCluster {
.thenThrow(new ServiceException("Server not running (2 of 3)"))
.thenThrow(new ServiceException("Server not running (3 of 3)"))
.thenAnswer(new Answer<ScanResponse>() {
+ @Override
public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
((HBaseRpcController) invocation.getArgument(0)).setCellScanner(CellUtil
.createCellScanner(cellScannables));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java
index 71492b1..63d2cc2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java
@@ -127,14 +127,14 @@ public class TestMetaTableLocator {
assertEquals(state, MetaTableLocator.getMetaRegionState(this.watcher).getState());
}
MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
- assertEquals(mtl.getMetaRegionLocation(this.watcher), SN);
+ assertEquals(SN, mtl.getMetaRegionLocation(this.watcher));
assertEquals(RegionState.State.OPEN,
MetaTableLocator.getMetaRegionState(this.watcher).getState());
mtl.deleteMetaLocation(this.watcher);
assertNull(MetaTableLocator.getMetaRegionState(this.watcher).getServerName());
- assertEquals(MetaTableLocator.getMetaRegionState(this.watcher).getState(),
- RegionState.State.OFFLINE);
+ assertEquals(RegionState.State.OFFLINE,
+ MetaTableLocator.getMetaRegionState(this.watcher).getState());
assertNull(mtl.getMetaRegionLocation(this.watcher));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java
index bdb74a4..24a8830 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMovedRegionsCleaner.java
@@ -48,6 +48,7 @@ import java.io.IOException;
super(conf);
}
+ @Override
protected int movedRegionCleanerPeriod() {
return 500;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java
index 1a0215e..acf7861 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMultiVersions.java
@@ -101,6 +101,7 @@ public class TestMultiVersions {
// TODO: Remove these deprecated classes or pull them in here if this is
// only test using them.
TimestampTestBase.doTestDelete(table, new FlushCache() {
+ @Override
public void flushcache() throws IOException {
UTIL.getHBaseCluster().flushcache();
}
@@ -109,6 +110,7 @@ public class TestMultiVersions {
// Perhaps drop and readd the table between tests so the former does
// not pollute this latter? Or put into separate tests.
TimestampTestBase.doTestTimestampScanning(table, new FlushCache() {
+ @Override
public void flushcache() throws IOException {
UTIL.getMiniHBaseCluster().flushcache();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java
index c1b5dac..cfc5c2f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerName.java
@@ -94,9 +94,8 @@ public class TestServerName {
ServerName.valueOf("www.example.org", 1234, 5678).toString());
assertEquals(sn.toString(),
ServerName.valueOf("www.example.org:1234", 5678).toString());
- assertEquals(sn.toString(),
- "www.example.org" + ServerName.SERVERNAME_SEPARATOR + "1234" +
- ServerName.SERVERNAME_SEPARATOR + "5678");
+ assertEquals("www.example.org" + ServerName.SERVERNAME_SEPARATOR + "1234"
+ + ServerName.SERVERNAME_SEPARATOR + "5678", sn.toString());
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java
index 2639821..4aeedb9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestServerSideScanMetricsFromClientSide.java
@@ -205,7 +205,8 @@ public class TestServerSideScanMetricsFromClientSide {
}
// The filter should filter out all rows, but we still expect to see every row.
- Filter filter = new RowFilter(CompareOperator.EQUAL, new BinaryComparator("xyz".getBytes()));
+ Filter filter =
+ new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("xyz")));
scan = new Scan(baseScan);
scan.setFilter(filter);
testMetric(scan, ServerSideScanMetrics.COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME, ROWS.length);
@@ -255,7 +256,8 @@ public class TestServerSideScanMetricsFromClientSide {
testRowsFilteredMetric(baseScan, null, 0);
// Row filter doesn't match any row key. All rows should be filtered
- Filter filter = new RowFilter(CompareOperator.EQUAL, new BinaryComparator("xyz".getBytes()));
+ Filter filter =
+ new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("xyz")));
testRowsFilteredMetric(baseScan, filter, ROWS.length);
// Filter will return results containing only the first key. Number of entire rows filtered
@@ -269,7 +271,7 @@ public class TestServerSideScanMetricsFromClientSide {
testRowsFilteredMetric(baseScan, filter, 0);
// Column prefix will NOT find any matching qualifier on any row. All rows should be filtered
- filter = new ColumnPrefixFilter("xyz".getBytes());
+ filter = new ColumnPrefixFilter(Bytes.toBytes("xyz"));
testRowsFilteredMetric(baseScan, filter, ROWS.length);
// Matching column value should exist in each row. No rows should be filtered.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
index fba429a..f81a36d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
@@ -421,7 +421,7 @@ public class TestAdmin1 {
this.admin.createTable(htd);
Table table = TEST_UTIL.getConnection().getTable(htd.getTableName());
TableDescriptor confirmedHtd = table.getDescriptor();
- assertEquals(TableDescriptor.COMPARATOR.compare(htd, confirmedHtd), 0);
+ assertEquals(0, TableDescriptor.COMPARATOR.compare(htd, confirmedHtd));
MetaTableAccessor.fullScanMetaAndPrint(TEST_UTIL.getConnection());
table.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
index 05b8edc..57bd158 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java
@@ -303,7 +303,7 @@ public class TestAdmin2 {
TableName tableName = TableName
.valueOf("testTableNotFoundExceptionWithoutAnyTables");
Table ht = TEST_UTIL.getConnection().getTable(tableName);
- ht.get(new Get("e".getBytes()));
+ ht.get(new Get(Bytes.toBytes("e")));
}
@Test (timeout=300000)
@@ -582,8 +582,9 @@ public class TestAdmin2 {
}
// Before the fix for HBASE-6146, the below table creation was failing as the hbase:meta table
// actually getting disabled by the disableTable() call.
- HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName().getBytes()));
- HColumnDescriptor hcd = new HColumnDescriptor("cf1".getBytes());
+ HTableDescriptor htd =
+ new HTableDescriptor(TableName.valueOf(Bytes.toBytes(name.getMethodName())));
+ HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("cf1"));
htd.addFamily(hcd);
TEST_UTIL.getHBaseAdmin().createTable(htd);
}
@@ -695,13 +696,13 @@ public class TestAdmin2 {
assertTrue(decommissionedRegionServers.isEmpty());
final TableName tableName = TableName.valueOf(name.getMethodName());
- TEST_UTIL.createMultiRegionTable(tableName, "f".getBytes(), 6);
+ TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 6);
ArrayList<ServerName> clusterRegionServers =
new ArrayList<>(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
.getLiveServerMetrics().keySet());
- assertEquals(clusterRegionServers.size(), 3);
+ assertEquals(3, clusterRegionServers.size());
HashMap<ServerName, List<RegionInfo>> serversToDecommssion = new HashMap<>();
// Get a server that has regions. We will decommission two of the servers,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncClusterAdminApi2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncClusterAdminApi2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncClusterAdminApi2.java
index ab8ebb5..3344c4b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncClusterAdminApi2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncClusterAdminApi2.java
@@ -56,6 +56,7 @@ public class TestAsyncClusterAdminApi2 extends TestAsyncAdminBase {
}
@Before
+ @Override
public void setUp() throws Exception {
TEST_UTIL.startMiniCluster(1, 3);
ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get();
@@ -63,6 +64,7 @@ public class TestAsyncClusterAdminApi2 extends TestAsyncAdminBase {
}
@After
+ @Override
public void tearDown() throws Exception {
IOUtils.closeQuietly(ASYNC_CONN);
TEST_UTIL.shutdownMiniCluster();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncDecommissionAdminApi.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncDecommissionAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncDecommissionAdminApi.java
index 8c2b060..8968b39 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncDecommissionAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncDecommissionAdminApi.java
@@ -49,7 +49,7 @@ public class TestAsyncDecommissionAdminApi extends TestAsyncAdminBase {
new ArrayList<>(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).get()
.getLiveServerMetrics().keySet());
- assertEquals(clusterRegionServers.size(), 2);
+ assertEquals(2, clusterRegionServers.size());
HashMap<ServerName, List<RegionInfo>> serversToDecommssion = new HashMap<>();
// Get a server that has regions. We will decommission one of the servers,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcedureAdminApi.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcedureAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcedureAdminApi.java
index 7a2c00f..d50e039 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcedureAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcedureAdminApi.java
@@ -90,7 +90,7 @@ public class TestAsyncProcedureAdminApi extends TestAsyncAdminBase {
byte[] result = admin.execProcedureWithReturn(SimpleMasterProcedureManager.SIMPLE_SIGNATURE,
"myTest2", new HashMap<>()).get();
assertArrayEquals("Incorrect return data from execProcedure",
- SimpleMasterProcedureManager.SIMPLE_DATA.getBytes(), result);
+ Bytes.toBytes(SimpleMasterProcedureManager.SIMPLE_DATA), result);
}
@Test
[20/50] [abbrv] hbase git commit: HBASE-19520 Add UTs for the new
lock type PEER
Posted by zh...@apache.org.
HBASE-19520 Add UTs for the new lock type PEER
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/211e6f53
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/211e6f53
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/211e6f53
Branch: refs/heads/HBASE-19397-branch-2
Commit: 211e6f5394bf4c2ae5be9c2ea41211ede8968df5
Parents: 521065e
Author: Guanghao Zhang <zg...@apache.org>
Authored: Wed Dec 20 16:43:38 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../procedure/MasterProcedureScheduler.java | 9 +-
.../procedure/TestMasterProcedureScheduler.java | 65 ++++++++-
...TestMasterProcedureSchedulerConcurrency.java | 135 +++++++++++++++++++
3 files changed, 201 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/211e6f53/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index 8ff2d12..a25217c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -389,6 +389,13 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
while (tableIter.hasNext()) {
count += tableIter.next().size();
}
+
+ // Peer queues
+ final AvlTreeIterator<PeerQueue> peerIter = new AvlTreeIterator<>(peerMap);
+ while (peerIter.hasNext()) {
+ count += peerIter.next().size();
+ }
+
return count;
}
@@ -1041,7 +1048,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
* @see #wakePeerExclusiveLock(Procedure, String)
* @param procedure the procedure trying to acquire the lock
* @param peerId peer to lock
- * @return true if the procedure has to wait for the per to be available
+ * @return true if the procedure has to wait for the peer to be available
*/
public boolean waitPeerExclusiveLock(Procedure<?> procedure, String peerId) {
schedLock();
http://git-wip-us.apache.org/repos/asf/hbase/blob/211e6f53/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
index 0291165..fd77e1f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
@@ -905,6 +905,27 @@ public class TestMasterProcedureScheduler {
}
}
+ public static class TestPeerProcedure extends TestProcedure implements PeerProcedureInterface {
+ private final String peerId;
+ private final PeerOperationType opType;
+
+ public TestPeerProcedure(long procId, String peerId, PeerOperationType opType) {
+ super(procId);
+ this.peerId = peerId;
+ this.opType = opType;
+ }
+
+ @Override
+ public String getPeerId() {
+ return peerId;
+ }
+
+ @Override
+ public PeerOperationType getPeerOperationType() {
+ return opType;
+ }
+ }
+
private static LockProcedure createLockProcedure(LockType lockType, long procId) throws Exception {
LockProcedure procedure = new LockProcedure();
@@ -927,22 +948,19 @@ public class TestMasterProcedureScheduler {
return createLockProcedure(LockType.SHARED, procId);
}
- private static void assertLockResource(LockedResource resource,
- LockedResourceType resourceType, String resourceName)
- {
+ private static void assertLockResource(LockedResource resource, LockedResourceType resourceType,
+ String resourceName) {
assertEquals(resourceType, resource.getResourceType());
assertEquals(resourceName, resource.getResourceName());
}
- private static void assertExclusiveLock(LockedResource resource, Procedure<?> procedure)
- {
+ private static void assertExclusiveLock(LockedResource resource, Procedure<?> procedure) {
assertEquals(LockType.EXCLUSIVE, resource.getLockType());
assertEquals(procedure, resource.getExclusiveLockOwnerProcedure());
assertEquals(0, resource.getSharedLockCount());
}
- private static void assertSharedLock(LockedResource resource, int lockCount)
- {
+ private static void assertSharedLock(LockedResource resource, int lockCount) {
assertEquals(LockType.SHARED, resource.getLockType());
assertEquals(lockCount, resource.getSharedLockCount());
}
@@ -1027,6 +1045,39 @@ public class TestMasterProcedureScheduler {
}
@Test
+ public void testListLocksPeer() throws Exception {
+ String peerId = "1";
+ LockProcedure procedure = createExclusiveLockProcedure(4);
+ queue.waitPeerExclusiveLock(procedure, peerId);
+
+ List<LockedResource> locks = queue.getLocks();
+ assertEquals(1, locks.size());
+
+ LockedResource resource = locks.get(0);
+ assertLockResource(resource, LockedResourceType.PEER, peerId);
+ assertExclusiveLock(resource, procedure);
+ assertTrue(resource.getWaitingProcedures().isEmpty());
+
+ // Try to acquire the exclusive lock again with same procedure
+ assertFalse(queue.waitPeerExclusiveLock(procedure, peerId));
+
+ // Try to acquire the exclusive lock again with new procedure
+ LockProcedure procedure2 = createExclusiveLockProcedure(5);
+ assertTrue(queue.waitPeerExclusiveLock(procedure2, peerId));
+
+ // Same peerId, still only has 1 LockedResource
+ locks = queue.getLocks();
+ assertEquals(1, locks.size());
+
+ resource = locks.get(0);
+ assertLockResource(resource, LockedResourceType.PEER, peerId);
+ // LockedResource owner still is the origin procedure
+ assertExclusiveLock(resource, procedure);
+ // The new procedure should in the waiting list
+ assertEquals(1, resource.getWaitingProcedures().size());
+ }
+
+ @Test
public void testListLocksWaiting() throws Exception {
LockProcedure procedure1 = createExclusiveLockProcedure(1);
queue.waitTableExclusiveLock(procedure1, TableName.valueOf("ns4", "table4"));
http://git-wip-us.apache.org/repos/asf/hbase/blob/211e6f53/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java
index 2e8e52a..4e67a63 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java
@@ -26,6 +26,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface.PeerOperationType;
+import org.apache.hadoop.hbase.master.procedure.TestMasterProcedureScheduler.TestPeerProcedure;
import org.apache.hadoop.hbase.master.procedure.TestMasterProcedureScheduler.TestTableProcedure;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -63,6 +65,85 @@ public class TestMasterProcedureSchedulerConcurrency {
queue.clear();
}
+ @Test
+ public void testConcurrentPeerOperations() throws Exception {
+ TestPeerProcedureSet procSet = new TestPeerProcedureSet(queue);
+
+ int NUM_ITEMS = 10;
+ int NUM_PEERS = 5;
+ AtomicInteger opsCount = new AtomicInteger(0);
+ for (int i = 0; i < NUM_PEERS; ++i) {
+ String peerId = String.format("test-peer-%04d", i);
+ for (int j = 1; j < NUM_ITEMS; ++j) {
+ procSet.addBack(new TestPeerProcedure(i * 100 + j, peerId, PeerOperationType.ADD));
+ opsCount.incrementAndGet();
+ }
+ }
+ assertEquals(opsCount.get(), queue.size());
+
+ Thread[] threads = new Thread[NUM_PEERS * 2];
+ HashSet<String> concurrentPeers = new HashSet<>();
+ ArrayList<String> failures = new ArrayList<>();
+ AtomicInteger concurrentCount = new AtomicInteger(0);
+ for (int i = 0; i < threads.length; ++i) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ while (opsCount.get() > 0) {
+ try {
+ TestPeerProcedure proc = procSet.acquire();
+ if (proc == null) {
+ queue.signalAll();
+ if (opsCount.get() > 0) {
+ continue;
+ }
+ break;
+ }
+
+ String peerId = proc.getPeerId();
+ synchronized (concurrentPeers) {
+ assertTrue("unexpected concurrency on " + peerId, concurrentPeers.add(peerId));
+ }
+ assertTrue(opsCount.decrementAndGet() >= 0);
+
+ try {
+ long procId = proc.getProcId();
+ int concurrent = concurrentCount.incrementAndGet();
+ assertTrue("inc-concurrent="+ concurrent +" 1 <= concurrent <= "+ NUM_PEERS,
+ concurrent >= 1 && concurrent <= NUM_PEERS);
+ LOG.debug("[S] peerId="+ peerId +" procId="+ procId +" concurrent="+ concurrent);
+ Thread.sleep(2000);
+ concurrent = concurrentCount.decrementAndGet();
+ LOG.debug("[E] peerId="+ peerId +" procId="+ procId +" concurrent="+ concurrent);
+ assertTrue("dec-concurrent=" + concurrent, concurrent < NUM_PEERS);
+ } finally {
+ synchronized (concurrentPeers) {
+ assertTrue(concurrentPeers.remove(peerId));
+ }
+ procSet.release(proc);
+ }
+ } catch (Throwable e) {
+ LOG.error("Failed " + e.getMessage(), e);
+ synchronized (failures) {
+ failures.add(e.getMessage());
+ }
+ } finally {
+ queue.signalAll();
+ }
+ }
+ }
+ };
+ threads[i].start();
+ }
+
+ for (int i = 0; i < threads.length; ++i) {
+ threads[i].join();
+ }
+ assertTrue(failures.toString(), failures.isEmpty());
+ assertEquals(0, opsCount.get());
+ assertEquals(0, queue.size());
+ }
+
/**
* Verify that "write" operations for a single table are serialized,
* but different tables can be executed in parallel.
@@ -221,4 +302,58 @@ public class TestMasterProcedureSchedulerConcurrency {
return ((TableProcedureInterface)proc).getTableOperationType();
}
}
+
+ public static class TestPeerProcedureSet {
+ private final MasterProcedureScheduler queue;
+
+ public TestPeerProcedureSet(final MasterProcedureScheduler queue) {
+ this.queue = queue;
+ }
+
+ public void addBack(TestPeerProcedure proc) {
+ queue.addBack(proc);
+ }
+
+ public TestPeerProcedure acquire() {
+ TestPeerProcedure proc = null;
+ boolean waiting = true;
+ while (waiting && queue.size() > 0) {
+ proc = (TestPeerProcedure) queue.poll(100000000L);
+ if (proc == null) {
+ continue;
+ }
+ switch (proc.getPeerOperationType()) {
+ case ADD:
+ case REMOVE:
+ case ENABLE:
+ case DISABLE:
+ case UPDATE_CONFIG:
+ waiting = queue.waitPeerExclusiveLock(proc, proc.getPeerId());
+ break;
+ case REFRESH:
+ waiting = false;
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+ return proc;
+ }
+
+ public void release(TestPeerProcedure proc) {
+ switch (proc.getPeerOperationType()) {
+ case ADD:
+ case REMOVE:
+ case ENABLE:
+ case DISABLE:
+ case UPDATE_CONFIG:
+ queue.wakePeerExclusiveLock(proc, proc.getPeerId());
+ break;
+ case REFRESH:
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+ }
}
[42/50] [abbrv] hbase git commit: HBASE-19687 Move the logic in
ReplicationZKNodeCleaner to ReplicationChecker and remove
ReplicationZKNodeCleanerChore
Posted by zh...@apache.org.
HBASE-19687 Move the logic in ReplicationZKNodeCleaner to ReplicationChecker and remove ReplicationZKNodeCleanerChore
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ba414a70
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ba414a70
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ba414a70
Branch: refs/heads/HBASE-19397-branch-2
Commit: ba414a70922aa80a26e4751daf9d951b8391c87f
Parents: 4caceda
Author: zhangduo <zh...@apache.org>
Authored: Wed Jan 3 09:39:44 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../replication/VerifyReplication.java | 6 +-
.../hbase/replication/ReplicationPeers.java | 26 +--
.../hbase/replication/ReplicationUtils.java | 69 +++++++
.../replication/TestReplicationStateBasic.java | 2 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 13 --
.../cleaner/ReplicationZKNodeCleaner.java | 192 -------------------
.../cleaner/ReplicationZKNodeCleanerChore.java | 54 ------
.../replication/ReplicationPeerManager.java | 18 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 13 +-
.../hbase/util/hbck/ReplicationChecker.java | 109 +++++++----
.../cleaner/TestReplicationZKNodeCleaner.java | 109 -----------
.../hbase/util/TestHBaseFsckReplication.java | 101 ++++++++++
.../hadoop/hbase/util/hbck/HbckTestingUtil.java | 6 +-
13 files changed, 259 insertions(+), 459 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index fe45762..fac4875 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -50,8 +50,8 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -345,10 +345,10 @@ public class VerifyReplication extends Configured implements Tool {
}
});
ReplicationPeerStorage storage =
- ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf);
+ ReplicationStorageFactory.getReplicationPeerStorage(localZKW, conf);
ReplicationPeerConfig peerConfig = storage.getPeerConfig(peerId);
return Pair.newPair(peerConfig,
- ReplicationPeers.getPeerClusterConfiguration(peerConfig, conf));
+ ReplicationUtils.getPeerClusterConfiguration(peerConfig, conf));
} catch (ReplicationException e) {
throw new IOException("An error occurred while trying to connect to the remove peer cluster",
e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 45940a5..fcbc350 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -17,14 +17,11 @@
*/
package org.apache.hadoop.hbase.replication;
-import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CompoundConfiguration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -106,25 +103,6 @@ public class ReplicationPeers {
return Collections.unmodifiableSet(peerCache.keySet());
}
- public static Configuration getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
- Configuration baseConf) throws ReplicationException {
- Configuration otherConf;
- try {
- otherConf = HBaseConfiguration.createClusterConf(baseConf, peerConfig.getClusterKey());
- } catch (IOException e) {
- throw new ReplicationException("Can't get peer configuration for peer " + peerConfig, e);
- }
-
- if (!peerConfig.getConfiguration().isEmpty()) {
- CompoundConfiguration compound = new CompoundConfiguration();
- compound.add(otherConf);
- compound.addStringMap(peerConfig.getConfiguration());
- return compound;
- }
-
- return otherConf;
- }
-
public PeerState refreshPeerState(String peerId) throws ReplicationException {
ReplicationPeerImpl peer = peerCache.get(peerId);
if (peer == null) {
@@ -158,7 +136,7 @@ public class ReplicationPeers {
private ReplicationPeerImpl createPeer(String peerId) throws ReplicationException {
ReplicationPeerConfig peerConfig = peerStorage.getPeerConfig(peerId);
boolean enabled = peerStorage.isPeerEnabled(peerId);
- return new ReplicationPeerImpl(getPeerClusterConfiguration(peerConfig, conf), peerId, enabled,
- peerConfig);
+ return new ReplicationPeerImpl(ReplicationUtils.getPeerClusterConfiguration(peerConfig, conf),
+ peerId, enabled, peerConfig);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
new file mode 100644
index 0000000..7b676ca
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CompoundConfiguration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Helper class for replication.
+ */
+@InterfaceAudience.Private
+public final class ReplicationUtils {
+
+ private ReplicationUtils() {
+ }
+
+ public static Configuration getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
+ Configuration baseConf) throws ReplicationException {
+ Configuration otherConf;
+ try {
+ otherConf = HBaseConfiguration.createClusterConf(baseConf, peerConfig.getClusterKey());
+ } catch (IOException e) {
+ throw new ReplicationException("Can't get peer configuration for peer " + peerConfig, e);
+ }
+
+ if (!peerConfig.getConfiguration().isEmpty()) {
+ CompoundConfiguration compound = new CompoundConfiguration();
+ compound.add(otherConf);
+ compound.addStringMap(peerConfig.getConfiguration());
+ return compound;
+ }
+
+ return otherConf;
+ }
+
+ public static void removeAllQueues(ReplicationQueueStorage queueStorage, String peerId)
+ throws ReplicationException {
+ for (ServerName replicator : queueStorage.getListOfReplicators()) {
+ List<String> queueIds = queueStorage.getAllQueues(replicator);
+ for (String queueId : queueIds) {
+ ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
+ if (queueInfo.getPeerId().equals(peerId)) {
+ queueStorage.removeQueue(replicator, queueId);
+ }
+ }
+ queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
index f3eeccc..fccffb5 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
@@ -240,7 +240,7 @@ public abstract class TestReplicationStateBasic {
rp.getPeerStorage().addPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO), true);
assertNumberOfPeers(2);
- assertEquals(KEY_ONE, ZKConfig.getZooKeeperClusterKey(ReplicationPeers
+ assertEquals(KEY_ONE, ZKConfig.getZooKeeperClusterKey(ReplicationUtils
.getPeerClusterConfiguration(rp.getPeerStorage().getPeerConfig(ID_ONE), rp.getConf())));
rp.getPeerStorage().removePeer(ID_ONE);
rp.removePeer(ID_ONE);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index b0a820e..bfbc6a2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -109,8 +109,6 @@ import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
import org.apache.hadoop.hbase.master.cleaner.ReplicationMetaCleaner;
-import org.apache.hadoop.hbase.master.cleaner.ReplicationZKNodeCleaner;
-import org.apache.hadoop.hbase.master.cleaner.ReplicationZKNodeCleanerChore;
import org.apache.hadoop.hbase.master.locking.LockManager;
import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
@@ -364,7 +362,6 @@ public class HMaster extends HRegionServer implements MasterServices {
CatalogJanitor catalogJanitorChore;
private ReplicationMetaCleaner replicationMetaCleaner;
- private ReplicationZKNodeCleanerChore replicationZKNodeCleanerChore;
private LogCleaner logCleaner;
private HFileCleaner hfileCleaner;
private ExpiredMobFileCleanerChore expiredMobFileCleanerChore;
@@ -1176,15 +1173,6 @@ public class HMaster extends HRegionServer implements MasterServices {
if (LOG.isTraceEnabled()) {
LOG.trace("Started service threads");
}
-
- // Start replication zk node cleaner
- try {
- replicationZKNodeCleanerChore = new ReplicationZKNodeCleanerChore(this, cleanerInterval,
- new ReplicationZKNodeCleaner(this.conf, this.getZooKeeper(), this));
- getChoreService().scheduleChore(replicationZKNodeCleanerChore);
- } catch (Exception e) {
- LOG.error("start replicationZKNodeCleanerChore failed", e);
- }
replicationMetaCleaner = new ReplicationMetaCleaner(this, this, cleanerInterval);
getChoreService().scheduleChore(replicationMetaCleaner);
}
@@ -1209,7 +1197,6 @@ public class HMaster extends HRegionServer implements MasterServices {
// Clean up and close up shop
if (this.logCleaner != null) this.logCleaner.cancel(true);
if (this.hfileCleaner != null) this.hfileCleaner.cancel(true);
- if (this.replicationZKNodeCleanerChore != null) this.replicationZKNodeCleanerChore.cancel(true);
if (this.replicationMetaCleaner != null) this.replicationMetaCleaner.cancel(true);
if (this.quotaManager != null) this.quotaManager.stop();
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
deleted file mode 100644
index f2c3ec9..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.cleaner;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Used to clean the replication queues belonging to the peer which does not exist.
- */
-@InterfaceAudience.Private
-public class ReplicationZKNodeCleaner {
- private static final Logger LOG = LoggerFactory.getLogger(ReplicationZKNodeCleaner.class);
- private final ReplicationQueueStorage queueStorage;
- private final ReplicationPeerStorage peerStorage;
- private final ReplicationQueueDeletor queueDeletor;
-
- public ReplicationZKNodeCleaner(Configuration conf, ZKWatcher zkw, Abortable abortable)
- throws IOException {
- this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
- this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zkw, conf);
- this.queueDeletor = new ReplicationQueueDeletor(zkw, conf, abortable);
- }
-
- /**
- * @return undeletedQueues replicator with its queueIds for removed peers
- * @throws IOException
- */
- public Map<ServerName, List<String>> getUnDeletedQueues() throws IOException {
- Map<ServerName, List<String>> undeletedQueues = new HashMap<>();
- try {
- Set<String> peerIds = new HashSet<>(peerStorage.listPeerIds());
- List<ServerName> replicators = this.queueStorage.getListOfReplicators();
- if (replicators == null || replicators.isEmpty()) {
- return undeletedQueues;
- }
- for (ServerName replicator : replicators) {
- List<String> queueIds = this.queueStorage.getAllQueues(replicator);
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (!peerIds.contains(queueInfo.getPeerId())) {
- undeletedQueues.computeIfAbsent(replicator, (key) -> new ArrayList<>()).add(queueId);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Undeleted replication queue for removed peer found: "
- + String.format("[removedPeerId=%s, replicator=%s, queueId=%s]",
- queueInfo.getPeerId(), replicator, queueId));
- }
- }
- }
- }
- } catch (ReplicationException ke) {
- throw new IOException("Failed to get the replication queues of all replicators", ke);
- }
- return undeletedQueues;
- }
-
- /**
- * @return undeletedHFileRefsQueue replicator with its undeleted queueIds for removed peers in
- * hfile-refs queue
- */
- public Set<String> getUnDeletedHFileRefsQueues() throws IOException {
- Set<String> undeletedHFileRefsQueue = new HashSet<>();
- String hfileRefsZNode = queueDeletor.getHfileRefsZNode();
- try {
- Set<String> peerIds = new HashSet<>(peerStorage.listPeerIds());
- List<String> listOfPeers = this.queueStorage.getAllPeersFromHFileRefsQueue();
- Set<String> peers = new HashSet<>(listOfPeers);
- peers.removeAll(peerIds);
- if (!peers.isEmpty()) {
- undeletedHFileRefsQueue.addAll(peers);
- }
- } catch (ReplicationException e) {
- throw new IOException("Failed to get list of all peers from hfile-refs znode "
- + hfileRefsZNode, e);
- }
- return undeletedHFileRefsQueue;
- }
-
- private class ReplicationQueueDeletor extends ReplicationStateZKBase {
-
- ReplicationQueueDeletor(ZKWatcher zk, Configuration conf, Abortable abortable) {
- super(zk, conf, abortable);
- }
-
- /**
- * @param replicator The regionserver which has undeleted queue
- * @param queueId The undeleted queue id
- */
- void removeQueue(final ServerName replicator, final String queueId) throws IOException {
- String queueZnodePath =
- ZNodePaths.joinZNode(ZNodePaths.joinZNode(this.queuesZNode, replicator.getServerName()),
- queueId);
- try {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (!peerStorage.listPeerIds().contains(queueInfo.getPeerId())) {
- ZKUtil.deleteNodeRecursively(this.zookeeper, queueZnodePath);
- LOG.info("Successfully removed replication queue, replicator: " + replicator
- + ", queueId: " + queueId);
- }
- } catch (ReplicationException | KeeperException e) {
- throw new IOException("Failed to delete queue, replicator: " + replicator + ", queueId: "
- + queueId);
- }
- }
-
- /**
- * @param hfileRefsQueueId The undeleted hfile-refs queue id
- * @throws IOException
- */
- void removeHFileRefsQueue(final String hfileRefsQueueId) throws IOException {
- String node = ZNodePaths.joinZNode(this.hfileRefsZNode, hfileRefsQueueId);
- try {
- if (!peerStorage.listPeerIds().contains(hfileRefsQueueId)) {
- ZKUtil.deleteNodeRecursively(this.zookeeper, node);
- LOG.info("Successfully removed hfile-refs queue " + hfileRefsQueueId + " from path "
- + hfileRefsZNode);
- }
- } catch (ReplicationException | KeeperException e) {
- throw new IOException("Failed to delete hfile-refs queue " + hfileRefsQueueId
- + " from path " + hfileRefsZNode, e);
- }
- }
-
- String getHfileRefsZNode() {
- return this.hfileRefsZNode;
- }
- }
-
- /**
- * Remove the undeleted replication queue's zk node for removed peers.
- * @param undeletedQueues replicator with its queueIds for removed peers
- * @throws IOException
- */
- public void removeQueues(final Map<ServerName, List<String>> undeletedQueues) throws IOException {
- for (Entry<ServerName, List<String>> replicatorAndQueueIds : undeletedQueues.entrySet()) {
- ServerName replicator = replicatorAndQueueIds.getKey();
- for (String queueId : replicatorAndQueueIds.getValue()) {
- queueDeletor.removeQueue(replicator, queueId);
- }
- }
- }
-
- /**
- * Remove the undeleted hfile-refs queue's zk node for removed peers.
- * @param undeletedHFileRefsQueues replicator with its undeleted queueIds for removed peers in
- * hfile-refs queue
- * @throws IOException
- */
- public void removeHFileRefsQueues(final Set<String> undeletedHFileRefsQueues) throws IOException {
- for (String hfileRefsQueueId : undeletedHFileRefsQueues) {
- queueDeletor.removeHFileRefsQueue(hfileRefsQueueId);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleanerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleanerChore.java
deleted file mode 100644
index 19ca804..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleanerChore.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.cleaner;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hbase.ScheduledChore;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.Stoppable;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Chore that will clean the replication queues belonging to the peer which does not exist.
- */
-@InterfaceAudience.Private
-public class ReplicationZKNodeCleanerChore extends ScheduledChore {
- private static final Logger LOG = LoggerFactory.getLogger(ReplicationZKNodeCleanerChore.class);
- private final ReplicationZKNodeCleaner cleaner;
-
- public ReplicationZKNodeCleanerChore(Stoppable stopper, int period,
- ReplicationZKNodeCleaner cleaner) {
- super("ReplicationZKNodeCleanerChore", stopper, period);
- this.cleaner = cleaner;
- }
-
- @Override
- protected void chore() {
- try {
- Map<ServerName, List<String>> undeletedQueues = cleaner.getUnDeletedQueues();
- cleaner.removeQueues(undeletedQueues);
- } catch (IOException e) {
- LOG.warn("Failed to clean replication zk node", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 1414d22..696b2d7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -216,19 +217,6 @@ public class ReplicationPeerManager {
return desc != null ? Optional.of(desc.getPeerConfig()) : Optional.empty();
}
- private void removeAllQueues0(String peerId) throws ReplicationException {
- for (ServerName replicator : queueStorage.getListOfReplicators()) {
- List<String> queueIds = queueStorage.getAllQueues(replicator);
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (queueInfo.getPeerId().equals(peerId)) {
- queueStorage.removeQueue(replicator, queueId);
- }
- }
- queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
- }
- }
-
public void removeAllQueuesAndHFileRefs(String peerId) throws ReplicationException {
// Here we need two passes to address the problem of claimQueue. Maybe a claimQueue is still
// on-going when the refresh peer config procedure is done, if a RS which has already been
@@ -241,8 +229,8 @@ public class ReplicationPeerManager {
// claimed once after the refresh peer procedure done(as the next claim queue will just delete
// it), so we can make sure that a two pass scan will finally find the queue and remove it,
// unless it has already been removed by others.
- removeAllQueues0(peerId);
- removeAllQueues0(peerId);
+ ReplicationUtils.removeAllQueues(queueStorage, peerId);
+ ReplicationUtils.removeAllQueues(queueStorage, peerId);
queueStorage.removePeerFromHFileRefs(peerId);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 3ee13c4..4c0630d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -116,6 +116,7 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.security.AccessDeniedException;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
@@ -142,6 +143,7 @@ import org.apache.yetus.audience.InterfaceStability;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
@@ -149,6 +151,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
import org.apache.hbase.thirdparty.com.google.common.collect.Ordering;
import org.apache.hbase.thirdparty.com.google.common.collect.TreeMultimap;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
@@ -752,7 +755,7 @@ public class HBaseFsck extends Configured implements Closeable {
* @return 0 on success, non-zero on failure
*/
public int onlineHbck()
- throws IOException, KeeperException, InterruptedException {
+ throws IOException, KeeperException, InterruptedException, ReplicationException {
// print hbase server version
errors.print("Version: " + status.getHBaseVersion());
@@ -3572,8 +3575,8 @@ public class HBaseFsck extends Configured implements Closeable {
return hbi;
}
- private void checkAndFixReplication() throws IOException {
- ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
+ private void checkAndFixReplication() throws ReplicationException {
+ ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, errors);
checker.checkUnDeletedQueues();
if (checker.hasUnDeletedQueues() && this.fixReplication) {
@@ -4861,8 +4864,8 @@ public class HBaseFsck extends Configured implements Closeable {
};
- public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
- InterruptedException {
+ public HBaseFsck exec(ExecutorService exec, String[] args)
+ throws KeeperException, IOException, InterruptedException, ReplicationException {
long sleepBeforeRerun = DEFAULT_SLEEP_BEFORE_RERUN;
boolean checkCorruptHFiles = false;
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index 85fa729..c08c654 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -17,84 +17,115 @@
*/
package org.apache.hadoop.hbase.util.hbck;
-import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.ClusterConnection;
-import org.apache.hadoop.hbase.master.cleaner.ReplicationZKNodeCleaner;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.util.HBaseFsck;
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Check and fix undeleted replication queues for removed peerId.
*/
@InterfaceAudience.Private
public class ReplicationChecker {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ReplicationChecker.class);
+
private final ErrorReporter errorReporter;
// replicator with its queueIds for removed peers
private Map<ServerName, List<String>> undeletedQueueIds = new HashMap<>();
// replicator with its undeleted queueIds for removed peers in hfile-refs queue
- private Set<String> undeletedHFileRefsQueueIds = new HashSet<>();
- private final ReplicationZKNodeCleaner cleaner;
+ private Set<String> undeletedHFileRefsPeerIds = new HashSet<>();
- public ReplicationChecker(Configuration conf, ZKWatcher zkw, ClusterConnection connection,
- ErrorReporter errorReporter) throws IOException {
- this.cleaner = new ReplicationZKNodeCleaner(conf, zkw, connection);
+ private final ReplicationPeerStorage peerStorage;
+ private final ReplicationQueueStorage queueStorage;
+
+ public ReplicationChecker(Configuration conf, ZKWatcher zkw, ErrorReporter errorReporter) {
+ this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zkw, conf);
+ this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
this.errorReporter = errorReporter;
}
public boolean hasUnDeletedQueues() {
- return errorReporter.getErrorList().contains(
- HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE);
+ return errorReporter.getErrorList()
+ .contains(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE);
}
- public void checkUnDeletedQueues() throws IOException {
- undeletedQueueIds = cleaner.getUnDeletedQueues();
- for (Entry<ServerName, List<String>> replicatorAndQueueIds : undeletedQueueIds.entrySet()) {
- ServerName replicator = replicatorAndQueueIds.getKey();
- for (String queueId : replicatorAndQueueIds.getValue()) {
+ private Map<ServerName, List<String>> getUnDeletedQueues() throws ReplicationException {
+ Map<ServerName, List<String>> undeletedQueues = new HashMap<>();
+ Set<String> peerIds = new HashSet<>(peerStorage.listPeerIds());
+ for (ServerName replicator : queueStorage.getListOfReplicators()) {
+ for (String queueId : queueStorage.getAllQueues(replicator)) {
ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- String msg = "Undeleted replication queue for removed peer found: "
- + String.format("[removedPeerId=%s, replicator=%s, queueId=%s]", queueInfo.getPeerId(),
- replicator, queueId);
- errorReporter.reportError(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE,
- msg);
+ if (!peerIds.contains(queueInfo.getPeerId())) {
+ undeletedQueues.computeIfAbsent(replicator, key -> new ArrayList<>()).add(queueId);
+ LOG.debug(
+ "Undeleted replication queue for removed peer found: " +
+ "[removedPeerId={}, replicator={}, queueId={}]",
+ queueInfo.getPeerId(), replicator, queueId);
+ }
}
}
-
- checkUnDeletedHFileRefsQueues();
+ return undeletedQueues;
}
- private void checkUnDeletedHFileRefsQueues() throws IOException {
- undeletedHFileRefsQueueIds = cleaner.getUnDeletedHFileRefsQueues();
- if (undeletedHFileRefsQueueIds != null && !undeletedHFileRefsQueueIds.isEmpty()) {
- String msg = "Undeleted replication hfile-refs queue for removed peer found: "
- + undeletedHFileRefsQueueIds + " under hfile-refs node";
- errorReporter
- .reportError(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE, msg);
+ private Set<String> getUndeletedHFileRefsPeers() throws ReplicationException {
+ Set<String> undeletedHFileRefsPeerIds =
+ new HashSet<>(queueStorage.getAllPeersFromHFileRefsQueue());
+ Set<String> peerIds = new HashSet<>(peerStorage.listPeerIds());
+ undeletedHFileRefsPeerIds.removeAll(peerIds);
+ if (LOG.isDebugEnabled()) {
+ for (String peerId : undeletedHFileRefsPeerIds) {
+ LOG.debug("Undeleted replication hfile-refs queue for removed peer {} found", peerId);
+ }
}
+ return undeletedHFileRefsPeerIds;
}
- public void fixUnDeletedQueues() throws IOException {
- if (!undeletedQueueIds.isEmpty()) {
- cleaner.removeQueues(undeletedQueueIds);
- }
- fixUnDeletedHFileRefsQueue();
+ public void checkUnDeletedQueues() throws ReplicationException {
+ undeletedQueueIds = getUnDeletedQueues();
+ undeletedQueueIds.forEach((replicator, queueIds) -> {
+ queueIds.forEach(queueId -> {
+ ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
+ String msg = "Undeleted replication queue for removed peer found: " +
+ String.format("[removedPeerId=%s, replicator=%s, queueId=%s]", queueInfo.getPeerId(),
+ replicator, queueId);
+ errorReporter.reportError(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE,
+ msg);
+ });
+ });
+ undeletedHFileRefsPeerIds = getUndeletedHFileRefsPeers();
+ undeletedHFileRefsPeerIds.stream()
+ .map(
+ peerId -> "Undeleted replication hfile-refs queue for removed peer " + peerId + " found")
+ .forEach(msg -> errorReporter
+ .reportError(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE, msg));
}
- private void fixUnDeletedHFileRefsQueue() throws IOException {
- if (undeletedHFileRefsQueueIds != null && !undeletedHFileRefsQueueIds.isEmpty()) {
- cleaner.removeHFileRefsQueues(undeletedHFileRefsQueueIds);
+ public void fixUnDeletedQueues() throws ReplicationException {
+ for (Map.Entry<ServerName, List<String>> replicatorAndQueueIds : undeletedQueueIds.entrySet()) {
+ ServerName replicator = replicatorAndQueueIds.getKey();
+ for (String queueId : replicatorAndQueueIds.getValue()) {
+ queueStorage.removeQueue(replicator, queueId);
+ }
+ queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
+ }
+ for (String peerId : undeletedHFileRefsPeerIds) {
+ queueStorage.removePeerFromHFileRefs(peerId);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
deleted file mode 100644
index 2ad8bd7..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.cleaner;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
-import org.apache.hadoop.hbase.testclassification.MasterTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({ MasterTests.class, MediumTests.class })
-public class TestReplicationZKNodeCleaner {
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
- private final String ID_ONE = "1";
- private final ServerName SERVER_ONE = ServerName.valueOf("server1", 8000, 1234);
- private final String ID_TWO = "2";
- private final ServerName SERVER_TWO = ServerName.valueOf("server2", 8000, 1234);
-
- private final Configuration conf;
- private final ZKWatcher zkw;
- private final ReplicationQueueStorage repQueues;
-
- public TestReplicationZKNodeCleaner() throws Exception {
- conf = TEST_UTIL.getConfiguration();
- zkw = new ZKWatcher(conf, "TestReplicationZKNodeCleaner", null);
- repQueues = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
- }
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- TEST_UTIL.getConfiguration().setInt("hbase.master.cleaner.interval", 10000);
- TEST_UTIL.startMiniCluster();
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
- }
-
- @Test
- public void testReplicationZKNodeCleaner() throws Exception {
- // add queue for ID_ONE which isn't exist
- repQueues.addWAL(SERVER_ONE, ID_ONE, "file1");
-
- ReplicationZKNodeCleaner cleaner = new ReplicationZKNodeCleaner(conf, zkw, null);
- Map<ServerName, List<String>> undeletedQueues = cleaner.getUnDeletedQueues();
- assertEquals(1, undeletedQueues.size());
- assertTrue(undeletedQueues.containsKey(SERVER_ONE));
- assertEquals(1, undeletedQueues.get(SERVER_ONE).size());
- assertTrue(undeletedQueues.get(SERVER_ONE).contains(ID_ONE));
-
- // add a recovery queue for ID_TWO which isn't exist
- repQueues.addWAL(SERVER_ONE, ID_TWO + "-" + SERVER_TWO, "file2");
-
- undeletedQueues = cleaner.getUnDeletedQueues();
- assertEquals(1, undeletedQueues.size());
- assertTrue(undeletedQueues.containsKey(SERVER_ONE));
- assertEquals(2, undeletedQueues.get(SERVER_ONE).size());
- assertTrue(undeletedQueues.get(SERVER_ONE).contains(ID_ONE));
- assertTrue(undeletedQueues.get(SERVER_ONE).contains(ID_TWO + "-" + SERVER_TWO));
-
- cleaner.removeQueues(undeletedQueues);
- undeletedQueues = cleaner.getUnDeletedQueues();
- assertEquals(0, undeletedQueues.size());
- }
-
- @Test
- public void testReplicationZKNodeCleanerChore() throws Exception {
- // add queue for ID_ONE which isn't exist
- repQueues.addWAL(SERVER_ONE, ID_ONE, "file1");
- // add a recovery queue for ID_TWO which isn't exist
- repQueues.addWAL(SERVER_ONE, ID_TWO + "-" + SERVER_TWO, "file2");
-
- // Wait the cleaner chore to run
- Thread.sleep(20000);
-
- ReplicationZKNodeCleaner cleaner = new ReplicationZKNodeCleaner(conf, zkw, null);
- assertEquals(0, cleaner.getUnDeletedQueues().size());
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
new file mode 100644
index 0000000..e64255c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ ReplicationTests.class, MediumTests.class })
+public class TestHBaseFsckReplication {
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void test() throws Exception {
+ ReplicationPeerStorage peerStorage = ReplicationStorageFactory
+ .getReplicationPeerStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
+ ReplicationQueueStorage queueStorage = ReplicationStorageFactory
+ .getReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
+
+ String peerId1 = "1";
+ String peerId2 = "2";
+ peerStorage.addPeer(peerId1, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
+ true);
+ peerStorage.addPeer(peerId2, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
+ true);
+ for (int i = 0; i < 10; i++) {
+ queueStorage.addWAL(ServerName.valueOf("localhost", 10000 + i, 100000 + i), peerId1,
+ "file-" + i);
+ }
+ queueStorage.addWAL(ServerName.valueOf("localhost", 10000, 100000), peerId2, "file");
+ HBaseFsck fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
+ HbckTestingUtil.assertNoErrors(fsck);
+
+ // should not remove anything since the replication peer is still alive
+ assertEquals(10, queueStorage.getListOfReplicators().size());
+ peerStorage.removePeer(peerId1);
+ // there should be orphan queues
+ assertEquals(10, queueStorage.getListOfReplicators().size());
+ fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), false);
+ HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
+ return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
+ }).limit(10).toArray(ERROR_CODE[]::new));
+
+ // should not delete anything when fix is false
+ assertEquals(10, queueStorage.getListOfReplicators().size());
+
+ fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
+ HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
+ return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
+ }).limit(10).toArray(ERROR_CODE[]::new));
+
+ List<ServerName> replicators = queueStorage.getListOfReplicators();
+ // should not remove the server with queue for peerId2
+ assertEquals(1, replicators.size());
+ assertEquals(ServerName.valueOf("localhost", 10000, 100000), replicators.get(0));
+ for (String queueId : queueStorage.getAllQueues(replicators.get(0))) {
+ assertEquals(peerId2, queueId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba414a70/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index 60d7324..99e4f08 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -46,7 +46,7 @@ public class HbckTestingUtil {
public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments, boolean fixMeta,
boolean fixHdfsHoles, boolean fixHdfsOverlaps, boolean fixHdfsOrphans,
boolean fixTableOrphans, boolean fixVersionFile, boolean fixReferenceFiles, boolean fixHFileLinks,
- boolean fixEmptyMetaRegionInfo, boolean fixTableLocks, Boolean fixReplication,
+ boolean fixEmptyMetaRegionInfo, boolean fixTableLocks, boolean fixReplication,
TableName table) throws Exception {
HBaseFsck fsck = new HBaseFsck(conf, exec);
try {
@@ -78,10 +78,8 @@ public class HbckTestingUtil {
/**
* Runs hbck with the -sidelineCorruptHFiles option
- * @param conf
* @param table table constraint
- * @return <returncode, hbckInstance>
- * @throws Exception
+ * @return hbckInstance
*/
public static HBaseFsck doHFileQuarantine(Configuration conf, TableName table) throws Exception {
String[] args = {"-sidelineCorruptHFiles", "-ignorePreCheckPermission", table.getNameAsString()};
[07/50] [abbrv] hbase git commit: HBASE-19811 Fix findbugs and
error-prone warnings in hbase-server (branch-2)
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java
index 21d914a..e5d3a79 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java
@@ -102,6 +102,7 @@ public class MasterProcedureSchedulerPerformanceEvaluation extends AbstractHBase
this.hri = hri;
}
+ @Override
public Procedure newProcedure(long procId) {
return new RegionProcedure(procId, hri);
}
@@ -132,6 +133,7 @@ public class MasterProcedureSchedulerPerformanceEvaluation extends AbstractHBase
this.tableName = tableName;
}
+ @Override
public Procedure newProcedure(long procId) {
return new TableProcedure(procId, tableName);
}
@@ -196,6 +198,7 @@ public class MasterProcedureSchedulerPerformanceEvaluation extends AbstractHBase
private final AtomicLong completed = new AtomicLong(0);
private class AddProcsWorker extends Thread {
+ @Override
public void run() {
final Random rand = new Random(System.currentTimeMillis());
long procId = procIds.incrementAndGet();
@@ -209,6 +212,7 @@ public class MasterProcedureSchedulerPerformanceEvaluation extends AbstractHBase
}
private class PollAndLockWorker extends Thread {
+ @Override
public void run() {
while (completed.get() < numOps) {
// With lock/unlock being ~100ns, and no other workload, 1000ns wait seams reasonable.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java
index 8dec59d..4adab53 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java
@@ -99,7 +99,7 @@ public class TestModifyNamespaceProcedure {
// Before modify
NamespaceDescriptor currentNsDescriptor =
UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
- assertEquals(currentNsDescriptor.getConfigurationValue(nsKey1), nsValue1before);
+ assertEquals(nsValue1before, currentNsDescriptor.getConfigurationValue(nsKey1));
assertNull(currentNsDescriptor.getConfigurationValue(nsKey2));
// Update
@@ -115,8 +115,8 @@ public class TestModifyNamespaceProcedure {
// Verify the namespace is updated.
currentNsDescriptor =
UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
- assertEquals(nsd.getConfigurationValue(nsKey1), nsValue1after);
- assertEquals(currentNsDescriptor.getConfigurationValue(nsKey2), nsValue2);
+ assertEquals(nsValue1after, nsd.getConfigurationValue(nsKey1));
+ assertEquals(nsValue2, currentNsDescriptor.getConfigurationValue(nsKey2));
}
@Test(timeout=60000)
@@ -219,7 +219,7 @@ public class TestModifyNamespaceProcedure {
// Validate
NamespaceDescriptor currentNsDescriptor =
UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
- assertEquals(currentNsDescriptor.getConfigurationValue(nsKey), nsValue);
+ assertEquals(nsValue, currentNsDescriptor.getConfigurationValue(nsKey));
}
@Test(timeout = 60000)
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java
index 8b58646..24a6bc5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -103,7 +104,7 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(2, currentHtd.getFamiliesKeys().size());
- assertTrue(currentHtd.hasFamily(cf2.getBytes()));
+ assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf2)));
// Test 2: Modify the table descriptor offline
UTIL.getAdmin().disableTable(tableName);
@@ -119,7 +120,7 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
- assertTrue(currentHtd.hasFamily(cf3.getBytes()));
+ assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf3)));
assertEquals(3, currentHtd.getFamiliesKeys().size());
}
@@ -137,7 +138,7 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
// Test 1: Modify the table descriptor
HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
- htd.removeFamily(cf2.getBytes());
+ htd.removeFamily(Bytes.toBytes(cf2));
long procId = ProcedureTestingUtility.submitAndWait(
procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
@@ -145,7 +146,7 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(2, currentHtd.getFamiliesKeys().size());
- assertFalse(currentHtd.hasFamily(cf2.getBytes()));
+ assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf2)));
// Test 2: Modify the table descriptor offline
UTIL.getAdmin().disableTable(tableName);
@@ -153,7 +154,7 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
HTableDescriptor htd2 =
new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
- htd2.removeFamily(cf3.getBytes());
+ htd2.removeFamily(Bytes.toBytes(cf3));
// Disable Sanity check
htd2.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
@@ -164,12 +165,12 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(1, currentHtd.getFamiliesKeys().size());
- assertFalse(currentHtd.hasFamily(cf3.getBytes()));
+ assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf3)));
//Removing the last family will fail
HTableDescriptor htd3 =
new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
- htd3.removeFamily(cf1.getBytes());
+ htd3.removeFamily(Bytes.toBytes(cf1));
long procId3 =
ProcedureTestingUtility.submitAndWait(procExec,
new ModifyTableProcedure(procExec.getEnvironment(), htd3));
@@ -179,7 +180,7 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
assertTrue("expected DoNotRetryIOException, got " + cause,
cause instanceof DoNotRetryIOException);
assertEquals(1, currentHtd.getFamiliesKeys().size());
- assertTrue(currentHtd.hasFamily(cf1.getBytes()));
+ assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf1)));
}
@Test(timeout=60000)
@@ -202,7 +203,7 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
htd.setCompactionEnabled(newCompactionEnableOption);
htd.addFamily(new HColumnDescriptor(cf2));
- htd.removeFamily(cf3.getBytes());
+ htd.removeFamily(Bytes.toBytes(cf3));
htd.setRegionReplication(3);
// Start the Modify procedure && kill the executor
@@ -240,7 +241,7 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
htd.setCompactionEnabled(newCompactionEnableOption);
htd.addFamily(new HColumnDescriptor(cf2));
- htd.removeFamily(cf3.getBytes());
+ htd.removeFamily(Bytes.toBytes(cf3));
// Start the Modify procedure && kill the executor
long procId = procExec.submitProcedure(
@@ -253,8 +254,8 @@ public class TestModifyTableProcedure extends TestTableDDLProcedureBase {
HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
assertEquals(2, currentHtd.getFamiliesKeys().size());
- assertTrue(currentHtd.hasFamily(cf2.getBytes()));
- assertFalse(currentHtd.hasFamily(cf3.getBytes()));
+ assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf2)));
+ assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf3)));
// cf2 should be added cf3 should be removed
MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java
index 2834b8f..2140d5e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestWALProcedureStoreOnHDFS.java
@@ -135,7 +135,7 @@ public class TestWALProcedureStoreOnHDFS {
final AtomicInteger reCount = new AtomicInteger(0);
Thread[] thread = new Thread[store.getNumThreads() * 2 + 1];
for (int i = 0; i < thread.length; ++i) {
- final long procId = i + 1;
+ final long procId = i + 1L;
thread[i] = new Thread(() -> {
try {
LOG.debug("[S] INSERT " + procId);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
index 60c9c4b..382625c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotFileCache.java
@@ -203,6 +203,7 @@ public class TestSnapshotFileCache {
}
class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
+ @Override
public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
Collection<String> files = new HashSet<>();
files.addAll(SnapshotReferenceUtil.getHFileNames(UTIL.getConfiguration(), fs, snapshotDir));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
index ad6c58e..8f81946 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotHFileCleaner.java
@@ -114,7 +114,8 @@ public class TestSnapshotHFileCleaner {
assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
}
- class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
+ static class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
+ @Override
public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
Collection<String> files = new HashSet<>();
files.addAll(SnapshotReferenceUtil.getHFileNames(TEST_UTIL.getConfiguration(), fs, snapshotDir));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
index 32e6522..65f50c9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
@@ -18,7 +18,7 @@
*/
package org.apache.hadoop.hbase.mob;
-import junit.framework.TestCase;
+import static org.junit.Assert.assertEquals;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -34,13 +34,15 @@ import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Category(SmallTests.class)
-public class TestCachedMobFile extends TestCase{
+public class TestCachedMobFile {
static final Logger LOG = LoggerFactory.getLogger(TestCachedMobFile.class);
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private Configuration conf = TEST_UTIL.getConfiguration();
@@ -50,10 +52,12 @@ public class TestCachedMobFile extends TestCase{
private static final long EXPECTED_REFERENCE_ZERO = 0;
private static final long EXPECTED_REFERENCE_ONE = 1;
private static final long EXPECTED_REFERENCE_TWO = 2;
+ @Rule
+ public TestName testName = new TestName();
@Test
public void testOpenClose() throws Exception {
- String caseName = getName();
+ String caseName = testName.getMethodName();
Path testDir = TEST_UTIL.getDataTestDir();
FileSystem fs = testDir.getFileSystem(conf);
HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
@@ -61,21 +65,21 @@ public class TestCachedMobFile extends TestCase{
.withOutputDir(testDir).withFileContext(meta).build();
MobTestUtil.writeStoreFile(writer, caseName);
CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf);
- Assert.assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
+ assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
cachedMobFile.open();
- Assert.assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile.getReferenceCount());
+ assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile.getReferenceCount());
cachedMobFile.open();
- Assert.assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile.getReferenceCount());
+ assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile.getReferenceCount());
cachedMobFile.close();
- Assert.assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile.getReferenceCount());
+ assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile.getReferenceCount());
cachedMobFile.close();
- Assert.assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
+ assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
}
@SuppressWarnings("SelfComparison")
@Test
public void testCompare() throws Exception {
- String caseName = getName();
+ String caseName = testName.getMethodName();
Path testDir = TEST_UTIL.getDataTestDir();
FileSystem fs = testDir.getFileSystem(conf);
Path outputDir1 = new Path(testDir, FAMILY1);
@@ -86,16 +90,16 @@ public class TestCachedMobFile extends TestCase{
CachedMobFile cachedMobFile1 = CachedMobFile.create(fs, writer1.getPath(), conf, cacheConf);
Path outputDir2 = new Path(testDir, FAMILY2);
StoreFileWriter writer2 = new StoreFileWriter.Builder(conf, cacheConf, fs)
- .withOutputDir(outputDir2)
- .withFileContext(meta)
- .build();
+ .withOutputDir(outputDir2)
+ .withFileContext(meta)
+ .build();
MobTestUtil.writeStoreFile(writer2, caseName);
CachedMobFile cachedMobFile2 = CachedMobFile.create(fs, writer2.getPath(), conf, cacheConf);
cachedMobFile1.access(1);
cachedMobFile2.access(2);
- Assert.assertEquals(cachedMobFile1.compareTo(cachedMobFile2), 1);
- Assert.assertEquals(cachedMobFile2.compareTo(cachedMobFile1), -1);
- Assert.assertEquals(cachedMobFile1.compareTo(cachedMobFile1), 0);
+ assertEquals(1, cachedMobFile1.compareTo(cachedMobFile2));
+ assertEquals(-1, cachedMobFile2.compareTo(cachedMobFile1));
+ assertEquals(0, cachedMobFile1.compareTo(cachedMobFile1));
}
@Test
@@ -105,7 +109,7 @@ public class TestCachedMobFile extends TestCase{
HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, fs)
.withOutputDir(testDir).withFileContext(meta).build();
- String caseName = getName();
+ String caseName = testName.getMethodName();
MobTestUtil.writeStoreFile(writer, caseName);
CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf);
byte[] family = Bytes.toBytes(caseName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
index f894fb2..cc2aa5c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
@@ -18,7 +18,9 @@
*/
package org.apache.hadoop.hbase.mob;
-import junit.framework.TestCase;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -36,17 +38,21 @@ import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Category(SmallTests.class)
-public class TestMobFile extends TestCase {
+public class TestMobFile {
static final Logger LOG = LoggerFactory.getLogger(TestMobFile.class);
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private Configuration conf = TEST_UTIL.getConfiguration();
private CacheConfig cacheConf = new CacheConfig(conf);
+ @Rule
+ public TestName testName = new TestName();
@Test
public void testReadKeyValue() throws Exception {
@@ -57,7 +63,7 @@ public class TestMobFile extends TestCase {
.withOutputDir(testDir)
.withFileContext(meta)
.build();
- String caseName = getName();
+ String caseName = testName.getMethodName();
MobTestUtil.writeStoreFile(writer, caseName);
MobFile mobFile =
@@ -110,7 +116,7 @@ public class TestMobFile extends TestCase {
.withOutputDir(testDir)
.withFileContext(meta)
.build();
- MobTestUtil.writeStoreFile(writer, getName());
+ MobTestUtil.writeStoreFile(writer, testName.getMethodName());
MobFile mobFile =
new MobFile(new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
index 42e6528..5077728 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
@@ -18,11 +18,12 @@
*/
package org.apache.hadoop.hbase.mob;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
import java.io.IOException;
import java.util.Date;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -38,13 +39,15 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Category(SmallTests.class)
-public class TestMobFileCache extends TestCase {
+public class TestMobFileCache {
static final Logger LOG = LoggerFactory.getLogger(TestMobFileCache.class);
private HBaseTestingUtility UTIL;
private HRegion region;
@@ -73,7 +76,7 @@ public class TestMobFileCache extends TestCase {
private static final byte[] QF2 = Bytes.toBytes("qf2");
private static final byte[] QF3 = Bytes.toBytes("qf3");
- @Override
+ @Before
public void setUp() throws Exception {
UTIL = HBaseTestingUtility.createLocalHTU();
conf = UTIL.getConfiguration();
@@ -93,8 +96,8 @@ public class TestMobFileCache extends TestCase {
region = UTIL.createLocalHRegion(htd, null, null);
}
- @Override
- protected void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
region.close();
region.getFilesystem().delete(UTIL.getDataTestDir(), true);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileName.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileName.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileName.java
index 8478e20..5ca73a6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileName.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileName.java
@@ -18,26 +18,31 @@
*/
package org.apache.hadoop.hbase.mob;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
import java.util.Date;
import java.util.Random;
import java.util.UUID;
-import junit.framework.TestCase;
-
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.MD5Hash;
+import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category(SmallTests.class)
-public class TestMobFileName extends TestCase {
+public class TestMobFileName {
private String uuid;
private Date date;
private String dateStr;
private byte[] startKey;
+ @Before
public void setUp() {
Random random = new Random();
uuid = UUID.randomUUID().toString().replaceAll("-", "");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 94680f2..214fe49 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -146,7 +146,7 @@ public class TestNamespaceAuditor {
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
ADMIN.createNamespace(nspDesc);
assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
- assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
+ assertEquals(3, ADMIN.listNamespaceDescriptors().length);
HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
HTableDescriptor tableDescOne =
@@ -592,7 +592,7 @@ public class TestNamespaceAuditor {
.build();
ADMIN.createNamespace(nspDesc);
assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
- assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
+ assertEquals(3, ADMIN.listNamespaceDescriptors().length);
HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
HTableDescriptor tableDescOne =
new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
index 0603b21..5ceb8e6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedure.java
@@ -52,7 +52,7 @@ public class TestProcedure {
when(coord.getRpcs()).thenReturn(comms); // make it not null
}
- class LatchedProcedure extends Procedure {
+ static class LatchedProcedure extends Procedure {
CountDownLatch startedAcquireBarrier = new CountDownLatch(1);
CountDownLatch startedDuringBarrier = new CountDownLatch(1);
CountDownLatch completedProcedure = new CountDownLatch(1);
@@ -93,6 +93,7 @@ public class TestProcedure {
final LatchedProcedure procspy = spy(proc);
// coordinator: start the barrier procedure
new Thread() {
+ @Override
public void run() {
procspy.call();
}
@@ -139,6 +140,7 @@ public class TestProcedure {
final LatchedProcedure procspy = spy(proc);
// start the barrier procedure
new Thread() {
+ @Override
public void run() {
procspy.call();
}
@@ -192,6 +194,7 @@ public class TestProcedure {
// start the barrier procedure
Thread t = new Thread() {
+ @Override
public void run() {
procspy.call();
}
@@ -214,6 +217,7 @@ public class TestProcedure {
// start the barrier procedure
Thread t = new Thread() {
+ @Override
public void run() {
procspy.call();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
index 1678a50..e2b1a79 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestProcedureCoordinator.java
@@ -213,6 +213,7 @@ public class TestProcedureCoordinator {
final Procedure spy = spy(task);
AcquireBarrierAnswer prepare = new AcquireBarrierAnswer(procName, cohort) {
+ @Override
public void doWork() {
// then do some fun where we commit before all nodes have prepared
// "one" commits before anyone else is done
@@ -288,7 +289,7 @@ public class TestProcedureCoordinator {
inorder.verify(controller).sendGlobalBarrierReached(eq(task), anyListOf(String.class));
}
- private abstract class OperationAnswer implements Answer<Void> {
+ private static abstract class OperationAnswer implements Answer<Void> {
private boolean ran = false;
public void ensureRan() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
index 1f5cbe6..c139862 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure/TestZKProcedure.java
@@ -244,7 +244,7 @@ public class TestZKProcedure {
Subprocedure r = ((Subprocedure) invocation.getMock());
LOG.error("Remote commit failure, not propagating error:" + remoteCause);
comms.receiveAbortProcedure(r.getName(), remoteCause);
- assertEquals(r.isComplete(), true);
+ assertEquals(true, r.isComplete());
// don't complete the error phase until the coordinator has gotten the error
// notification (which ensures that we never progress past prepare)
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
index a44ad74..ce0d2f2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
@@ -142,7 +142,7 @@ public class TestQuotaStatusRPCs {
assertTrue(
"Observed table usage was " + snapshot.getUsage(),
snapshot.getUsage() >= tableSize);
- assertEquals(snapshot.getLimit(), sizeLimit);
+ assertEquals(sizeLimit, snapshot.getLimit());
SpaceQuotaStatus pbStatus = snapshot.getQuotaStatus();
assertFalse(pbStatus.isInViolation());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSuperUserQuotaPermissions.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSuperUserQuotaPermissions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSuperUserQuotaPermissions.java
index 300268f..9ecde78 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSuperUserQuotaPermissions.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSuperUserQuotaPermissions.java
@@ -267,6 +267,7 @@ public class TestSuperUserQuotaPermissions {
private <T> T doAsUser(UserGroupInformation ugi, Callable<T> task) throws Exception {
return ugi.doAs(new PrivilegedExceptionAction<T>() {
+ @Override
public T run() throws Exception {
return task.call();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
index 82e1755..57948b6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
@@ -51,7 +51,7 @@ public class EncodedSeekPerformanceTest {
/** Use this benchmark with default options */
public EncodedSeekPerformanceTest() {
configuration.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f);
- randomizer = new Random(42l);
+ randomizer = new Random(42L);
numberOfSeeks = DEFAULT_NUMBER_OF_SEEKS;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java
index e159678..8199b07 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MockHStoreFile.java
@@ -111,11 +111,13 @@ public class MockHStoreFile extends HStoreFile {
this.entryCount = entryCount;
}
+ @Override
public OptionalLong getMinimumTimestamp() {
return timeRangeTracker == null ? OptionalLong.empty()
: OptionalLong.of(timeRangeTracker.getMin());
}
+ @Override
public OptionalLong getMaximumTimestamp() {
return timeRangeTracker == null ? OptionalLong.empty()
: OptionalLong.of(timeRangeTracker.getMax());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java
index 5af7d96..f89be43 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/StatefulStoreMockMaker.java
@@ -46,6 +46,7 @@ public class StatefulStoreMockMaker {
return 0;
}
private class CancelAnswer implements Answer<Object> {
+ @Override
public CompactionContext answer(InvocationOnMock invocation) throws Throwable {
cancelCompaction(invocation.getArgument(0));
return null;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
index 774888c..c8a75f0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestAtomicOperation.java
@@ -364,9 +364,9 @@ public class TestAtomicOperation {
assertEquals(0, failures.get());
Get g = new Get(row);
Result result = region.get(g);
- assertEquals(result.getValue(fam1, qual1).length, 10000);
- assertEquals(result.getValue(fam1, qual2).length, 10000);
- assertEquals(result.getValue(fam2, qual3).length, 10000);
+ assertEquals(10000, result.getValue(fam1, qual1).length);
+ assertEquals(10000, result.getValue(fam1, qual2).length);
+ assertEquals(10000, result.getValue(fam2, qual3).length);
}
/**
* Test multi-threaded row mutations.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
index 06cbf7a..a0babe8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
@@ -50,6 +50,7 @@ public class TestBlocksScanned extends HBaseTestCase {
private static HBaseTestingUtility TEST_UTIL = null;
+ @Override
@Before
public void setUp() throws Exception {
super.setUp();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
index f3c5da6..0121aef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
@@ -112,6 +112,7 @@ public class TestBulkLoad {
argThat(bulkLogWalEdit(WALEdit.BULK_LOAD, tableName.toBytes(),
familyName, storeFileNames)),
anyBoolean())).thenAnswer(new Answer() {
+ @Override
public Object answer(InvocationOnMock invocation) {
WALKeyImpl walKey = invocation.getArgument(1);
MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
@@ -137,6 +138,7 @@ public class TestBulkLoad {
when(log.append(any(),
any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)),
anyBoolean())).thenAnswer(new Answer() {
+ @Override
public Object answer(InvocationOnMock invocation) {
WALKeyImpl walKey = invocation.getArgument(1);
MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
@@ -156,6 +158,7 @@ public class TestBulkLoad {
when(log.append(any(),
any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)),
anyBoolean())).thenAnswer(new Answer() {
+ @Override
public Object answer(InvocationOnMock invocation) {
WALKeyImpl walKey = invocation.getArgument(1);
MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
@@ -176,6 +179,7 @@ public class TestBulkLoad {
when(log.append(any(),
any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)),
anyBoolean())).thenAnswer(new Answer() {
+ @Override
public Object answer(InvocationOnMock invocation) {
WALKeyImpl walKey = invocation.getArgument(1);
MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
@@ -281,7 +285,7 @@ public class TestBulkLoad {
writer.append(new KeyValue(CellUtil.createCell(randomBytes,
family,
randomBytes,
- 0l,
+ 0L,
KeyValue.Type.Put.getCode(),
randomBytes)));
} finally {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java
index 0717b4d..734b930 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java
@@ -18,25 +18,25 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import java.lang.management.ManagementFactory;
-
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.NavigableMap;
import java.util.NavigableSet;
import java.util.SortedSet;
-import junit.framework.TestCase;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
-
import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
-
-
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
@@ -48,11 +48,9 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import static org.junit.Assert.assertTrue;
-
@Category({RegionServerTests.class, SmallTests.class})
@RunWith(Parameterized.class)
-public class TestCellFlatSet extends TestCase {
+public class TestCellFlatSet {
@Parameterized.Parameters
public static Object[] data() {
return new Object[] { "SMALL_CHUNKS", "NORMAL_CHUNKS" }; // test with different chunk sizes
@@ -77,25 +75,22 @@ public class TestCellFlatSet extends TestCase {
public TestCellFlatSet(String chunkType){
long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
.getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(CONF, false));
- if (chunkType == "NORMAL_CHUNKS") {
+ if (chunkType.equals("NORMAL_CHUNKS")) {
chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false,
globalMemStoreLimit, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null);
- assertTrue(chunkCreator != null);
+ assertNotNull(chunkCreator);
smallChunks = false;
} else {
// chunkCreator with smaller chunk size, so only 3 cell-representations can accommodate a chunk
chunkCreator = ChunkCreator.initialize(SMALL_CHUNK_SIZE, false,
globalMemStoreLimit, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null);
- assertTrue(chunkCreator != null);
+ assertNotNull(chunkCreator);
smallChunks = true;
}
}
@Before
- @Override
public void setUp() throws Exception {
- super.setUp();
-
// create array of Cells to bass to the CellFlatMap under CellSet
final byte[] one = Bytes.toBytes(15);
final byte[] two = Bytes.toBytes(25);
@@ -126,7 +121,7 @@ public class TestCellFlatSet extends TestCase {
ascCCM = setUpCellChunkMap(true);
descCCM = setUpCellChunkMap(false);
- if (smallChunks == true) { // check jumbo chunks as well
+ if (smallChunks) { // check jumbo chunks as well
ascCCM = setUpJumboCellChunkMap(true);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java
index 4fa06b0..a58a22e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java
@@ -36,6 +36,7 @@ public class TestCellSkipListSet extends TestCase {
private final CellSet csls =
new CellSet(CellComparatorImpl.COMPARATOR);
+ @Override
protected void setUp() throws Exception {
super.setUp();
this.csls.clear();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
index c0ba621..87e4aff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
@@ -56,6 +56,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
/**
@@ -197,7 +198,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
Cell nr = ((CompactingMemStore)this.memstore).getNextRow(new KeyValue(Bytes.toBytes(i),
System.currentTimeMillis()));
if (i + 1 == ROW_COUNT) {
- assertEquals(nr, null);
+ assertNull(nr);
} else {
assertTrue(CellComparator.getInstance().compareRows(nr,
new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
@@ -317,7 +318,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
EnvironmentEdgeManager.injectEdge(edge);
long t = memstore.timeOfOldestEdit();
- assertEquals(t, Long.MAX_VALUE);
+ assertEquals(Long.MAX_VALUE, t);
// test the case that the timeOfOldestEdit is updated after a KV add
memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
index 1a04c8e..030658e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
@@ -454,7 +454,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
count++;
}
}
- assertEquals("the count should be ", count, 150);
+ assertEquals("the count should be ", 150, count);
for(int i = 0; i < scanners.size(); i++) {
scanners.get(i).close();
}
@@ -481,7 +481,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
} finally {
itr.close();
}
- assertEquals("the count should be ", cnt, 150);
+ assertEquals("the count should be ", 150, cnt);
}
private void addRowsByKeysWith50Cols(AbstractMemStore hmc, String[] keys) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index afe3228..f017617 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -116,7 +116,8 @@ public class TestCompaction {
// Increment the least significant character so we get to next row.
secondRowBytes[START_KEY_BYTES.length - 1]++;
thirdRowBytes = START_KEY_BYTES.clone();
- thirdRowBytes[START_KEY_BYTES.length - 1] += 2;
+ thirdRowBytes[START_KEY_BYTES.length - 1] =
+ (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
}
@Before
@@ -264,7 +265,7 @@ public class TestCompaction {
FileSystem fs = store.getFileSystem();
// default compaction policy created one and only one new compacted file
Path dstPath = store.getRegionFileSystem().createTempName();
- FSDataOutputStream stream = fs.create(dstPath, null, true, 512, (short)3, (long)1024, null);
+ FSDataOutputStream stream = fs.create(dstPath, null, true, 512, (short)3, 1024L, null);
stream.writeChars("CORRUPT FILE!!!!");
stream.close();
Path origPath = store.getRegionFileSystem().commitStoreFile(
@@ -390,7 +391,7 @@ public class TestCompaction {
class StoreMockMaker extends StatefulStoreMockMaker {
public ArrayList<HStoreFile> compacting = new ArrayList<>();
public ArrayList<HStoreFile> notCompacting = new ArrayList<>();
- private ArrayList<Integer> results;
+ private final ArrayList<Integer> results;
public StoreMockMaker(ArrayList<Integer> results) {
this.results = results;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java
index d2e1866..e9f381e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionArchiveConcurrentClose.java
@@ -125,6 +125,7 @@ public class TestCompactionArchiveConcurrentClose {
// now run the cleaner with a concurrent close
Thread cleanerThread = new Thread() {
+ @Override
public void run() {
cleaner.chore();
}
@@ -138,6 +139,7 @@ public class TestCompactionArchiveConcurrentClose {
}
final AtomicReference<Exception> closeException = new AtomicReference<>();
Thread closeThread = new Thread() {
+ @Override
public void run() {
// wait for the chore to complete and call close
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java
index 112fe4d..0dafd80 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionFileNotFound.java
@@ -141,7 +141,7 @@ public class TestCompactionFileNotFound {
}
});
// Split at this point should not result in the RS being aborted
- assertEquals(util.getMiniHBaseCluster().getLiveRegionServerThreads().size(), 3);
+ assertEquals(3, util.getMiniHBaseCluster().getLiveRegionServerThreads().size());
} finally {
if (admin != null) {
admin.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
index b89fb0e..53e04e0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
@@ -575,7 +575,7 @@ public class TestDefaultMemStore {
Cell nr = ((DefaultMemStore) this.memstore).getNextRow(new KeyValue(Bytes.toBytes(i),
System.currentTimeMillis()));
if (i + 1 == ROW_COUNT) {
- assertEquals(nr, null);
+ assertNull(nr);
} else {
assertTrue(CellComparatorImpl.COMPARATOR.compareRows(nr,
new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
@@ -863,7 +863,7 @@ public class TestDefaultMemStore {
EnvironmentEdgeManager.injectEdge(edge);
DefaultMemStore memstore = new DefaultMemStore();
long t = memstore.timeOfOldestEdit();
- assertEquals(t, Long.MAX_VALUE);
+ assertEquals(Long.MAX_VALUE, t);
// test the case that the timeOfOldestEdit is updated after a KV add
memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
index 642b471..aa38b7a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -168,7 +169,7 @@ public class TestGetClosestAtOrBefore {
byte [] metaKey = HRegionInfo.createRegionName(
tableb, tofindBytes,
HConstants.NINES, false);
- LOG.info("find=" + new String(metaKey));
+ LOG.info("find=" + new String(metaKey, StandardCharsets.UTF_8));
Result r = UTIL.getClosestRowBefore(mr, metaKey, HConstants.CATALOG_FAMILY);
if (answer == -1) {
assertNull(r);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 3c11b31..ee11075 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -43,6 +43,7 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
@@ -669,7 +670,7 @@ public class TestHRegion {
MonitoredTask status = TaskMonitor.get().createStatus(method);
Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
for (HStore store : region.getStores()) {
- maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId - 1);
+ maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId - 1);
}
long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
assertEquals(maxSeqId, seqId);
@@ -721,7 +722,7 @@ public class TestHRegion {
MonitoredTask status = TaskMonitor.get().createStatus(method);
Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
for (HStore store : region.getStores()) {
- maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
+ maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1);
}
long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
assertEquals(maxSeqId, seqId);
@@ -766,7 +767,7 @@ public class TestHRegion {
Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
for (HStore store : region.getStores()) {
- maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), minSeqId);
+ maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId);
}
long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, null);
assertEquals(minSeqId, seqId);
@@ -824,7 +825,7 @@ public class TestHRegion {
Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
MonitoredTask status = TaskMonitor.get().createStatus(method);
for (HStore store : region.getStores()) {
- maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
+ maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), recoverSeqId - 1);
}
long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
assertEquals(maxSeqId, seqId);
@@ -1070,7 +1071,7 @@ public class TestHRegion {
}
}
- class IsFlushWALMarker implements ArgumentMatcher<WALEdit> {
+ static class IsFlushWALMarker implements ArgumentMatcher<WALEdit> {
volatile FlushAction[] actions;
public IsFlushWALMarker(FlushAction... actions) {
this.actions = actions;
@@ -2192,7 +2193,7 @@ public class TestHRegion {
deleteMap.put(family, kvs);
region.delete(deleteMap, Durability.SYNC_WAL);
} catch (Exception e) {
- assertTrue("Family " + new String(family) + " does not exist", false);
+ fail("Family " + new String(family, StandardCharsets.UTF_8) + " does not exist");
}
// testing non existing family
@@ -2205,7 +2206,8 @@ public class TestHRegion {
} catch (Exception e) {
ok = true;
}
- assertEquals("Family " + new String(family) + " does exist", true, ok);
+ assertEquals("Family " + new String(family, StandardCharsets.UTF_8) + " does exist",
+ true, ok);
} finally {
HBaseTestingUtility.closeRegionAndWAL(this.region);
this.region = null;
@@ -3467,18 +3469,18 @@ public class TestHRegion {
List<Cell> results = new ArrayList<>();
assertTrue(s.next(results));
- assertEquals(results.size(), 1);
+ assertEquals(1, results.size());
results.clear();
assertTrue(s.next(results));
- assertEquals(results.size(), 3);
+ assertEquals(3, results.size());
assertTrue("orderCheck", CellUtil.matchingFamily(results.get(0), cf_alpha));
assertTrue("orderCheck", CellUtil.matchingFamily(results.get(1), cf_essential));
assertTrue("orderCheck", CellUtil.matchingFamily(results.get(2), cf_joined));
results.clear();
assertFalse(s.next(results));
- assertEquals(results.size(), 0);
+ assertEquals(0, results.size());
} finally {
HBaseTestingUtility.closeRegionAndWAL(this.region);
this.region = null;
@@ -3564,16 +3566,19 @@ public class TestHRegion {
while (true) {
boolean more = s.next(results, scannerContext);
if ((index >> 1) < 5) {
- if (index % 2 == 0)
- assertEquals(results.size(), 3);
- else
- assertEquals(results.size(), 1);
- } else
- assertEquals(results.size(), 1);
+ if (index % 2 == 0) {
+ assertEquals(3, results.size());
+ } else {
+ assertEquals(1, results.size());
+ }
+ } else {
+ assertEquals(1, results.size());
+ }
results.clear();
index++;
- if (!more)
+ if (!more) {
break;
+ }
}
} finally {
HBaseTestingUtility.closeRegionAndWAL(this.region);
@@ -4448,7 +4453,7 @@ public class TestHRegion {
// after all increment finished, the row will increment to 20*100 = 2000
int threadNum = 20;
int incCounter = 100;
- long expected = threadNum * incCounter;
+ long expected = (long) threadNum * incCounter;
Thread[] incrementers = new Thread[threadNum];
Thread flushThread = new Thread(flusher);
for (int i = 0; i < threadNum; i++) {
@@ -4470,7 +4475,7 @@ public class TestHRegion {
List<Cell> kvs = res.getColumnCells(Incrementer.family, Incrementer.qualifier);
// we just got the latest version
- assertEquals(kvs.size(), 1);
+ assertEquals(1, kvs.size());
Cell kv = kvs.get(0);
assertEquals(expected, Bytes.toLong(kv.getValueArray(), kv.getValueOffset()));
this.region = null;
@@ -4561,7 +4566,7 @@ public class TestHRegion {
List<Cell> kvs = res.getColumnCells(Appender.family, Appender.qualifier);
// we just got the latest version
- assertEquals(kvs.size(), 1);
+ assertEquals(1, kvs.size());
Cell kv = kvs.get(0);
byte[] appendResult = new byte[kv.getValueLength()];
System.arraycopy(kv.getValueArray(), kv.getValueOffset(), appendResult, 0, kv.getValueLength());
@@ -6150,7 +6155,7 @@ public class TestHRegion {
r = region.get(new Get(row));
byte[] val = r.getValue(fam1, q1);
assertNotNull(val);
- assertEquals(Bytes.toLong(val), 1L);
+ assertEquals(1L, Bytes.toLong(val));
// Increment with a TTL of 5 seconds
Increment incr = new Increment(row).addColumn(fam1, q1, 1L);
@@ -6161,7 +6166,7 @@ public class TestHRegion {
r = region.get(new Get(row));
val = r.getValue(fam1, q1);
assertNotNull(val);
- assertEquals(Bytes.toLong(val), 2L);
+ assertEquals(2L, Bytes.toLong(val));
// Increment time to T+25 seconds
edge.incrementTime(5000);
@@ -6170,7 +6175,7 @@ public class TestHRegion {
r = region.get(new Get(row));
val = r.getValue(fam1, q1);
assertNotNull(val);
- assertEquals(Bytes.toLong(val), 1L);
+ assertEquals(1L, Bytes.toLong(val));
// Increment time to T+30 seconds
edge.incrementTime(5000);
@@ -6199,14 +6204,14 @@ public class TestHRegion {
Result result = region.get(new Get(row));
Cell c = result.getColumnLatestCell(fam1, qual1);
assertNotNull(c);
- assertEquals(c.getTimestamp(), 10L);
+ assertEquals(10L, c.getTimestamp());
edge.setValue(1); // clock goes back
region.increment(inc);
result = region.get(new Get(row));
c = result.getColumnLatestCell(fam1, qual1);
- assertEquals(c.getTimestamp(), 11L);
- assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L);
+ assertEquals(11L, c.getTimestamp());
+ assertEquals(2L, Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()));
}
@Test
@@ -6224,13 +6229,13 @@ public class TestHRegion {
Result result = region.get(new Get(row));
Cell c = result.getColumnLatestCell(fam1, qual1);
assertNotNull(c);
- assertEquals(c.getTimestamp(), 10L);
+ assertEquals(10L, c.getTimestamp());
edge.setValue(1); // clock goes back
region.append(a);
result = region.get(new Get(row));
c = result.getColumnLatestCell(fam1, qual1);
- assertEquals(c.getTimestamp(), 11L);
+ assertEquals(11L, c.getTimestamp());
byte[] expected = new byte[qual1.length*2];
System.arraycopy(qual1, 0, expected, 0, qual1.length);
@@ -6255,7 +6260,7 @@ public class TestHRegion {
Result result = region.get(new Get(row));
Cell c = result.getColumnLatestCell(fam1, qual1);
assertNotNull(c);
- assertEquals(c.getTimestamp(), 10L);
+ assertEquals(10L, c.getTimestamp());
edge.setValue(1); // clock goes back
p = new Put(row);
@@ -6264,7 +6269,7 @@ public class TestHRegion {
region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), p, false);
result = region.get(new Get(row));
c = result.getColumnLatestCell(fam1, qual1);
- assertEquals(c.getTimestamp(), 10L);
+ assertEquals(10L, c.getTimestamp());
assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(),
qual2, 0, qual2.length));
@@ -6304,9 +6309,9 @@ public class TestHRegion {
};
OperationStatus[] status = region.batchMutate(mutations);
- assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
- assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SANITY_CHECK_FAILURE);
- assertEquals(status[2].getOperationStatusCode(), OperationStatusCode.SUCCESS);
+ assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
+ assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, status[1].getOperationStatusCode());
+ assertEquals(OperationStatusCode.SUCCESS, status[2].getOperationStatusCode());
// test with a row lock held for a long time
@@ -6347,8 +6352,8 @@ public class TestHRegion {
// this will wait for the row lock, and it will eventually succeed
OperationStatus[] status = region.batchMutate(mutations);
- assertEquals(status[0].getOperationStatusCode(), OperationStatusCode.SUCCESS);
- assertEquals(status[1].getOperationStatusCode(), OperationStatusCode.SUCCESS);
+ assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode());
+ assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode());
return null;
}
});
@@ -6374,7 +6379,7 @@ public class TestHRegion {
Result result = region.get(new Get(row));
Cell c = result.getColumnLatestCell(fam1, qual1);
assertNotNull(c);
- assertEquals(c.getTimestamp(), 10L);
+ assertEquals(10L, c.getTimestamp());
edge.setValue(1); // clock goes back
p = new Put(row);
@@ -6386,7 +6391,7 @@ public class TestHRegion {
new BinaryComparator(qual1), rm, false));
result = region.get(new Get(row));
c = result.getColumnLatestCell(fam1, qual1);
- assertEquals(c.getTimestamp(), 10L);
+ assertEquals(10L, c.getTimestamp());
LOG.info("c value " +
Bytes.toStringBinary(c.getValueArray(), c.getValueOffset(), c.getValueLength()));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index 3c25f6b..dcd7ff7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -232,7 +232,7 @@ public class TestHRegionReplayEvents {
// flush region
FlushResultImpl flush = (FlushResultImpl)secondaryRegion.flush(true);
- assertEquals(flush.result, FlushResultImpl.Result.CANNOT_FLUSH);
+ assertEquals(FlushResultImpl.Result.CANNOT_FLUSH, flush.result);
verifyData(secondaryRegion, 0, 1000, cq, families);
@@ -1207,13 +1207,13 @@ public class TestHRegionReplayEvents {
// primary region is empty at this point. Request a flush with writeFlushRequestWalMarker=false
FlushResultImpl result = primaryRegion.flushcache(true, false, FlushLifeCycleTracker.DUMMY);
assertNotNull(result);
- assertEquals(result.result, FlushResultImpl.Result.CANNOT_FLUSH_MEMSTORE_EMPTY);
+ assertEquals(FlushResultImpl.Result.CANNOT_FLUSH_MEMSTORE_EMPTY, result.result);
assertFalse(result.wroteFlushWalMarker);
// request flush again, but this time with writeFlushRequestWalMarker = true
result = primaryRegion.flushcache(true, true, FlushLifeCycleTracker.DUMMY);
assertNotNull(result);
- assertEquals(result.result, FlushResultImpl.Result.CANNOT_FLUSH_MEMSTORE_EMPTY);
+ assertEquals(FlushResultImpl.Result.CANNOT_FLUSH_MEMSTORE_EMPTY, result.result);
assertTrue(result.wroteFlushWalMarker);
List<FlushDescriptor> flushes = Lists.newArrayList();
@@ -1644,7 +1644,7 @@ public class TestHRegionReplayEvents {
hFileFactory.withFileContext(new HFileContext());
HFile.Writer writer = hFileFactory.create();
try {
- writer.append(new KeyValue(CellUtil.createCell(valueBytes, family, valueBytes, 0l,
+ writer.append(new KeyValue(CellUtil.createCell(valueBytes, family, valueBytes, 0L,
KeyValue.Type.Put.getCode(), valueBytes)));
} finally {
writer.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
index b6b39bc..034d9d7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionServerBulkLoad.java
@@ -186,6 +186,7 @@ public class TestHRegionServerBulkLoad {
this.tableName = tableName;
}
+ @Override
public void doAnAction() throws Exception {
long iteration = numBulkLoads.getAndIncrement();
Path dir = UTIL.getDataTestDirOnTestFS(String.format("bulkLoad_%08d",
@@ -295,6 +296,7 @@ public class TestHRegionServerBulkLoad {
table = UTIL.getConnection().getTable(TABLE_NAME);
}
+ @Override
public void doAnAction() throws Exception {
Scan s = new Scan();
for (byte[] family : targetFamilies) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
index b8d3ec7..288333b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
@@ -756,7 +756,7 @@ public class TestHStore {
}
@Override
- public void write(byte[] buf, int offset, int length) throws IOException {
+ public synchronized void write(byte[] buf, int offset, int length) throws IOException {
System.err.println("faulty stream write at pos " + getPos());
injectFault();
super.write(buf, offset, length);
@@ -1551,7 +1551,7 @@ public class TestHStore {
ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(5).build(), hook);
}
- private class MyStore extends HStore {
+ private static class MyStore extends HStore {
private final MyStoreHook hook;
MyStore(final HRegion region, final ColumnFamilyDescriptor family, final Configuration
@@ -1576,7 +1576,7 @@ public class TestHStore {
}
}
- private abstract class MyStoreHook {
+ private abstract static class MyStoreHook {
void getScanners(MyStore store) throws IOException {
}
@@ -1595,7 +1595,7 @@ public class TestHStore {
MyStore store = initMyStore(name.getMethodName(), conf, new MyStoreHook() {});
MemStoreSizing memStoreSizing = new MemStoreSizing();
long ts = System.currentTimeMillis();
- long seqID = 1l;
+ long seqID = 1L;
// Add some data to the region and do some flushes
for (int i = 1; i < 10; i++) {
store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")),
@@ -1663,6 +1663,7 @@ public class TestHStore {
return this.heap;
}
+ @Override
public void run() {
scanner.trySwitchToStreamRead();
heap = scanner.heap;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
index 31f16ea..1a38d7f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
@@ -90,11 +90,13 @@ public class TestHStoreFile extends HBaseTestCase {
private static final int CKBYTES = 512;
private static String TEST_FAMILY = "cf";
+ @Override
@Before
public void setUp() throws Exception {
super.setUp();
}
+ @Override
@After
public void tearDown() throws Exception {
super.tearDown();
@@ -505,8 +507,8 @@ public class TestHStoreFile extends HBaseTestCase {
long now = System.currentTimeMillis();
for (int i = 0; i < 2000; i += 2) {
String row = String.format(localFormatter, i);
- KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(),
- "col".getBytes(), now, "value".getBytes());
+ KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
+ Bytes.toBytes("col"), now, Bytes.toBytes("value"));
writer.append(kv);
}
writer.close();
@@ -523,12 +525,13 @@ public class TestHStoreFile extends HBaseTestCase {
for (int i = 0; i < 2000; i++) {
String row = String.format(localFormatter, i);
TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);
- columns.add("family:col".getBytes());
+ columns.add(Bytes.toBytes("family:col"));
- Scan scan = new Scan(row.getBytes(),row.getBytes());
- scan.addColumn("family".getBytes(), "family:col".getBytes());
+ Scan scan = new Scan(Bytes.toBytes(row),Bytes.toBytes(row));
+ scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes("family:col"));
HStore store = mock(HStore.class);
- when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of("family"));
+ when(store.getColumnFamilyDescriptor())
+ .thenReturn(ColumnFamilyDescriptorBuilder.of("family"));
boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);
if (i % 2 == 0) {
if (!exists) falseNeg++;
@@ -592,8 +595,8 @@ public class TestHStoreFile extends HBaseTestCase {
long now = System.currentTimeMillis();
for (int i = 0; i < 2000; i += 2) {
String row = String.format(localFormatter, i);
- KeyValue kv = new KeyValue(row.getBytes(), "family".getBytes(),
- "col".getBytes(), now, KeyValue.Type.DeleteFamily, "value".getBytes());
+ KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
+ Bytes.toBytes("col"), now, KeyValue.Type.DeleteFamily, Bytes.toBytes("value"));
writer.append(kv);
}
writer.close();
@@ -696,9 +699,8 @@ public class TestHStoreFile extends HBaseTestCase {
String row = String.format(localFormatter, i);
String col = String.format(localFormatter, j);
for (int k= 0; k < versions; ++k) { // versions
- KeyValue kv = new KeyValue(row.getBytes(),
- "family".getBytes(), ("col" + col).getBytes(),
- now-k, Bytes.toBytes((long)-1));
+ KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
+ Bytes.toBytes("col" + col), now-k, Bytes.toBytes(-1L));
writer.append(kv);
}
}
@@ -713,7 +715,8 @@ public class TestHStoreFile extends HBaseTestCase {
assertEquals(expKeys[x], reader.generalBloomFilter.getKeyCount());
HStore store = mock(HStore.class);
- when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of("family"));
+ when(store.getColumnFamilyDescriptor())
+ .thenReturn(ColumnFamilyDescriptorBuilder.of("family"));
// check false positives rate
int falsePos = 0;
int falseNeg = 0;
@@ -722,10 +725,10 @@ public class TestHStoreFile extends HBaseTestCase {
String row = String.format(localFormatter, i);
String col = String.format(localFormatter, j);
TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);
- columns.add(("col" + col).getBytes());
+ columns.add(Bytes.toBytes("col" + col));
- Scan scan = new Scan(row.getBytes(),row.getBytes());
- scan.addColumn("family".getBytes(), ("col"+col).getBytes());
+ Scan scan = new Scan(Bytes.toBytes(row),Bytes.toBytes(row));
+ scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes(("col"+col)));
boolean exists =
scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
index 24ab7a1..d9956de 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
@@ -88,6 +88,7 @@ public class TestKeyValueHeap extends HBaseTestCase {
return actual;
}
+ @Override
@Before
public void setUp() throws Exception {
super.setUp();
@@ -185,7 +186,7 @@ public class TestKeyValueHeap extends HBaseTestCase {
for (KeyValueScanner scanner : scanners) {
// Verify that close is called and only called once for each scanner
assertTrue(((SeekTestScanner) scanner).isClosed());
- assertEquals(((SeekTestScanner) scanner).getClosedNum(), 1);
+ assertEquals(1, ((SeekTestScanner) scanner).getClosedNum());
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
index 2684bdf..0840a63 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
@@ -108,7 +108,8 @@ public class TestMajorCompaction {
// Increment the least significant character so we get to next row.
secondRowBytes[START_KEY_BYTES.length - 1]++;
thirdRowBytes = START_KEY_BYTES.clone();
- thirdRowBytes[START_KEY_BYTES.length - 1] += 2;
+ thirdRowBytes[START_KEY_BYTES.length - 1] =
+ (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
}
@Before
@@ -277,7 +278,7 @@ public class TestMajorCompaction {
// Force major compaction.
r.compact(true);
- assertEquals(r.getStore(COLUMN_FAMILY_TEXT).getStorefiles().size(), 1);
+ assertEquals(1, r.getStore(COLUMN_FAMILY_TEXT).getStorefiles().size());
result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).readVersions(100));
assertTrue("Second row should still be deleted", result.isEmpty());
@@ -398,8 +399,8 @@ public class TestMajorCompaction {
private void createSmallerStoreFile(final HRegion region) throws IOException {
Table loader = new RegionAsTable(region);
- HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY), ("" +
- "bbb").getBytes(), null);
+ HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY), Bytes.toBytes("" +
+ "bbb"), null);
region.flush(true);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
index 8d74d8d..795fa2f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
@@ -62,8 +62,8 @@ public class TestMemStoreLAB {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- ChunkCreator.initialize(1 * 1024, false, 50*1024000l, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT,
- null);
+ ChunkCreator.initialize(1 * 1024, false, 50 * 1024000L, 0.2f,
+ MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null);
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
index 98b0761..4ae92a4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
@@ -76,7 +76,8 @@ public class TestMinorCompaction {
// Increment the least significant character so we get to next row.
secondRowBytes[START_KEY_BYTES.length - 1]++;
thirdRowBytes = START_KEY_BYTES.clone();
- thirdRowBytes[START_KEY_BYTES.length - 1] += 2;
+ thirdRowBytes[START_KEY_BYTES.length - 1] =
+ (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
col1 = Bytes.toBytes("column1");
col2 = Bytes.toBytes("column2");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConcurrencyControl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConcurrencyControl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConcurrencyControl.java
index 084c4c91..a09c8cd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConcurrencyControl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConcurrencyControl.java
@@ -46,6 +46,7 @@ public class TestMultiVersionConcurrencyControl {
private Random rnd = new Random();
public boolean failed = false;
+ @Override
public void run() {
while (!finished.get()) {
MultiVersionConcurrencyControl.WriteEntry e =
@@ -82,6 +83,7 @@ public class TestMultiVersionConcurrencyControl {
final AtomicBoolean readerFailed = new AtomicBoolean(false);
final AtomicLong failedAt = new AtomicLong();
Runnable reader = new Runnable() {
+ @Override
public void run() {
long prev = mvcc.getReadPoint();
while (!finished.get()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
index a8b0f22..82b0251 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
@@ -24,30 +24,31 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.ipc.PriorityFunction;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ipc.PriorityFunction;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.Get;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.RequestHeader;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
-
-import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
-import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
/**
* Tests that verify certain RPCs get a higher QoS.
@@ -85,7 +86,7 @@ public class TestPriorityRpc {
RegionSpecifier regionSpecifier = regionSpecifierBuilder.build();
getRequestBuilder.setRegion(regionSpecifier);
Get.Builder getBuilder = Get.newBuilder();
- getBuilder.setRow(UnsafeByteOperations.unsafeWrap("somerow".getBytes()));
+ getBuilder.setRow(UnsafeByteOperations.unsafeWrap(Bytes.toBytes("somerow")));
getRequestBuilder.setGet(getBuilder.build());
GetRequest getRequest = getRequestBuilder.build();
RequestHeader header = headerBuilder.build();
@@ -96,7 +97,8 @@ public class TestPriorityRpc {
RegionInfo mockRegionInfo = Mockito.mock(RegionInfo.class);
Mockito.when(mockRpc.getRegion(Mockito.any())).thenReturn(mockRegion);
Mockito.when(mockRegion.getRegionInfo()).thenReturn(mockRegionInfo);
- Mockito.when(mockRegionInfo.getTable()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO.getTable());
+ Mockito.when(mockRegionInfo.getTable())
+ .thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO.getTable());
// Presume type.
((AnnotationReadingPriorityFunction)priority).setRegionServer(mockRS);
assertEquals(HConstants.SYSTEMTABLE_QOS, priority.getPriority(header, getRequest,
[38/50] [abbrv] hbase git commit: HBASE-19634 Add permission check
for executeProcedures in AccessController
Posted by zh...@apache.org.
HBASE-19634 Add permission check for executeProcedures in AccessController
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6bd7fd8c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6bd7fd8c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6bd7fd8c
Branch: refs/heads/HBASE-19397-branch-2
Commit: 6bd7fd8c9e803489b4b19c2796bf331d96de220b
Parents: ca70668
Author: zhangduo <zh...@apache.org>
Authored: Thu Jan 4 16:18:21 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../hbase/coprocessor/RegionServerObserver.java | 14 +++++
.../hbase/regionserver/RSRpcServices.java | 54 +++++++++++---------
.../RegionServerCoprocessorHost.java | 18 +++++++
.../hbase/security/access/AccessController.java | 30 ++++++-----
.../hadoop/hbase/TestJMXConnectorServer.java | 7 +++
.../security/access/TestAccessController.java | 18 +++++--
6 files changed, 101 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bd7fd8c/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
index c1af3fb..5b751df 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerObserver.java
@@ -126,4 +126,18 @@ public interface RegionServerObserver {
default void postClearCompactionQueues(
final ObserverContext<RegionServerCoprocessorEnvironment> ctx)
throws IOException {}
+
+ /**
+ * This will be called before executing procedures
+ * @param ctx the environment to interact with the framework and region server.
+ */
+ default void preExecuteProcedures(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
+ throws IOException {}
+
+ /**
+ * This will be called after executing procedures
+ * @param ctx the environment to interact with the framework and region server.
+ */
+ default void postExecuteProcedures(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
+ throws IOException {}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bd7fd8c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 5391a82..e540464 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -41,7 +41,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
-
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -142,6 +141,7 @@ import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter;
@@ -3454,36 +3454,40 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
}
@Override
+ @QosPriority(priority = HConstants.ADMIN_QOS)
public ExecuteProceduresResponse executeProcedures(RpcController controller,
ExecuteProceduresRequest request) throws ServiceException {
- if (request.getOpenRegionCount() > 0) {
- for (OpenRegionRequest req : request.getOpenRegionList()) {
- openRegion(controller, req);
+ try {
+ checkOpen();
+ regionServer.getRegionServerCoprocessorHost().preExecuteProcedures();
+ if (request.getOpenRegionCount() > 0) {
+ for (OpenRegionRequest req : request.getOpenRegionList()) {
+ openRegion(controller, req);
+ }
}
- }
- if (request.getCloseRegionCount() > 0) {
- for (CloseRegionRequest req : request.getCloseRegionList()) {
- closeRegion(controller, req);
+ if (request.getCloseRegionCount() > 0) {
+ for (CloseRegionRequest req : request.getCloseRegionList()) {
+ closeRegion(controller, req);
+ }
}
- }
- if (request.getProcCount() > 0) {
- for (RemoteProcedureRequest req : request.getProcList()) {
- RSProcedureCallable callable;
- try {
- callable =
- Class.forName(req.getProcClass()).asSubclass(RSProcedureCallable.class).newInstance();
- } catch (Exception e) {
- // here we just ignore the error as this should not happen and we do not provide a general
- // way to report errors for all types of remote procedure. The procedure will hang at
- // master side but after you solve the problem and restart master it will be executed
- // again and pass.
- LOG.warn("create procedure of type " + req.getProcClass() + " failed, give up", e);
- continue;
+ if (request.getProcCount() > 0) {
+ for (RemoteProcedureRequest req : request.getProcList()) {
+ RSProcedureCallable callable;
+ try {
+ callable =
+ Class.forName(req.getProcClass()).asSubclass(RSProcedureCallable.class).newInstance();
+ } catch (Exception e) {
+ regionServer.remoteProcedureComplete(req.getProcId(), e);
+ continue;
+ }
+ callable.init(req.getProcData().toByteArray(), regionServer);
+ regionServer.executeProcedure(req.getProcId(), callable);
}
- callable.init(req.getProcData().toByteArray(), regionServer);
- regionServer.executeProcedure(req.getProcId(), callable);
}
+ regionServer.getRegionServerCoprocessorHost().postExecuteProcedures();
+ return ExecuteProceduresResponse.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
}
- return ExecuteProceduresResponse.getDefaultInstance();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bd7fd8c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
index 1986668..f4122ce 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
@@ -205,6 +205,24 @@ public class RegionServerCoprocessorHost extends
});
}
+ public void preExecuteProcedures() throws IOException {
+ execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() {
+ @Override
+ public void call(RegionServerObserver observer) throws IOException {
+ observer.preExecuteProcedures(this);
+ }
+ });
+ }
+
+ public void postExecuteProcedures() throws IOException {
+ execOperation(coprocEnvironments.isEmpty() ? null : new RegionServerObserverOperation() {
+ @Override
+ public void call(RegionServerObserver observer) throws IOException {
+ observer.postExecuteProcedures(this);
+ }
+ });
+ }
+
/**
* Coprocessor environment extension providing access to region server
* related services.
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bd7fd8c/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index f191c9d..6acc133 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -117,13 +117,6 @@ import org.apache.hadoop.hbase.security.Superusers;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.access.Permission.Action;
-import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
-import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-import org.apache.hbase.thirdparty.com.google.common.collect.MapMaker;
-import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.ByteRange;
import org.apache.hadoop.hbase.util.Bytes;
@@ -136,6 +129,14 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
+import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+import org.apache.hbase.thirdparty.com.google.common.collect.MapMaker;
+import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+
/**
* Provides basic authorization checks for data access and administrative
* operations.
@@ -2455,7 +2456,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
throws IOException {
requirePermission(ctx, "replicateLogEntries", Action.WRITE);
}
-
+
@Override
public void preClearCompactionQueues(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
throws IOException {
@@ -2507,8 +2508,7 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
@Override
public void preRequestLock(ObserverContext<MasterCoprocessorEnvironment> ctx, String namespace,
- TableName tableName, RegionInfo[] regionInfos, String description)
- throws IOException {
+ TableName tableName, RegionInfo[] regionInfos, String description) throws IOException {
// There are operations in the CREATE and ADMIN domain which may require lock, READ
// or WRITE. So for any lock request, we check for these two perms irrespective of lock type.
String reason = String.format("Description=%s", description);
@@ -2521,12 +2521,18 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
checkLockPermissions(ctx, null, tableName, null, description);
}
+ @Override
+ public void preExecuteProcedures(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
+ throws IOException {
+ checkSystemOrSuperUser(getActiveUser(ctx));
+ }
+
/**
* Returns the active user to which authorization checks should be applied.
* If we are in the context of an RPC call, the remote user is used,
* otherwise the currently logged in user is used.
*/
- public User getActiveUser(ObserverContext<?> ctx) throws IOException {
+ private User getActiveUser(ObserverContext<?> ctx) throws IOException {
// for non-rpc handling, fallback to system user
Optional<User> optionalUser = ctx.getCaller();
if (optionalUser.isPresent()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bd7fd8c/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXConnectorServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXConnectorServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXConnectorServer.java
index d09b0a4..6b743b5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXConnectorServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXConnectorServer.java
@@ -210,5 +210,12 @@ public class TestJMXConnectorServer {
throw new AccessDeniedException("Insufficient permissions to shut down cluster.");
}
}
+
+ @Override
+ public void preExecuteProcedures(ObserverContext<RegionServerCoprocessorEnvironment> ctx)
+ throws IOException {
+ // FIXME: ignore the procedure permission check since in our UT framework master is neither
+ // the systemuser nor the superuser so we can not call executeProcedures...
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6bd7fd8c/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 9a1bd74..073d481 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -30,7 +30,6 @@ import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
import com.google.protobuf.ServiceException;
-
import java.io.IOException;
import java.security.PrivilegedAction;
import java.util.ArrayList;
@@ -38,7 +37,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -3067,4 +3065,18 @@ public class TestAccessController extends SecureTestUtil {
verifyAllowed(
action, SUPERUSER, USER_ADMIN, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER);
}
+
+ @Test
+ public void testExecuteProcedures() throws Exception {
+ AccessTestAction action = new AccessTestAction() {
+ @Override
+ public Object run() throws Exception {
+ ACCESS_CONTROLLER.preExecuteProcedures(ObserverContextImpl.createAndPrepare(RSCP_ENV));
+ return null;
+ }
+ };
+
+ verifyAllowed(action, SUPERUSER);
+ verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER, USER_ADMIN);
+ }
}
[15/50] [abbrv] hbase git commit: HBASE-19564 Procedure id is missing
in the response of peer related operations
Posted by zh...@apache.org.
HBASE-19564 Procedure id is missing in the response of peer related operations
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/521065e5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/521065e5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/521065e5
Branch: refs/heads/HBASE-19397-branch-2
Commit: 521065e544ce1ca696c43dc640138f13eb2b2243
Parents: 4482adc
Author: zhangduo <zh...@apache.org>
Authored: Wed Dec 20 20:57:37 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../hadoop/hbase/master/MasterRpcServices.java | 24 ++++++++++----------
.../master/replication/ModifyPeerProcedure.java | 4 +---
2 files changed, 13 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/521065e5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 8025a51..72bf2d1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1886,10 +1886,10 @@ public class MasterRpcServices extends RSRpcServices
public AddReplicationPeerResponse addReplicationPeer(RpcController controller,
AddReplicationPeerRequest request) throws ServiceException {
try {
- master.addReplicationPeer(request.getPeerId(),
- ReplicationPeerConfigUtil.convert(request.getPeerConfig()), request.getPeerState()
- .getState().equals(ReplicationState.State.ENABLED));
- return AddReplicationPeerResponse.newBuilder().build();
+ long procId = master.addReplicationPeer(request.getPeerId(),
+ ReplicationPeerConfigUtil.convert(request.getPeerConfig()),
+ request.getPeerState().getState().equals(ReplicationState.State.ENABLED));
+ return AddReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1899,8 +1899,8 @@ public class MasterRpcServices extends RSRpcServices
public RemoveReplicationPeerResponse removeReplicationPeer(RpcController controller,
RemoveReplicationPeerRequest request) throws ServiceException {
try {
- master.removeReplicationPeer(request.getPeerId());
- return RemoveReplicationPeerResponse.newBuilder().build();
+ long procId = master.removeReplicationPeer(request.getPeerId());
+ return RemoveReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1910,8 +1910,8 @@ public class MasterRpcServices extends RSRpcServices
public EnableReplicationPeerResponse enableReplicationPeer(RpcController controller,
EnableReplicationPeerRequest request) throws ServiceException {
try {
- master.enableReplicationPeer(request.getPeerId());
- return EnableReplicationPeerResponse.newBuilder().build();
+ long procId = master.enableReplicationPeer(request.getPeerId());
+ return EnableReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1921,8 +1921,8 @@ public class MasterRpcServices extends RSRpcServices
public DisableReplicationPeerResponse disableReplicationPeer(RpcController controller,
DisableReplicationPeerRequest request) throws ServiceException {
try {
- master.disableReplicationPeer(request.getPeerId());
- return DisableReplicationPeerResponse.newBuilder().build();
+ long procId = master.disableReplicationPeer(request.getPeerId());
+ return DisableReplicationPeerResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
@@ -1948,9 +1948,9 @@ public class MasterRpcServices extends RSRpcServices
public UpdateReplicationPeerConfigResponse updateReplicationPeerConfig(RpcController controller,
UpdateReplicationPeerConfigRequest request) throws ServiceException {
try {
- master.updateReplicationPeerConfig(request.getPeerId(),
+ long procId = master.updateReplicationPeerConfig(request.getPeerId(),
ReplicationPeerConfigUtil.convert(request.getPeerConfig()));
- return UpdateReplicationPeerConfigResponse.newBuilder().build();
+ return UpdateReplicationPeerConfigResponse.newBuilder().setProcId(procId).build();
} catch (ReplicationException | IOException e) {
throw new ServiceException(e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/521065e5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 7076bab..23f6f87 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -54,9 +54,7 @@ public abstract class ModifyPeerProcedure
protected ModifyPeerProcedure(String peerId) {
this.peerId = peerId;
- // TODO: temporarily set a 4.0 here to always wait for the procedure exection completed. Change
- // to 3.0 or 2.0 after the client modification is done.
- this.latch = ProcedurePrepareLatch.createLatch(4, 0);
+ this.latch = ProcedurePrepareLatch.createLatch(2, 0);
}
public ProcedurePrepareLatch getLatch() {
[28/50] [abbrv] hbase git commit: HBASE-19573 Rewrite ReplicationPeer
with the new replication storage interface
Posted by zh...@apache.org.
HBASE-19573 Rewrite ReplicationPeer with the new replication storage interface
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dcc77931
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dcc77931
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dcc77931
Branch: refs/heads/HBASE-19397-branch-2
Commit: dcc779312f0f4423636730e8549ba035a7dbfba9
Parents: 0541dbe
Author: Guanghao Zhang <zg...@apache.org>
Authored: Tue Dec 26 11:39:34 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../replication/VerifyReplication.java | 5 -
.../hbase/replication/ReplicationPeer.java | 42 ++--
.../hbase/replication/ReplicationPeerImpl.java | 169 ++++++++++++++
.../replication/ReplicationPeerZKImpl.java | 233 -------------------
.../hbase/replication/ReplicationPeers.java | 4 +-
.../replication/ReplicationPeersZKImpl.java | 23 +-
.../replication/TestReplicationStateBasic.java | 7 +-
.../regionserver/PeerProcedureHandlerImpl.java | 29 +--
8 files changed, 216 insertions(+), 296 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcc77931/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 9065f4e..09d4b4b 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
import org.apache.hadoop.hbase.util.Bytes;
@@ -333,7 +332,6 @@ public class VerifyReplication extends Configured implements Tool {
private static Pair<ReplicationPeerConfig, Configuration> getPeerQuorumConfig(
final Configuration conf, String peerId) throws IOException {
ZKWatcher localZKW = null;
- ReplicationPeerZKImpl peer = null;
try {
localZKW = new ZKWatcher(conf, "VerifyReplication",
new Abortable() {
@@ -354,9 +352,6 @@ public class VerifyReplication extends Configured implements Tool {
throw new IOException(
"An error occurred while trying to connect to the remove peer cluster", e);
} finally {
- if (peer != null) {
- peer.close();
- }
if (localZKW != null) {
localZKW.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcc77931/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index b66d76d..4846018 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
-
/**
* ReplicationPeer manages enabled / disabled state for the peer.
*/
@@ -49,65 +48,52 @@ public interface ReplicationPeer {
String getId();
/**
- * Get the peer config object
- * @return the ReplicationPeerConfig for this peer
- */
- public ReplicationPeerConfig getPeerConfig();
-
- /**
- * Get the peer config object. if loadFromBackingStore is true, it will load from backing store
- * directly and update its load peer config. otherwise, just return the local cached peer config.
- * @return the ReplicationPeerConfig for this peer
- */
- public ReplicationPeerConfig getPeerConfig(boolean loadFromBackingStore)
- throws ReplicationException;
-
- /**
* Returns the state of the peer by reading local cache.
* @return the enabled state
*/
PeerState getPeerState();
/**
- * Returns the state of peer, if loadFromBackingStore is true, it will load from backing store
- * directly and update its local peer state. otherwise, just return the local cached peer state.
- * @return the enabled state
+ * Get the peer config object
+ * @return the ReplicationPeerConfig for this peer
*/
- PeerState getPeerState(boolean loadFromBackingStore) throws ReplicationException;
+ ReplicationPeerConfig getPeerConfig();
/**
* Get the configuration object required to communicate with this peer
* @return configuration object
*/
- public Configuration getConfiguration();
+ Configuration getConfiguration();
/**
* Get replicable (table, cf-list) map of this peer
* @return the replicable (table, cf-list) map
*/
- public Map<TableName, List<String>> getTableCFs();
+ Map<TableName, List<String>> getTableCFs();
/**
* Get replicable namespace set of this peer
* @return the replicable namespaces set
*/
- public Set<String> getNamespaces();
+ Set<String> getNamespaces();
/**
* Get the per node bandwidth upper limit for this peer
* @return the bandwidth upper limit
*/
- public long getPeerBandwidth();
+ long getPeerBandwidth();
/**
* Register a peer config listener to catch the peer config change event.
* @param listener listener to catch the peer config change event.
*/
- public void registerPeerConfigListener(ReplicationPeerConfigListener listener);
+ void registerPeerConfigListener(ReplicationPeerConfigListener listener);
/**
- * Notify all the registered ReplicationPeerConfigListener to update their peer config.
- * @param newPeerConfig the new peer config.
+ * @deprecated Use {@link #registerPeerConfigListener(ReplicationPeerConfigListener)} instead.
*/
- public void triggerPeerConfigChange(ReplicationPeerConfig newPeerConfig);
-}
+ @Deprecated
+ default void trackPeerConfigChanges(ReplicationPeerConfigListener listener) {
+ registerPeerConfigListener(listener);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcc77931/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
new file mode 100644
index 0000000..2c7ea9b
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -0,0 +1,169 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+
+@InterfaceAudience.Private
+public class ReplicationPeerImpl implements ReplicationPeer {
+ private static final Logger LOG = LoggerFactory.getLogger(ReplicationPeerImpl.class);
+
+ private final ReplicationPeerStorage peerStorage;
+
+ private final Configuration conf;
+
+ private final String id;
+
+ private volatile ReplicationPeerConfig peerConfig;
+
+ private volatile PeerState peerState;
+
+ private final List<ReplicationPeerConfigListener> peerConfigListeners;
+
+ /**
+ * Constructor that takes all the objects required to communicate with the specified peer, except
+ * for the region server addresses.
+ * @param conf configuration object to this peer
+ * @param id string representation of this peer's identifier
+ * @param peerConfig configuration for the replication peer
+ */
+ public ReplicationPeerImpl(ZKWatcher zkWatcher, Configuration conf, String id,
+ ReplicationPeerConfig peerConfig) {
+ this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zkWatcher, conf);
+ this.conf = conf;
+ this.peerConfig = peerConfig;
+ this.id = id;
+ this.peerConfigListeners = new ArrayList<>();
+ }
+
+ public void refreshPeerState() throws ReplicationException {
+ this.peerState = peerStorage.isPeerEnabled(id) ? PeerState.ENABLED : PeerState.DISABLED;
+ }
+
+ public void refreshPeerConfig() throws ReplicationException {
+ this.peerConfig = peerStorage.getPeerConfig(id).orElse(peerConfig);
+ peerConfigListeners.forEach(listener -> listener.peerConfigUpdated(peerConfig));
+ }
+
+ /**
+ * Get the identifier of this peer
+ * @return string representation of the id (short)
+ */
+ @Override
+ public String getId() {
+ return id;
+ }
+
+ @Override
+ public PeerState getPeerState() {
+ return peerState;
+ }
+
+ /**
+ * Get the peer config object
+ * @return the ReplicationPeerConfig for this peer
+ */
+ @Override
+ public ReplicationPeerConfig getPeerConfig() {
+ return peerConfig;
+ }
+
+ /**
+ * Get the configuration object required to communicate with this peer
+ * @return configuration object
+ */
+ @Override
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ /**
+ * Get replicable (table, cf-list) map of this peer
+ * @return the replicable (table, cf-list) map
+ */
+ @Override
+ public Map<TableName, List<String>> getTableCFs() {
+ return this.peerConfig.getTableCFsMap();
+ }
+
+ /**
+ * Get replicable namespace set of this peer
+ * @return the replicable namespaces set
+ */
+ @Override
+ public Set<String> getNamespaces() {
+ return this.peerConfig.getNamespaces();
+ }
+
+ @Override
+ public long getPeerBandwidth() {
+ return this.peerConfig.getBandwidth();
+ }
+
+ @Override
+ public void registerPeerConfigListener(ReplicationPeerConfigListener listener) {
+ this.peerConfigListeners.add(listener);
+ }
+
+ /**
+ * Parse the raw data from ZK to get a peer's state
+ * @param bytes raw ZK data
+ * @return True if the passed in <code>bytes</code> are those of a pb serialized ENABLED state.
+ * @throws DeserializationException
+ */
+ public static boolean isStateEnabled(final byte[] bytes) throws DeserializationException {
+ ReplicationProtos.ReplicationState.State state = parseStateFrom(bytes);
+ return ReplicationProtos.ReplicationState.State.ENABLED == state;
+ }
+
+ /**
+ * @param bytes Content of a state znode.
+ * @return State parsed from the passed bytes.
+ * @throws DeserializationException
+ */
+ private static ReplicationProtos.ReplicationState.State parseStateFrom(final byte[] bytes)
+ throws DeserializationException {
+ ProtobufUtil.expectPBMagicPrefix(bytes);
+ int pbLen = ProtobufUtil.lengthOfPBMagic();
+ ReplicationProtos.ReplicationState.Builder builder =
+ ReplicationProtos.ReplicationState.newBuilder();
+ ReplicationProtos.ReplicationState state;
+ try {
+ ProtobufUtil.mergeFrom(builder, bytes, pbLen, bytes.length - pbLen);
+ state = builder.build();
+ return state.getState();
+ } catch (IOException e) {
+ throw new DeserializationException(e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcc77931/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
deleted file mode 100644
index 49b9460..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.log.HBaseMarkers;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-
-@InterfaceAudience.Private
-public class ReplicationPeerZKImpl extends ReplicationStateZKBase
- implements ReplicationPeer, Abortable, Closeable {
- private static final Logger LOG = LoggerFactory.getLogger(ReplicationPeerZKImpl.class);
-
- private volatile ReplicationPeerConfig peerConfig;
- private final String id;
- private volatile PeerState peerState;
- private volatile Map<TableName, List<String>> tableCFs = new HashMap<>();
- private final Configuration conf;
-
- private final List<ReplicationPeerConfigListener> peerConfigListeners;
-
- /**
- * Constructor that takes all the objects required to communicate with the specified peer, except
- * for the region server addresses.
- * @param conf configuration object to this peer
- * @param id string representation of this peer's identifier
- * @param peerConfig configuration for the replication peer
- */
- public ReplicationPeerZKImpl(ZKWatcher zkWatcher, Configuration conf, String id,
- ReplicationPeerConfig peerConfig, Abortable abortable) throws ReplicationException {
- super(zkWatcher, conf, abortable);
- this.conf = conf;
- this.peerConfig = peerConfig;
- this.id = id;
- this.peerConfigListeners = new ArrayList<>();
- }
-
- private PeerState readPeerState() throws ReplicationException {
- try {
- byte[] data = ZKUtil.getData(zookeeper, this.getPeerStateNode(id));
- this.peerState = isStateEnabled(data) ? PeerState.ENABLED : PeerState.DISABLED;
- } catch (DeserializationException | KeeperException | InterruptedException e) {
- throw new ReplicationException("Get and deserialize peer state data from zookeeper failed: ",
- e);
- }
- return this.peerState;
- }
-
- private ReplicationPeerConfig readPeerConfig() throws ReplicationException {
- try {
- byte[] data = ZKUtil.getData(zookeeper, this.getPeerNode(id));
- if (data != null) {
- this.peerConfig = ReplicationPeerConfigUtil.parsePeerFrom(data);
- }
- } catch (DeserializationException | KeeperException | InterruptedException e) {
- throw new ReplicationException("Get and deserialize peer config date from zookeeper failed: ",
- e);
- }
- return this.peerConfig;
- }
-
- @Override
- public PeerState getPeerState() {
- return peerState;
- }
-
- @Override
- public PeerState getPeerState(boolean loadFromBackingStore) throws ReplicationException {
- if (loadFromBackingStore) {
- return readPeerState();
- } else {
- return peerState;
- }
- }
-
- /**
- * Get the identifier of this peer
- * @return string representation of the id (short)
- */
- @Override
- public String getId() {
- return id;
- }
-
- /**
- * Get the peer config object
- * @return the ReplicationPeerConfig for this peer
- */
- @Override
- public ReplicationPeerConfig getPeerConfig() {
- return peerConfig;
- }
-
- @Override
- public ReplicationPeerConfig getPeerConfig(boolean loadFromBackingStore)
- throws ReplicationException {
- if (loadFromBackingStore) {
- return readPeerConfig();
- } else {
- return peerConfig;
- }
- }
-
- /**
- * Get the configuration object required to communicate with this peer
- * @return configuration object
- */
- @Override
- public Configuration getConfiguration() {
- return conf;
- }
-
- /**
- * Get replicable (table, cf-list) map of this peer
- * @return the replicable (table, cf-list) map
- */
- @Override
- public Map<TableName, List<String>> getTableCFs() {
- this.tableCFs = peerConfig.getTableCFsMap();
- return this.tableCFs;
- }
-
- /**
- * Get replicable namespace set of this peer
- * @return the replicable namespaces set
- */
- @Override
- public Set<String> getNamespaces() {
- return this.peerConfig.getNamespaces();
- }
-
- @Override
- public long getPeerBandwidth() {
- return this.peerConfig.getBandwidth();
- }
-
- @Override
- public void registerPeerConfigListener(ReplicationPeerConfigListener listener) {
- this.peerConfigListeners.add(listener);
- }
-
- @Override
- public void triggerPeerConfigChange(ReplicationPeerConfig newPeerConfig) {
- for (ReplicationPeerConfigListener listener : this.peerConfigListeners) {
- listener.peerConfigUpdated(newPeerConfig);
- }
- }
-
- @Override
- public void abort(String why, Throwable e) {
- LOG.error(HBaseMarkers.FATAL, "The ReplicationPeer corresponding to peer " +
- peerConfig + " was aborted for the following reason(s):" + why, e);
- }
-
- @Override
- public boolean isAborted() {
- // Currently the replication peer is never "Aborted", we just log when the
- // abort method is called.
- return false;
- }
-
- @Override
- public void close() throws IOException {
- // TODO: stop zkw?
- }
-
- /**
- * Parse the raw data from ZK to get a peer's state
- * @param bytes raw ZK data
- * @return True if the passed in <code>bytes</code> are those of a pb serialized ENABLED state.
- * @throws DeserializationException
- */
- public static boolean isStateEnabled(final byte[] bytes) throws DeserializationException {
- ReplicationProtos.ReplicationState.State state = parseStateFrom(bytes);
- return ReplicationProtos.ReplicationState.State.ENABLED == state;
- }
-
- /**
- * @param bytes Content of a state znode.
- * @return State parsed from the passed bytes.
- * @throws DeserializationException
- */
- private static ReplicationProtos.ReplicationState.State parseStateFrom(final byte[] bytes)
- throws DeserializationException {
- ProtobufUtil.expectPBMagicPrefix(bytes);
- int pbLen = ProtobufUtil.lengthOfPBMagic();
- ReplicationProtos.ReplicationState.Builder builder =
- ReplicationProtos.ReplicationState.newBuilder();
- ReplicationProtos.ReplicationState state;
- try {
- ProtobufUtil.mergeFrom(builder, bytes, pbLen, bytes.length - pbLen);
- state = builder.build();
- return state.getState();
- } catch (IOException e) {
- throw new DeserializationException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcc77931/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 10936bf..afc19bd 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -116,13 +116,13 @@ public interface ReplicationPeers {
throws ReplicationException;
/**
- * Returns the ReplicationPeer for the specified connected peer. This ReplicationPeer will
+ * Returns the ReplicationPeerImpl for the specified connected peer. This ReplicationPeer will
* continue to track changes to the Peer's state and config. This method returns null if no
* peer has been connected with the given peerId.
* @param peerId id for the peer
* @return ReplicationPeer object
*/
- ReplicationPeer getConnectedPeer(String peerId);
+ ReplicationPeerImpl getConnectedPeer(String peerId);
/**
* Returns the set of peerIds of the clusters that have been connected and have an underlying
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcc77931/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 7de4619..7f6498d 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -80,7 +80,7 @@ import org.slf4j.LoggerFactory;
public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements ReplicationPeers {
// Map of peer clusters keyed by their id
- private Map<String, ReplicationPeerZKImpl> peerClusters;
+ private ConcurrentMap<String, ReplicationPeerImpl> peerClusters;
private final ReplicationQueueStorage queueStorage;
private Abortable abortable;
@@ -232,7 +232,7 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
}
String peerStateZNode = getPeerStateNode(id);
try {
- return ReplicationPeerZKImpl.isStateEnabled(ZKUtil.getData(this.zookeeper, peerStateZNode));
+ return ReplicationPeerImpl.isStateEnabled(ZKUtil.getData(this.zookeeper, peerStateZNode));
} catch (KeeperException e) {
throw new ReplicationException(e);
} catch (DeserializationException e) {
@@ -270,7 +270,7 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
}
@Override
- public ReplicationPeer getConnectedPeer(String peerId) {
+ public ReplicationPeerImpl getConnectedPeer(String peerId) {
return peerClusters.get(peerId);
}
@@ -423,7 +423,7 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
public void peerDisconnected(String peerId) {
ReplicationPeer rp = this.peerClusters.get(peerId);
if (rp != null) {
- ((ConcurrentMap<String, ReplicationPeerZKImpl>) peerClusters).remove(peerId, rp);
+ peerClusters.remove(peerId, rp);
}
}
@@ -440,7 +440,7 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
return false;
}
- ReplicationPeerZKImpl peer = null;
+ ReplicationPeerImpl peer = null;
try {
peer = createPeer(peerId);
} catch (Exception e) {
@@ -449,8 +449,7 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
if (peer == null) {
return false;
}
- ReplicationPeerZKImpl previous =
- ((ConcurrentMap<String, ReplicationPeerZKImpl>) peerClusters).putIfAbsent(peerId, peer);
+ ReplicationPeerImpl previous = peerClusters.putIfAbsent(peerId, peer);
if (previous == null) {
LOG.info("Added new peer cluster=" + peer.getPeerConfig().getClusterKey());
} else {
@@ -493,19 +492,19 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
* @return object representing the peer
* @throws ReplicationException
*/
- private ReplicationPeerZKImpl createPeer(String peerId) throws ReplicationException {
+ private ReplicationPeerImpl createPeer(String peerId) throws ReplicationException {
Pair<ReplicationPeerConfig, Configuration> pair = getPeerConf(peerId);
if (pair == null) {
return null;
}
Configuration peerConf = pair.getSecond();
- ReplicationPeerZKImpl peer =
- new ReplicationPeerZKImpl(zookeeper, peerConf, peerId, pair.getFirst(), abortable);
+ ReplicationPeerImpl peer =
+ new ReplicationPeerImpl(zookeeper, peerConf, peerId, pair.getFirst());
// Load peer state and peer config by reading zookeeper directly.
- peer.getPeerState(true);
- peer.getPeerConfig(true);
+ peer.refreshPeerState();
+ peer.refreshPeerConfig();
return peer;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcc77931/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
index 6fe869c..8905d43 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
@@ -312,12 +312,15 @@ public abstract class TestReplicationStateBasic {
rp.disablePeer(ID_ONE);
// now we do not rely on zk watcher to trigger the state change so we need to trigger it
// manually...
- assertEquals(PeerState.DISABLED, rp.getConnectedPeer(ID_ONE).getPeerState(true));
+ ReplicationPeerImpl peer = rp.getConnectedPeer(ID_ONE);
+ peer.refreshPeerState();
+ assertEquals(PeerState.DISABLED, peer.getPeerState());
assertConnectedPeerStatus(false, ID_ONE);
rp.enablePeer(ID_ONE);
// now we do not rely on zk watcher to trigger the state change so we need to trigger it
// manually...
- assertEquals(PeerState.ENABLED, rp.getConnectedPeer(ID_ONE).getPeerState(true));
+ peer.refreshPeerState();
+ assertEquals(PeerState.ENABLED, peer.getPeerState());
assertConnectedPeerStatus(true, ID_ONE);
// Disconnect peer
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcc77931/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index 9b493d9..598357c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -21,15 +21,14 @@ package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationPeer;
-import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.log4j.Logger;
+import org.apache.hadoop.hbase.replication.ReplicationPeerImpl;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
- private static final Logger LOG = Logger.getLogger(PeerProcedureHandlerImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(PeerProcedureHandlerImpl.class);
private ReplicationSourceManager replicationSourceManager;
@@ -49,10 +48,11 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
@Override
public void disablePeer(String peerId) throws ReplicationException, IOException {
- ReplicationPeer peer = replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
+ ReplicationPeerImpl peer =
+ replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
if (peer != null) {
- PeerState peerState = peer.getPeerState(true);
- LOG.info("disablePeer state, peer id: " + peerId + ", state: " + peerState);
+ peer.refreshPeerState();
+ LOG.info("disable replication peer, id: " + peerId + ", new state: " + peer.getPeerState());
} else {
throw new ReplicationException("No connected peer found, peerId=" + peerId);
}
@@ -60,10 +60,11 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
@Override
public void enablePeer(String peerId) throws ReplicationException, IOException {
- ReplicationPeer peer = replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
+ ReplicationPeerImpl peer =
+ replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
if (peer != null) {
- PeerState peerState = peer.getPeerState(true);
- LOG.info("enablePeer state, peer id: " + peerId + ", state: " + peerState);
+ peer.refreshPeerState();
+ LOG.info("enable replication peer, id: " + peerId + ", new state: " + peer.getPeerState());
} else {
throw new ReplicationException("No connected peer found, peerId=" + peerId);
}
@@ -71,11 +72,11 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
@Override
public void updatePeerConfig(String peerId) throws ReplicationException, IOException {
- ReplicationPeer peer = replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
+ ReplicationPeerImpl peer =
+ replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
if (peer == null) {
throw new ReplicationException("No connected peer found, peerId=" + peerId);
}
- ReplicationPeerConfig rpc = peer.getPeerConfig(true);
- peer.triggerPeerConfigChange(rpc);
+ peer.refreshPeerConfig();
}
}
[18/50] [abbrv] hbase git commit: HBASE-19536 Client side changes for
moving peer modification from zk watcher to procedure
Posted by zh...@apache.org.
HBASE-19536 Client side changes for moving peer modification from zk watcher to procedure
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4482adc0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4482adc0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4482adc0
Branch: refs/heads/HBASE-19397-branch-2
Commit: 4482adc00fd429d6c84d939b8e83484a5a00b5b1
Parents: e8dbb5d
Author: Guanghao Zhang <zg...@apache.org>
Authored: Tue Dec 19 15:50:57 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Admin.java | 87 ++++++++++-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 149 ++++++++++++++-----
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 82 +++++-----
3 files changed, 238 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4482adc0/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 40dac2f..b8546fa 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -2473,7 +2473,7 @@ public interface Admin extends Abortable, Closeable {
/**
* Add a new replication peer for replicating data to slave cluster.
* @param peerId a short name that identifies the peer
- * @param peerConfig configuration for the replication slave cluster
+ * @param peerConfig configuration for the replication peer
* @throws IOException if a remote or network exception occurs
*/
default void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig)
@@ -2484,7 +2484,7 @@ public interface Admin extends Abortable, Closeable {
/**
* Add a new replication peer for replicating data to slave cluster.
* @param peerId a short name that identifies the peer
- * @param peerConfig configuration for the replication slave cluster
+ * @param peerConfig configuration for the replication peer
* @param enabled peer state, true if ENABLED and false if DISABLED
* @throws IOException if a remote or network exception occurs
*/
@@ -2492,6 +2492,37 @@ public interface Admin extends Abortable, Closeable {
throws IOException;
/**
+ * Add a new replication peer but does not block and wait for it.
+ * <p>
+ * You can use Future.get(long, TimeUnit) to wait on the operation to complete. It may throw
+ * ExecutionException if there was an error while executing the operation or TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to complete.
+ * @param peerId a short name that identifies the peer
+ * @param peerConfig configuration for the replication peer
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ default Future<Void> addReplicationPeerAsync(String peerId, ReplicationPeerConfig peerConfig)
+ throws IOException {
+ return addReplicationPeerAsync(peerId, peerConfig, true);
+ }
+
+ /**
+ * Add a new replication peer but does not block and wait for it.
+ * <p>
+ * You can use Future.get(long, TimeUnit) to wait on the operation to complete. It may throw
+ * ExecutionException if there was an error while executing the operation or TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to complete.
+ * @param peerId a short name that identifies the peer
+ * @param peerConfig configuration for the replication peer
+ * @param enabled peer state, true if ENABLED and false if DISABLED
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ Future<Void> addReplicationPeerAsync(String peerId, ReplicationPeerConfig peerConfig,
+ boolean enabled) throws IOException;
+
+ /**
* Remove a peer and stop the replication.
* @param peerId a short name that identifies the peer
* @throws IOException if a remote or network exception occurs
@@ -2499,6 +2530,18 @@ public interface Admin extends Abortable, Closeable {
void removeReplicationPeer(String peerId) throws IOException;
/**
+ * Remove a replication peer but does not block and wait for it.
+ * <p>
+ * You can use Future.get(long, TimeUnit) to wait on the operation to complete. It may throw
+ * ExecutionException if there was an error while executing the operation or TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to complete.
+ * @param peerId a short name that identifies the peer
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ Future<Void> removeReplicationPeerAsync(String peerId) throws IOException;
+
+ /**
* Restart the replication stream to the specified peer.
* @param peerId a short name that identifies the peer
* @throws IOException if a remote or network exception occurs
@@ -2506,6 +2549,18 @@ public interface Admin extends Abortable, Closeable {
void enableReplicationPeer(String peerId) throws IOException;
/**
+ * Enable a replication peer but does not block and wait for it.
+ * <p>
+ * You can use Future.get(long, TimeUnit) to wait on the operation to complete. It may throw
+ * ExecutionException if there was an error while executing the operation or TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to complete.
+ * @param peerId a short name that identifies the peer
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ Future<Void> enableReplicationPeerAsync(String peerId) throws IOException;
+
+ /**
* Stop the replication stream to the specified peer.
* @param peerId a short name that identifies the peer
* @throws IOException if a remote or network exception occurs
@@ -2513,6 +2568,18 @@ public interface Admin extends Abortable, Closeable {
void disableReplicationPeer(String peerId) throws IOException;
/**
+ * Disable a replication peer but does not block and wait for it.
+ * <p>
+ * You can use Future.get(long, TimeUnit) to wait on the operation to complete. It may throw
+ * ExecutionException if there was an error while executing the operation or TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to complete.
+ * @param peerId a short name that identifies the peer
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ Future<Void> disableReplicationPeerAsync(String peerId) throws IOException;
+
+ /**
* Returns the configured ReplicationPeerConfig for the specified peer.
* @param peerId a short name that identifies the peer
* @return ReplicationPeerConfig for the peer
@@ -2523,13 +2590,27 @@ public interface Admin extends Abortable, Closeable {
/**
* Update the peerConfig for the specified peer.
* @param peerId a short name that identifies the peer
- * @param peerConfig new config for the peer
+ * @param peerConfig new config for the replication peer
* @throws IOException if a remote or network exception occurs
*/
void updateReplicationPeerConfig(String peerId,
ReplicationPeerConfig peerConfig) throws IOException;
/**
+ * Update the peerConfig for the specified peer but does not block and wait for it.
+ * <p>
+ * You can use Future.get(long, TimeUnit) to wait on the operation to complete. It may throw
+ * ExecutionException if there was an error while executing the operation or TimeoutException in
+ * case the wait timeout was not long enough to allow the operation to complete.
+ * @param peerId a short name that identifies the peer
+ * @param peerConfig new config for the replication peer
+ * @return the result of the async operation
+ * @throws IOException IOException if a remote or network exception occurs
+ */
+ Future<Void> updateReplicationPeerConfigAsync(String peerId, ReplicationPeerConfig peerConfig)
+ throws IOException;
+
+ /**
* Append the replicable table column family config from the specified peer.
* @param id a short that identifies the cluster
* @param tableCfs A map from tableName to column family names
http://git-wip-us.apache.org/repos/asf/hbase/blob/4482adc0/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index c137383..8685984 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -39,6 +39,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -200,7 +201,12 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.DisableReplicationPeerResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.EnableReplicationPeerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.GetReplicationPeerConfigResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.RemoveReplicationPeerResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.UpdateReplicationPeerConfigResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
/**
@@ -3779,6 +3785,25 @@ public class HBaseAdmin implements Admin {
}
}
+ @InterfaceAudience.Private
+ @InterfaceStability.Evolving
+ private static class ReplicationFuture extends ProcedureFuture<Void> {
+ private final String peerId;
+ private final Supplier<String> getOperation;
+
+ public ReplicationFuture(HBaseAdmin admin, String peerId, Long procId,
+ Supplier<String> getOperation) {
+ super(admin, procId);
+ this.peerId = peerId;
+ this.getOperation = getOperation;
+ }
+
+ @Override
+ public String toString() {
+ return "Operation: " + getOperation.get() + ", peerId: " + peerId;
+ }
+ }
+
@Override
public List<SecurityCapability> getSecurityCapabilities() throws IOException {
try {
@@ -3851,50 +3876,82 @@ public class HBaseAdmin implements Admin {
@Override
public void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
throws IOException {
- executeCallable(new MasterCallable<Void>(getConnection(), getRpcControllerFactory()) {
- @Override
- protected Void rpcCall() throws Exception {
- master.addReplicationPeer(getRpcController(),
- RequestConverter.buildAddReplicationPeerRequest(peerId, peerConfig, enabled));
- return null;
- }
- });
+ get(addReplicationPeerAsync(peerId, peerConfig, enabled), this.syncWaitTimeout,
+ TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public Future<Void> addReplicationPeerAsync(String peerId, ReplicationPeerConfig peerConfig,
+ boolean enabled) throws IOException {
+ AddReplicationPeerResponse response = executeCallable(
+ new MasterCallable<AddReplicationPeerResponse>(getConnection(), getRpcControllerFactory()) {
+ @Override
+ protected AddReplicationPeerResponse rpcCall() throws Exception {
+ return master.addReplicationPeer(getRpcController(),
+ RequestConverter.buildAddReplicationPeerRequest(peerId, peerConfig, enabled));
+ }
+ });
+ return new ReplicationFuture(this, peerId, response.getProcId(), () -> "ADD_REPLICATION_PEER");
}
@Override
public void removeReplicationPeer(String peerId) throws IOException {
- executeCallable(new MasterCallable<Void>(getConnection(), getRpcControllerFactory()) {
- @Override
- protected Void rpcCall() throws Exception {
- master.removeReplicationPeer(getRpcController(),
- RequestConverter.buildRemoveReplicationPeerRequest(peerId));
- return null;
- }
- });
+ get(removeReplicationPeerAsync(peerId), this.syncWaitTimeout, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public Future<Void> removeReplicationPeerAsync(String peerId) throws IOException {
+ RemoveReplicationPeerResponse response =
+ executeCallable(new MasterCallable<RemoveReplicationPeerResponse>(getConnection(),
+ getRpcControllerFactory()) {
+ @Override
+ protected RemoveReplicationPeerResponse rpcCall() throws Exception {
+ return master.removeReplicationPeer(getRpcController(),
+ RequestConverter.buildRemoveReplicationPeerRequest(peerId));
+ }
+ });
+ return new ReplicationFuture(this, peerId, response.getProcId(),
+ () -> "REMOVE_REPLICATION_PEER");
}
@Override
public void enableReplicationPeer(final String peerId) throws IOException {
- executeCallable(new MasterCallable<Void>(getConnection(), getRpcControllerFactory()) {
- @Override
- protected Void rpcCall() throws Exception {
- master.enableReplicationPeer(getRpcController(),
- RequestConverter.buildEnableReplicationPeerRequest(peerId));
- return null;
- }
- });
+ get(enableReplicationPeerAsync(peerId), this.syncWaitTimeout, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public Future<Void> enableReplicationPeerAsync(final String peerId) throws IOException {
+ EnableReplicationPeerResponse response =
+ executeCallable(new MasterCallable<EnableReplicationPeerResponse>(getConnection(),
+ getRpcControllerFactory()) {
+ @Override
+ protected EnableReplicationPeerResponse rpcCall() throws Exception {
+ return master.enableReplicationPeer(getRpcController(),
+ RequestConverter.buildEnableReplicationPeerRequest(peerId));
+ }
+ });
+ return new ReplicationFuture(this, peerId, response.getProcId(),
+ () -> "ENABLE_REPLICATION_PEER");
}
@Override
public void disableReplicationPeer(final String peerId) throws IOException {
- executeCallable(new MasterCallable<Void>(getConnection(), getRpcControllerFactory()) {
- @Override
- protected Void rpcCall() throws Exception {
- master.disableReplicationPeer(getRpcController(),
- RequestConverter.buildDisableReplicationPeerRequest(peerId));
- return null;
- }
- });
+ get(disableReplicationPeerAsync(peerId), this.syncWaitTimeout, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public Future<Void> disableReplicationPeerAsync(final String peerId) throws IOException {
+ DisableReplicationPeerResponse response =
+ executeCallable(new MasterCallable<DisableReplicationPeerResponse>(getConnection(),
+ getRpcControllerFactory()) {
+ @Override
+ protected DisableReplicationPeerResponse rpcCall() throws Exception {
+ return master.disableReplicationPeer(getRpcController(),
+ RequestConverter.buildDisableReplicationPeerRequest(peerId));
+ }
+ });
+ return new ReplicationFuture(this, peerId, response.getProcId(),
+ () -> "DISABLE_REPLICATION_PEER");
}
@Override
@@ -3913,14 +3970,24 @@ public class HBaseAdmin implements Admin {
@Override
public void updateReplicationPeerConfig(final String peerId,
final ReplicationPeerConfig peerConfig) throws IOException {
- executeCallable(new MasterCallable<Void>(getConnection(), getRpcControllerFactory()) {
- @Override
- protected Void rpcCall() throws Exception {
- master.updateReplicationPeerConfig(getRpcController(),
- RequestConverter.buildUpdateReplicationPeerConfigRequest(peerId, peerConfig));
- return null;
- }
- });
+ get(updateReplicationPeerConfigAsync(peerId, peerConfig), this.syncWaitTimeout,
+ TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public Future<Void> updateReplicationPeerConfigAsync(final String peerId,
+ final ReplicationPeerConfig peerConfig) throws IOException {
+ UpdateReplicationPeerConfigResponse response =
+ executeCallable(new MasterCallable<UpdateReplicationPeerConfigResponse>(getConnection(),
+ getRpcControllerFactory()) {
+ @Override
+ protected UpdateReplicationPeerConfigResponse rpcCall() throws Exception {
+ return master.updateReplicationPeerConfig(getRpcController(),
+ RequestConverter.buildUpdateReplicationPeerConfigRequest(peerId, peerConfig));
+ }
+ });
+ return new ReplicationFuture(this, peerId, response.getProcId(),
+ () -> "UPDATE_REPLICATION_PEER_CONFIG");
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/4482adc0/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
index a826f8c..050bfe2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -36,6 +36,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -1558,47 +1559,34 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
@Override
public CompletableFuture<Void> addReplicationPeer(String peerId,
ReplicationPeerConfig peerConfig, boolean enabled) {
- return this
- .<Void> newMasterCaller()
- .action(
- (controller, stub) -> this
- .<AddReplicationPeerRequest, AddReplicationPeerResponse, Void> call(controller, stub,
- RequestConverter.buildAddReplicationPeerRequest(peerId, peerConfig, enabled), (s,
- c, req, done) -> s.addReplicationPeer(c, req, done), (resp) -> null)).call();
+ return this.<AddReplicationPeerRequest, AddReplicationPeerResponse> procedureCall(
+ RequestConverter.buildAddReplicationPeerRequest(peerId, peerConfig, enabled),
+ (s, c, req, done) -> s.addReplicationPeer(c, req, done), (resp) -> resp.getProcId(),
+ new ReplicationProcedureBiConsumer(peerId, () -> "ADD_REPLICATION_PEER"));
}
@Override
public CompletableFuture<Void> removeReplicationPeer(String peerId) {
- return this
- .<Void> newMasterCaller()
- .action(
- (controller, stub) -> this
- .<RemoveReplicationPeerRequest, RemoveReplicationPeerResponse, Void> call(controller,
- stub, RequestConverter.buildRemoveReplicationPeerRequest(peerId),
- (s, c, req, done) -> s.removeReplicationPeer(c, req, done), (resp) -> null)).call();
+ return this.<RemoveReplicationPeerRequest, RemoveReplicationPeerResponse> procedureCall(
+ RequestConverter.buildRemoveReplicationPeerRequest(peerId),
+ (s, c, req, done) -> s.removeReplicationPeer(c, req, done), (resp) -> resp.getProcId(),
+ new ReplicationProcedureBiConsumer(peerId, () -> "REMOVE_REPLICATION_PEER"));
}
@Override
public CompletableFuture<Void> enableReplicationPeer(String peerId) {
- return this
- .<Void> newMasterCaller()
- .action(
- (controller, stub) -> this
- .<EnableReplicationPeerRequest, EnableReplicationPeerResponse, Void> call(controller,
- stub, RequestConverter.buildEnableReplicationPeerRequest(peerId),
- (s, c, req, done) -> s.enableReplicationPeer(c, req, done), (resp) -> null)).call();
+ return this.<EnableReplicationPeerRequest, EnableReplicationPeerResponse> procedureCall(
+ RequestConverter.buildEnableReplicationPeerRequest(peerId),
+ (s, c, req, done) -> s.enableReplicationPeer(c, req, done), (resp) -> resp.getProcId(),
+ new ReplicationProcedureBiConsumer(peerId, () -> "ENABLE_REPLICATION_PEER"));
}
@Override
public CompletableFuture<Void> disableReplicationPeer(String peerId) {
- return this
- .<Void> newMasterCaller()
- .action(
- (controller, stub) -> this
- .<DisableReplicationPeerRequest, DisableReplicationPeerResponse, Void> call(
- controller, stub, RequestConverter.buildDisableReplicationPeerRequest(peerId), (s,
- c, req, done) -> s.disableReplicationPeer(c, req, done), (resp) -> null))
- .call();
+ return this.<DisableReplicationPeerRequest, DisableReplicationPeerResponse> procedureCall(
+ RequestConverter.buildDisableReplicationPeerRequest(peerId),
+ (s, c, req, done) -> s.disableReplicationPeer(c, req, done), (resp) -> resp.getProcId(),
+ new ReplicationProcedureBiConsumer(peerId, () -> "DISABLE_REPLICATION_PEER"));
}
@Override
@@ -1617,13 +1605,11 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
public CompletableFuture<Void> updateReplicationPeerConfig(String peerId,
ReplicationPeerConfig peerConfig) {
return this
- .<Void> newMasterCaller()
- .action(
- (controller, stub) -> this
- .<UpdateReplicationPeerConfigRequest, UpdateReplicationPeerConfigResponse, Void> call(
- controller, stub, RequestConverter.buildUpdateReplicationPeerConfigRequest(peerId,
- peerConfig), (s, c, req, done) -> s.updateReplicationPeerConfig(c, req, done), (
- resp) -> null)).call();
+ .<UpdateReplicationPeerConfigRequest, UpdateReplicationPeerConfigResponse> procedureCall(
+ RequestConverter.buildUpdateReplicationPeerConfigRequest(peerId, peerConfig),
+ (s, c, req, done) -> s.updateReplicationPeerConfig(c, req, done),
+ (resp) -> resp.getProcId(),
+ new ReplicationProcedureBiConsumer(peerId, () -> "UPDATE_REPLICATION_PEER_CONFIG"));
}
@Override
@@ -2582,6 +2568,30 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
}
}
+ private class ReplicationProcedureBiConsumer extends ProcedureBiConsumer {
+ private final String peerId;
+ private final Supplier<String> getOperation;
+
+ ReplicationProcedureBiConsumer(String peerId, Supplier<String> getOperation) {
+ this.peerId = peerId;
+ this.getOperation = getOperation;
+ }
+
+ String getDescription() {
+ return "Operation: " + getOperation.get() + ", peerId: " + peerId;
+ }
+
+ @Override
+ void onFinished() {
+ LOG.info(getDescription() + " completed");
+ }
+
+ @Override
+ void onError(Throwable error) {
+ LOG.info(getDescription() + " failed with " + error.getMessage());
+ }
+ }
+
private CompletableFuture<Void> waitProcedureResult(CompletableFuture<Long> procFuture) {
CompletableFuture<Void> future = new CompletableFuture<>();
procFuture.whenComplete((procId, error) -> {
[22/50] [abbrv] hbase git commit: HBASE-19543 Abstract a replication
storage interface to extract the zk specific code
Posted by zh...@apache.org.
HBASE-19543 Abstract a replication storage interface to extract the zk specific code
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7213a0dc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7213a0dc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7213a0dc
Branch: refs/heads/HBASE-19397-branch-2
Commit: 7213a0dc74b4ff90e05e3e40f2cc3362d9d3fd6c
Parents: f8fa2fc
Author: zhangduo <zh...@apache.org>
Authored: Fri Dec 22 14:37:28 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../hadoop/hbase/util/CollectionUtils.java | 3 +
hbase-replication/pom.xml | 12 +
.../replication/ReplicationPeerStorage.java | 74 ++++
.../replication/ReplicationQueueStorage.java | 164 +++++++
.../replication/ReplicationStateZKBase.java | 1 -
.../replication/ReplicationStorageFactory.java | 49 +++
.../replication/ZKReplicationPeerStorage.java | 164 +++++++
.../replication/ZKReplicationQueueStorage.java | 425 +++++++++++++++++++
.../replication/ZKReplicationStorageBase.java | 75 ++++
.../TestZKReplicationPeerStorage.java | 171 ++++++++
.../TestZKReplicationQueueStorage.java | 171 ++++++++
.../org/apache/hadoop/hbase/master/HMaster.java | 36 +-
.../hadoop/hbase/master/MasterServices.java | 6 +-
.../master/procedure/MasterProcedureEnv.java | 24 +-
.../master/replication/AddPeerProcedure.java | 6 +-
.../replication/DisablePeerProcedure.java | 7 +-
.../master/replication/EnablePeerProcedure.java | 6 +-
.../master/replication/ModifyPeerProcedure.java | 41 +-
.../master/replication/RemovePeerProcedure.java | 6 +-
.../master/replication/ReplicationManager.java | 199 ---------
.../replication/ReplicationPeerManager.java | 331 +++++++++++++++
.../replication/UpdatePeerConfigProcedure.java | 7 +-
.../replication/TestReplicationAdmin.java | 63 ++-
.../hbase/master/MockNoopMasterServices.java | 13 +-
.../hbase/master/TestMasterNoCluster.java | 4 +-
.../TestReplicationDisableInactivePeer.java | 6 +-
26 files changed, 1750 insertions(+), 314 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
index 875b124..8bbb6f1 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CollectionUtils.java
@@ -107,6 +107,9 @@ public class CollectionUtils {
return list.get(list.size() - 1);
}
+ public static <T> List<T> nullToEmpty(List<T> list) {
+ return list != null ? list : Collections.emptyList();
+ }
/**
* In HBASE-16648 we found that ConcurrentHashMap.get is much faster than computeIfAbsent if the
* value already exists. Notice that the implementation does not guarantee that the supplier will
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml
index bd593d3..b28e852 100644
--- a/hbase-replication/pom.xml
+++ b/hbase-replication/pom.xml
@@ -121,6 +121,18 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-zookeeper</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-zookeeper</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<!-- General dependencies -->
<dependency>
<groupId>org.apache.commons</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
new file mode 100644
index 0000000..e00cd0d
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Perform read/write to the replication peer storage.
+ */
+@InterfaceAudience.Private
+public interface ReplicationPeerStorage {
+
+ /**
+ * Add a replication peer.
+ * @throws ReplicationException if there are errors accessing the storage service.
+ */
+ void addPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
+ throws ReplicationException;
+
+ /**
+ * Remove a replication peer.
+ * @throws ReplicationException if there are errors accessing the storage service.
+ */
+ void removePeer(String peerId) throws ReplicationException;
+
+ /**
+ * Set the state of peer, {@code true} to {@code ENABLED}, otherwise to {@code DISABLED}.
+ * @throws ReplicationException if there are errors accessing the storage service.
+ */
+ void setPeerState(String peerId, boolean enabled) throws ReplicationException;
+
+ /**
+ * Update the config a replication peer.
+ * @throws ReplicationException if there are errors accessing the storage service.
+ */
+ void updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)
+ throws ReplicationException;
+
+ /**
+ * Return the peer ids of all replication peers.
+ * @throws ReplicationException if there are errors accessing the storage service.
+ */
+ List<String> listPeerIds() throws ReplicationException;
+
+ /**
+ * Test whether a replication peer is enabled.
+ * @throws ReplicationException if there are errors accessing the storage service.
+ */
+ boolean isPeerEnabled(String peerId) throws ReplicationException;
+
+ /**
+ * Get the peer config of a replication peer.
+ * @throws ReplicationException if there are errors accessing the storage service.
+ */
+ Optional<ReplicationPeerConfig> getPeerConfig(String peerId) throws ReplicationException;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
new file mode 100644
index 0000000..7210d9a
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Perform read/write to the replication queue storage.
+ */
+@InterfaceAudience.Private
+public interface ReplicationQueueStorage {
+
+ /**
+ * Remove a replication queue for a given regionserver.
+ * @param serverName the name of the regionserver
+ * @param queueId a String that identifies the queue.
+ */
+ void removeQueue(ServerName serverName, String queueId) throws ReplicationException;
+
+ /**
+ * Add a new WAL file to the given queue for a given regionserver. If the queue does not exist it
+ * is created.
+ * @param serverName the name of the regionserver
+ * @param queueId a String that identifies the queue.
+ * @param fileName name of the WAL
+ */
+ void addWAL(ServerName serverName, String queueId, String fileName) throws ReplicationException;
+
+ /**
+ * Remove an WAL file from the given queue for a given regionserver.
+ * @param serverName the name of the regionserver
+ * @param queueId a String that identifies the queue.
+ * @param fileName name of the WAL
+ */
+ void removeWAL(ServerName serverName, String queueId, String fileName)
+ throws ReplicationException;
+
+ /**
+ * Set the current position for a specific WAL in a given queue for a given regionserver.
+ * @param serverName the name of the regionserver
+ * @param queueId a String that identifies the queue
+ * @param fileName name of the WAL
+ * @param position the current position in the file
+ */
+ void setWALPosition(ServerName serverName, String queueId, String fileName, long position)
+ throws ReplicationException;
+
+ /**
+ * Get the current position for a specific WAL in a given queue for a given regionserver.
+ * @param serverName the name of the regionserver
+ * @param queueId a String that identifies the queue
+ * @param fileName name of the WAL
+ * @return the current position in the file
+ */
+ long getWALPosition(ServerName serverName, String queueId, String fileName)
+ throws ReplicationException;
+
+ /**
+ * Get a list of all queues for the specified region server.
+ * @param serverName the server name of the region server that owns the set of queues
+ * @return a list of queueIds
+ */
+ List<String> getAllQueues(ServerName serverName) throws ReplicationException;
+
+ /**
+ * Change ownership for the queue identified by queueId and belongs to a dead region server.
+ * @param sourceServerName the name of the dead region server
+ * @param destServerName the name of the target region server
+ * @param queueId the id of the queue
+ * @return the new PeerId and A SortedSet of WALs in its queue
+ */
+ Pair<String, SortedSet<String>> claimQueue(ServerName sourceServerName, String queueId,
+ ServerName destServerName) throws ReplicationException;
+
+ /**
+ * Remove the record of region server if the queue is empty.
+ */
+ void removeReplicatorIfQueueIsEmpty(ServerName serverName) throws ReplicationException;
+
+ /**
+ * Get a list of all region servers that have outstanding replication queues. These servers could
+ * be alive, dead or from a previous run of the cluster.
+ * @return a list of server names
+ */
+ List<ServerName> getListOfReplicators() throws ReplicationException;
+
+ /**
+ * Load all wals in all replication queues. This method guarantees to return a snapshot which
+ * contains all WALs in the zookeeper at the start of this call even there is concurrent queue
+ * failover. However, some newly created WALs during the call may not be included.
+ */
+ Set<String> getAllWALs() throws ReplicationException;
+
+ /**
+ * Add a peer to hfile reference queue if peer does not exist.
+ * @param peerId peer cluster id to be added
+ * @throws ReplicationException if fails to add a peer id to hfile reference queue
+ */
+ void addPeerToHFileRefs(String peerId) throws ReplicationException;
+
+ /**
+ * Remove a peer from hfile reference queue.
+ * @param peerId peer cluster id to be removed
+ */
+ void removePeerFromHFileRefs(String peerId) throws ReplicationException;
+
+ /**
+ * Add new hfile references to the queue.
+ * @param peerId peer cluster id to which the hfiles need to be replicated
+ * @param pairs list of pairs of { HFile location in staging dir, HFile path in region dir which
+ * will be added in the queue }
+ * @throws ReplicationException if fails to add a hfile reference
+ */
+ void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs) throws ReplicationException;
+
+ /**
+ * Remove hfile references from the queue.
+ * @param peerId peer cluster id from which this hfile references needs to be removed
+ * @param files list of hfile references to be removed
+ */
+ void removeHFileRefs(String peerId, List<String> files) throws ReplicationException;
+
+ /**
+ * Get the change version number of replication hfile references node. This can be used as
+ * optimistic locking to get a consistent snapshot of the replication queues of hfile references.
+ * @return change version number of hfile references node
+ */
+ int getHFileRefsNodeChangeVersion() throws ReplicationException;
+
+ /**
+ * Get list of all peers from hfile reference queue.
+ * @return a list of peer ids
+ */
+ List<String> getAllPeersFromHFileRefsQueue() throws ReplicationException;
+
+ /**
+ * Get a list of all hfile references in the given peer.
+ * @param peerId a String that identifies the peer
+ * @return a list of hfile references
+ */
+ List<String> getReplicableHFiles(String peerId) throws ReplicationException;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
index 05bbc84..f49537c 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
@@ -63,7 +63,6 @@ public abstract class ReplicationStateZKBase {
protected final Configuration conf;
protected final Abortable abortable;
- // Public for testing
public static final byte[] ENABLED_ZNODE_BYTES =
toByteArray(ReplicationProtos.ReplicationState.State.ENABLED);
public static final byte[] DISABLED_ZNODE_BYTES =
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
new file mode 100644
index 0000000..60d0749
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Used to create replication storage(peer, queue) classes.
+ * <p>
+ * For now we only have zk based implementation.
+ */
+@InterfaceAudience.Private
+public class ReplicationStorageFactory {
+
+ private ReplicationStorageFactory() {
+ }
+
+ /**
+ * Create a new {@link ReplicationPeerStorage}.
+ */
+ public static ReplicationPeerStorage getReplicationPeerStorage(ZKWatcher zk, Configuration conf) {
+ return new ZKReplicationPeerStorage(zk, conf);
+ }
+
+ /**
+ * Create a new {@link ReplicationQueueStorage}.
+ */
+ public static ReplicationQueueStorage getReplicationQueueStorage(ZKWatcher zk,
+ Configuration conf) {
+ return new ZKReplicationQueueStorage(zk, conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
new file mode 100644
index 0000000..49af4c3
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.util.CollectionUtils;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+
+/**
+ * ZK based replication peer storage.
+ */
+@InterfaceAudience.Private
+class ZKReplicationPeerStorage extends ZKReplicationStorageBase implements ReplicationPeerStorage {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZKReplicationPeerStorage.class);
+
+ public static final byte[] ENABLED_ZNODE_BYTES =
+ toByteArray(ReplicationProtos.ReplicationState.State.ENABLED);
+ public static final byte[] DISABLED_ZNODE_BYTES =
+ toByteArray(ReplicationProtos.ReplicationState.State.DISABLED);
+
+ /**
+ * The name of the znode that contains the replication status of a remote slave (i.e. peer)
+ * cluster.
+ */
+ private final String peerStateNodeName;
+
+ /**
+ * The name of the znode that contains a list of all remote slave (i.e. peer) clusters.
+ */
+ private final String peersZNode;
+
+ public ZKReplicationPeerStorage(ZKWatcher zookeeper, Configuration conf) {
+ super(zookeeper, conf);
+ this.peerStateNodeName = conf.get("zookeeper.znode.replication.peers.state", "peer-state");
+ String peersZNodeName = conf.get("zookeeper.znode.replication.peers", "peers");
+ this.peersZNode = ZNodePaths.joinZNode(replicationZNode, peersZNodeName);
+ }
+
+ private String getPeerStateNode(String peerId) {
+ return ZNodePaths.joinZNode(getPeerNode(peerId), peerStateNodeName);
+ }
+
+ private String getPeerNode(String peerId) {
+ return ZNodePaths.joinZNode(peersZNode, peerId);
+ }
+
+ @Override
+ public void addPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
+ throws ReplicationException {
+ try {
+ ZKUtil.createWithParents(zookeeper, peersZNode);
+ ZKUtil.multiOrSequential(zookeeper,
+ Arrays.asList(
+ ZKUtilOp.createAndFailSilent(getPeerNode(peerId),
+ ReplicationPeerConfigUtil.toByteArray(peerConfig)),
+ ZKUtilOp.createAndFailSilent(getPeerStateNode(peerId),
+ enabled ? ENABLED_ZNODE_BYTES : DISABLED_ZNODE_BYTES)),
+ false);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Could not add peer with id=" + peerId + ", peerConfif=>" +
+ peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"), e);
+ }
+ }
+
+ @Override
+ public void removePeer(String peerId) throws ReplicationException {
+ try {
+ ZKUtil.deleteNodeRecursively(zookeeper, getPeerNode(peerId));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Could not remove peer with id=" + peerId, e);
+ }
+ }
+
+ @Override
+ public void setPeerState(String peerId, boolean enabled) throws ReplicationException {
+ byte[] stateBytes = enabled ? ENABLED_ZNODE_BYTES : DISABLED_ZNODE_BYTES;
+ try {
+ ZKUtil.setData(zookeeper, getPeerStateNode(peerId), stateBytes);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Unable to change state of the peer with id=" + peerId, e);
+ }
+ }
+
+ @Override
+ public void updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)
+ throws ReplicationException {
+ try {
+ ZKUtil.setData(this.zookeeper, getPeerNode(peerId),
+ ReplicationPeerConfigUtil.toByteArray(peerConfig));
+ } catch (KeeperException e) {
+ throw new ReplicationException(
+ "There was a problem trying to save changes to the " + "replication peer " + peerId, e);
+ }
+ }
+
+ @Override
+ public List<String> listPeerIds() throws ReplicationException {
+ try {
+ return CollectionUtils.nullToEmpty(ZKUtil.listChildrenAndWatchThem(zookeeper, peersZNode));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Cannot get the list of peers", e);
+ }
+ }
+
+ @Override
+ public boolean isPeerEnabled(String peerId) throws ReplicationException {
+ try {
+ return Arrays.equals(ENABLED_ZNODE_BYTES,
+ ZKUtil.getData(zookeeper, getPeerStateNode(peerId)));
+ } catch (KeeperException | InterruptedException e) {
+ throw new ReplicationException("Unable to get status of the peer with id=" + peerId, e);
+ }
+ }
+
+ @Override
+ public Optional<ReplicationPeerConfig> getPeerConfig(String peerId) throws ReplicationException {
+ byte[] data;
+ try {
+ data = ZKUtil.getData(zookeeper, getPeerNode(peerId));
+ } catch (KeeperException | InterruptedException e) {
+ throw new ReplicationException("Error getting configuration for peer with id=" + peerId, e);
+ }
+ if (data == null || data.length == 0) {
+ return Optional.empty();
+ }
+ try {
+ return Optional.of(ReplicationPeerConfigUtil.parsePeerFrom(data));
+ } catch (DeserializationException e) {
+ LOG.warn("Failed to parse replication peer config for peer with id=" + peerId, e);
+ return Optional.empty();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
new file mode 100644
index 0000000..7015d7f
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
@@ -0,0 +1,425 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static java.util.stream.Collectors.toList;
+import static org.apache.hadoop.hbase.util.CollectionUtils.nullToEmpty;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CollectionUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.BadVersionException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.apache.zookeeper.KeeperException.NotEmptyException;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.Sets;
+
+/**
+ * ZK based replication queue storage.
+ */
+@InterfaceAudience.Private
+class ZKReplicationQueueStorage extends ZKReplicationStorageBase
+ implements ReplicationQueueStorage {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZKReplicationQueueStorage.class);
+
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY =
+ "zookeeper.znode.replication.hfile.refs";
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT = "hfile-refs";
+
+ /**
+ * The name of the znode that contains all replication queues
+ */
+ private final String queuesZNode;
+
+ /**
+ * The name of the znode that contains queues of hfile references to be replicated
+ */
+ private final String hfileRefsZNode;
+
+ public ZKReplicationQueueStorage(ZKWatcher zookeeper, Configuration conf) {
+ super(zookeeper, conf);
+
+ String queuesZNodeName = conf.get("zookeeper.znode.replication.rs", "rs");
+ String hfileRefsZNodeName = conf.get(ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+ ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
+ this.queuesZNode = ZNodePaths.joinZNode(replicationZNode, queuesZNodeName);
+ this.hfileRefsZNode = ZNodePaths.joinZNode(replicationZNode, hfileRefsZNodeName);
+ }
+
+ private String getRsNode(ServerName serverName) {
+ return ZNodePaths.joinZNode(queuesZNode, serverName.getServerName());
+ }
+
+ private String getQueueNode(ServerName serverName, String queueId) {
+ return ZNodePaths.joinZNode(getRsNode(serverName), queueId);
+ }
+
+ private String getFileNode(String queueNode, String fileName) {
+ return ZNodePaths.joinZNode(queueNode, fileName);
+ }
+
+ private String getFileNode(ServerName serverName, String queueId, String fileName) {
+ return getFileNode(getQueueNode(serverName, queueId), fileName);
+ }
+
+ @Override
+ public void removeQueue(ServerName serverName, String queueId) throws ReplicationException {
+ try {
+ ZKUtil.deleteNodeRecursively(zookeeper, getQueueNode(serverName, queueId));
+ } catch (KeeperException e) {
+ throw new ReplicationException(
+ "Failed to delete queue (serverName=" + serverName + ", queueId=" + queueId + ")", e);
+ }
+ }
+
+ @Override
+ public void addWAL(ServerName serverName, String queueId, String fileName)
+ throws ReplicationException {
+ try {
+ ZKUtil.createWithParents(zookeeper, getFileNode(serverName, queueId, fileName));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to add wal to queue (serverName=" + serverName +
+ ", queueId=" + queueId + ", fileName=" + fileName + ")", e);
+ }
+ }
+
+ @Override
+ public void removeWAL(ServerName serverName, String queueId, String fileName)
+ throws ReplicationException {
+ String fileNode = getFileNode(serverName, queueId, fileName);
+ try {
+ ZKUtil.deleteNode(zookeeper, fileNode);
+ } catch (NoNodeException e) {
+ LOG.warn(fileNode + " has already been deleted when removing log");
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to remove wal from queue (serverName=" + serverName +
+ ", queueId=" + queueId + ", fileName=" + fileName + ")", e);
+ }
+ }
+
+ @Override
+ public void setWALPosition(ServerName serverName, String queueId, String fileName, long position)
+ throws ReplicationException {
+ try {
+ ZKUtil.setData(zookeeper, getFileNode(serverName, queueId, fileName),
+ ZKUtil.positionToByteArray(position));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to set log position (serverName=" + serverName +
+ ", queueId=" + queueId + ", fileName=" + fileName + ", position=" + position + ")", e);
+ }
+ }
+
+ @Override
+ public long getWALPosition(ServerName serverName, String queueId, String fileName)
+ throws ReplicationException {
+ byte[] bytes;
+ try {
+ bytes = ZKUtil.getData(zookeeper, getFileNode(serverName, queueId, fileName));
+ } catch (KeeperException | InterruptedException e) {
+ throw new ReplicationException("Failed to get log position (serverName=" + serverName +
+ ", queueId=" + queueId + ", fileName=" + fileName + ")", e);
+ }
+ try {
+ return ZKUtil.parseWALPositionFrom(bytes);
+ } catch (DeserializationException de) {
+ LOG.warn("Failed to parse log position (serverName=" + serverName + ", queueId=" + queueId +
+ ", fileName=" + fileName + ")");
+ }
+ // if we can not parse the position, start at the beginning of the wal file again
+ return 0;
+ }
+
+ @Override
+ public Pair<String, SortedSet<String>> claimQueue(ServerName sourceServerName, String queueId,
+ ServerName destServerName) throws ReplicationException {
+ LOG.info(
+ "Atomically moving " + sourceServerName + "/" + queueId + "'s WALs to " + destServerName);
+ try {
+ ZKUtil.createWithParents(zookeeper, getRsNode(destServerName));
+ } catch (KeeperException e) {
+ throw new ReplicationException(
+ "Claim queue queueId=" + queueId + " from " + sourceServerName + " to " + destServerName +
+ " failed when creating the node for " + destServerName,
+ e);
+ }
+ try {
+ String oldQueueNode = getQueueNode(sourceServerName, queueId);
+ List<String> wals = ZKUtil.listChildrenNoWatch(zookeeper, oldQueueNode);
+ String newQueueId = queueId + "-" + sourceServerName;
+ if (CollectionUtils.isEmpty(wals)) {
+ ZKUtil.deleteNodeFailSilent(zookeeper, oldQueueNode);
+ LOG.info("Removed " + sourceServerName + "/" + queueId + " since it's empty");
+ return new Pair<>(newQueueId, Collections.emptySortedSet());
+ }
+ String newQueueNode = getQueueNode(destServerName, newQueueId);
+ List<ZKUtilOp> listOfOps = new ArrayList<>();
+ SortedSet<String> logQueue = new TreeSet<>();
+ // create the new cluster znode
+ listOfOps.add(ZKUtilOp.createAndFailSilent(newQueueNode, HConstants.EMPTY_BYTE_ARRAY));
+ // get the offset of the logs and set it to new znodes
+ for (String wal : wals) {
+ String oldWalNode = getFileNode(oldQueueNode, wal);
+ byte[] logOffset = ZKUtil.getData(this.zookeeper, oldWalNode);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Creating " + wal + " with data " + Bytes.toStringBinary(logOffset));
+ }
+ String newWalNode = getFileNode(newQueueNode, wal);
+ listOfOps.add(ZKUtilOp.createAndFailSilent(newWalNode, logOffset));
+ listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalNode));
+ logQueue.add(wal);
+ }
+ // add delete op for peer
+ listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldQueueNode));
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("The multi list size is: " + listOfOps.size());
+ }
+ ZKUtil.multiOrSequential(zookeeper, listOfOps, false);
+
+ LOG.info(
+ "Atomically moved " + sourceServerName + "/" + queueId + "'s WALs to " + destServerName);
+ return new Pair<>(newQueueId, logQueue);
+ } catch (NoNodeException | NodeExistsException | NotEmptyException | BadVersionException e) {
+ // Multi call failed; it looks like some other regionserver took away the logs.
+ // These exceptions mean that zk tells us the request can not be execute so it is safe to just
+ // return a null. For other types of exception should be thrown out to notify the upper layer.
+ LOG.info(
+ "Claim queue queueId=" + queueId + " from " + sourceServerName + " to " + destServerName +
+ " failed with " + e.toString() + ", maybe someone else has already took away the logs");
+ return null;
+ } catch (KeeperException | InterruptedException e) {
+ throw new ReplicationException("Claim queue queueId=" + queueId + " from " +
+ sourceServerName + " to " + destServerName + " failed", e);
+ }
+ }
+
+ @Override
+ public void removeReplicatorIfQueueIsEmpty(ServerName serverName) throws ReplicationException {
+ try {
+ ZKUtil.deleteNodeFailSilent(zookeeper, getRsNode(serverName));
+ } catch (NotEmptyException e) {
+ // keep silence to avoid logging too much.
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to remove replicator for " + serverName, e);
+ }
+ }
+
+ private List<ServerName> getListOfReplicators0() throws KeeperException {
+ return nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, queuesZNode)).stream()
+ .map(ServerName::parseServerName).collect(toList());
+ }
+
+ @Override
+ public List<ServerName> getListOfReplicators() throws ReplicationException {
+ try {
+ return getListOfReplicators0();
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to get list of replicators", e);
+ }
+ }
+
+ private List<String> getLogsInQueue0(ServerName serverName, String queueId)
+ throws KeeperException {
+ return nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, getQueueNode(serverName, queueId)));
+ }
+
+ private List<String> getAllQueues0(ServerName serverName) throws KeeperException {
+ return nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, getRsNode(serverName)));
+ }
+
+ @Override
+ public List<String> getAllQueues(ServerName serverName) throws ReplicationException {
+ try {
+ return getAllQueues0(serverName);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to get all queues (serverName=" + serverName + ")", e);
+ }
+ }
+
+ private int getQueuesZNodeCversion() throws KeeperException {
+ Stat stat = new Stat();
+ ZKUtil.getDataNoWatch(this.zookeeper, this.queuesZNode, stat);
+ return stat.getCversion();
+ }
+
+ @Override
+ public Set<String> getAllWALs() throws ReplicationException {
+ try {
+ for (int retry = 0;; retry++) {
+ int v0 = getQueuesZNodeCversion();
+ List<ServerName> rss = getListOfReplicators0();
+ if (rss.isEmpty()) {
+ LOG.debug("Didn't find any region server that replicates, won't prevent any deletions.");
+ return Collections.emptySet();
+ }
+ Set<String> wals = Sets.newHashSet();
+ for (ServerName rs : rss) {
+ for (String queueId : getAllQueues0(rs)) {
+ wals.addAll(getLogsInQueue0(rs, queueId));
+ }
+ }
+ int v1 = getQueuesZNodeCversion();
+ if (v0 == v1) {
+ return wals;
+ }
+ LOG.info(String.format("Replication queue node cversion changed from %d to %d, retry = %d",
+ v0, v1, retry));
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to get all wals", e);
+ }
+ }
+
+ private String getHFileRefsPeerNode(String peerId) {
+ return ZNodePaths.joinZNode(hfileRefsZNode, peerId);
+ }
+
+ private String getHFileNode(String peerNode, String fileName) {
+ return ZNodePaths.joinZNode(peerNode, fileName);
+ }
+
+ @Override
+ public void addPeerToHFileRefs(String peerId) throws ReplicationException {
+ String peerNode = getHFileRefsPeerNode(peerId);
+ try {
+ if (ZKUtil.checkExists(zookeeper, peerNode) == -1) {
+ LOG.info("Adding peer " + peerId + " to hfile reference queue.");
+ ZKUtil.createWithParents(zookeeper, peerNode);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to add peer " + peerId + " to hfile reference queue.",
+ e);
+ }
+ }
+
+ @Override
+ public void removePeerFromHFileRefs(String peerId) throws ReplicationException {
+ String peerNode = getHFileRefsPeerNode(peerId);
+ try {
+ if (ZKUtil.checkExists(zookeeper, peerNode) == -1) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Peer " + peerNode + " not found in hfile reference queue.");
+ }
+ } else {
+ LOG.info("Removing peer " + peerNode + " from hfile reference queue.");
+ ZKUtil.deleteNodeRecursively(zookeeper, peerNode);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException(
+ "Failed to remove peer " + peerId + " from hfile reference queue.", e);
+ }
+ }
+
+ @Override
+ public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
+ throws ReplicationException {
+ String peerNode = getHFileRefsPeerNode(peerId);
+ boolean debugEnabled = LOG.isDebugEnabled();
+ if (debugEnabled) {
+ LOG.debug("Adding hfile references " + pairs + " in queue " + peerNode);
+ }
+ List<ZKUtilOp> listOfOps =
+ pairs.stream().map(p -> p.getSecond().getName()).map(n -> getHFileNode(peerNode, n))
+ .map(f -> ZKUtilOp.createAndFailSilent(f, HConstants.EMPTY_BYTE_ARRAY)).collect(toList());
+ if (debugEnabled) {
+ LOG.debug("The multi list size for adding hfile references in zk for node " + peerNode +
+ " is " + listOfOps.size());
+ }
+ try {
+ ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to add hfile reference to peer " + peerId, e);
+ }
+ }
+
+ @Override
+ public void removeHFileRefs(String peerId, List<String> files) throws ReplicationException {
+ String peerNode = getHFileRefsPeerNode(peerId);
+ boolean debugEnabled = LOG.isDebugEnabled();
+ if (debugEnabled) {
+ LOG.debug("Removing hfile references " + files + " from queue " + peerNode);
+ }
+
+ List<ZKUtilOp> listOfOps = files.stream().map(n -> getHFileNode(peerNode, n))
+ .map(ZKUtilOp::deleteNodeFailSilent).collect(toList());
+ if (debugEnabled) {
+ LOG.debug("The multi list size for removing hfile references in zk for node " + peerNode +
+ " is " + listOfOps.size());
+ }
+ try {
+ ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to remove hfile reference from peer " + peerId, e);
+ }
+ }
+
+ @Override
+ public int getHFileRefsNodeChangeVersion() throws ReplicationException {
+ Stat stat = new Stat();
+ try {
+ ZKUtil.getDataNoWatch(zookeeper, hfileRefsZNode, stat);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to get stat of replication hfile references node.", e);
+ }
+ return stat.getCversion();
+ }
+
+ @Override
+ public List<String> getAllPeersFromHFileRefsQueue() throws ReplicationException {
+ try {
+ return nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, hfileRefsZNode));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to get list of all peers in hfile references node.",
+ e);
+ }
+ }
+
+ @Override
+ public List<String> getReplicableHFiles(String peerId) throws ReplicationException {
+ try {
+ return nullToEmpty(ZKUtil.listChildrenNoWatch(this.zookeeper, getHFileRefsPeerNode(peerId)));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to get list of hfile references for peer " + peerId,
+ e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
new file mode 100644
index 0000000..b8a2044
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+
+/**
+ * This is a base class for maintaining replication related data,for example, peer, queue, etc, in
+ * zookeeper.
+ */
+@InterfaceAudience.Private
+class ZKReplicationStorageBase {
+
+ /** The name of the base znode that contains all replication state. */
+ protected final String replicationZNode;
+
+ protected final ZKWatcher zookeeper;
+ protected final Configuration conf;
+
+ protected ZKReplicationStorageBase(ZKWatcher zookeeper, Configuration conf) {
+ this.zookeeper = zookeeper;
+ this.conf = conf;
+ String replicationZNodeName = conf.get("zookeeper.znode.replication", "replication");
+
+ this.replicationZNode =
+ ZNodePaths.joinZNode(this.zookeeper.znodePaths.baseZNode, replicationZNodeName);
+
+ }
+
+ /**
+ * Serialized protobuf of <code>state</code> with pb magic prefix prepended suitable for use as
+ * content of a peer-state znode under a peer cluster id as in
+ * /hbase/replication/peers/PEER_ID/peer-state.
+ */
+ protected static byte[] toByteArray(final ReplicationProtos.ReplicationState.State state) {
+ ReplicationProtos.ReplicationState msg =
+ ReplicationProtos.ReplicationState.newBuilder().setState(state).build();
+ // There is no toByteArray on this pb Message?
+ // 32 bytes is default which seems fair enough here.
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ CodedOutputStream cos = CodedOutputStream.newInstance(baos, 16);
+ msg.writeTo(cos);
+ cos.flush();
+ baos.flush();
+ return ProtobufUtil.prependPBMagic(baos.toByteArray());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
new file mode 100644
index 0000000..a3be1e6
--- /dev/null
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.hbase.HBaseZKTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ ReplicationTests.class, MediumTests.class })
+public class TestZKReplicationPeerStorage {
+
+ private static final HBaseZKTestingUtility UTIL = new HBaseZKTestingUtility();
+
+ private static ZKReplicationPeerStorage STORAGE;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.startMiniZKCluster();
+ STORAGE = new ZKReplicationPeerStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ UTIL.shutdownMiniZKCluster();
+ }
+
+ private Set<String> randNamespaces(Random rand) {
+ return Stream.generate(() -> Long.toHexString(rand.nextLong())).limit(rand.nextInt(5))
+ .collect(toSet());
+ }
+
+ private Map<TableName, List<String>> randTableCFs(Random rand) {
+ int size = rand.nextInt(5);
+ Map<TableName, List<String>> map = new HashMap<>();
+ for (int i = 0; i < size; i++) {
+ TableName tn = TableName.valueOf(Long.toHexString(rand.nextLong()));
+ List<String> cfs = Stream.generate(() -> Long.toHexString(rand.nextLong()))
+ .limit(rand.nextInt(5)).collect(toList());
+ map.put(tn, cfs);
+ }
+ return map;
+ }
+
+ private ReplicationPeerConfig getConfig(int seed) {
+ Random rand = new Random(seed);
+ ReplicationPeerConfig config = new ReplicationPeerConfig();
+ config.setClusterKey(Long.toHexString(rand.nextLong()));
+ config.setReplicationEndpointImpl(Long.toHexString(rand.nextLong()));
+ config.setNamespaces(randNamespaces(rand));
+ config.setExcludeNamespaces(randNamespaces(rand));
+ config.setTableCFsMap(randTableCFs(rand));
+ config.setReplicateAllUserTables(rand.nextBoolean());
+ config.setBandwidth(rand.nextInt(1000));
+ return config;
+ }
+
+ private void assertSetEquals(Set<String> expected, Set<String> actual) {
+ if (expected == null || expected.size() == 0) {
+ assertTrue(actual == null || actual.size() == 0);
+ return;
+ }
+ assertEquals(expected.size(), actual.size());
+ expected.forEach(s -> assertTrue(actual.contains(s)));
+ }
+
+ private void assertMapEquals(Map<TableName, List<String>> expected,
+ Map<TableName, List<String>> actual) {
+ if (expected == null || expected.size() == 0) {
+ assertTrue(actual == null || actual.size() == 0);
+ return;
+ }
+ assertEquals(expected.size(), actual.size());
+ expected.forEach((expectedTn, expectedCFs) -> {
+ List<String> actualCFs = actual.get(expectedTn);
+ if (expectedCFs == null || expectedCFs.size() == 0) {
+ assertTrue(actual.containsKey(expectedTn));
+ assertTrue(actualCFs == null || actualCFs.size() == 0);
+ } else {
+ assertNotNull(actualCFs);
+ assertEquals(expectedCFs.size(), actualCFs.size());
+ for (Iterator<String> expectedIt = expectedCFs.iterator(), actualIt = actualCFs.iterator();
+ expectedIt.hasNext();) {
+ assertEquals(expectedIt.next(), actualIt.next());
+ }
+ }
+ });
+ }
+
+ private void assertConfigEquals(ReplicationPeerConfig expected, ReplicationPeerConfig actual) {
+ assertEquals(expected.getClusterKey(), actual.getClusterKey());
+ assertEquals(expected.getReplicationEndpointImpl(), actual.getReplicationEndpointImpl());
+ assertSetEquals(expected.getNamespaces(), actual.getNamespaces());
+ assertSetEquals(expected.getExcludeNamespaces(), actual.getExcludeNamespaces());
+ assertMapEquals(expected.getTableCFsMap(), actual.getTableCFsMap());
+ assertMapEquals(expected.getExcludeTableCFsMap(), actual.getExcludeTableCFsMap());
+ assertEquals(expected.replicateAllUserTables(), actual.replicateAllUserTables());
+ assertEquals(expected.getBandwidth(), actual.getBandwidth());
+ }
+
+ @Test
+ public void test() throws ReplicationException {
+ int peerCount = 10;
+ for (int i = 0; i < peerCount; i++) {
+ STORAGE.addPeer(Integer.toString(i), getConfig(i), i % 2 == 0);
+ }
+ List<String> peerIds = STORAGE.listPeerIds();
+ assertEquals(peerCount, peerIds.size());
+ for (String peerId : peerIds) {
+ int seed = Integer.parseInt(peerId);
+ assertConfigEquals(getConfig(seed), STORAGE.getPeerConfig(peerId).get());
+ }
+ for (int i = 0; i < peerCount; i++) {
+ STORAGE.updatePeerConfig(Integer.toString(i), getConfig(i + 1));
+ }
+ for (String peerId : peerIds) {
+ int seed = Integer.parseInt(peerId);
+ assertConfigEquals(getConfig(seed + 1), STORAGE.getPeerConfig(peerId).get());
+ }
+ for (int i = 0; i < peerCount; i++) {
+ assertEquals(i % 2 == 0, STORAGE.isPeerEnabled(Integer.toString(i)));
+ }
+ for (int i = 0; i < peerCount; i++) {
+ STORAGE.setPeerState(Integer.toString(i), i % 2 != 0);
+ }
+ for (int i = 0; i < peerCount; i++) {
+ assertEquals(i % 2 != 0, STORAGE.isPeerEnabled(Integer.toString(i)));
+ }
+ String toRemove = Integer.toString(peerCount / 2);
+ STORAGE.removePeer(toRemove);
+ peerIds = STORAGE.listPeerIds();
+ assertEquals(peerCount - 1, peerIds.size());
+ assertFalse(peerIds.contains(toRemove));
+ assertFalse(STORAGE.getPeerConfig(toRemove).isPresent());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
new file mode 100644
index 0000000..d5bba0d
--- /dev/null
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+
+import org.apache.hadoop.hbase.HBaseZKTestingUtility;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ ReplicationTests.class, MediumTests.class })
+public class TestZKReplicationQueueStorage {
+ private static final HBaseZKTestingUtility UTIL = new HBaseZKTestingUtility();
+
+ private static ZKReplicationQueueStorage STORAGE;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.startMiniZKCluster();
+ STORAGE = new ZKReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ UTIL.shutdownMiniZKCluster();
+ }
+
+ @After
+ public void tearDownAfterTest() throws ReplicationException {
+ for (ServerName serverName : STORAGE.getListOfReplicators()) {
+ for (String queue : STORAGE.getAllQueues(serverName)) {
+ STORAGE.removeQueue(serverName, queue);
+ }
+ STORAGE.removeReplicatorIfQueueIsEmpty(serverName);
+ }
+ for (String peerId : STORAGE.getAllPeersFromHFileRefsQueue()) {
+ STORAGE.removePeerFromHFileRefs(peerId);
+ }
+ }
+
+ private ServerName getServerName(int i) {
+ return ServerName.valueOf("127.0.0.1", 8000 + i, 10000 + i);
+ }
+
+ @Test
+ public void testReplicator() throws ReplicationException {
+ assertTrue(STORAGE.getListOfReplicators().isEmpty());
+ String queueId = "1";
+ for (int i = 0; i < 10; i++) {
+ STORAGE.addWAL(getServerName(i), queueId, "file" + i);
+ }
+ List<ServerName> replicators = STORAGE.getListOfReplicators();
+ assertEquals(10, replicators.size());
+ for (int i = 0; i < 10; i++) {
+ assertThat(replicators, hasItems(getServerName(i)));
+ }
+ for (int i = 0; i < 5; i++) {
+ STORAGE.removeQueue(getServerName(i), queueId);
+ }
+ for (int i = 0; i < 10; i++) {
+ STORAGE.removeReplicatorIfQueueIsEmpty(getServerName(i));
+ }
+ replicators = STORAGE.getListOfReplicators();
+ assertEquals(5, replicators.size());
+ for (int i = 5; i < 10; i++) {
+ assertThat(replicators, hasItems(getServerName(i)));
+ }
+ }
+
+ private String getFileName(String base, int i) {
+ return String.format(base + "-%04d", i);
+ }
+
+ @Test
+ public void testAddRemoveLog() throws ReplicationException {
+ ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
+ assertTrue(STORAGE.getAllQueues(serverName1).isEmpty());
+ String queue1 = "1";
+ String queue2 = "2";
+ for (int i = 0; i < 10; i++) {
+ STORAGE.addWAL(serverName1, queue1, getFileName("file1", i));
+ STORAGE.addWAL(serverName1, queue2, getFileName("file2", i));
+ }
+ List<String> queueIds = STORAGE.getAllQueues(serverName1);
+ assertEquals(2, queueIds.size());
+ assertThat(queueIds, hasItems("1", "2"));
+
+ for (int i = 0; i < 10; i++) {
+ assertEquals(0, STORAGE.getWALPosition(serverName1, queue1, getFileName("file1", i)));
+ assertEquals(0, STORAGE.getWALPosition(serverName1, queue2, getFileName("file2", i)));
+ STORAGE.setWALPosition(serverName1, queue1, getFileName("file1", i), (i + 1) * 100);
+ STORAGE.setWALPosition(serverName1, queue2, getFileName("file2", i), (i + 1) * 100 + 10);
+ }
+
+ for (int i = 0; i < 10; i++) {
+ assertEquals((i + 1) * 100,
+ STORAGE.getWALPosition(serverName1, queue1, getFileName("file1", i)));
+ assertEquals((i + 1) * 100 + 10,
+ STORAGE.getWALPosition(serverName1, queue2, getFileName("file2", i)));
+ }
+
+ for (int i = 0; i < 10; i++) {
+ if (i % 2 == 0) {
+ STORAGE.removeWAL(serverName1, queue1, getFileName("file1", i));
+ } else {
+ STORAGE.removeWAL(serverName1, queue2, getFileName("file2", i));
+ }
+ }
+
+ queueIds = STORAGE.getAllQueues(serverName1);
+ assertEquals(2, queueIds.size());
+ assertThat(queueIds, hasItems("1", "2"));
+
+ ServerName serverName2 = ServerName.valueOf("127.0.0.1", 8001, 10001);
+ Pair<String, SortedSet<String>> peer1 = STORAGE.claimQueue(serverName1, "1", serverName2);
+
+ assertEquals("1-" + serverName1.getServerName(), peer1.getFirst());
+ assertEquals(5, peer1.getSecond().size());
+ int i = 1;
+ for (String wal : peer1.getSecond()) {
+ assertEquals(getFileName("file1", i), wal);
+ assertEquals((i + 1) * 100,
+ STORAGE.getWALPosition(serverName2, peer1.getFirst(), getFileName("file1", i)));
+ i += 2;
+ }
+
+ queueIds = STORAGE.getAllQueues(serverName1);
+ assertEquals(1, queueIds.size());
+ assertThat(queueIds, hasItems("2"));
+
+ queueIds = STORAGE.getAllQueues(serverName2);
+ assertEquals(1, queueIds.size());
+ assertThat(queueIds, hasItems(peer1.getFirst()));
+
+ Set<String> allWals = STORAGE.getAllWALs();
+ assertEquals(10, allWals.size());
+ for (i = 0; i < 10; i++) {
+ assertThat(allWals, hasItems(i % 2 == 0 ? getFileName("file2", i) : getFileName("file1", i)));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index da3464a..b0a820e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -133,7 +134,7 @@ import org.apache.hadoop.hbase.master.replication.DisablePeerProcedure;
import org.apache.hadoop.hbase.master.replication.EnablePeerProcedure;
import org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure;
import org.apache.hadoop.hbase.master.replication.RemovePeerProcedure;
-import org.apache.hadoop.hbase.master.replication.ReplicationManager;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
import org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.mob.MobConstants;
@@ -326,7 +327,7 @@ public class HMaster extends HRegionServer implements MasterServices {
private AssignmentManager assignmentManager;
// manager of replication
- private ReplicationManager replicationManager;
+ private ReplicationPeerManager replicationPeerManager;
// buffer for "fatal error" notices from region servers
// in the cluster. This is only used for assisting
@@ -717,8 +718,8 @@ public class HMaster extends HRegionServer implements MasterServices {
/**
* Initialize all ZK based system trackers.
*/
- void initializeZKBasedSystemTrackers() throws IOException,
- InterruptedException, KeeperException, CoordinatedStateException {
+ void initializeZKBasedSystemTrackers() throws IOException, InterruptedException, KeeperException,
+ CoordinatedStateException, ReplicationException {
this.balancer = LoadBalancerFactory.getLoadBalancer(conf);
this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf);
this.normalizer.setMasterServices(this);
@@ -736,7 +737,7 @@ public class HMaster extends HRegionServer implements MasterServices {
this.assignmentManager = new AssignmentManager(this);
this.assignmentManager.start();
- this.replicationManager = new ReplicationManager(conf, zooKeeper, this);
+ this.replicationPeerManager = ReplicationPeerManager.create(zooKeeper, conf);
this.regionServerTracker = new RegionServerTracker(zooKeeper, this, this.serverManager);
this.regionServerTracker.start();
@@ -781,9 +782,8 @@ public class HMaster extends HRegionServer implements MasterServices {
* <li>Handle either fresh cluster start or master failover</li>
* </ol>
*/
- private void finishActiveMasterInitialization(MonitoredTask status)
- throws IOException, InterruptedException, KeeperException, CoordinatedStateException {
-
+ private void finishActiveMasterInitialization(MonitoredTask status) throws IOException,
+ InterruptedException, KeeperException, CoordinatedStateException, ReplicationException {
Thread zombieDetector = new Thread(new InitializationMonitor(this),
"ActiveMasterInitializationMonitor-" + System.currentTimeMillis());
zombieDetector.setDaemon(true);
@@ -3385,18 +3385,19 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
- public ReplicationPeerConfig getReplicationPeerConfig(String peerId) throws ReplicationException,
- IOException {
+ public ReplicationPeerConfig getReplicationPeerConfig(String peerId)
+ throws ReplicationException, IOException {
if (cpHost != null) {
cpHost.preGetReplicationPeerConfig(peerId);
}
- final ReplicationPeerConfig peerConfig = this.replicationManager.getPeerConfig(peerId);
- LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" + peerId + ", config="
- + peerConfig);
+ LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" + peerId);
+ Optional<ReplicationPeerConfig> peerConfig =
+ this.replicationPeerManager.getPeerConfig(peerId);
+
if (cpHost != null) {
cpHost.postGetReplicationPeerConfig(peerId);
}
- return peerConfig;
+ return peerConfig.orElse(null);
}
@Override
@@ -3415,7 +3416,8 @@ public class HMaster extends HRegionServer implements MasterServices {
}
LOG.info(getClientIdAuditPrefix() + " list replication peers, regex=" + regex);
Pattern pattern = regex == null ? null : Pattern.compile(regex);
- List<ReplicationPeerDescription> peers = this.replicationManager.listReplicationPeers(pattern);
+ List<ReplicationPeerDescription> peers =
+ this.replicationPeerManager.listPeers(pattern);
if (cpHost != null) {
cpHost.postListReplicationPeers(regex);
}
@@ -3565,7 +3567,7 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
- public ReplicationManager getReplicationManager() {
- return replicationManager;
+ public ReplicationPeerManager getReplicationPeerManager() {
+ return replicationPeerManager;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index e798455..9d371bd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.locking.LockManager;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.master.replication.ReplicationManager;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
import org.apache.hadoop.hbase.procedure2.LockedResource;
@@ -459,9 +459,9 @@ public interface MasterServices extends Server {
IOException;
/**
- * Returns the {@link ReplicationManager}.
+ * Returns the {@link ReplicationPeerManager}.
*/
- ReplicationManager getReplicationManager();
+ ReplicationPeerManager getReplicationPeerManager();
/**
* Update the peerConfig for the specified peer
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
index fa4d371..ae038a5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
@@ -24,24 +24,24 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.conf.ConfigurationObserver;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.yetus.audience.InterfaceStability;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
-import org.apache.hadoop.hbase.master.replication.ReplicationManager;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
-import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.Superusers;
+import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -138,8 +138,8 @@ public class MasterProcedureEnv implements ConfigurationObserver {
return remoteDispatcher;
}
- public ReplicationManager getReplicationManager() {
- return master.getReplicationManager();
+ public ReplicationPeerManager getReplicationPeerManager() {
+ return master.getReplicationPeerManager();
}
public boolean isRunning() {
@@ -151,22 +151,22 @@ public class MasterProcedureEnv implements ConfigurationObserver {
return master.isInitialized();
}
- public boolean waitInitialized(Procedure proc) {
+ public boolean waitInitialized(Procedure<?> proc) {
return master.getInitializedEvent().suspendIfNotReady(proc);
}
- public boolean waitServerCrashProcessingEnabled(Procedure proc) {
+ public boolean waitServerCrashProcessingEnabled(Procedure<?> proc) {
if (master instanceof HMaster) {
return ((HMaster)master).getServerCrashProcessingEnabledEvent().suspendIfNotReady(proc);
}
return false;
}
- public boolean waitFailoverCleanup(Procedure proc) {
+ public boolean waitFailoverCleanup(Procedure<?> proc) {
return master.getAssignmentManager().getFailoverCleanupEvent().suspendIfNotReady(proc);
}
- public void setEventReady(ProcedureEvent event, boolean isReady) {
+ public void setEventReady(ProcedureEvent<?> event, boolean isReady) {
if (isReady) {
event.wake(procSched);
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
index 066c3e7..a4f9b32 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
@@ -58,16 +58,18 @@ public class AddPeerProcedure extends ModifyPeerProcedure {
}
@Override
- protected void prePeerModification(MasterProcedureEnv env) throws IOException {
+ protected void prePeerModification(MasterProcedureEnv env)
+ throws IOException, ReplicationException {
MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preAddReplicationPeer(peerId, peerConfig);
}
+ env.getReplicationPeerManager().preAddPeer(peerId, peerConfig);
}
@Override
protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
- env.getReplicationManager().addReplicationPeer(peerId, peerConfig, enabled);
+ env.getReplicationPeerManager().addPeer(peerId, peerConfig, enabled);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
index 9a28de6..10e35a8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,12 +52,12 @@ public class DisablePeerProcedure extends ModifyPeerProcedure {
if (cpHost != null) {
cpHost.preDisableReplicationPeer(peerId);
}
+ env.getReplicationPeerManager().preDisablePeer(peerId);
}
@Override
- protected void updatePeerStorage(MasterProcedureEnv env)
- throws IllegalArgumentException, Exception {
- env.getReplicationManager().disableReplicationPeer(peerId);
+ protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
+ env.getReplicationPeerManager().disablePeer(peerId);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
index 4855901..f2a9f01 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,11 +52,12 @@ public class EnablePeerProcedure extends ModifyPeerProcedure {
if (cpHost != null) {
cpHost.preEnableReplicationPeer(peerId);
}
+ env.getReplicationPeerManager().preEnablePeer(peerId);
}
@Override
- protected void updatePeerStorage(MasterProcedureEnv env) throws Exception {
- env.getReplicationManager().enableReplicationPeer(peerId);
+ protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
+ env.getReplicationPeerManager().enablePeer(peerId);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index c4552ed..279fbc7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
@@ -27,6 +26,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,17 +67,16 @@ public abstract class ModifyPeerProcedure
}
/**
- * Called before we start the actual processing. If an exception is thrown then we will give up
- * and mark the procedure as failed directly.
+ * Called before we start the actual processing. The implementation should call the pre CP hook,
+ * and also the pre-check for the peer modification.
+ * <p>
+ * If an IOException is thrown then we will give up and mark the procedure as failed directly. If
+ * all checks passes then the procedure can not be rolled back any more.
*/
- protected abstract void prePeerModification(MasterProcedureEnv env) throws IOException;
+ protected abstract void prePeerModification(MasterProcedureEnv env)
+ throws IOException, ReplicationException;
- /**
- * We will give up and mark the procedure as failure if {@link IllegalArgumentException} is
- * thrown, for other type of Exception we will retry.
- */
- protected abstract void updatePeerStorage(MasterProcedureEnv env)
- throws IllegalArgumentException, Exception;
+ protected abstract void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException;
/**
* Called before we finish the procedure. The implementation can do some logging work, and also
@@ -100,23 +99,24 @@ public abstract class ModifyPeerProcedure
try {
prePeerModification(env);
} catch (IOException e) {
- LOG.warn(getClass().getName() + " failed to call prePeerModification for peer " + peerId +
- ", mark the procedure as failure and give up", e);
- setFailure("prePeerModification", e);
+ LOG.warn(
+ getClass().getName() + " failed to call CP hook or the pre check is failed for peer " +
+ peerId + ", mark the procedure as failure and give up",
+ e);
+ setFailure("master-" + getPeerOperationType().name().toLowerCase() + "-peer", e);
releaseLatch();
return Flow.NO_MORE_STATE;
+ } catch (ReplicationException e) {
+ LOG.warn(getClass().getName() + " failed to call prePeerModification for peer " + peerId +
+ ", retry", e);
+ throw new ProcedureYieldException();
}
setNextState(PeerModificationState.UPDATE_PEER_STORAGE);
return Flow.HAS_MORE_STATE;
case UPDATE_PEER_STORAGE:
try {
updatePeerStorage(env);
- } catch (IllegalArgumentException e) {
- setFailure("master-" + getPeerOperationType().name().toLowerCase() + "-peer",
- new DoNotRetryIOException(e));
- releaseLatch();
- return Flow.NO_MORE_STATE;
- } catch (Exception e) {
+ } catch (ReplicationException e) {
LOG.warn(
getClass().getName() + " update peer storage for peer " + peerId + " failed, retry", e);
throw new ProcedureYieldException();
@@ -158,8 +158,7 @@ public abstract class ModifyPeerProcedure
@Override
protected void rollbackState(MasterProcedureEnv env, PeerModificationState state)
throws IOException, InterruptedException {
- if (state == PeerModificationState.PRE_PEER_MODIFICATION ||
- state == PeerModificationState.UPDATE_PEER_STORAGE) {
+ if (state == PeerModificationState.PRE_PEER_MODIFICATION) {
// actually the peer related operations has no rollback, but if we haven't done any
// modifications on the peer storage, we can just return.
return;
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
index d40df02..6e9c384 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,11 +52,12 @@ public class RemovePeerProcedure extends ModifyPeerProcedure {
if (cpHost != null) {
cpHost.preRemoveReplicationPeer(peerId);
}
+ env.getReplicationPeerManager().preRemovePeer(peerId);
}
@Override
- protected void updatePeerStorage(MasterProcedureEnv env) throws Exception {
- env.getReplicationManager().removeReplicationPeer(peerId);
+ protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
+ env.getReplicationPeerManager().removePeer(peerId);
}
@Override
[50/50] [abbrv] hbase git commit: HBASE-19748
TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint UT hangs
Posted by zh...@apache.org.
HBASE-19748 TestRegionReplicaFailover and TestRegionReplicaReplicationEndpoint UT hangs
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/21986839
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/21986839
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/21986839
Branch: refs/heads/HBASE-19397-branch-2
Commit: 21986839374c65f88900a09b88ca434c93ba1e6f
Parents: 2aff1ed
Author: huzheng <op...@gmail.com>
Authored: Wed Jan 10 15:00:30 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:20:38 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/master/HMaster.java | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/21986839/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index a7ef275..c7e67ac 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -38,7 +38,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -70,6 +69,7 @@ import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.PleaseHoldException;
+import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerMetricsBuilder;
import org.apache.hadoop.hbase.ServerName;
@@ -3378,13 +3378,12 @@ public class HMaster extends HRegionServer implements MasterServices {
cpHost.preGetReplicationPeerConfig(peerId);
}
LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" + peerId);
- Optional<ReplicationPeerConfig> peerConfig =
- this.replicationPeerManager.getPeerConfig(peerId);
-
+ ReplicationPeerConfig peerConfig = this.replicationPeerManager.getPeerConfig(peerId)
+ .orElseThrow(() -> new ReplicationPeerNotFoundException(peerId));
if (cpHost != null) {
cpHost.postGetReplicationPeerConfig(peerId);
}
- return peerConfig.orElse(null);
+ return peerConfig;
}
@Override
[34/50] [abbrv] hbase git commit: HBASE-19622 Reimplement
ReplicationPeers with the new replication storage interface
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index bf9cd30..1359575 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -166,7 +166,6 @@ public class ReplicationSourceManager implements ReplicationListener {
this.clusterId = clusterId;
this.walFileLengthProvider = walFileLengthProvider;
this.replicationTracker.registerListener(this);
- this.replicationPeers.getAllPeerIds();
// It's preferable to failover 1 RS at a time, but with good zk servers
// more could be processed at the same time.
int nbWorkers = conf.getInt("replication.executor.workers", 1);
@@ -270,8 +269,8 @@ public class ReplicationSourceManager implements ReplicationListener {
}
List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers().stream()
.map(ServerName::valueOf).collect(Collectors.toList());
- LOG.info(
- "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);
+ LOG.info("Current list of replicators: " + currentReplicators + " other RSs: "
+ + otherRegionServers);
// Look if there's anything to process after a restart
for (ServerName rs : currentReplicators) {
@@ -288,7 +287,7 @@ public class ReplicationSourceManager implements ReplicationListener {
* The returned future is for adoptAbandonedQueues task.
*/
Future<?> init() throws IOException, ReplicationException {
- for (String id : this.replicationPeers.getConnectedPeerIds()) {
+ for (String id : this.replicationPeers.getAllPeerIds()) {
addSource(id);
if (replicationForBulkLoadDataEnabled) {
// Check if peer exists in hfile-refs queue, if not add it. This can happen in the case
@@ -307,8 +306,8 @@ public class ReplicationSourceManager implements ReplicationListener {
*/
@VisibleForTesting
ReplicationSourceInterface addSource(String id) throws IOException, ReplicationException {
- ReplicationPeerConfig peerConfig = replicationPeers.getReplicationPeerConfig(id);
- ReplicationPeer peer = replicationPeers.getConnectedPeer(id);
+ ReplicationPeerConfig peerConfig = replicationPeers.getPeerConfig(id);
+ ReplicationPeer peer = replicationPeers.getPeer(id);
ReplicationSourceInterface src = getReplicationSource(id, peerConfig, peer);
synchronized (this.walsById) {
this.sources.add(src);
@@ -354,7 +353,7 @@ public class ReplicationSourceManager implements ReplicationListener {
public void deleteSource(String peerId, boolean closeConnection) {
abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), peerId));
if (closeConnection) {
- this.replicationPeers.peerDisconnected(peerId);
+ this.replicationPeers.removePeer(peerId);
}
}
@@ -445,12 +444,12 @@ public class ReplicationSourceManager implements ReplicationListener {
// update replication queues on ZK
// synchronize on replicationPeers to avoid adding source for the to-be-removed peer
synchronized (replicationPeers) {
- for (String id : replicationPeers.getConnectedPeerIds()) {
+ for (String id : replicationPeers.getAllPeerIds()) {
try {
this.queueStorage.addWAL(server.getServerName(), id, logName);
} catch (ReplicationException e) {
- throw new IOException("Cannot add log to replication queue" +
- " when creating a new source, queueId=" + id + ", filename=" + logName, e);
+ throw new IOException("Cannot add log to replication queue"
+ + " when creating a new source, queueId=" + id + ", filename=" + logName, e);
}
}
}
@@ -593,7 +592,7 @@ public class ReplicationSourceManager implements ReplicationListener {
public void addPeer(String id) throws ReplicationException, IOException {
LOG.info("Trying to add peer, peerId: " + id);
- boolean added = this.replicationPeers.peerConnected(id);
+ boolean added = this.replicationPeers.addPeer(id);
if (added) {
LOG.info("Peer " + id + " connected success, trying to start the replication source thread.");
addSource(id);
@@ -729,19 +728,25 @@ public class ReplicationSourceManager implements ReplicationListener {
// there is not an actual peer defined corresponding to peerId for the failover.
ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(peerId);
String actualPeerId = replicationQueueInfo.getPeerId();
- ReplicationPeer peer = replicationPeers.getConnectedPeer(actualPeerId);
+
+ ReplicationPeer peer = replicationPeers.getPeer(actualPeerId);
+ if (peer == null) {
+ LOG.warn("Skipping failover for peer:" + actualPeerId + " of node " + deadRS
+ + ", peer is null");
+ abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), peerId));
+ continue;
+ }
+
ReplicationPeerConfig peerConfig = null;
try {
- peerConfig = replicationPeers.getReplicationPeerConfig(actualPeerId);
- } catch (ReplicationException ex) {
- LOG.warn("Received exception while getting replication peer config, skipping replay"
- + ex);
- }
- if (peer == null || peerConfig == null) {
- LOG.warn("Skipping failover for peer:" + actualPeerId + " of node " + deadRS);
+ peerConfig = replicationPeers.getPeerConfig(actualPeerId);
+ } catch (Exception e) {
+ LOG.warn("Skipping failover for peer:" + actualPeerId + " of node " + deadRS
+ + ", failed to read peer config", e);
abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), peerId));
continue;
}
+
// track sources in walsByIdRecoveredQueues
Map<String, SortedSet<String>> walsByGroup = new HashMap<>();
walsByIdRecoveredQueues.put(peerId, walsByGroup);
@@ -760,7 +765,7 @@ public class ReplicationSourceManager implements ReplicationListener {
// synchronized on oldsources to avoid adding recovered source for the to-be-removed peer
// see removePeer
synchronized (oldsources) {
- if (!replicationPeers.getConnectedPeerIds().contains(src.getPeerId())) {
+ if (!replicationPeers.getAllPeerIds().contains(src.getPeerId())) {
src.terminate("Recovered queue doesn't belong to any current peer");
closeRecoveredQueue(src);
continue;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
index 201055e..82cc388 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
@@ -87,7 +87,7 @@ public class TestReplicationHFileCleaner {
server = new DummyServer();
conf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
Replication.decorateMasterConfiguration(conf);
- rp = ReplicationFactory.getReplicationPeers(server.getZooKeeper(), conf, server);
+ rp = ReplicationFactory.getReplicationPeers(server.getZooKeeper(), conf);
rp.init();
rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
fs = FileSystem.get(conf);
@@ -101,7 +101,8 @@ public class TestReplicationHFileCleaner {
@Before
public void setup() throws ReplicationException, IOException {
root = TEST_UTIL.getDataTestDirOnTestFS();
- rp.registerPeer(peerId, new ReplicationPeerConfig().setClusterKey(TEST_UTIL.getClusterKey()));
+ rp.getPeerStorage().addPeer(peerId,
+ ReplicationPeerConfig.newBuilder().setClusterKey(TEST_UTIL.getClusterKey()).build(), true);
rq.addPeerToHFileRefs(peerId);
}
@@ -112,7 +113,7 @@ public class TestReplicationHFileCleaner {
} catch (IOException e) {
LOG.warn("Failed to delete files recursively from path " + root);
}
- rp.unregisterPeer(peerId);
+ rp.getPeerStorage().removePeer(peerId);
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
index c57d9bb..ca4369e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestMultiSlaveReplication.java
@@ -21,9 +21,7 @@ package org.apache.hadoop.hbase.replication;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
index f118ca3..fdfa6b7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
@@ -22,8 +22,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
@@ -44,6 +42,7 @@ import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -73,10 +72,6 @@ public class TestReplicationTrackerZKImpl {
private ReplicationTracker rt;
private AtomicInteger rsRemovedCount;
private String rsRemovedData;
- private AtomicInteger plChangedCount;
- private List<String> plChangedData;
- private AtomicInteger peerRemovedCount;
- private String peerRemovedData;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -93,7 +88,7 @@ public class TestReplicationTrackerZKImpl {
String fakeRs1 = ZNodePaths.joinZNode(zkw.znodePaths.rsZNode, "hostname1.example.org:1234");
try {
ZKClusterId.setClusterId(zkw, new ClusterId());
- rp = ReplicationFactory.getReplicationPeers(zkw, conf, zkw);
+ rp = ReplicationFactory.getReplicationPeers(zkw, conf);
rp.init();
rt = ReplicationFactory.getReplicationTracker(zkw, rp, conf, zkw, new DummyServer(fakeRs1));
} catch (Exception e) {
@@ -101,10 +96,6 @@ public class TestReplicationTrackerZKImpl {
}
rsRemovedCount = new AtomicInteger(0);
rsRemovedData = "";
- plChangedCount = new AtomicInteger(0);
- plChangedData = new ArrayList<>();
- peerRemovedCount = new AtomicInteger(0);
- peerRemovedData = "";
}
@AfterClass
@@ -157,25 +148,22 @@ public class TestReplicationTrackerZKImpl {
@Test(timeout = 30000)
public void testPeerNameControl() throws Exception {
int exists = 0;
- int hyphen = 0;
- rp.registerPeer("6", new ReplicationPeerConfig().setClusterKey(utility.getClusterKey()));
+ rp.getPeerStorage().addPeer("6",
+ ReplicationPeerConfig.newBuilder().setClusterKey(utility.getClusterKey()).build(), true);
- try{
- rp.registerPeer("6", new ReplicationPeerConfig().setClusterKey(utility.getClusterKey()));
- }catch(IllegalArgumentException e){
- exists++;
+ try {
+ rp.getPeerStorage().addPeer("6",
+ ReplicationPeerConfig.newBuilder().setClusterKey(utility.getClusterKey()).build(), true);
+ } catch (ReplicationException e) {
+ if (e.getCause() instanceof KeeperException.NodeExistsException) {
+ exists++;
+ }
}
- try{
- rp.registerPeer("6-ec2", new ReplicationPeerConfig().setClusterKey(utility.getClusterKey()));
- }catch(IllegalArgumentException e){
- hyphen++;
- }
assertEquals(1, exists);
- assertEquals(1, hyphen);
// clean up
- rp.unregisterPeer("6");
+ rp.getPeerStorage().removePeer("6");
}
private class DummyReplicationListener implements ReplicationListener {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 325012d..dd56a76 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -380,7 +380,7 @@ public abstract class TestReplicationSourceManager {
}
Server s1 = new DummyServer("dummyserver1.example.org");
ReplicationPeers rp1 =
- ReplicationFactory.getReplicationPeers(s1.getZooKeeper(), s1.getConfiguration(), s1);
+ ReplicationFactory.getReplicationPeers(s1.getZooKeeper(), s1.getConfiguration());
rp1.init();
NodeFailoverWorker w1 =
manager.new NodeFailoverWorker(server.getServerName());
@@ -561,7 +561,7 @@ public abstract class TestReplicationSourceManager {
private void addPeerAndWait(final String peerId, final ReplicationPeerConfig peerConfig,
final boolean waitForSource) throws Exception {
final ReplicationPeers rp = manager.getReplicationPeers();
- rp.registerPeer(peerId, peerConfig);
+ rp.getPeerStorage().addPeer(peerId, peerConfig, true);
try {
manager.addPeer(peerId);
} catch (Exception e) {
@@ -588,7 +588,7 @@ public abstract class TestReplicationSourceManager {
}
return true;
} else {
- return (rp.getConnectedPeer(peerId) != null);
+ return (rp.getPeer(peerId) != null);
}
});
}
@@ -600,8 +600,8 @@ public abstract class TestReplicationSourceManager {
*/
private void removePeerAndWait(final String peerId) throws Exception {
final ReplicationPeers rp = manager.getReplicationPeers();
- if (rp.getAllPeerIds().contains(peerId)) {
- rp.unregisterPeer(peerId);
+ if (rp.getPeerStorage().listPeerIds().contains(peerId)) {
+ rp.getPeerStorage().removePeer(peerId);
try {
manager.removePeer(peerId);
} catch (Exception e) {
@@ -611,10 +611,9 @@ public abstract class TestReplicationSourceManager {
Waiter.waitFor(conf, 20000, new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
- List<String> peers = rp.getAllPeerIds();
- return (!manager.getAllQueues().contains(peerId)) &&
- (rp.getConnectedPeer(peerId) == null) && (!peers.contains(peerId)) &&
- manager.getSource(peerId) == null;
+ Collection<String> peers = rp.getPeerStorage().listPeerIds();
+ return (!manager.getAllQueues().contains(peerId)) && (rp.getPeer(peerId) == null)
+ && (!peers.contains(peerId)) && manager.getSource(peerId) == null;
}
});
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/HBaseZKTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/HBaseZKTestingUtility.java b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/HBaseZKTestingUtility.java
index fc31c37..b755c32 100644
--- a/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/HBaseZKTestingUtility.java
+++ b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/HBaseZKTestingUtility.java
@@ -182,8 +182,7 @@ public class HBaseZKTestingUtility extends HBaseCommonTestingUtility {
/**
* Gets a ZKWatcher.
*/
- public static ZKWatcher getZooKeeperWatcher(HBaseZKTestingUtility testUtil)
- throws ZooKeeperConnectionException, IOException {
+ public static ZKWatcher getZooKeeperWatcher(HBaseZKTestingUtility testUtil) throws IOException {
ZKWatcher zkw = new ZKWatcher(testUtil.getConfiguration(), "unittest", new Abortable() {
boolean aborted = false;
[21/50] [abbrv] hbase git commit: HBASE-19543 Abstract a replication
storage interface to extract the zk specific code
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
deleted file mode 100644
index b6f8784..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.replication;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
-import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
-
-/**
- * Manages and performs all replication admin operations.
- * <p>
- * Used to add/remove a replication peer.
- */
-@InterfaceAudience.Private
-public class ReplicationManager {
- private final ReplicationQueuesClient replicationQueuesClient;
- private final ReplicationPeers replicationPeers;
-
- public ReplicationManager(Configuration conf, ZKWatcher zkw, Abortable abortable)
- throws IOException {
- try {
- this.replicationQueuesClient = ReplicationFactory
- .getReplicationQueuesClient(new ReplicationQueuesClientArguments(conf, abortable, zkw));
- this.replicationQueuesClient.init();
- this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf,
- this.replicationQueuesClient, abortable);
- this.replicationPeers.init();
- } catch (Exception e) {
- throw new IOException("Failed to construct ReplicationManager", e);
- }
- }
-
- public void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
- throws ReplicationException {
- checkPeerConfig(peerConfig);
- replicationPeers.registerPeer(peerId, peerConfig, enabled);
- replicationPeers.peerConnected(peerId);
- }
-
- public void removeReplicationPeer(String peerId) throws ReplicationException {
- replicationPeers.peerDisconnected(peerId);
- replicationPeers.unregisterPeer(peerId);
- }
-
- public void enableReplicationPeer(String peerId) throws ReplicationException {
- this.replicationPeers.enablePeer(peerId);
- }
-
- public void disableReplicationPeer(String peerId) throws ReplicationException {
- this.replicationPeers.disablePeer(peerId);
- }
-
- public ReplicationPeerConfig getPeerConfig(String peerId)
- throws ReplicationException, ReplicationPeerNotFoundException {
- ReplicationPeerConfig peerConfig = replicationPeers.getReplicationPeerConfig(peerId);
- if (peerConfig == null) {
- throw new ReplicationPeerNotFoundException(peerId);
- }
- return peerConfig;
- }
-
- public void updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)
- throws ReplicationException, IOException {
- checkPeerConfig(peerConfig);
- this.replicationPeers.updatePeerConfig(peerId, peerConfig);
- }
-
- public List<ReplicationPeerDescription> listReplicationPeers(Pattern pattern)
- throws ReplicationException {
- List<ReplicationPeerDescription> peers = new ArrayList<>();
- List<String> peerIds = replicationPeers.getAllPeerIds();
- for (String peerId : peerIds) {
- if (pattern == null || (pattern != null && pattern.matcher(peerId).matches())) {
- peers.add(new ReplicationPeerDescription(peerId,
- replicationPeers.getStatusOfPeerFromBackingStore(peerId),
- replicationPeers.getReplicationPeerConfig(peerId)));
- }
- }
- return peers;
- }
-
- /**
- * If replicate_all flag is true, it means all user tables will be replicated to peer cluster.
- * Then allow config exclude namespaces or exclude table-cfs which can't be replicated to
- * peer cluster.
- *
- * If replicate_all flag is false, it means all user tables can't be replicated to peer cluster.
- * Then allow to config namespaces or table-cfs which will be replicated to peer cluster.
- */
- private void checkPeerConfig(ReplicationPeerConfig peerConfig) {
- if (peerConfig.replicateAllUserTables()) {
- if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty()) ||
- (peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) {
- throw new IllegalArgumentException("Need clean namespaces or table-cfs config firstly " +
- "when you want replicate all cluster");
- }
- checkNamespacesAndTableCfsConfigConflict(peerConfig.getExcludeNamespaces(),
- peerConfig.getExcludeTableCFsMap());
- } else {
- if ((peerConfig.getExcludeNamespaces() != null
- && !peerConfig.getExcludeNamespaces().isEmpty())
- || (peerConfig.getExcludeTableCFsMap() != null
- && !peerConfig.getExcludeTableCFsMap().isEmpty())) {
- throw new IllegalArgumentException(
- "Need clean exclude-namespaces or exclude-table-cfs config firstly"
- + " when replicate_all flag is false");
- }
- checkNamespacesAndTableCfsConfigConflict(peerConfig.getNamespaces(),
- peerConfig.getTableCFsMap());
- }
- checkConfiguredWALEntryFilters(peerConfig);
- }
-
- /**
- * Set a namespace in the peer config means that all tables in this namespace will be replicated
- * to the peer cluster.
- * <ol>
- * <li>If peer config already has a namespace, then not allow set any table of this namespace to
- * the peer config.</li>
- * <li>If peer config already has a table, then not allow set this table's namespace to the peer
- * config.</li>
- * </ol>
- * <p>
- * Set a exclude namespace in the peer config means that all tables in this namespace can't be
- * replicated to the peer cluster.
- * <ol>
- * <li>If peer config already has a exclude namespace, then not allow set any exclude table of
- * this namespace to the peer config.</li>
- * <li>If peer config already has a exclude table, then not allow set this table's namespace as a
- * exclude namespace.</li>
- * </ol>
- */
- private void checkNamespacesAndTableCfsConfigConflict(Set<String> namespaces,
- Map<TableName, ? extends Collection<String>> tableCfs) {
- if (namespaces == null || namespaces.isEmpty()) {
- return;
- }
- if (tableCfs == null || tableCfs.isEmpty()) {
- return;
- }
- for (Map.Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
- TableName table = entry.getKey();
- if (namespaces.contains(table.getNamespaceAsString())) {
- throw new IllegalArgumentException("Table-cfs " + table + " is conflict with namespaces "
- + table.getNamespaceAsString() + " in peer config");
- }
- }
- }
-
- private void checkConfiguredWALEntryFilters(ReplicationPeerConfig peerConfig) {
- String filterCSV = peerConfig.getConfiguration()
- .get(BaseReplicationEndpoint.REPLICATION_WALENTRYFILTER_CONFIG_KEY);
- if (filterCSV != null && !filterCSV.isEmpty()) {
- String[] filters = filterCSV.split(",");
- for (String filter : filters) {
- try {
- Class.forName(filter).newInstance();
- } catch (Exception e) {
- throw new IllegalArgumentException("Configured WALEntryFilter " + filter +
- " could not be created. Failing add/update " + "peer operation.", e);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
new file mode 100644
index 0000000..5abd874
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -0,0 +1,331 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
+import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * Manages and performs all replication admin operations.
+ * <p>
+ * Used to add/remove a replication peer.
+ */
+@InterfaceAudience.Private
+public final class ReplicationPeerManager {
+
+ private final ReplicationPeerStorage peerStorage;
+
+ private final ReplicationQueueStorage queueStorage;
+
+ private final ConcurrentMap<String, ReplicationPeerDescription> peers;
+
+ private ReplicationPeerManager(ReplicationPeerStorage peerStorage,
+ ReplicationQueueStorage queueStorage,
+ ConcurrentMap<String, ReplicationPeerDescription> peers) {
+ this.peerStorage = peerStorage;
+ this.queueStorage = queueStorage;
+ this.peers = peers;
+ }
+
+ private void checkQueuesDeleted(String peerId)
+ throws ReplicationException, DoNotRetryIOException {
+ for (ServerName replicator : queueStorage.getListOfReplicators()) {
+ List<String> queueIds = queueStorage.getAllQueues(replicator);
+ for (String queueId : queueIds) {
+ ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
+ if (queueInfo.getPeerId().equals(peerId)) {
+ throw new DoNotRetryIOException("undeleted queue for peerId: " + peerId +
+ ", replicator: " + replicator + ", queueId: " + queueId);
+ }
+ }
+ }
+ if (queueStorage.getAllPeersFromHFileRefsQueue().contains(peerId)) {
+ throw new DoNotRetryIOException("Undeleted queue for peer " + peerId + " in hfile-refs");
+ }
+ }
+
+ public void preAddPeer(String peerId, ReplicationPeerConfig peerConfig)
+ throws DoNotRetryIOException, ReplicationException {
+ if (peerId.contains("-")) {
+ throw new DoNotRetryIOException("Found invalid peer name: " + peerId);
+ }
+ checkPeerConfig(peerConfig);
+ if (peers.containsKey(peerId)) {
+ throw new DoNotRetryIOException("Replication peer " + peerId + " already exists");
+ }
+ // make sure that there is no queues with the same peer id. This may happen when we create a
+ // peer with the same id with a old deleted peer. If the replication queues for the old peer
+ // have not been cleaned up yet then we should not create the new peer, otherwise the old wal
+ // file may also be replicated.
+ checkQueuesDeleted(peerId);
+ }
+
+ private ReplicationPeerDescription checkPeerExists(String peerId) throws DoNotRetryIOException {
+ ReplicationPeerDescription desc = peers.get(peerId);
+ if (desc == null) {
+ throw new DoNotRetryIOException("Replication peer " + peerId + " does not exist");
+ }
+ return desc;
+ }
+
+ public void preRemovePeer(String peerId) throws DoNotRetryIOException {
+ checkPeerExists(peerId);
+ }
+
+ public void preEnablePeer(String peerId) throws DoNotRetryIOException {
+ ReplicationPeerDescription desc = checkPeerExists(peerId);
+ if (desc.isEnabled()) {
+ throw new DoNotRetryIOException("Replication peer " + peerId + " has already been enabled");
+ }
+ }
+
+ public void preDisablePeer(String peerId) throws DoNotRetryIOException {
+ ReplicationPeerDescription desc = checkPeerExists(peerId);
+ if (!desc.isEnabled()) {
+ throw new DoNotRetryIOException("Replication peer " + peerId + " has already been disabled");
+ }
+ }
+
+ public void preUpdatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)
+ throws DoNotRetryIOException {
+ checkPeerConfig(peerConfig);
+ ReplicationPeerDescription desc = checkPeerExists(peerId);
+ ReplicationPeerConfig oldPeerConfig = desc.getPeerConfig();
+ if (!StringUtils.isBlank(peerConfig.getClusterKey()) &&
+ !peerConfig.getClusterKey().equals(oldPeerConfig.getClusterKey())) {
+ throw new DoNotRetryIOException(
+ "Changing the cluster key on an existing peer is not allowed. Existing key '" +
+ oldPeerConfig.getClusterKey() + "' for peer " + peerId + " does not match new key '" +
+ peerConfig.getClusterKey() + "'");
+ }
+
+ if (!StringUtils.isBlank(peerConfig.getReplicationEndpointImpl()) &&
+ !peerConfig.getReplicationEndpointImpl().equals(oldPeerConfig.getReplicationEndpointImpl())) {
+ throw new DoNotRetryIOException("Changing the replication endpoint implementation class " +
+ "on an existing peer is not allowed. Existing class '" +
+ oldPeerConfig.getReplicationEndpointImpl() + "' for peer " + peerId +
+ " does not match new class '" + peerConfig.getReplicationEndpointImpl() + "'");
+ }
+ }
+
+ private ReplicationPeerConfig copy(ReplicationPeerConfig peerConfig) {
+ ReplicationPeerConfig copiedPeerConfig = new ReplicationPeerConfig();
+ copiedPeerConfig.getConfiguration().putAll(peerConfig.getConfiguration());
+ copiedPeerConfig.getPeerData().putAll(peerConfig.getPeerData());
+ copiedPeerConfig.setTableCFsMap(peerConfig.getTableCFsMap());
+ copiedPeerConfig.setNamespaces(peerConfig.getNamespaces());
+ copiedPeerConfig.setExcludeTableCFsMap(peerConfig.getExcludeTableCFsMap());
+ copiedPeerConfig.setExcludeNamespaces(peerConfig.getExcludeNamespaces());
+ copiedPeerConfig.setBandwidth(peerConfig.getBandwidth());
+ copiedPeerConfig.setReplicateAllUserTables(peerConfig.replicateAllUserTables());
+ copiedPeerConfig.setClusterKey(peerConfig.getClusterKey());
+ copiedPeerConfig.setReplicationEndpointImpl(peerConfig.getReplicationEndpointImpl());
+ return copiedPeerConfig;
+ }
+
+ public void addPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
+ throws ReplicationException {
+ if (peers.containsKey(peerId)) {
+ // this should be a retry, just return
+ return;
+ }
+ ReplicationPeerConfig copiedPeerConfig = copy(peerConfig);
+ peerStorage.addPeer(peerId, copiedPeerConfig, enabled);
+ peers.put(peerId, new ReplicationPeerDescription(peerId, enabled, copiedPeerConfig));
+ }
+
+ public void removePeer(String peerId) throws ReplicationException {
+ if (!peers.containsKey(peerId)) {
+ // this should be a retry, just return
+ return;
+ }
+ peerStorage.removePeer(peerId);
+ peers.remove(peerId);
+ }
+
+ private void setPeerState(String peerId, boolean enabled) throws ReplicationException {
+ ReplicationPeerDescription desc = peers.get(peerId);
+ if (desc.isEnabled() == enabled) {
+ // this should be a retry, just return
+ return;
+ }
+ peerStorage.setPeerState(peerId, enabled);
+ peers.put(peerId, new ReplicationPeerDescription(peerId, enabled, desc.getPeerConfig()));
+ }
+
+ public void enablePeer(String peerId) throws ReplicationException {
+ setPeerState(peerId, true);
+ }
+
+ public void disablePeer(String peerId) throws ReplicationException {
+ setPeerState(peerId, false);
+ }
+
+ public void updatePeerConfig(String peerId, ReplicationPeerConfig peerConfig)
+ throws ReplicationException {
+ // the checking rules are too complicated here so we give up checking whether this is a retry.
+ ReplicationPeerDescription desc = peers.get(peerId);
+ ReplicationPeerConfig oldPeerConfig = desc.getPeerConfig();
+ ReplicationPeerConfig newPeerConfig = copy(peerConfig);
+ // we need to use the new conf to overwrite the old one.
+ newPeerConfig.getConfiguration().putAll(oldPeerConfig.getConfiguration());
+ newPeerConfig.getConfiguration().putAll(peerConfig.getConfiguration());
+ newPeerConfig.getPeerData().putAll(oldPeerConfig.getPeerData());
+ newPeerConfig.getPeerData().putAll(peerConfig.getPeerData());
+
+ peerStorage.updatePeerConfig(peerId, newPeerConfig);
+ peers.put(peerId, new ReplicationPeerDescription(peerId, desc.isEnabled(), newPeerConfig));
+ }
+
+ public List<ReplicationPeerDescription> listPeers(Pattern pattern) {
+ if (pattern == null) {
+ return new ArrayList<>(peers.values());
+ }
+ return peers.values().stream().filter(r -> pattern.matcher(r.getPeerId()).matches())
+ .collect(Collectors.toList());
+ }
+
+ public Optional<ReplicationPeerConfig> getPeerConfig(String peerId) {
+ ReplicationPeerDescription desc = peers.get(peerId);
+ return desc != null ? Optional.of(desc.getPeerConfig()) : Optional.empty();
+ }
+
+ /**
+ * If replicate_all flag is true, it means all user tables will be replicated to peer cluster.
+ * Then allow config exclude namespaces or exclude table-cfs which can't be replicated to peer
+ * cluster.
+ * <p>
+ * If replicate_all flag is false, it means all user tables can't be replicated to peer cluster.
+ * Then allow to config namespaces or table-cfs which will be replicated to peer cluster.
+ */
+ private static void checkPeerConfig(ReplicationPeerConfig peerConfig)
+ throws DoNotRetryIOException {
+ if (peerConfig.replicateAllUserTables()) {
+ if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty()) ||
+ (peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) {
+ throw new DoNotRetryIOException("Need clean namespaces or table-cfs config firstly " +
+ "when you want replicate all cluster");
+ }
+ checkNamespacesAndTableCfsConfigConflict(peerConfig.getExcludeNamespaces(),
+ peerConfig.getExcludeTableCFsMap());
+ } else {
+ if ((peerConfig.getExcludeNamespaces() != null &&
+ !peerConfig.getExcludeNamespaces().isEmpty()) ||
+ (peerConfig.getExcludeTableCFsMap() != null &&
+ !peerConfig.getExcludeTableCFsMap().isEmpty())) {
+ throw new DoNotRetryIOException(
+ "Need clean exclude-namespaces or exclude-table-cfs config firstly" +
+ " when replicate_all flag is false");
+ }
+ checkNamespacesAndTableCfsConfigConflict(peerConfig.getNamespaces(),
+ peerConfig.getTableCFsMap());
+ }
+ checkConfiguredWALEntryFilters(peerConfig);
+ }
+
+ /**
+ * Set a namespace in the peer config means that all tables in this namespace will be replicated
+ * to the peer cluster.
+ * <ol>
+ * <li>If peer config already has a namespace, then not allow set any table of this namespace to
+ * the peer config.</li>
+ * <li>If peer config already has a table, then not allow set this table's namespace to the peer
+ * config.</li>
+ * </ol>
+ * <p>
+ * Set a exclude namespace in the peer config means that all tables in this namespace can't be
+ * replicated to the peer cluster.
+ * <ol>
+ * <li>If peer config already has a exclude namespace, then not allow set any exclude table of
+ * this namespace to the peer config.</li>
+ * <li>If peer config already has a exclude table, then not allow set this table's namespace as a
+ * exclude namespace.</li>
+ * </ol>
+ */
+ private static void checkNamespacesAndTableCfsConfigConflict(Set<String> namespaces,
+ Map<TableName, ? extends Collection<String>> tableCfs) throws DoNotRetryIOException {
+ if (namespaces == null || namespaces.isEmpty()) {
+ return;
+ }
+ if (tableCfs == null || tableCfs.isEmpty()) {
+ return;
+ }
+ for (Map.Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
+ TableName table = entry.getKey();
+ if (namespaces.contains(table.getNamespaceAsString())) {
+ throw new DoNotRetryIOException("Table-cfs " + table + " is conflict with namespaces " +
+ table.getNamespaceAsString() + " in peer config");
+ }
+ }
+ }
+
+ private static void checkConfiguredWALEntryFilters(ReplicationPeerConfig peerConfig)
+ throws DoNotRetryIOException {
+ String filterCSV = peerConfig.getConfiguration()
+ .get(BaseReplicationEndpoint.REPLICATION_WALENTRYFILTER_CONFIG_KEY);
+ if (filterCSV != null && !filterCSV.isEmpty()) {
+ String[] filters = filterCSV.split(",");
+ for (String filter : filters) {
+ try {
+ Class.forName(filter).newInstance();
+ } catch (Exception e) {
+ throw new DoNotRetryIOException("Configured WALEntryFilter " + filter +
+ " could not be created. Failing add/update " + "peer operation.", e);
+ }
+ }
+ }
+ }
+
+ public static ReplicationPeerManager create(ZKWatcher zk, Configuration conf)
+ throws ReplicationException {
+ ReplicationPeerStorage peerStorage =
+ ReplicationStorageFactory.getReplicationPeerStorage(zk, conf);
+ ConcurrentMap<String, ReplicationPeerDescription> peers = new ConcurrentHashMap<>();
+ for (String peerId : peerStorage.listPeerIds()) {
+ Optional<ReplicationPeerConfig> peerConfig = peerStorage.getPeerConfig(peerId);
+ boolean enabled = peerStorage.isPeerEnabled(peerId);
+ peers.put(peerId, new ReplicationPeerDescription(peerId, enabled, peerConfig.get()));
+ }
+ return new ReplicationPeerManager(peerStorage,
+ ReplicationStorageFactory.getReplicationQueueStorage(zk, conf), peers);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
index d8154dc..a43532d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -59,12 +60,12 @@ public class UpdatePeerConfigProcedure extends ModifyPeerProcedure {
if (cpHost != null) {
cpHost.preUpdateReplicationPeerConfig(peerId, peerConfig);
}
+ env.getReplicationPeerManager().preUpdatePeerConfig(peerId, peerConfig);
}
@Override
- protected void updatePeerStorage(MasterProcedureEnv env)
- throws IllegalArgumentException, Exception {
- env.getReplicationManager().updatePeerConfig(peerId, peerConfig);
+ protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
+ env.getReplicationPeerManager().updatePeerConfig(peerId, peerConfig);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index aac57dc..d612eab 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -17,6 +17,12 @@
*/
package org.apache.hadoop.hbase.client.replication;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,25 +32,23 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.TestReplicationEndpoint.InterClusterReplicationEndpointForTest;
import org.apache.hadoop.hbase.replication.TestReplicationEndpoint.ReplicationEndpointForTest;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -52,15 +56,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
/**
* Unit testing of ReplicationAdmin
@@ -68,8 +63,6 @@ import static org.junit.Assert.fail;
@Category({MediumTests.class, ClientTests.class})
public class TestReplicationAdmin {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestReplicationAdmin.class);
private final static HBaseTestingUtility TEST_UTIL =
new HBaseTestingUtility();
@@ -104,16 +97,17 @@ public class TestReplicationAdmin {
}
@After
- public void cleanupPeer() {
- try {
- hbaseAdmin.removeReplicationPeer(ID_ONE);
- } catch (Exception e) {
- LOG.debug("Replication peer " + ID_ONE + " may already be removed");
+ public void tearDown() throws Exception {
+ for (ReplicationPeerDescription desc : hbaseAdmin.listReplicationPeers()) {
+ hbaseAdmin.removeReplicationPeer(desc.getPeerId());
}
- try {
- hbaseAdmin.removeReplicationPeer(ID_SECOND);
- } catch (Exception e) {
- LOG.debug("Replication peer " + ID_SECOND + " may already be removed");
+ ReplicationQueueStorage queueStorage = ReplicationStorageFactory
+ .getReplicationQueueStorage(TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConfiguration());
+ for (ServerName serverName : queueStorage.getListOfReplicators()) {
+ for (String queue : queueStorage.getAllQueues(serverName)) {
+ queueStorage.removeQueue(serverName, queue);
+ }
+ queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
}
}
@@ -203,32 +197,29 @@ public class TestReplicationAdmin {
ReplicationPeerConfig rpc2 = new ReplicationPeerConfig();
rpc2.setClusterKey(KEY_SECOND);
Configuration conf = TEST_UTIL.getConfiguration();
- ZKWatcher zkw = new ZKWatcher(conf, "Test HBaseAdmin", null);
- ReplicationQueues repQueues =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, null, zkw));
- repQueues.init("server1");
+ ReplicationQueueStorage queueStorage =
+ ReplicationStorageFactory.getReplicationQueueStorage(TEST_UTIL.getZooKeeperWatcher(), conf);
+ ServerName serverName = ServerName.valueOf("server1", 8000, 1234);
// add queue for ID_ONE
- repQueues.addLog(ID_ONE, "file1");
+ queueStorage.addWAL(serverName, ID_ONE, "file1");
try {
admin.addPeer(ID_ONE, rpc1, null);
fail();
} catch (Exception e) {
// OK!
}
- repQueues.removeQueue(ID_ONE);
- assertEquals(0, repQueues.getAllQueues().size());
+ queueStorage.removeQueue(serverName, ID_ONE);
+ assertEquals(0, queueStorage.getAllQueues(serverName).size());
// add recovered queue for ID_ONE
- repQueues.addLog(ID_ONE + "-server2", "file1");
+ queueStorage.addWAL(serverName, ID_ONE + "-server2", "file1");
try {
admin.addPeer(ID_ONE, rpc2, null);
fail();
} catch (Exception e) {
// OK!
}
- repQueues.removeAllQueues();
- zkw.close();
}
/**
@@ -424,7 +415,7 @@ public class TestReplicationAdmin {
tableCFs.clear();
tableCFs.put(tableName2, null);
admin.removePeerTableCFs(ID_ONE, tableCFs);
- assertTrue(false);
+ fail();
} catch (ReplicationException e) {
}
tableCFs.clear();
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index e88710e..4e66676 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.master;
import static org.mockito.Mockito.mock;
+import com.google.protobuf.Service;
import java.io.IOException;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.ChoreService;
@@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.locking.LockManager;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.master.replication.ReplicationManager;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
import org.apache.hadoop.hbase.procedure2.LockedResource;
@@ -55,9 +55,8 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import com.google.protobuf.Service;
-
public class MockNoopMasterServices implements MasterServices {
+
private final Configuration conf;
private final MetricsMaster metricsMaster;
@@ -461,7 +460,7 @@ public class MockNoopMasterServices implements MasterServices {
}
@Override
- public ProcedureEvent getInitializedEvent() {
+ public ProcedureEvent<?> getInitializedEvent() {
return null;
}
@@ -476,7 +475,7 @@ public class MockNoopMasterServices implements MasterServices {
}
@Override
- public ReplicationManager getReplicationManager() {
+ public ReplicationPeerManager getReplicationPeerManager() {
return null;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
index 15c8b6a..a541516 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
@@ -44,6 +44,8 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -271,7 +273,7 @@ public class TestMasterNoCluster {
@Override
void initializeZKBasedSystemTrackers() throws IOException, InterruptedException,
- KeeperException, CoordinatedStateException {
+ KeeperException, CoordinatedStateException, ReplicationException {
super.initializeZKBasedSystemTrackers();
// Record a newer server in server manager at first
getServerManager().recordNewServerWithLock(newServer,
http://git-wip-us.apache.org/repos/asf/hbase/blob/7213a0dc/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
index 7b9dea4..384daed 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -47,9 +46,6 @@ public class TestReplicationDisableInactivePeer extends TestReplicationBase {
*/
@Test(timeout = 600000)
public void testDisableInactivePeer() throws Exception {
-
- // enabling and shutdown the peer
- admin.enablePeer("2");
utility2.shutdownMiniHBaseCluster();
byte[] rowkey = Bytes.toBytes("disable inactive peer");
[08/50] [abbrv] hbase git commit: HBASE-19811 Fix findbugs and
error-prone warnings in hbase-server (branch-2)
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
index 2a35365..bd0efd8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
@@ -93,7 +93,7 @@ public class TestMasterCoprocessorExceptionWithAbort {
fail("BuggyMasterObserver failed to throw an exception.");
} catch (IOException e) {
assertEquals("HBaseAdmin threw an interrupted IOException as expected.",
- e.getClass().getName(), "java.io.InterruptedIOException");
+ "java.io.InterruptedIOException", e.getClass().getName());
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index 431e73e..ea817ff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -932,6 +932,7 @@ public class TestMasterObserver {
return preModifyTableActionCalled && !postCompletedModifyTableActionCalled;
}
+ @Override
public void preEnableTableAction(
final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
index 166dfdd..09aa4ff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
@@ -341,6 +341,7 @@ public class TestWALObserver {
User user = HBaseTestingUtility.getDifferentUser(newConf,
".replay.wal.secondtime");
user.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
Path p = runWALSplit(newConf);
LOG.info("WALSplit path == " + p);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java
index b0b17f9..a3f2f1c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/executor/TestExecutorService.java
@@ -145,7 +145,7 @@ public class TestExecutorService {
}
public static class TestEventHandler extends EventHandler {
- private AtomicBoolean lock;
+ private final AtomicBoolean lock;
private AtomicInteger counter;
public TestEventHandler(Server server, EventType eventType,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
index c31eebf..0e5fdb2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestBitComparator.java
@@ -103,13 +103,13 @@ public class TestBitComparator {
private void testOperation(byte[] data, byte[] comparatorBytes, BitComparator.BitwiseOp operator,
int expected) {
BitComparator comparator = new BitComparator(comparatorBytes, operator);
- assertEquals(comparator.compareTo(data), expected);
+ assertEquals(expected, comparator.compareTo(data));
}
private void testOperation(ByteBuffer data, byte[] comparatorBytes,
BitComparator.BitwiseOp operator, int expected) {
BitComparator comparator = new BitComparator(comparatorBytes, operator);
- assertEquals(comparator.compareTo(data, 0, data.capacity()), expected);
+ assertEquals(expected, comparator.compareTo(data, 0, data.capacity()));
}
@Test
@@ -142,13 +142,13 @@ public class TestBitComparator {
private void testOperationWithOffset(byte[] data, byte[] comparatorBytes,
BitComparator.BitwiseOp operator, int expected) {
BitComparator comparator = new BitComparator(comparatorBytes, operator);
- assertEquals(comparator.compareTo(data, 1, comparatorBytes.length), expected);
+ assertEquals(expected, comparator.compareTo(data, 1, comparatorBytes.length));
}
private void testOperationWithOffset(ByteBuffer data, byte[] comparatorBytes,
BitComparator.BitwiseOp operator, int expected) {
BitComparator comparator = new BitComparator(comparatorBytes, operator);
- assertEquals(comparator.compareTo(data, 1, comparatorBytes.length), expected);
+ assertEquals(expected, comparator.compareTo(data, 1, comparatorBytes.length));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index ca2c88b..ec11ce0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -136,8 +136,8 @@ public class TestFilter {
Bytes.toBytes("f1"), Bytes.toBytes("f2")
};
- private long numRows = ROWS_ONE.length + ROWS_TWO.length;
- private long colsPerRow = FAMILIES.length * QUALIFIERS_ONE.length;
+ private long numRows = (long) ROWS_ONE.length + ROWS_TWO.length;
+ private long colsPerRow = (long) FAMILIES.length * QUALIFIERS_ONE.length;
@Before
public void setUp() throws Exception {
@@ -1756,15 +1756,14 @@ public class TestFilter {
assertTrue("Qualifier mismatch", CellUtil.matchingQualifier(kv, kvs[idx]));
assertFalse("Should not have returned whole value", CellUtil.matchingValue(kv, kvs[idx]));
if (useLen) {
- assertEquals("Value in result is not SIZEOF_INT",
- kv.getValueLength(), Bytes.SIZEOF_INT);
+ assertEquals("Value in result is not SIZEOF_INT", Bytes.SIZEOF_INT, kv.getValueLength());
LOG.info("idx = " + idx + ", len=" + kvs[idx].getValueLength()
+ ", actual=" + Bytes.toInt(CellUtil.cloneValue(kv)));
assertEquals("Scan value should be the length of the actual value. ",
kvs[idx].getValueLength(), Bytes.toInt(CellUtil.cloneValue(kv)) );
LOG.info("good");
} else {
- assertEquals("Value in result is not empty", kv.getValueLength(), 0);
+ assertEquals("Value in result is not empty", 0, kv.getValueLength());
}
idx++;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java
index fdd7e77..ad5ee99 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterFromRegionSide.java
@@ -136,6 +136,7 @@ public class TestFilterFromRegionSide {
public static class FirstSeveralCellsFilter extends FilterBase{
private int count = 0;
+ @Override
public void reset() {
count = 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 6574d04..2d2a425 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -768,7 +768,7 @@ public class TestFilterList {
MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
- assertEquals(filterList.filterCell(kv1), ReturnCode.INCLUDE);
+ assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter2, filter3);
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));
@@ -936,6 +936,7 @@ public class TestFilterList {
private static class MockNextRowFilter extends FilterBase {
private int hitCount = 0;
+ @Override
public ReturnCode filterCell(final Cell v) throws IOException {
hitCount++;
return ReturnCode.NEXT_ROW;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterSerialization.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterSerialization.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterSerialization.java
index 8fa41e3..d470fac 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterSerialization.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterSerialization.java
@@ -301,8 +301,8 @@ public class TestFilterSerialization {
// Non-empty timestamp list
LinkedList<Long> list = new LinkedList<>();
- list.add(new Long(System.currentTimeMillis()));
- list.add(new Long(System.currentTimeMillis()));
+ list.add(System.currentTimeMillis());
+ list.add(System.currentTimeMillis());
timestampsFilter = new TimestampsFilter(list);
assertTrue(timestampsFilter.areSerializedFieldsEqual(
ProtobufUtil.toFilter(ProtobufUtil.toFilter(timestampsFilter))));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
index 5f25b49..25ea358 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
@@ -86,7 +86,7 @@ public class TestFuzzyRowFilterEndToEnd {
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
ConstantSizeRegionSplitPolicy.class.getName());
// set no splits
- conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
+ conf.setLong(HConstants.HREGION_MAX_FILESIZE, (1024L) * 1024 * 1024 * 10);
TEST_UTIL.startMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
index ebccc34..768ab7a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestInvocationRecordFilter.java
@@ -162,20 +162,24 @@ public class TestInvocationRecordFilter {
private List<Cell> visitedKeyValues = new ArrayList<>();
+ @Override
public void reset() {
visitedKeyValues.clear();
}
+ @Override
public ReturnCode filterCell(final Cell ignored) {
visitedKeyValues.add(ignored);
return ReturnCode.INCLUDE;
}
+ @Override
public void filterRowCells(List<Cell> kvs) {
kvs.clear();
kvs.addAll(visitedKeyValues);
}
+ @Override
public boolean hasFilterRow() {
return true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java
index bcd239d..c5200f9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestParseFilter.java
@@ -18,9 +18,12 @@
*/
package org.apache.hadoop.hbase.filter;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@@ -89,7 +92,7 @@ public class TestParseFilter {
String filterString = " PrefixFilter('row' ) ";
PrefixFilter prefixFilter = doTestFilter(filterString, PrefixFilter.class);
byte [] prefix = prefixFilter.getPrefix();
- assertEquals(new String(prefix), "row");
+ assertEquals("row", new String(prefix, StandardCharsets.UTF_8));
filterString = " PrefixFilter(row)";
@@ -107,7 +110,7 @@ public class TestParseFilter {
ColumnPrefixFilter columnPrefixFilter =
doTestFilter(filterString, ColumnPrefixFilter.class);
byte [] columnPrefix = columnPrefixFilter.getPrefix();
- assertEquals(new String(columnPrefix), "qualifier");
+ assertEquals("qualifier", new String(columnPrefix, StandardCharsets.UTF_8));
}
@Test
@@ -116,8 +119,8 @@ public class TestParseFilter {
MultipleColumnPrefixFilter multipleColumnPrefixFilter =
doTestFilter(filterString, MultipleColumnPrefixFilter.class);
byte [][] prefixes = multipleColumnPrefixFilter.getPrefix();
- assertEquals(new String(prefixes[0]), "qualifier1");
- assertEquals(new String(prefixes[1]), "qualifier2");
+ assertEquals("qualifier1", new String(prefixes[0], StandardCharsets.UTF_8));
+ assertEquals("qualifier2", new String(prefixes[1], StandardCharsets.UTF_8));
}
@Test
@@ -126,7 +129,7 @@ public class TestParseFilter {
ColumnCountGetFilter columnCountGetFilter =
doTestFilter(filterString, ColumnCountGetFilter.class);
int limit = columnCountGetFilter.getLimit();
- assertEquals(limit, 4);
+ assertEquals(4, limit);
filterString = " ColumnCountGetFilter('abc')";
try {
@@ -151,7 +154,7 @@ public class TestParseFilter {
PageFilter pageFilter =
doTestFilter(filterString, PageFilter.class);
long pageSize = pageFilter.getPageSize();
- assertEquals(pageSize, 4);
+ assertEquals(4, pageSize);
filterString = " PageFilter('123')";
try {
@@ -168,9 +171,9 @@ public class TestParseFilter {
ColumnPaginationFilter columnPaginationFilter =
doTestFilter(filterString, ColumnPaginationFilter.class);
int limit = columnPaginationFilter.getLimit();
- assertEquals(limit, 4);
+ assertEquals(4, limit);
int offset = columnPaginationFilter.getOffset();
- assertEquals(offset, 6);
+ assertEquals(6, offset);
filterString = " ColumnPaginationFilter('124')";
try {
@@ -203,7 +206,7 @@ public class TestParseFilter {
InclusiveStopFilter inclusiveStopFilter =
doTestFilter(filterString, InclusiveStopFilter.class);
byte [] stopRowKey = inclusiveStopFilter.getStopRowKey();
- assertEquals(new String(stopRowKey), "row 3");
+ assertEquals("row 3", new String(stopRowKey, StandardCharsets.UTF_8));
}
@@ -213,13 +216,13 @@ public class TestParseFilter {
TimestampsFilter timestampsFilter =
doTestFilter(filterString, TimestampsFilter.class);
List<Long> timestamps = timestampsFilter.getTimestamps();
- assertEquals(timestamps.size(), 2);
- assertEquals(timestamps.get(0), new Long(6));
+ assertEquals(2, timestamps.size());
+ assertEquals(Long.valueOf(6), timestamps.get(0));
filterString = "TimestampsFilter()";
timestampsFilter = doTestFilter(filterString, TimestampsFilter.class);
timestamps = timestampsFilter.getTimestamps();
- assertEquals(timestamps.size(), 0);
+ assertEquals(0, timestamps.size());
filterString = "TimestampsFilter(9223372036854775808, 6)";
try {
@@ -246,7 +249,7 @@ public class TestParseFilter {
assertEquals(CompareOperator.EQUAL, rowFilter.getCompareOperator());
assertTrue(rowFilter.getComparator() instanceof BinaryComparator);
BinaryComparator binaryComparator = (BinaryComparator) rowFilter.getComparator();
- assertEquals("regionse", new String(binaryComparator.getValue()));
+ assertEquals("regionse", new String(binaryComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -258,7 +261,7 @@ public class TestParseFilter {
assertTrue(familyFilter.getComparator() instanceof BinaryPrefixComparator);
BinaryPrefixComparator binaryPrefixComparator =
(BinaryPrefixComparator) familyFilter.getComparator();
- assertEquals("pre", new String(binaryPrefixComparator.getValue()));
+ assertEquals("pre", new String(binaryPrefixComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -270,7 +273,7 @@ public class TestParseFilter {
assertTrue(qualifierFilter.getComparator() instanceof RegexStringComparator);
RegexStringComparator regexStringComparator =
(RegexStringComparator) qualifierFilter.getComparator();
- assertEquals("pre*", new String(regexStringComparator.getValue()));
+ assertEquals("pre*", new String(regexStringComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -282,7 +285,7 @@ public class TestParseFilter {
assertTrue(valueFilter.getComparator() instanceof SubstringComparator);
SubstringComparator substringComparator =
(SubstringComparator) valueFilter.getComparator();
- assertEquals("pre", new String(substringComparator.getValue()));
+ assertEquals("pre", new String(substringComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -290,8 +293,8 @@ public class TestParseFilter {
String filterString = "ColumnRangeFilter('abc', true, 'xyz', false)";
ColumnRangeFilter columnRangeFilter =
doTestFilter(filterString, ColumnRangeFilter.class);
- assertEquals("abc", new String(columnRangeFilter.getMinColumn()));
- assertEquals("xyz", new String(columnRangeFilter.getMaxColumn()));
+ assertEquals("abc", new String(columnRangeFilter.getMinColumn(), StandardCharsets.UTF_8));
+ assertEquals("xyz", new String(columnRangeFilter.getMaxColumn(), StandardCharsets.UTF_8));
assertTrue(columnRangeFilter.isMinColumnInclusive());
assertFalse(columnRangeFilter.isMaxColumnInclusive());
}
@@ -301,13 +304,14 @@ public class TestParseFilter {
String filterString = "DependentColumnFilter('family', 'qualifier', true, =, 'binary:abc')";
DependentColumnFilter dependentColumnFilter =
doTestFilter(filterString, DependentColumnFilter.class);
- assertEquals("family", new String(dependentColumnFilter.getFamily()));
- assertEquals("qualifier", new String(dependentColumnFilter.getQualifier()));
+ assertEquals("family", new String(dependentColumnFilter.getFamily(), StandardCharsets.UTF_8));
+ assertEquals("qualifier",
+ new String(dependentColumnFilter.getQualifier(), StandardCharsets.UTF_8));
assertTrue(dependentColumnFilter.getDropDependentColumn());
assertEquals(CompareOperator.EQUAL, dependentColumnFilter.getCompareOperator());
assertTrue(dependentColumnFilter.getComparator() instanceof BinaryComparator);
BinaryComparator binaryComparator = (BinaryComparator)dependentColumnFilter.getComparator();
- assertEquals("abc", new String(binaryComparator.getValue()));
+ assertEquals("abc", new String(binaryComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -316,25 +320,27 @@ public class TestParseFilter {
"('family', 'qualifier', >=, 'binary:a', true, false)";
SingleColumnValueFilter singleColumnValueFilter =
doTestFilter(filterString, SingleColumnValueFilter.class);
- assertEquals("family", new String(singleColumnValueFilter.getFamily()));
- assertEquals("qualifier", new String(singleColumnValueFilter.getQualifier()));
- assertEquals(singleColumnValueFilter.getCompareOperator(), CompareOperator.GREATER_OR_EQUAL);
+ assertEquals("family", new String(singleColumnValueFilter.getFamily(), StandardCharsets.UTF_8));
+ assertEquals("qualifier",
+ new String(singleColumnValueFilter.getQualifier(), StandardCharsets.UTF_8));
+ assertEquals(CompareOperator.GREATER_OR_EQUAL, singleColumnValueFilter.getCompareOperator());
assertTrue(singleColumnValueFilter.getComparator() instanceof BinaryComparator);
BinaryComparator binaryComparator = (BinaryComparator) singleColumnValueFilter.getComparator();
- assertEquals(new String(binaryComparator.getValue()), "a");
+ assertEquals("a", new String(binaryComparator.getValue(), StandardCharsets.UTF_8));
assertTrue(singleColumnValueFilter.getFilterIfMissing());
assertFalse(singleColumnValueFilter.getLatestVersionOnly());
filterString = "SingleColumnValueFilter ('family', 'qualifier', >, 'binaryprefix:a')";
singleColumnValueFilter = doTestFilter(filterString, SingleColumnValueFilter.class);
- assertEquals("family", new String(singleColumnValueFilter.getFamily()));
- assertEquals("qualifier", new String(singleColumnValueFilter.getQualifier()));
- assertEquals(singleColumnValueFilter.getCompareOperator(), CompareOperator.GREATER);
+ assertEquals("family", new String(singleColumnValueFilter.getFamily(), StandardCharsets.UTF_8));
+ assertEquals("qualifier",
+ new String(singleColumnValueFilter.getQualifier(), StandardCharsets.UTF_8));
+ assertEquals(CompareOperator.GREATER, singleColumnValueFilter.getCompareOperator());
assertTrue(singleColumnValueFilter.getComparator() instanceof BinaryPrefixComparator);
BinaryPrefixComparator binaryPrefixComparator =
(BinaryPrefixComparator) singleColumnValueFilter.getComparator();
- assertEquals(new String(binaryPrefixComparator.getValue()), "a");
+ assertEquals("a", new String(binaryPrefixComparator.getValue(), StandardCharsets.UTF_8));
assertFalse(singleColumnValueFilter.getFilterIfMissing());
assertTrue(singleColumnValueFilter.getLatestVersionOnly());
}
@@ -345,10 +351,13 @@ public class TestParseFilter {
"SingleColumnValueExcludeFilter ('family', 'qualifier', <, 'binaryprefix:a')";
SingleColumnValueExcludeFilter singleColumnValueExcludeFilter =
doTestFilter(filterString, SingleColumnValueExcludeFilter.class);
- assertEquals(singleColumnValueExcludeFilter.getCompareOperator(), CompareOperator.LESS);
- assertEquals("family", new String(singleColumnValueExcludeFilter.getFamily()));
- assertEquals("qualifier", new String(singleColumnValueExcludeFilter.getQualifier()));
- assertEquals(new String(singleColumnValueExcludeFilter.getComparator().getValue()), "a");
+ assertEquals(CompareOperator.LESS, singleColumnValueExcludeFilter.getCompareOperator());
+ assertEquals("family",
+ new String(singleColumnValueExcludeFilter.getFamily(), StandardCharsets.UTF_8));
+ assertEquals("qualifier",
+ new String(singleColumnValueExcludeFilter.getQualifier(), StandardCharsets.UTF_8));
+ assertEquals("a", new String(singleColumnValueExcludeFilter.getComparator().getValue(),
+ StandardCharsets.UTF_8));
assertFalse(singleColumnValueExcludeFilter.getFilterIfMissing());
assertTrue(singleColumnValueExcludeFilter.getLatestVersionOnly());
@@ -356,14 +365,16 @@ public class TestParseFilter {
"('family', 'qualifier', <=, 'binaryprefix:a', true, false)";
singleColumnValueExcludeFilter =
doTestFilter(filterString, SingleColumnValueExcludeFilter.class);
- assertEquals("family", new String(singleColumnValueExcludeFilter.getFamily()));
- assertEquals("qualifier", new String(singleColumnValueExcludeFilter.getQualifier()));
- assertEquals(singleColumnValueExcludeFilter.getCompareOperator(),
- CompareOperator.LESS_OR_EQUAL);
+ assertEquals("family",
+ new String(singleColumnValueExcludeFilter.getFamily(), StandardCharsets.UTF_8));
+ assertEquals("qualifier",
+ new String(singleColumnValueExcludeFilter.getQualifier(), StandardCharsets.UTF_8));
+ assertEquals(CompareOperator.LESS_OR_EQUAL,
+ singleColumnValueExcludeFilter.getCompareOperator());
assertTrue(singleColumnValueExcludeFilter.getComparator() instanceof BinaryPrefixComparator);
BinaryPrefixComparator binaryPrefixComparator =
(BinaryPrefixComparator) singleColumnValueExcludeFilter.getComparator();
- assertEquals(new String(binaryPrefixComparator.getValue()), "a");
+ assertEquals("a", new String(binaryPrefixComparator.getValue(), StandardCharsets.UTF_8));
assertTrue(singleColumnValueExcludeFilter.getFilterIfMissing());
assertFalse(singleColumnValueExcludeFilter.getLatestVersionOnly());
}
@@ -379,7 +390,7 @@ public class TestParseFilter {
assertEquals(CompareOperator.EQUAL, valueFilter.getCompareOperator());
assertTrue(valueFilter.getComparator() instanceof BinaryComparator);
BinaryComparator binaryComparator = (BinaryComparator) valueFilter.getComparator();
- assertEquals("0", new String(binaryComparator.getValue()));
+ assertEquals("0", new String(binaryComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -393,7 +404,7 @@ public class TestParseFilter {
assertEquals(CompareOperator.NOT_EQUAL, rowFilter.getCompareOperator());
assertTrue(rowFilter.getComparator() instanceof BinaryComparator);
BinaryComparator binaryComparator = (BinaryComparator) rowFilter.getComparator();
- assertEquals("row1", new String(binaryComparator.getValue()));
+ assertEquals("row1", new String(binaryComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -407,7 +418,7 @@ public class TestParseFilter {
assertTrue(filters.get(1) instanceof FirstKeyOnlyFilter);
PrefixFilter PrefixFilter = (PrefixFilter) filters.get(0);
byte [] prefix = PrefixFilter.getPrefix();
- assertEquals(new String(prefix), "realtime");
+ assertEquals("realtime", new String(prefix, StandardCharsets.UTF_8));
FirstKeyOnlyFilter firstKeyOnlyFilter = (FirstKeyOnlyFilter) filters.get(1);
}
@@ -420,7 +431,7 @@ public class TestParseFilter {
ArrayList<Filter> filterListFilters = (ArrayList<Filter>) filterList.getFilters();
assertTrue(filterListFilters.get(0) instanceof FilterList);
assertTrue(filterListFilters.get(1) instanceof FamilyFilter);
- assertEquals(filterList.getOperator(), FilterList.Operator.MUST_PASS_ONE);
+ assertEquals(FilterList.Operator.MUST_PASS_ONE, filterList.getOperator());
filterList = (FilterList) filterListFilters.get(0);
FamilyFilter familyFilter = (FamilyFilter) filterListFilters.get(1);
@@ -428,22 +439,22 @@ public class TestParseFilter {
filterListFilters = (ArrayList<Filter>)filterList.getFilters();
assertTrue(filterListFilters.get(0) instanceof PrefixFilter);
assertTrue(filterListFilters.get(1) instanceof QualifierFilter);
- assertEquals(filterList.getOperator(), FilterList.Operator.MUST_PASS_ALL);
+ assertEquals(FilterList.Operator.MUST_PASS_ALL, filterList.getOperator());
assertEquals(CompareOperator.EQUAL, familyFilter.getCompareOperator());
assertTrue(familyFilter.getComparator() instanceof BinaryComparator);
BinaryComparator binaryComparator = (BinaryComparator) familyFilter.getComparator();
- assertEquals("qualifier", new String(binaryComparator.getValue()));
+ assertEquals("qualifier", new String(binaryComparator.getValue(), StandardCharsets.UTF_8));
PrefixFilter prefixFilter = (PrefixFilter) filterListFilters.get(0);
byte [] prefix = prefixFilter.getPrefix();
- assertEquals(new String(prefix), "realtime");
+ assertEquals("realtime", new String(prefix, StandardCharsets.UTF_8));
QualifierFilter qualifierFilter = (QualifierFilter) filterListFilters.get(1);
assertEquals(CompareOperator.GREATER_OR_EQUAL, qualifierFilter.getCompareOperator());
assertTrue(qualifierFilter.getComparator() instanceof BinaryComparator);
binaryComparator = (BinaryComparator) qualifierFilter.getComparator();
- assertEquals("e", new String(binaryComparator.getValue()));
+ assertEquals("e", new String(binaryComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -466,7 +477,7 @@ public class TestParseFilter {
ColumnPrefixFilter columnPrefixFilter = (ColumnPrefixFilter) filters.get(0);
byte [] columnPrefix = columnPrefixFilter.getPrefix();
- assertEquals(new String(columnPrefix), "realtime");
+ assertEquals("realtime", new String(columnPrefix, StandardCharsets.UTF_8));
FirstKeyOnlyFilter firstKeyOnlyFilter = (FirstKeyOnlyFilter) filters.get(1);
@@ -477,7 +488,7 @@ public class TestParseFilter {
assertTrue(familyFilter.getComparator() instanceof SubstringComparator);
SubstringComparator substringComparator =
(SubstringComparator) familyFilter.getComparator();
- assertEquals("hihi", new String(substringComparator.getValue()));
+ assertEquals("hihi", new String(substringComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -497,7 +508,7 @@ public class TestParseFilter {
SkipFilter skipFilter = (SkipFilter) filters.get(2);
byte [] columnPrefix = columnPrefixFilter.getPrefix();
- assertEquals(new String(columnPrefix), "realtime");
+ assertEquals("realtime", new String(columnPrefix, StandardCharsets.UTF_8));
assertTrue(skipFilter.getFilter() instanceof FamilyFilter);
FamilyFilter familyFilter = (FamilyFilter) skipFilter.getFilter();
@@ -506,7 +517,7 @@ public class TestParseFilter {
assertTrue(familyFilter.getComparator() instanceof SubstringComparator);
SubstringComparator substringComparator =
(SubstringComparator) familyFilter.getComparator();
- assertEquals("hihi", new String(substringComparator.getValue()));
+ assertEquals("hihi", new String(substringComparator.getValue(), StandardCharsets.UTF_8));
}
@Test
@@ -537,7 +548,7 @@ public class TestParseFilter {
}
@Test
- public void testIncorrectComparatorType () throws IOException {
+ public void testIncorrectComparatorType() throws IOException {
String filterString = "RowFilter ('>=' , 'binaryoperator:region')";
try {
doTestFilter(filterString, RowFilter.class);
@@ -584,7 +595,7 @@ public class TestParseFilter {
PrefixFilter prefixFilter = (PrefixFilter)filters.get(0);
byte [] prefix = prefixFilter.getPrefix();
- assertEquals(new String(prefix), "realtime");
+ assertEquals("realtime", new String(prefix, StandardCharsets.UTF_8));
}
@Test
@@ -606,41 +617,40 @@ public class TestParseFilter {
PrefixFilter prefixFilter = (PrefixFilter)filters.get(0);
byte [] prefix = prefixFilter.getPrefix();
- assertEquals(new String(prefix), "realtime");
+ assertEquals("realtime", new String(prefix, StandardCharsets.UTF_8));
SkipFilter skipFilter = (SkipFilter)filters.get(1);
assertTrue(skipFilter.getFilter() instanceof FirstKeyOnlyFilter);
}
@Test
- public void testUnescapedQuote1 () throws IOException {
+ public void testUnescapedQuote1() throws IOException {
String filterString = "InclusiveStopFilter ('row''3')";
InclusiveStopFilter inclusiveStopFilter =
doTestFilter(filterString, InclusiveStopFilter.class);
byte [] stopRowKey = inclusiveStopFilter.getStopRowKey();
- assertEquals(new String(stopRowKey), "row'3");
+ assertEquals("row'3", new String(stopRowKey, StandardCharsets.UTF_8));
}
@Test
- public void testUnescapedQuote2 () throws IOException {
+ public void testUnescapedQuote2() throws IOException {
String filterString = "InclusiveStopFilter ('row''3''')";
InclusiveStopFilter inclusiveStopFilter =
doTestFilter(filterString, InclusiveStopFilter.class);
byte [] stopRowKey = inclusiveStopFilter.getStopRowKey();
- assertEquals(new String(stopRowKey), "row'3'");
+ assertEquals("row'3'", new String(stopRowKey, StandardCharsets.UTF_8));
}
@Test
- public void testUnescapedQuote3 () throws IOException {
+ public void testUnescapedQuote3() throws IOException {
String filterString = " InclusiveStopFilter ('''')";
- InclusiveStopFilter inclusiveStopFilter =
- doTestFilter(filterString, InclusiveStopFilter.class);
+ InclusiveStopFilter inclusiveStopFilter = doTestFilter(filterString, InclusiveStopFilter.class);
byte [] stopRowKey = inclusiveStopFilter.getStopRowKey();
- assertEquals(new String(stopRowKey), "'");
+ assertEquals("'", new String(stopRowKey, StandardCharsets.UTF_8));
}
@Test
- public void testIncorrectFilterString () throws IOException {
+ public void testIncorrectFilterString() throws IOException {
String filterString = "()";
byte [] filterStringAsByteArray = Bytes.toBytes(filterString);
try {
@@ -652,10 +662,9 @@ public class TestParseFilter {
}
@Test
- public void testCorrectFilterString () throws IOException {
+ public void testCorrectFilterString() throws IOException {
String filterString = "(FirstKeyOnlyFilter())";
- FirstKeyOnlyFilter firstKeyOnlyFilter =
- doTestFilter(filterString, FirstKeyOnlyFilter.class);
+ FirstKeyOnlyFilter firstKeyOnlyFilter = doTestFilter(filterString, FirstKeyOnlyFilter.class);
}
@Test
@@ -665,7 +674,8 @@ public class TestParseFilter {
assertTrue(f.getSupportedFilters().contains("MyFilter"));
}
- private <T extends Filter> T doTestFilter(String filterString, Class<T> clazz) throws IOException {
+ private <T extends Filter> T doTestFilter(String filterString, Class<T> clazz)
+ throws IOException {
byte [] filterStringAsByteArray = Bytes.toBytes(filterString);
filter = f.parseFilterString(filterStringAsByteArray);
assertEquals(clazz, filter.getClass());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
index 764d033..c6b1b5f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestSingleColumnValueExcludeFilter.java
@@ -68,7 +68,7 @@ public class TestSingleColumnValueExcludeFilter {
filter.filterRowCells(kvs);
- assertEquals("resultSize", kvs.size(), 2);
+ assertEquals("resultSize", 2, kvs.size());
assertTrue("leftKV1", CellComparatorImpl.COMPARATOR.compare(kvs.get(0), c) == 0);
assertTrue("leftKV2", CellComparatorImpl.COMPARATOR.compare(kvs.get(1), c) == 0);
assertFalse("allRemainingWhenMatch", filter.filterAllRemaining());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
index b24d30b..5ba7dfa 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorder.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hdfs.DFSClient;
@@ -248,7 +249,7 @@ public class TestBlockReorder {
*/
@Test()
public void testHBaseCluster() throws Exception {
- byte[] sb = "sb".getBytes();
+ byte[] sb = Bytes.toBytes("sb");
htu.startMiniZKCluster();
MiniHBaseCluster hbm = htu.startMiniHBaseCluster(1, 1);
@@ -442,7 +443,7 @@ public class TestBlockReorder {
do {
l = getNamenode(dfs.getClient()).getBlockLocations(fileName, 0, 1);
Assert.assertNotNull(l.getLocatedBlocks());
- Assert.assertEquals(l.getLocatedBlocks().size(), 1);
+ Assert.assertEquals(1, l.getLocatedBlocks().size());
Assert.assertTrue("Expecting " + repCount + " , got " + l.get(0).getLocations().length,
System.currentTimeMillis() < max);
} while (l.get(0).getLocations().length != repCount);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java
index 33bac39..7747bdb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java
@@ -125,7 +125,7 @@ public class TestHFileLink {
HFileLink.parseBackReferenceName(encodedRegion+"."+
tableName.getNameAsString().replace(TableName.NAMESPACE_DELIM, '='));
assertEquals(parsedRef.getFirst(), tableName);
- assertEquals(parsedRef.getSecond(), encodedRegion);
+ assertEquals(encodedRegion, parsedRef.getSecond());
//verify resolving back reference
Path storeFileDir = new Path(refLinkDir, encodedRegion+"."+
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
index 94df090..b3148c7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
@@ -546,9 +546,9 @@ public class TestHeapSize {
assertTrue(ClassSize.OBJECT == 12 || ClassSize.OBJECT == 16); // depending on CompressedOops
}
if (ClassSize.useUnsafeLayout()) {
- assertEquals(ClassSize.OBJECT + 4, ClassSize.ARRAY);
+ assertEquals(ClassSize.ARRAY, ClassSize.OBJECT + 4);
} else {
- assertEquals(ClassSize.OBJECT + 8, ClassSize.ARRAY);
+ assertEquals(ClassSize.ARRAY, ClassSize.OBJECT + 8);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
index f43f147..82a50c4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
@@ -72,16 +72,16 @@ public class TestBufferedDataBlockEncoder {
@Test
public void testCommonPrefixComparators() {
- KeyValue kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put);
- KeyValue kv2 = new KeyValue(row1, fam_1_2, qual1, 1l, Type.Maximum);
+ KeyValue kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put);
+ KeyValue kv2 = new KeyValue(row1, fam_1_2, qual1, 1L, Type.Maximum);
assertTrue((BufferedDataBlockEncoder.compareCommonFamilyPrefix(kv1, kv2, 4) < 0));
- kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put);
- kv2 = new KeyValue(row_1_0, fam_1_2, qual1, 1l, Type.Maximum);
+ kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put);
+ kv2 = new KeyValue(row_1_0, fam_1_2, qual1, 1L, Type.Maximum);
assertTrue((BufferedDataBlockEncoder.compareCommonRowPrefix(kv1, kv2, 4) < 0));
- kv1 = new KeyValue(row1, fam1, qual2, 1l, Type.Put);
- kv2 = new KeyValue(row1, fam1, qual1, 1l, Type.Maximum);
+ kv1 = new KeyValue(row1, fam1, qual2, 1L, Type.Put);
+ kv2 = new KeyValue(row1, fam1, qual1, 1L, Type.Maximum);
assertTrue((BufferedDataBlockEncoder.compareCommonQualifierPrefix(kv1, kv2, 4) > 0));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
index f41db93..cbbc9dc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
@@ -82,7 +82,7 @@ public class TestDataBlockEncoders {
static final byte[] HFILEBLOCK_DUMMY_HEADER = new byte[HConstants.HFILEBLOCK_HEADER_SIZE];
private RedundantKVGenerator generator = new RedundantKVGenerator();
- private Random randomizer = new Random(42l);
+ private Random randomizer = new Random(42L);
private final boolean includesMemstoreTS;
private final boolean includesTags;
@@ -129,14 +129,14 @@ public class TestDataBlockEncoders {
byte[] qualifier = new byte[0];
byte[] value = new byte[0];
if (!includesTags) {
- kvList.add(new KeyValue(row, family, qualifier, 0l, value));
- kvList.add(new KeyValue(row, family, qualifier, 0l, value));
+ kvList.add(new KeyValue(row, family, qualifier, 0L, value));
+ kvList.add(new KeyValue(row, family, qualifier, 0L, value));
} else {
byte[] metaValue1 = Bytes.toBytes("metaValue1");
byte[] metaValue2 = Bytes.toBytes("metaValue2");
- kvList.add(new KeyValue(row, family, qualifier, 0l, value,
+ kvList.add(new KeyValue(row, family, qualifier, 0L, value,
new Tag[] { new ArrayBackedTag((byte) 1, metaValue1) }));
- kvList.add(new KeyValue(row, family, qualifier, 0l, value,
+ kvList.add(new KeyValue(row, family, qualifier, 0L, value,
new Tag[] { new ArrayBackedTag((byte) 1, metaValue2) }));
}
testEncodersOnDataset(kvList, includesMemstoreTS, includesTags);
@@ -158,13 +158,13 @@ public class TestDataBlockEncoders {
if (includesTags) {
byte[] metaValue1 = Bytes.toBytes("metaValue1");
byte[] metaValue2 = Bytes.toBytes("metaValue2");
- kvList.add(new KeyValue(row, family, qualifier, 0l, value,
+ kvList.add(new KeyValue(row, family, qualifier, 0L, value,
new Tag[] { new ArrayBackedTag((byte) 1, metaValue1) }));
- kvList.add(new KeyValue(row, family, qualifier, 0l, value,
+ kvList.add(new KeyValue(row, family, qualifier, 0L, value,
new Tag[] { new ArrayBackedTag((byte) 1, metaValue2) }));
} else {
- kvList.add(new KeyValue(row, family, qualifier, -1l, Type.Put, value));
- kvList.add(new KeyValue(row, family, qualifier, -2l, Type.Put, value));
+ kvList.add(new KeyValue(row, family, qualifier, -1L, Type.Put, value));
+ kvList.add(new KeyValue(row, family, qualifier, -2L, Type.Put, value));
}
testEncodersOnDataset(kvList, includesMemstoreTS, includesTags);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestLoadAndSwitchEncodeOnDisk.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestLoadAndSwitchEncodeOnDisk.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestLoadAndSwitchEncodeOnDisk.java
index e62af9e..d46a553 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestLoadAndSwitchEncodeOnDisk.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestLoadAndSwitchEncodeOnDisk.java
@@ -61,10 +61,12 @@ public class TestLoadAndSwitchEncodeOnDisk extends
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true);
}
+ @Override
protected int numKeys() {
return 3000;
}
+ @Override
@Test(timeout=TIMEOUT_MS)
public void loadTest() throws Exception {
Admin admin = TEST_UTIL.getAdmin();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java
index e0d2a9b..d304e74 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekBeforeWithReverseScan.java
@@ -97,12 +97,12 @@ public class TestSeekBeforeWithReverseScan {
while (scanner.next(res)) {
count++;
}
- assertEquals(Bytes.toString(res.get(0).getRowArray(), res.get(0).getRowOffset(), res.get(0)
- .getRowLength()), "b");
- assertEquals(Bytes.toString(res.get(1).getRowArray(), res.get(1).getRowOffset(), res.get(1)
- .getRowLength()), "ab");
- assertEquals(Bytes.toString(res.get(2).getRowArray(), res.get(2).getRowOffset(), res.get(2)
- .getRowLength()), "a");
+ assertEquals("b", Bytes.toString(res.get(0).getRowArray(), res.get(0).getRowOffset(),
+ res.get(0).getRowLength()));
+ assertEquals("ab", Bytes.toString(res.get(1).getRowArray(), res.get(1).getRowOffset(),
+ res.get(1).getRowLength()));
+ assertEquals("a", Bytes.toString(res.get(2).getRowArray(), res.get(2).getRowOffset(),
+ res.get(2).getRowLength()));
assertEquals(3, count);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
index 462f77a..4300387 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
@@ -302,7 +302,7 @@ public class CacheTestUtils {
@Override
public long heapSize() {
- return 4 + buf.length;
+ return 4L + buf.length;
}
@Override
@@ -373,9 +373,10 @@ public class CacheTestUtils {
String strKey;
/* No conflicting keys */
- for (strKey = new Long(rand.nextLong()).toString(); !usedStrings
- .add(strKey); strKey = new Long(rand.nextLong()).toString())
- ;
+ strKey = Long.toString(rand.nextLong());
+ while (!usedStrings.add(strKey)) {
+ strKey = Long.toString(rand.nextLong());
+ }
returnedBlocks[i] = new HFileBlockPair();
returnedBlocks[i].blockName = new BlockCacheKey(strKey, 0);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/NanoTimer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/NanoTimer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/NanoTimer.java
index a133cb4..aaf1711 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/NanoTimer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/NanoTimer.java
@@ -98,6 +98,7 @@ public class NanoTimer {
*
* Note: If timer is never started, "ERR" will be returned.
*/
+ @Override
public String toString() {
if (!readable()) {
return "ERR";
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
index 611c524..6d3d4ab 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
@@ -297,7 +297,7 @@ public class TestCacheOnWrite {
// block we cached at write-time and block read from file should be identical
assertEquals(block.getChecksumType(), fromCache.getChecksumType());
assertEquals(block.getBlockType(), fromCache.getBlockType());
- assertNotEquals(block.getBlockType(), BlockType.ENCODED_DATA);
+ assertNotEquals(BlockType.ENCODED_DATA, block.getBlockType());
assertEquals(block.getOnDiskSizeWithHeader(), fromCache.getOnDiskSizeWithHeader());
assertEquals(block.getOnDiskSizeWithoutHeader(), fromCache.getOnDiskSizeWithoutHeader());
assertEquals(
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
index 9c36788..a8b7d1f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
@@ -124,7 +124,7 @@ public class TestHFileBlock {
static int writeTestKeyValues(HFileBlock.Writer hbw, int seed, boolean includesMemstoreTS,
boolean useTag) throws IOException {
List<KeyValue> keyValues = new ArrayList<>();
- Random randomizer = new Random(42l + seed); // just any fixed number
+ Random randomizer = new Random(42L + seed); // just any fixed number
// generate keyValues
for (int i = 0; i < NUM_KEYVALUES; ++i) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java
index bec774e..a049b329 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java
@@ -262,7 +262,7 @@ public class TestHFileEncryption {
assertTrue("Initial seekTo failed", scanner.seekTo());
for (i = 0; i < 100; i++) {
KeyValue kv = testKvs.get(RNG.nextInt(testKvs.size()));
- assertEquals("Unable to find KV as expected: " + kv, scanner.seekTo(kv), 0);
+ assertEquals("Unable to find KV as expected: " + kv, 0, scanner.seekTo(kv));
}
} finally {
scanner.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index af169f5..8429ee8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -280,10 +280,10 @@ public class TestLruBlockCache {
}
// A single eviction run should have occurred
- assertEquals(cache.getStats().getEvictionCount(), 1);
+ assertEquals(1, cache.getStats().getEvictionCount());
// We expect two entries evicted
- assertEquals(cache.getStats().getEvictedCount(), 2);
+ assertEquals(2, cache.getStats().getEvictedCount());
// Our expected size overruns acceptable limit
assertTrue(expectedCacheSize >
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java
index 9b2602f..3873a6c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestBucketCache.java
@@ -93,7 +93,7 @@ public class TestBucketCache {
String ioEngineName = "offheap";
String persistencePath = null;
- private class MockedBucketCache extends BucketCache {
+ private static class MockedBucketCache extends BucketCache {
public MockedBucketCache(String ioEngineName, long capacity, int blockSize, int[] bucketSizes,
int writerThreads, int writerQLen, String persistencePath) throws FileNotFoundException,
@@ -314,12 +314,18 @@ public class TestBucketCache {
BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize,
constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf);
- assertEquals(BucketCache.ACCEPT_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getAcceptableFactor(), 0.9f, 0);
- assertEquals(BucketCache.MIN_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getMinFactor(), 0.5f, 0);
- assertEquals(BucketCache.EXTRA_FREE_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getExtraFreeFactor(), 0.5f, 0);
- assertEquals(BucketCache.SINGLE_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getSingleFactor(), 0.1f, 0);
- assertEquals(BucketCache.MULTI_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getMultiFactor(), 0.7f, 0);
- assertEquals(BucketCache.MEMORY_FACTOR_CONFIG_NAME + " failed to propagate.", cache.getMemoryFactor(), 0.2f, 0);
+ assertEquals(BucketCache.ACCEPT_FACTOR_CONFIG_NAME + " failed to propagate.", 0.9f,
+ cache.getAcceptableFactor(), 0);
+ assertEquals(BucketCache.MIN_FACTOR_CONFIG_NAME + " failed to propagate.", 0.5f,
+ cache.getMinFactor(), 0);
+ assertEquals(BucketCache.EXTRA_FREE_FACTOR_CONFIG_NAME + " failed to propagate.", 0.5f,
+ cache.getExtraFreeFactor(), 0);
+ assertEquals(BucketCache.SINGLE_FACTOR_CONFIG_NAME + " failed to propagate.", 0.1f,
+ cache.getSingleFactor(), 0);
+ assertEquals(BucketCache.MULTI_FACTOR_CONFIG_NAME + " failed to propagate.", 0.7f,
+ cache.getMultiFactor(), 0);
+ assertEquals(BucketCache.MEMORY_FACTOR_CONFIG_NAME + " failed to propagate.", 0.2f,
+ cache.getMemoryFactor(), 0);
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyIPC.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyIPC.java
index 09429dd..0a168ba 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyIPC.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestNettyIPC.java
@@ -148,7 +148,7 @@ public class TestNettyIPC extends AbstractTestIPC {
super(server, name, services, bindAddress, conf, scheduler, true);
}
- final class FailingConnection extends NettyServerRpcConnection {
+ static final class FailingConnection extends NettyServerRpcConnection {
private FailingConnection(TestFailingRpcServer rpcServer, Channel channel) {
super(rpcServer, channel);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestProtoBufRpc.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestProtoBufRpc.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestProtoBufRpc.java
index 83a7acc..f6f6fc5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestProtoBufRpc.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestProtoBufRpc.java
@@ -112,7 +112,7 @@ public class TestProtoBufRpc {
// Test echo method
EchoRequestProto echoRequest = EchoRequestProto.newBuilder().setMessage("hello").build();
EchoResponseProto echoResponse = stub.echo(null, echoRequest);
- assertEquals(echoResponse.getMessage(), "hello");
+ assertEquals("hello", echoResponse.getMessage());
stub.error(null, emptyRequest);
fail("Expected exception is not thrown");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientLeaks.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientLeaks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientLeaks.java
index 9d2fd91..9e70c93 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientLeaks.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientLeaks.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hbase.ipc;
import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
import static org.junit.Assert.assertTrue;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-
import java.io.IOException;
import java.net.Socket;
import java.net.SocketAddress;
@@ -39,6 +37,7 @@ import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -50,6 +49,8 @@ import org.junit.rules.TestRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+
@Category(MediumTests.class)
public class TestRpcClientLeaks {
@Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()).
@@ -112,7 +113,7 @@ public class TestRpcClientLeaks {
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(TableName.valueOf(name.getMethodName()));
- table.get(new Get("asd".getBytes()));
+ table.get(new Get(Bytes.toBytes("asd")));
connection.close();
for (Socket socket : MyRpcClientImpl.savedSockets) {
assertTrue("Socket + " + socket + " is not closed", socket.isClosed());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java
index e4a8767..e646c14 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java
@@ -166,7 +166,8 @@ public class TestSimpleRpcScheduler {
for (String callQueueName:callQueueInfo.getCallQueueNames()) {
for (String calledMethod: callQueueInfo.getCalledMethodNames(callQueueName)) {
- assertEquals(callQueueInfo.getCallMethodCount(callQueueName, calledMethod), totalCallMethods);
+ assertEquals(totalCallMethods,
+ callQueueInfo.getCallMethodCount(callQueueName, calledMethod));
}
}
@@ -327,7 +328,7 @@ public class TestSimpleRpcScheduler {
RpcScheduler scheduler = new SimpleRpcScheduler(schedConf, 2, 1, 1, priority,
HConstants.QOS_THRESHOLD);
- assertNotEquals(scheduler, null);
+ assertNotEquals(null, scheduler);
}
@Test
@@ -571,6 +572,7 @@ public class TestSimpleRpcScheduler {
};
CallRunner cr = new CallRunner(null, putCall) {
+ @Override
public void run() {
if (sleepTime <= 0) return;
try {
@@ -581,10 +583,12 @@ public class TestSimpleRpcScheduler {
}
}
+ @Override
public RpcCall getRpcCall() {
return putCall;
}
+ @Override
public void drop() {
}
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
index b080d7f..6b6f0de 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
@@ -77,6 +77,7 @@ abstract public class MapreduceTestingShim {
}
private static class MapreduceV1Shim extends MapreduceTestingShim {
+ @Override
public JobContext newJobContext(Configuration jobConf) throws IOException {
// Implementing:
// return new JobContext(jobConf, new JobID());
@@ -105,6 +106,7 @@ abstract public class MapreduceTestingShim {
}
}
+ @Override
public JobConf obtainJobConf(MiniMRCluster cluster) {
if (cluster == null) return null;
try {
@@ -129,6 +131,7 @@ abstract public class MapreduceTestingShim {
};
private static class MapreduceV2Shim extends MapreduceTestingShim {
+ @Override
public JobContext newJobContext(Configuration jobConf) {
return newJob(jobConf);
}
@@ -147,6 +150,7 @@ abstract public class MapreduceTestingShim {
}
}
+ @Override
public JobConf obtainJobConf(MiniMRCluster cluster) {
try {
Method meth = MiniMRCluster.class.getMethod("getJobTrackerConf", emptyParam);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
index 16f98a0..53e80f3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java
@@ -365,10 +365,8 @@ public class TestAssignmentListener {
drainingServerTracker.start();
// Confirm our ServerManager lists are empty.
- Assert.assertEquals(serverManager.getOnlineServers(),
- new HashMap<ServerName, ServerLoad>());
- Assert.assertEquals(serverManager.getDrainingServersList(),
- new ArrayList<ServerName>());
+ Assert.assertEquals(new HashMap<ServerName, ServerLoad>(), serverManager.getOnlineServers());
+ Assert.assertEquals(new ArrayList<ServerName>(), serverManager.getDrainingServersList());
// checkAndRecordNewServer() is how servers are added to the ServerManager.
ArrayList<ServerName> onlineDrainingServers = new ArrayList<>();
@@ -381,8 +379,7 @@ public class TestAssignmentListener {
}
// Verify the ServerManager lists are correctly updated.
- Assert.assertEquals(serverManager.getOnlineServers(), onlineServers);
- Assert.assertEquals(serverManager.getDrainingServersList(),
- onlineDrainingServers);
+ Assert.assertEquals(onlineServers, serverManager.getOnlineServers());
+ Assert.assertEquals(onlineDrainingServers, serverManager.getDrainingServersList());
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index e8aa755..aa5ad1e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -190,7 +190,7 @@ public class TestMasterFailover {
RegionState metaState = MetaTableLocator.getMetaRegionState(hrs.getZooKeeper());
assertEquals("hbase:meta should be online on RS",
metaState.getServerName(), metaServerName);
- assertEquals("hbase:meta should be online on RS", metaState.getState(), State.OPEN);
+ assertEquals("hbase:meta should be online on RS", State.OPEN, metaState.getState());
// Start up a new master
LOG.info("Starting up a new master");
@@ -203,7 +203,7 @@ public class TestMasterFailover {
metaState = MetaTableLocator.getMetaRegionState(activeMaster.getZooKeeper());
assertEquals("hbase:meta should be online on RS",
metaState.getServerName(), metaServerName);
- assertEquals("hbase:meta should be online on RS", metaState.getState(), State.OPEN);
+ assertEquals("hbase:meta should be online on RS", State.OPEN, metaState.getState());
// Done, shutdown the cluster
} finally {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
index fd44c89..15c8b6a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
@@ -189,6 +189,7 @@ public class TestMasterNoCluster {
TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(),
HRegionInfo.FIRST_META_REGIONINFO);
HMaster master = new HMaster(conf) {
+ @Override
InetAddress getRemoteInetAddress(final int port, final long serverStartCode)
throws UnknownHostException {
// Return different address dependent on port passed.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
index cd7af50..e399f2e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
@@ -242,7 +243,7 @@ public class TestMasterOperationsForRegionReplicas {
}
assert(defaultReplicas.size() == numRegions);
Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
- assert(counts.size() == 1 && counts.contains(new Integer(numReplica)));
+ assert(counts.size() == 1 && counts.contains(numReplica));
} finally {
ADMIN.disableTable(tableName);
ADMIN.deleteTable(tableName);
@@ -336,7 +337,7 @@ public class TestMasterOperationsForRegionReplicas {
byte[] startKey = region.getStartKey();
if (region.getTable().equals(table)) {
setOfStartKeys.add(startKey); //ignore other tables
- LOG.info("--STARTKEY " + new String(startKey)+"--");
+ LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
}
}
// the number of startkeys will be equal to the number of regions hosted in each server
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
index 29c24f7..648ea64 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java
@@ -121,6 +121,7 @@ public class TestMasterShutdown {
master.start();
LOG.info("Called master start on " + master.getName());
Thread shutdownThread = new Thread("Shutdown-Thread") {
+ @Override
public void run() {
LOG.info("Before call to shutdown master");
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
index ebfec22..2e6c699 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java
@@ -88,8 +88,7 @@ public class TestMetaShutdownHandler {
}
RegionState metaState =
MetaTableLocator.getMetaRegionState(master.getZooKeeper());
- assertEquals("Meta should be not in transition",
- metaState.getState(), RegionState.State.OPEN);
+ assertEquals("Meta should be not in transition", RegionState.State.OPEN, metaState.getState());
assertNotEquals("Meta should be moved off master",
metaServerName, master.getServerName());
@@ -115,8 +114,7 @@ public class TestMetaShutdownHandler {
regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));
// Now, make sure meta is registered in zk
metaState = MetaTableLocator.getMetaRegionState(master.getZooKeeper());
- assertEquals("Meta should be not in transition",
- metaState.getState(), RegionState.State.OPEN);
+ assertEquals("Meta should be not in transition", RegionState.State.OPEN, metaState.getState());
assertEquals("Meta should be assigned", metaState.getServerName(),
regionStates.getRegionServerOfRegion(HRegionInfo.FIRST_META_REGIONINFO));
assertNotEquals("Meta should be assigned on a different server",
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
index e99d533..2794952 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
@@ -389,8 +389,7 @@ public class TestRegionPlacement {
lastRegionOpenedCount = currentRegionOpened;
assertEquals("There are only " + regionMovement + " instead of "
- + expected + " region movement for " + attempt + " attempts",
- regionMovement, expected);
+ + expected + " region movement for " + attempt + " attempts", expected, regionMovement);
}
/**
@@ -469,6 +468,7 @@ public class TestRegionPlacement {
final AtomicInteger totalRegionNum = new AtomicInteger(0);
LOG.info("The start of region placement verification");
MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
+ @Override
public boolean visit(Result result) throws IOException {
try {
@SuppressWarnings("deprecation")
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index cd5239e..128d7ee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -384,23 +384,23 @@ public class TestSplitLogManager {
@Test (timeout=180000)
public void testTaskResigned() throws Exception {
LOG.info("TestTaskResigned - resubmit task node once in RESIGNED state");
- assertEquals(tot_mgr_resubmit.sum(), 0);
+ assertEquals(0, tot_mgr_resubmit.sum());
slm = new SplitLogManager(master, conf);
- assertEquals(tot_mgr_resubmit.sum(), 0);
+ assertEquals(0, tot_mgr_resubmit.sum());
TaskBatch batch = new TaskBatch();
String tasknode = submitTaskAndWait(batch, "foo/1");
- assertEquals(tot_mgr_resubmit.sum(), 0);
+ assertEquals(0, tot_mgr_resubmit.sum());
final ServerName worker1 = ServerName.valueOf("worker1,1,1");
- assertEquals(tot_mgr_resubmit.sum(), 0);
+ assertEquals(0, tot_mgr_resubmit.sum());
SplitLogTask slt = new SplitLogTask.Resigned(worker1);
- assertEquals(tot_mgr_resubmit.sum(), 0);
+ assertEquals(0, tot_mgr_resubmit.sum());
ZKUtil.setData(zkw, tasknode, slt.toByteArray());
ZKUtil.checkExists(zkw, tasknode);
// Could be small race here.
if (tot_mgr_resubmit.sum() == 0) {
waitForCounter(tot_mgr_resubmit, 0, 1, to/2);
}
- assertEquals(tot_mgr_resubmit.sum(), 1);
+ assertEquals(1, tot_mgr_resubmit.sum());
byte[] taskstate = ZKUtil.getData(zkw, tasknode);
slt = SplitLogTask.parseFrom(taskstate);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableStateManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableStateManager.java
index 1f61ee7..5a75297 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableStateManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableStateManager.java
@@ -65,9 +65,8 @@ public class TestTableStateManager {
TEST_UTIL.restartHBaseCluster(1);
HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
- Assert.assertEquals(
- master.getTableStateManager().getTableState(tableName),
- TableState.State.DISABLED);
+ Assert.assertEquals(TableState.State.DISABLED,
+ master.getTableStateManager().getTableState(tableName));
}
private void setTableStateInZK(ZKWatcher watcher, final TableName tableName,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
index 83fafff..9bd4443 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
@@ -107,10 +107,12 @@ public class MockMasterServices extends MockNoopMasterServices {
this.walManager = new MasterWalManager(this);
// Mock an AM.
this.assignmentManager = new AssignmentManager(this, new MockRegionStateStore(this)) {
+ @Override
public boolean isTableEnabled(final TableName tableName) {
return true;
}
+ @Override
public boolean isTableDisabled(final TableName tableName) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java
index 37d9820..d2a4020 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java
@@ -501,8 +501,8 @@ public class TestSplitTableRegionProcedure {
daughters.get(i),
startRow,
numRows,
- ColumnFamilyName1.getBytes(),
- ColumnFamilyName2.getBytes());
+ Bytes.toBytes(ColumnFamilyName1),
+ Bytes.toBytes(ColumnFamilyName2));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
index adf56b8..e180fb5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
@@ -159,6 +159,7 @@ public class BalancerTestBase {
public MockMapping(Configuration conf) {
}
+ @Override
public List<String> resolve(List<String> names) {
List<String> ret = new ArrayList<>(names.size());
for (String name : names) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index 08b27ec..644de6a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -392,6 +392,7 @@ public class TestLogsCleaner {
.when(zk).getData("/hbase/replication/rs", null, new Stat());
}
+ @Override
public RecoverableZooKeeper getRecoverableZooKeeper() {
return zk;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
index 2948701..85f0d1f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
@@ -351,6 +351,7 @@ public class TestReplicationHFileCleaner {
.when(zk).getData("/hbase/replication/hfile-refs", null, new Stat());
}
+ @Override
public RecoverableZooKeeper getRecoverableZooKeeper() {
return zk;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
index 6069041..6a71df3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
@@ -183,7 +183,7 @@ public class TestSnapshotFromMaster {
DisabledTableSnapshotHandler mockHandler = Mockito.mock(DisabledTableSnapshotHandler.class);
Mockito.when(mockHandler.getException()).thenReturn(null);
Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
- Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true));
+ Mockito.when(mockHandler.isFinished()).thenReturn(Boolean.TRUE);
Mockito.when(mockHandler.getCompletionTimestamp())
.thenReturn(EnvironmentEdgeManager.currentTime());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/locking/TestLockProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/locking/TestLockProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/locking/TestLockProcedure.java
index 7f031cc..1f743db 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/locking/TestLockProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/locking/TestLockProcedure.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
import org.hamcrest.core.IsInstanceOf;
import org.hamcrest.core.StringStartsWith;
import org.junit.After;
@@ -60,6 +61,7 @@ import org.junit.rules.TestName;
import org.junit.rules.TestRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
import org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos.LockHeartbeatRequest;
@@ -105,8 +107,10 @@ public class TestLockProcedure {
setupConf(UTIL.getConfiguration());
UTIL.startMiniCluster(1);
UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(namespace).build());
- UTIL.createTable(tableName1, new byte[][]{"fam".getBytes()}, new byte[][] {"1".getBytes()});
- UTIL.createTable(tableName2, new byte[][]{"fam".getBytes()}, new byte[][] {"1".getBytes()});
+ UTIL.createTable(tableName1,
+ new byte[][]{ Bytes.toBytes("fam")}, new byte[][] {Bytes.toBytes("1")});
+ UTIL.createTable(tableName2,
+ new byte[][]{Bytes.toBytes("fam")}, new byte[][] {Bytes.toBytes("1")});
masterRpcService = UTIL.getHBaseCluster().getMaster().getMasterRpcServices();
procExec = UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
tableRegions1 = UTIL.getAdmin().getRegions(tableName1);
@@ -195,7 +199,7 @@ public class TestLockProcedure {
LockHeartbeatResponse response = masterRpcService.lockHeartbeat(null,
LockHeartbeatRequest.newBuilder().setProcId(procId).build());
if (response.getLockStatus() == LockHeartbeatResponse.LockStatus.LOCKED) {
- assertEquals(response.getTimeoutMs(), HEARTBEAT_TIMEOUT);
+ assertEquals(HEARTBEAT_TIMEOUT, response.getTimeoutMs());
LOG.debug(String.format("Proc id %s acquired lock.", procId));
return true;
}
@@ -349,7 +353,8 @@ public class TestLockProcedure {
CountDownLatch latch = new CountDownLatch(1);
// MasterRpcServices don't set latch with LockProcedure, so create one and submit it directly.
LockProcedure lockProc = new LockProcedure(UTIL.getConfiguration(),
- TableName.valueOf("table"), org.apache.hadoop.hbase.procedure2.LockType.EXCLUSIVE, "desc", latch);
+ TableName.valueOf("table"),
+ org.apache.hadoop.hbase.procedure2.LockType.EXCLUSIVE, "desc", latch);
procExec.submitProcedure(lockProc);
assertTrue(latch.await(2000, TimeUnit.MILLISECONDS));
releaseLock(lockProc.getProcId());
[32/50] [abbrv] hbase git commit: HBASE-19617 Remove
ReplicationQueues, use ReplicationQueueStorage directly
Posted by zh...@apache.org.
HBASE-19617 Remove ReplicationQueues, use ReplicationQueueStorage directly
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b3b92efa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b3b92efa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b3b92efa
Branch: refs/heads/HBASE-19397-branch-2
Commit: b3b92efaaad9d442c45ae56f0059aeb4e04cc176
Parents: f90fcc9
Author: zhangduo <zh...@apache.org>
Authored: Wed Dec 27 22:03:51 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:04 2018 +0800
----------------------------------------------------------------------
.../hbase/replication/ReplicationFactory.java | 9 +-
.../hbase/replication/ReplicationQueues.java | 160 -------
.../replication/ReplicationQueuesArguments.java | 70 ---
.../replication/ReplicationQueuesZKImpl.java | 407 -----------------
.../hbase/replication/ReplicationTableBase.java | 442 -------------------
.../replication/ReplicationTrackerZKImpl.java | 21 +-
.../replication/ZKReplicationQueueStorage.java | 22 +
.../replication/TestReplicationStateBasic.java | 131 +++---
.../replication/TestReplicationStateZKImpl.java | 41 +-
.../regionserver/DumpReplicationQueues.java | 15 +-
.../RecoveredReplicationSource.java | 17 +-
.../RecoveredReplicationSourceShipper.java | 22 +-
.../replication/regionserver/Replication.java | 41 +-
.../regionserver/ReplicationSource.java | 16 +-
.../ReplicationSourceInterface.java | 11 +-
.../regionserver/ReplicationSourceManager.java | 261 ++++++-----
.../regionserver/ReplicationSyncUp.java | 29 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 12 +-
.../cleaner/TestReplicationHFileCleaner.java | 26 +-
.../cleaner/TestReplicationZKNodeCleaner.java | 22 +-
.../replication/ReplicationSourceDummy.java | 6 +-
.../replication/TestReplicationSyncUpTool.java | 6 +-
.../TestReplicationSourceManager.java | 104 ++---
.../TestReplicationSourceManagerZkImpl.java | 58 +--
24 files changed, 381 insertions(+), 1568 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 6c1c213..5e70e57 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -17,12 +17,11 @@
*/
package org.apache.hadoop.hbase.replication;
-import org.apache.commons.lang3.reflect.ConstructorUtils;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* A factory class for instantiating replication objects that deal with replication state.
@@ -30,12 +29,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@InterfaceAudience.Private
public class ReplicationFactory {
- public static ReplicationQueues getReplicationQueues(ReplicationQueuesArguments args)
- throws Exception {
- return (ReplicationQueues) ConstructorUtils.invokeConstructor(ReplicationQueuesZKImpl.class,
- args);
- }
-
public static ReplicationPeers getReplicationPeers(ZKWatcher zk, Configuration conf,
Abortable abortable) {
return getReplicationPeers(zk, conf, null, abortable);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
deleted file mode 100644
index 7f440b1..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.List;
-import java.util.SortedSet;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * This provides an interface for maintaining a region server's replication queues. These queues
- * keep track of the WALs and HFile references (if hbase.replication.bulkload.enabled is enabled)
- * that still need to be replicated to remote clusters.
- */
-@InterfaceAudience.Private
-public interface ReplicationQueues {
-
- /**
- * Initialize the region server replication queue interface.
- * @param serverName The server name of the region server that owns the replication queues this
- * interface manages.
- */
- void init(String serverName) throws ReplicationException;
-
- /**
- * Remove a replication queue.
- * @param queueId a String that identifies the queue.
- */
- void removeQueue(String queueId);
-
- /**
- * Add a new WAL file to the given queue. If the queue does not exist it is created.
- * @param queueId a String that identifies the queue.
- * @param filename name of the WAL
- */
- void addLog(String queueId, String filename) throws ReplicationException;
-
- /**
- * Remove an WAL file from the given queue.
- * @param queueId a String that identifies the queue.
- * @param filename name of the WAL
- */
- void removeLog(String queueId, String filename);
-
- /**
- * Set the current position for a specific WAL in a given queue.
- * @param queueId a String that identifies the queue
- * @param filename name of the WAL
- * @param position the current position in the file
- */
- void setLogPosition(String queueId, String filename, long position);
-
- /**
- * Get the current position for a specific WAL in a given queue.
- * @param queueId a String that identifies the queue
- * @param filename name of the WAL
- * @return the current position in the file
- */
- long getLogPosition(String queueId, String filename) throws ReplicationException;
-
- /**
- * Remove all replication queues for this region server.
- */
- void removeAllQueues();
-
- /**
- * Get a list of all WALs in the given queue.
- * @param queueId a String that identifies the queue
- * @return a list of WALs, null if no such queue exists for this server
- */
- List<String> getLogsInQueue(String queueId);
-
- /**
- * Get a list of all queues for this region server.
- * @return a list of queueIds, an empty list if this region server is dead and has no outstanding queues
- */
- List<String> getAllQueues();
-
- /**
- * Get queueIds from a dead region server, whose queues has not been claimed by other region
- * servers.
- * @return empty if the queue exists but no children, null if the queue does not exist.
- */
- List<String> getUnClaimedQueueIds(String regionserver);
-
- /**
- * Take ownership for the queue identified by queueId and belongs to a dead region server.
- * @param regionserver the id of the dead region server
- * @param queueId the id of the queue
- * @return the new PeerId and A SortedSet of WALs in its queue, and null if no unclaimed queue.
- */
- Pair<String, SortedSet<String>> claimQueue(String regionserver, String queueId);
-
- /**
- * Remove the znode of region server if the queue is empty.
- * @param regionserver
- */
- void removeReplicatorIfQueueIsEmpty(String regionserver);
-
- /**
- * Get a list of all region servers that have outstanding replication queues. These servers could
- * be alive, dead or from a previous run of the cluster.
- * @return a list of server names
- */
- List<String> getListOfReplicators();
-
- /**
- * Checks if the provided znode is the same as this region server's
- * @param regionserver the id of the region server
- * @return if this is this rs's znode
- */
- boolean isThisOurRegionServer(String regionserver);
-
- /**
- * Add a peer to hfile reference queue if peer does not exist.
- * @param peerId peer cluster id to be added
- * @throws ReplicationException if fails to add a peer id to hfile reference queue
- */
- void addPeerToHFileRefs(String peerId) throws ReplicationException;
-
- /**
- * Remove a peer from hfile reference queue.
- * @param peerId peer cluster id to be removed
- */
- void removePeerFromHFileRefs(String peerId);
-
- /**
- * Add new hfile references to the queue.
- * @param peerId peer cluster id to which the hfiles need to be replicated
- * @param pairs list of pairs of { HFile location in staging dir, HFile path in region dir which
- * will be added in the queue }
- * @throws ReplicationException if fails to add a hfile reference
- */
- void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs) throws ReplicationException;
-
- /**
- * Remove hfile references from the queue.
- * @param peerId peer cluster id from which this hfile references needs to be removed
- * @param files list of hfile references to be removed
- */
- void removeHFileRefs(String peerId, List<String> files);
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
deleted file mode 100644
index c2a5df3..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
-
-/**
- * Wrapper around common arguments used to construct ReplicationQueues. Used to construct various
- * ReplicationQueues Implementations with different constructor arguments by reflection.
- */
-@InterfaceAudience.Private
-public class ReplicationQueuesArguments {
-
- private ZKWatcher zk;
- private Configuration conf;
- private Abortable abort;
-
- public ReplicationQueuesArguments(Configuration conf, Abortable abort) {
- this.conf = conf;
- this.abort = abort;
- }
-
- public ReplicationQueuesArguments(Configuration conf, Abortable abort, ZKWatcher zk) {
- this(conf, abort);
- setZk(zk);
- }
-
- public ZKWatcher getZk() {
- return zk;
- }
-
- public void setZk(ZKWatcher zk) {
- this.zk = zk;
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- public Abortable getAbortable() {
- return abort;
- }
-
- public void setAbortable(Abortable abort) {
- this.abort = abort;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
deleted file mode 100644
index 7551cb7..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++ /dev/null
@@ -1,407 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class provides an implementation of the
- * interface using ZooKeeper. The
- * base znode that this class works at is the myQueuesZnode. The myQueuesZnode contains a list of
- * all outstanding WAL files on this region server that need to be replicated. The myQueuesZnode is
- * the regionserver name (a concatenation of the region server’s hostname, client port and start
- * code). For example:
- *
- * /hbase/replication/rs/hostname.example.org,6020,1234
- *
- * Within this znode, the region server maintains a set of WAL replication queues. These queues are
- * represented by child znodes named using there give queue id. For example:
- *
- * /hbase/replication/rs/hostname.example.org,6020,1234/1
- * /hbase/replication/rs/hostname.example.org,6020,1234/2
- *
- * Each queue has one child znode for every WAL that still needs to be replicated. The value of
- * these WAL child znodes is the latest position that has been replicated. This position is updated
- * every time a WAL entry is replicated. For example:
- *
- * /hbase/replication/rs/hostname.example.org,6020,1234/1/23522342.23422 [VALUE: 254]
- */
-@InterfaceAudience.Private
-public class ReplicationQueuesZKImpl extends ReplicationStateZKBase implements ReplicationQueues {
-
- /** Znode containing all replication queues for this region server. */
- private String myQueuesZnode;
-
- private static final Logger LOG = LoggerFactory.getLogger(ReplicationQueuesZKImpl.class);
-
- public ReplicationQueuesZKImpl(ReplicationQueuesArguments args) {
- this(args.getZk(), args.getConf(), args.getAbortable());
- }
-
- public ReplicationQueuesZKImpl(final ZKWatcher zk, Configuration conf,
- Abortable abortable) {
- super(zk, conf, abortable);
- }
-
- @Override
- public void init(String serverName) throws ReplicationException {
- this.myQueuesZnode = ZNodePaths.joinZNode(this.queuesZNode, serverName);
- try {
- if (ZKUtil.checkExists(this.zookeeper, this.myQueuesZnode) < 0) {
- ZKUtil.createWithParents(this.zookeeper, this.myQueuesZnode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Could not initialize replication queues.", e);
- }
- if (conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
- HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT)) {
- try {
- if (ZKUtil.checkExists(this.zookeeper, this.hfileRefsZNode) < 0) {
- ZKUtil.createWithParents(this.zookeeper, this.hfileRefsZNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Could not initialize hfile references replication queue.",
- e);
- }
- }
- }
-
- @Override
- public void removeQueue(String queueId) {
- try {
- ZKUtil.deleteNodeRecursively(this.zookeeper,
- ZNodePaths.joinZNode(this.myQueuesZnode, queueId));
- } catch (KeeperException e) {
- this.abortable.abort("Failed to delete queue (queueId=" + queueId + ")", e);
- }
- }
-
- @Override
- public void addLog(String queueId, String filename) throws ReplicationException {
- String znode = ZNodePaths.joinZNode(this.myQueuesZnode, queueId);
- znode = ZNodePaths.joinZNode(znode, filename);
- try {
- ZKUtil.createWithParents(this.zookeeper, znode);
- } catch (KeeperException e) {
- throw new ReplicationException(
- "Could not add log because znode could not be created. queueId=" + queueId
- + ", filename=" + filename);
- }
- }
-
- @Override
- public void removeLog(String queueId, String filename) {
- try {
- String znode = ZNodePaths.joinZNode(this.myQueuesZnode, queueId);
- znode = ZNodePaths.joinZNode(znode, filename);
- ZKUtil.deleteNode(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to remove wal from queue (queueId=" + queueId + ", filename="
- + filename + ")", e);
- }
- }
-
- @Override
- public void setLogPosition(String queueId, String filename, long position) {
- try {
- String znode = ZNodePaths.joinZNode(this.myQueuesZnode, queueId);
- znode = ZNodePaths.joinZNode(znode, filename);
- // Why serialize String of Long and not Long as bytes?
- ZKUtil.setData(this.zookeeper, znode, ZKUtil.positionToByteArray(position));
- } catch (KeeperException e) {
- this.abortable.abort("Failed to write replication wal position (filename=" + filename
- + ", position=" + position + ")", e);
- }
- }
-
- @Override
- public long getLogPosition(String queueId, String filename) throws ReplicationException {
- String clusterZnode = ZNodePaths.joinZNode(this.myQueuesZnode, queueId);
- String znode = ZNodePaths.joinZNode(clusterZnode, filename);
- byte[] bytes = null;
- try {
- bytes = ZKUtil.getData(this.zookeeper, znode);
- } catch (KeeperException e) {
- throw new ReplicationException("Internal Error: could not get position in log for queueId="
- + queueId + ", filename=" + filename, e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return 0;
- }
- try {
- return ZKUtil.parseWALPositionFrom(bytes);
- } catch (DeserializationException de) {
- LOG.warn("Failed to parse WALPosition for queueId=" + queueId + " and wal=" + filename
- + " znode content, continuing.");
- }
- // if we can not parse the position, start at the beginning of the wal file
- // again
- return 0;
- }
-
- @Override
- public boolean isThisOurRegionServer(String regionserver) {
- return ZNodePaths.joinZNode(this.queuesZNode, regionserver).equals(this.myQueuesZnode);
- }
-
- @Override
- public List<String> getUnClaimedQueueIds(String regionserver) {
- if (isThisOurRegionServer(regionserver)) {
- return null;
- }
- String rsZnodePath = ZNodePaths.joinZNode(this.queuesZNode, regionserver);
- List<String> queues = null;
- try {
- queues = ZKUtil.listChildrenNoWatch(this.zookeeper, rsZnodePath);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to getUnClaimedQueueIds for RS" + regionserver, e);
- }
- return queues;
- }
-
- @Override
- public Pair<String, SortedSet<String>> claimQueue(String regionserver, String queueId) {
- LOG.info("Atomically moving " + regionserver + "/" + queueId + "'s WALs to my queue");
- return moveQueueUsingMulti(regionserver, queueId);
- }
-
- @Override
- public void removeReplicatorIfQueueIsEmpty(String regionserver) {
- String rsPath = ZNodePaths.joinZNode(this.queuesZNode, regionserver);
- try {
- List<String> list = ZKUtil.listChildrenNoWatch(this.zookeeper, rsPath);
- if (list != null && list.isEmpty()){
- ZKUtil.deleteNode(this.zookeeper, rsPath);
- }
- } catch (KeeperException e) {
- LOG.warn("Got error while removing replicator", e);
- }
- }
-
- @Override
- public void removeAllQueues() {
- try {
- ZKUtil.deleteNodeRecursively(this.zookeeper, this.myQueuesZnode);
- } catch (KeeperException e) {
- // if the znode is already expired, don't bother going further
- if (e instanceof KeeperException.SessionExpiredException) {
- return;
- }
- this.abortable.abort("Failed to delete replication queues for region server: "
- + this.myQueuesZnode, e);
- }
- }
-
- @Override
- public List<String> getLogsInQueue(String queueId) {
- String znode = ZNodePaths.joinZNode(this.myQueuesZnode, queueId);
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of wals for queueId=" + queueId, e);
- }
- return result;
- }
-
- @Override
- public List<String> getAllQueues() {
- List<String> listOfQueues = null;
- try {
- listOfQueues = ZKUtil.listChildrenNoWatch(this.zookeeper, this.myQueuesZnode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get a list of queues for region server: "
- + this.myQueuesZnode, e);
- }
- return listOfQueues == null ? new ArrayList<>() : listOfQueues;
- }
-
- /**
- * It "atomically" copies one peer's wals queue from another dead region server and returns them
- * all sorted. The new peer id is equal to the old peer id appended with the dead server's znode.
- * @param znode pertaining to the region server to copy the queues from
- * @peerId peerId pertaining to the queue need to be copied
- */
- private Pair<String, SortedSet<String>> moveQueueUsingMulti(String znode, String peerId) {
- try {
- // hbase/replication/rs/deadrs
- String deadRSZnodePath = ZNodePaths.joinZNode(this.queuesZNode, znode);
- List<ZKUtilOp> listOfOps = new ArrayList<>();
- ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(peerId);
-
- String newPeerId = peerId + "-" + znode;
- String newPeerZnode = ZNodePaths.joinZNode(this.myQueuesZnode, newPeerId);
- // check the logs queue for the old peer cluster
- String oldClusterZnode = ZNodePaths.joinZNode(deadRSZnodePath, peerId);
- List<String> wals = ZKUtil.listChildrenNoWatch(this.zookeeper, oldClusterZnode);
-
- if (!peerExists(replicationQueueInfo.getPeerId())) {
- LOG.warn("Peer " + replicationQueueInfo.getPeerId() +
- " didn't exist, will move its queue to avoid the failure of multi op");
- for (String wal : wals) {
- String oldWalZnode = ZNodePaths.joinZNode(oldClusterZnode, wal);
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalZnode));
- }
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
- return null;
- }
-
- SortedSet<String> logQueue = new TreeSet<>();
- if (wals == null || wals.isEmpty()) {
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
- } else {
- // create the new cluster znode
- ZKUtilOp op = ZKUtilOp.createAndFailSilent(newPeerZnode, HConstants.EMPTY_BYTE_ARRAY);
- listOfOps.add(op);
- // get the offset of the logs and set it to new znodes
- for (String wal : wals) {
- String oldWalZnode = ZNodePaths.joinZNode(oldClusterZnode, wal);
- byte[] logOffset = ZKUtil.getData(this.zookeeper, oldWalZnode);
- LOG.debug("Creating " + wal + " with data " + Bytes.toString(logOffset));
- String newLogZnode = ZNodePaths.joinZNode(newPeerZnode, wal);
- listOfOps.add(ZKUtilOp.createAndFailSilent(newLogZnode, logOffset));
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalZnode));
- logQueue.add(wal);
- }
- // add delete op for peer
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
-
- if (LOG.isTraceEnabled())
- LOG.trace(" The multi list size is: " + listOfOps.size());
- }
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
-
- LOG.info("Atomically moved " + znode + "/" + peerId + "'s WALs to my queue");
- return new Pair<>(newPeerId, logQueue);
- } catch (KeeperException e) {
- // Multi call failed; it looks like some other regionserver took away the logs.
- LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e);
- } catch (InterruptedException e) {
- LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e);
- Thread.currentThread().interrupt();
- }
- return null;
- }
-
- @Override
- public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
- throws ReplicationException {
- String peerZnode = ZNodePaths.joinZNode(this.hfileRefsZNode, peerId);
- boolean debugEnabled = LOG.isDebugEnabled();
- if (debugEnabled) {
- LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode);
- }
-
- int size = pairs.size();
- List<ZKUtilOp> listOfOps = new ArrayList<>(size);
-
- for (int i = 0; i < size; i++) {
- listOfOps.add(ZKUtilOp.createAndFailSilent(
- ZNodePaths.joinZNode(peerZnode, pairs.get(i).getSecond().getName()),
- HConstants.EMPTY_BYTE_ARRAY));
- }
- if (debugEnabled) {
- LOG.debug(" The multi list size for adding hfile references in zk for node " + peerZnode
- + " is " + listOfOps.size());
- }
- try {
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to create hfile reference znode=" + e.getPath(), e);
- }
- }
-
- @Override
- public void removeHFileRefs(String peerId, List<String> files) {
- String peerZnode = ZNodePaths.joinZNode(this.hfileRefsZNode, peerId);
- boolean debugEnabled = LOG.isDebugEnabled();
- if (debugEnabled) {
- LOG.debug("Removing hfile references " + files + " from queue " + peerZnode);
- }
-
- int size = files.size();
- List<ZKUtilOp> listOfOps = new ArrayList<>(size);
-
- for (int i = 0; i < size; i++) {
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(ZNodePaths.joinZNode(peerZnode, files.get(i))));
- }
- if (debugEnabled) {
- LOG.debug(" The multi list size for removing hfile references in zk for node " + peerZnode
- + " is " + listOfOps.size());
- }
- try {
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- LOG.error("Failed to remove hfile reference znode=" + e.getPath(), e);
- }
- }
-
- @Override
- public void addPeerToHFileRefs(String peerId) throws ReplicationException {
- String peerZnode = ZNodePaths.joinZNode(this.hfileRefsZNode, peerId);
- try {
- if (ZKUtil.checkExists(this.zookeeper, peerZnode) == -1) {
- LOG.info("Adding peer " + peerId + " to hfile reference queue.");
- ZKUtil.createWithParents(this.zookeeper, peerZnode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to add peer " + peerId + " to hfile reference queue.",
- e);
- }
- }
-
- @Override
- public void removePeerFromHFileRefs(String peerId) {
- final String peerZnode = ZNodePaths.joinZNode(this.hfileRefsZNode, peerId);
- try {
- if (ZKUtil.checkExists(this.zookeeper, peerZnode) == -1) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Peer " + peerZnode + " not found in hfile reference queue.");
- }
- return;
- } else {
- LOG.info("Removing peer " + peerZnode + " from hfile reference queue.");
- ZKUtil.deleteNodeRecursively(this.zookeeper, peerZnode);
- }
- } catch (KeeperException e) {
- LOG.error("Ignoring the exception to remove peer " + peerId + " from hfile reference queue.",
- e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
deleted file mode 100644
index 0d8427c..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
+++ /dev/null
@@ -1,442 +0,0 @@
-/*
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.hadoop.hbase.CompareOperator;
-import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.RetryCounter;
-import org.apache.hadoop.hbase.util.RetryCounterFactory;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executor;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/*
- * Abstract class that provides an interface to the Replication Table. Which is currently
- * being used for WAL offset tracking.
- * The basic schema of this table will store each individual queue as a
- * seperate row. The row key will be a unique identifier of the creating server's name and the
- * queueId. Each queue must have the following two columns:
- * COL_QUEUE_OWNER: tracks which server is currently responsible for tracking the queue
- * COL_QUEUE_OWNER_HISTORY: a "|" delimited list of the previous server's that have owned this
- * queue. The most recent previous owner is the leftmost entry.
- * They will also have columns mapping [WAL filename : offset]
- * The most flexible method of interacting with the Replication Table is by calling
- * getOrBlockOnReplicationTable() which will return a new copy of the Replication Table. It is up
- * to the caller to close the returned table.
- */
-@InterfaceAudience.Private
-abstract class ReplicationTableBase {
-
- /** Name of the HBase Table used for tracking replication*/
- public static final TableName REPLICATION_TABLE_NAME =
- TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "replication");
-
- // Column family and column names for Queues in the Replication Table
- public static final byte[] CF_QUEUE = Bytes.toBytes("q");
- public static final byte[] COL_QUEUE_OWNER = Bytes.toBytes("o");
- public static final byte[] COL_QUEUE_OWNER_HISTORY = Bytes.toBytes("h");
-
- // Column Descriptor for the Replication Table
- private static final HColumnDescriptor REPLICATION_COL_DESCRIPTOR =
- new HColumnDescriptor(CF_QUEUE).setMaxVersions(1)
- .setInMemory(true)
- .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
- // TODO: Figure out which bloom filter to use
- .setBloomFilterType(BloomType.NONE);
-
- // The value used to delimit the queueId and server name inside of a queue's row key. Currently a
- // hyphen, because it is guaranteed that queueId (which is a cluster id) cannot contain hyphens.
- // See HBASE-11394.
- public static final String ROW_KEY_DELIMITER = "-";
-
- // The value used to delimit server names in the queue history list
- public static final String QUEUE_HISTORY_DELIMITER = "|";
-
- /*
- * Make sure that HBase table operations for replication have a high number of retries. This is
- * because the server is aborted if any HBase table operation fails. Each RPC will be attempted
- * 3600 times before exiting. This provides each operation with 2 hours of retries
- * before the server is aborted.
- */
- private static final int CLIENT_RETRIES = 3600;
- private static final int RPC_TIMEOUT = 2000;
- private static final int OPERATION_TIMEOUT = CLIENT_RETRIES * RPC_TIMEOUT;
-
- // We only need a single thread to initialize the Replication Table
- private static final int NUM_INITIALIZE_WORKERS = 1;
-
- protected final Configuration conf;
- protected final Abortable abortable;
- private final Connection connection;
- private final Executor executor;
- private volatile CountDownLatch replicationTableInitialized;
-
- public ReplicationTableBase(Configuration conf, Abortable abort) throws IOException {
- this.conf = new Configuration(conf);
- this.abortable = abort;
- decorateConf();
- this.connection = ConnectionFactory.createConnection(this.conf);
- this.executor = setUpExecutor();
- this.replicationTableInitialized = new CountDownLatch(1);
- createReplicationTableInBackground();
- }
-
- /**
- * Modify the connection's config so that operations run on the Replication Table have longer and
- * a larger number of retries
- */
- private void decorateConf() {
- this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, CLIENT_RETRIES);
- }
-
- /**
- * Sets up the thread pool executor used to build the Replication Table in the background
- * @return the configured executor
- */
- private Executor setUpExecutor() {
- ThreadPoolExecutor tempExecutor = new ThreadPoolExecutor(NUM_INITIALIZE_WORKERS,
- NUM_INITIALIZE_WORKERS, 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
- ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
- tfb.setNameFormat("ReplicationTableExecutor-%d");
- tfb.setDaemon(true);
- tempExecutor.setThreadFactory(tfb.build());
- return tempExecutor;
- }
-
- /**
- * Get whether the Replication Table has been successfully initialized yet
- * @return whether the Replication Table is initialized
- */
- public boolean getInitializationStatus() {
- return replicationTableInitialized.getCount() == 0;
- }
-
- /**
- * Increases the RPC and operations timeouts for the Replication Table
- */
- private Table setReplicationTableTimeOuts(Table replicationTable) {
- replicationTable.setRpcTimeout(RPC_TIMEOUT);
- replicationTable.setOperationTimeout(OPERATION_TIMEOUT);
- return replicationTable;
- }
-
- /**
- * Build the row key for the given queueId. This will uniquely identify it from all other queues
- * in the cluster.
- * @param serverName The owner of the queue
- * @param queueId String identifier of the queue
- * @return String representation of the queue's row key
- */
- protected String buildQueueRowKey(String serverName, String queueId) {
- return queueId + ROW_KEY_DELIMITER + serverName;
- }
-
- /**
- * Parse the original queueId from a row key
- * @param rowKey String representation of a queue's row key
- * @return the original queueId
- */
- protected String getRawQueueIdFromRowKey(String rowKey) {
- return rowKey.split(ROW_KEY_DELIMITER)[0];
- }
-
- /**
- * Returns a queue's row key given either its raw or reclaimed queueId
- *
- * @param queueId queueId of the queue
- * @return byte representation of the queue's row key
- */
- protected byte[] queueIdToRowKey(String serverName, String queueId) {
- // Cluster id's are guaranteed to have no hyphens, so if the passed in queueId has no hyphen
- // then this is not a reclaimed queue.
- if (!queueId.contains(ROW_KEY_DELIMITER)) {
- return Bytes.toBytes(buildQueueRowKey(serverName, queueId));
- // If the queueId contained some hyphen it was reclaimed. In this case, the queueId is the
- // queue's row key
- } else {
- return Bytes.toBytes(queueId);
- }
- }
-
- /**
- * Creates a "|" delimited record of the queue's past region server owners.
- *
- * @param originalHistory the queue's original owner history
- * @param oldServer the name of the server that used to own the queue
- * @return the queue's new owner history
- */
- protected String buildClaimedQueueHistory(String originalHistory, String oldServer) {
- return oldServer + QUEUE_HISTORY_DELIMITER + originalHistory;
- }
-
- /**
- * Get a list of all region servers that have outstanding replication queues. These servers could
- * be alive, dead or from a previous run of the cluster.
- * @return a list of server names
- */
- protected List<String> getListOfReplicators() {
- // scan all of the queues and return a list of all unique OWNER values
- Set<String> peerServers = new HashSet<>();
- ResultScanner allQueuesInCluster = null;
- try (Table replicationTable = getOrBlockOnReplicationTable()){
- Scan scan = new Scan();
- scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER);
- allQueuesInCluster = replicationTable.getScanner(scan);
- for (Result queue : allQueuesInCluster) {
- peerServers.add(Bytes.toString(queue.getValue(CF_QUEUE, COL_QUEUE_OWNER)));
- }
- } catch (IOException e) {
- String errMsg = "Failed getting list of replicators";
- abortable.abort(errMsg, e);
- } finally {
- if (allQueuesInCluster != null) {
- allQueuesInCluster.close();
- }
- }
- return new ArrayList<>(peerServers);
- }
-
- protected List<String> getAllQueues(String serverName) {
- List<String> allQueues = new ArrayList<>();
- ResultScanner queueScanner = null;
- try {
- queueScanner = getQueuesBelongingToServer(serverName);
- for (Result queue : queueScanner) {
- String rowKey = Bytes.toString(queue.getRow());
- // If the queue does not have a Owner History, then we must be its original owner. So we
- // want to return its queueId in raw form
- if (Bytes.toString(queue.getValue(CF_QUEUE, COL_QUEUE_OWNER_HISTORY)).length() == 0) {
- allQueues.add(getRawQueueIdFromRowKey(rowKey));
- } else {
- allQueues.add(rowKey);
- }
- }
- return allQueues;
- } catch (IOException e) {
- String errMsg = "Failed getting list of all replication queues for serverName=" + serverName;
- abortable.abort(errMsg, e);
- return null;
- } finally {
- if (queueScanner != null) {
- queueScanner.close();
- }
- }
- }
-
- protected List<String> getLogsInQueue(String serverName, String queueId) {
- String rowKey = queueId;
- if (!queueId.contains(ROW_KEY_DELIMITER)) {
- rowKey = buildQueueRowKey(serverName, queueId);
- }
- return getLogsInQueue(Bytes.toBytes(rowKey));
- }
-
- protected List<String> getLogsInQueue(byte[] rowKey) {
- String errMsg = "Failed getting logs in queue queueId=" + Bytes.toString(rowKey);
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- Get getQueue = new Get(rowKey);
- Result queue = replicationTable.get(getQueue);
- if (queue == null || queue.isEmpty()) {
- abortable.abort(errMsg, new ReplicationException(errMsg));
- return null;
- }
- return readWALsFromResult(queue);
- } catch (IOException e) {
- abortable.abort(errMsg, e);
- return null;
- }
- }
-
- /**
- * Read all of the WAL's from a queue into a list
- *
- * @param queue HBase query result containing the queue
- * @return a list of all the WAL filenames
- */
- protected List<String> readWALsFromResult(Result queue) {
- List<String> wals = new ArrayList<>();
- Map<byte[], byte[]> familyMap = queue.getFamilyMap(CF_QUEUE);
- for (byte[] cQualifier : familyMap.keySet()) {
- // Ignore the meta data fields of the queue
- if (Arrays.equals(cQualifier, COL_QUEUE_OWNER) || Arrays.equals(cQualifier,
- COL_QUEUE_OWNER_HISTORY)) {
- continue;
- }
- wals.add(Bytes.toString(cQualifier));
- }
- return wals;
- }
-
- /**
- * Get the queue id's and meta data (Owner and History) for the queues belonging to the named
- * server
- *
- * @param server name of the server
- * @return a ResultScanner over the QueueIds belonging to the server
- * @throws IOException
- */
- protected ResultScanner getQueuesBelongingToServer(String server) throws IOException {
- Scan scan = new Scan();
- SingleColumnValueFilter filterMyQueues = new SingleColumnValueFilter(CF_QUEUE, COL_QUEUE_OWNER,
- CompareOperator.EQUAL, Bytes.toBytes(server));
- scan.setFilter(filterMyQueues);
- scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER);
- scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER_HISTORY);
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- ResultScanner results = replicationTable.getScanner(scan);
- return results;
- }
- }
-
- /**
- * Attempts to acquire the Replication Table. This operation will block until it is assigned by
- * the CreateReplicationWorker thread. It is up to the caller of this method to close the
- * returned Table
- * @return the Replication Table when it is created
- * @throws IOException
- */
- protected Table getOrBlockOnReplicationTable() throws IOException {
- // Sleep until the Replication Table becomes available
- try {
- replicationTableInitialized.await();
- } catch (InterruptedException e) {
- String errMsg = "Unable to acquire the Replication Table due to InterruptedException: " +
- e.getMessage();
- throw new InterruptedIOException(errMsg);
- }
- return getAndSetUpReplicationTable();
- }
-
- /**
- * Creates a new copy of the Replication Table and sets up the proper Table time outs for it
- *
- * @return the Replication Table
- * @throws IOException
- */
- private Table getAndSetUpReplicationTable() throws IOException {
- Table replicationTable = connection.getTable(REPLICATION_TABLE_NAME);
- setReplicationTableTimeOuts(replicationTable);
- return replicationTable;
- }
-
- /**
- * Builds the Replication Table in a background thread. Any method accessing the Replication Table
- * should do so through getOrBlockOnReplicationTable()
- *
- * @return the Replication Table
- * @throws IOException if the Replication Table takes too long to build
- */
- private void createReplicationTableInBackground() throws IOException {
- executor.execute(new CreateReplicationTableWorker());
- }
-
- /**
- * Attempts to build the Replication Table. Will continue blocking until we have a valid
- * Table for the Replication Table.
- */
- private class CreateReplicationTableWorker implements Runnable {
-
- private Admin admin;
-
- @Override
- public void run() {
- try {
- admin = connection.getAdmin();
- if (!replicationTableExists()) {
- createReplicationTable();
- }
- int maxRetries = conf.getInt("hbase.replication.queues.createtable.retries.number",
- CLIENT_RETRIES);
- RetryCounterFactory counterFactory = new RetryCounterFactory(maxRetries, RPC_TIMEOUT);
- RetryCounter retryCounter = counterFactory.create();
- while (!replicationTableExists()) {
- retryCounter.sleepUntilNextRetry();
- if (!retryCounter.shouldRetry()) {
- throw new IOException("Unable to acquire the Replication Table");
- }
- }
- replicationTableInitialized.countDown();
- } catch (IOException | InterruptedException e) {
- abortable.abort("Failed building Replication Table", e);
- }
- }
-
- /**
- * Create the replication table with the provided HColumnDescriptor REPLICATION_COL_DESCRIPTOR
- * in TableBasedReplicationQueuesImpl
- *
- * @throws IOException
- */
- private void createReplicationTable() throws IOException {
- HTableDescriptor replicationTableDescriptor = new HTableDescriptor(REPLICATION_TABLE_NAME);
- replicationTableDescriptor.addFamily(REPLICATION_COL_DESCRIPTOR);
- try {
- admin.createTable(replicationTableDescriptor);
- } catch (TableExistsException e) {
- // In this case we can just continue as normal
- }
- }
-
- /**
- * Checks whether the Replication Table exists yet
- *
- * @return whether the Replication Table exists
- * @throws IOException
- */
- private boolean replicationTableExists() {
- try {
- return admin.tableExists(REPLICATION_TABLE_NAME);
- } catch (IOException e) {
- return false;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
index 2c522f6..5659e4b 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -54,6 +53,8 @@ public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements
super(zookeeper, conf, abortable);
this.stopper = stopper;
this.zookeeper.registerListener(new OtherRegionServerWatcher(this.zookeeper));
+ // watch the changes
+ refreshOtherRegionServersList(true);
}
@Override
@@ -71,7 +72,7 @@ public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements
*/
@Override
public List<String> getListOfRegionServers() {
- refreshOtherRegionServersList();
+ refreshOtherRegionServersList(false);
List<String> list = null;
synchronized (otherRegionServers) {
@@ -137,7 +138,7 @@ public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements
if (!path.startsWith(this.watcher.znodePaths.rsZNode)) {
return false;
}
- return refreshOtherRegionServersList();
+ return refreshOtherRegionServersList(true);
}
}
@@ -157,8 +158,8 @@ public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements
* @return true if the local list of the other region servers was updated with the ZK data (even
* if it was empty), false if the data was missing in ZK
*/
- private boolean refreshOtherRegionServersList() {
- List<String> newRsList = getRegisteredRegionServers();
+ private boolean refreshOtherRegionServersList(boolean watch) {
+ List<String> newRsList = getRegisteredRegionServers(watch);
if (newRsList == null) {
return false;
} else {
@@ -174,10 +175,14 @@ public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements
* Get a list of all the other region servers in this cluster and set a watch
* @return a list of server nanes
*/
- private List<String> getRegisteredRegionServers() {
+ private List<String> getRegisteredRegionServers(boolean watch) {
List<String> result = null;
try {
- result = ZKUtil.listChildrenAndWatchThem(this.zookeeper, this.zookeeper.znodePaths.rsZNode);
+ if (watch) {
+ result = ZKUtil.listChildrenAndWatchThem(this.zookeeper, this.zookeeper.znodePaths.rsZNode);
+ } else {
+ result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.zookeeper.znodePaths.rsZNode);
+ }
} catch (KeeperException e) {
this.abortable.abort("Get list of registered region servers", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
index 0275d52..41f50d8 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
@@ -54,6 +54,28 @@ import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTe
/**
* ZK based replication queue storage.
+ * <p>
+ * The base znode for each regionserver is the regionserver name. For example:
+ *
+ * <pre>
+ * /hbase/replication/rs/hostname.example.org,6020,1234
+ * </pre>
+ *
+ * Within this znode, the region server maintains a set of WAL replication queues. These queues are
+ * represented by child znodes named using there give queue id. For example:
+ *
+ * <pre>
+ * /hbase/replication/rs/hostname.example.org,6020,1234/1
+ * /hbase/replication/rs/hostname.example.org,6020,1234/2
+ * </pre>
+ *
+ * Each queue has one child znode for every WAL that still needs to be replicated. The value of
+ * these WAL child znodes is the latest position that has been replicated. This position is updated
+ * every time a WAL entry is replicated. For example:
+ *
+ * <pre>
+ * /hbase/replication/rs/hostname.example.org,6020,1234/1/23522342.23422 [VALUE: 254]
+ * </pre>
*/
@InterfaceAudience.Private
class ZKReplicationQueueStorage extends ZKReplicationStorageBase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
index 8905d43..4afda5d 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
@@ -42,9 +42,8 @@ import org.slf4j.LoggerFactory;
*/
public abstract class TestReplicationStateBasic {
- protected ReplicationQueues rq1;
- protected ReplicationQueues rq2;
- protected ReplicationQueues rq3;
+ private static final Logger LOG = LoggerFactory.getLogger(TestReplicationStateBasic.class);
+
protected ReplicationQueueStorage rqs;
protected ServerName server1 = ServerName.valueOf("hostname1.example.org", 1234, 12345);
protected ServerName server2 = ServerName.valueOf("hostname2.example.org", 1234, 12345);
@@ -63,8 +62,6 @@ public abstract class TestReplicationStateBasic {
protected static final int ZK_MAX_COUNT = 300;
protected static final int ZK_SLEEP_INTERVAL = 100; // millis
- private static final Logger LOG = LoggerFactory.getLogger(TestReplicationStateBasic.class);
-
@Test
public void testReplicationQueueStorage() throws ReplicationException {
// Test methods with empty state
@@ -76,15 +73,13 @@ public abstract class TestReplicationStateBasic {
* Set up data Two replicators: -- server1: three queues with 0, 1 and 2 log files each --
* server2: zero queues
*/
- rq1.init(server1.getServerName());
- rq2.init(server2.getServerName());
- rq1.addLog("qId1", "trash");
- rq1.removeLog("qId1", "trash");
- rq1.addLog("qId2", "filename1");
- rq1.addLog("qId3", "filename2");
- rq1.addLog("qId3", "filename3");
- rq2.addLog("trash", "trash");
- rq2.removeQueue("trash");
+ rqs.addWAL(server1, "qId1", "trash");
+ rqs.removeWAL(server1, "qId1", "trash");
+ rqs.addWAL(server1,"qId2", "filename1");
+ rqs.addWAL(server1,"qId3", "filename2");
+ rqs.addWAL(server1,"qId3", "filename3");
+ rqs.addWAL(server2,"trash", "trash");
+ rqs.removeQueue(server2,"trash");
List<ServerName> reps = rqs.getListOfReplicators();
assertEquals(2, reps.size());
@@ -105,62 +100,55 @@ public abstract class TestReplicationStateBasic {
assertTrue(list.contains("qId3"));
}
+ private void removeAllQueues(ServerName serverName) throws ReplicationException {
+ for (String queue: rqs.getAllQueues(serverName)) {
+ rqs.removeQueue(serverName, queue);
+ }
+ }
@Test
public void testReplicationQueues() throws ReplicationException {
- rq1.init(server1.getServerName());
- rq2.init(server2.getServerName());
- rq3.init(server3.getServerName());
// Initialize ReplicationPeer so we can add peers (we don't transfer lone queues)
rp.init();
- // 3 replicators should exist
- assertEquals(3, rq1.getListOfReplicators().size());
- rq1.removeQueue("bogus");
- rq1.removeLog("bogus", "bogus");
- rq1.removeAllQueues();
- assertEquals(0, rq1.getAllQueues().size());
- assertEquals(0, rq1.getLogPosition("bogus", "bogus"));
- assertNull(rq1.getLogsInQueue("bogus"));
- assertNull(rq1.getUnClaimedQueueIds(ServerName.valueOf("bogus", 1234, -1L).toString()));
-
- rq1.setLogPosition("bogus", "bogus", 5L);
+ rqs.removeQueue(server1, "bogus");
+ rqs.removeWAL(server1, "bogus", "bogus");
+ removeAllQueues(server1);
+ assertEquals(0, rqs.getAllQueues(server1).size());
+ assertEquals(0, rqs.getWALPosition(server1, "bogus", "bogus"));
+ assertTrue(rqs.getWALsInQueue(server1, "bogus").isEmpty());
+ assertTrue(rqs.getAllQueues(ServerName.valueOf("bogus", 1234, 12345)).isEmpty());
populateQueues();
- assertEquals(3, rq1.getListOfReplicators().size());
- assertEquals(0, rq2.getLogsInQueue("qId1").size());
- assertEquals(5, rq3.getLogsInQueue("qId5").size());
- assertEquals(0, rq3.getLogPosition("qId1", "filename0"));
- rq3.setLogPosition("qId5", "filename4", 354L);
- assertEquals(354L, rq3.getLogPosition("qId5", "filename4"));
+ assertEquals(3, rqs.getListOfReplicators().size());
+ assertEquals(0, rqs.getWALsInQueue(server2, "qId1").size());
+ assertEquals(5, rqs.getWALsInQueue(server3, "qId5").size());
+ assertEquals(0, rqs.getWALPosition(server3, "qId1", "filename0"));
+ rqs.setWALPosition(server3, "qId5", "filename4", 354L);
+ assertEquals(354L, rqs.getWALPosition(server3, "qId5", "filename4"));
- assertEquals(5, rq3.getLogsInQueue("qId5").size());
- assertEquals(0, rq2.getLogsInQueue("qId1").size());
- assertEquals(0, rq1.getAllQueues().size());
- assertEquals(1, rq2.getAllQueues().size());
- assertEquals(5, rq3.getAllQueues().size());
+ assertEquals(5, rqs.getWALsInQueue(server3, "qId5").size());
+ assertEquals(0, rqs.getWALsInQueue(server2, "qId1").size());
+ assertEquals(0, rqs.getAllQueues(server1).size());
+ assertEquals(1, rqs.getAllQueues(server2).size());
+ assertEquals(5, rqs.getAllQueues(server3).size());
- assertEquals(0, rq3.getUnClaimedQueueIds(server1.getServerName()).size());
- rq3.removeReplicatorIfQueueIsEmpty(server1.getServerName());
- assertEquals(2, rq3.getListOfReplicators().size());
+ assertEquals(0, rqs.getAllQueues(server1).size());
+ rqs.removeReplicatorIfQueueIsEmpty(server1);
+ assertEquals(2, rqs.getListOfReplicators().size());
- List<String> queues = rq2.getUnClaimedQueueIds(server3.getServerName());
+ List<String> queues = rqs.getAllQueues(server3);
assertEquals(5, queues.size());
for (String queue : queues) {
- rq2.claimQueue(server3.getServerName(), queue);
+ rqs.claimQueue(server3, queue, server2);
}
- rq2.removeReplicatorIfQueueIsEmpty(server3.getServerName());
- assertEquals(1, rq2.getListOfReplicators().size());
-
- // Try to claim our own queues
- assertNull(rq2.getUnClaimedQueueIds(server2.getServerName()));
- rq2.removeReplicatorIfQueueIsEmpty(server2.getServerName());
-
- assertEquals(6, rq2.getAllQueues().size());
+ rqs.removeReplicatorIfQueueIsEmpty(server3);
+ assertEquals(1, rqs.getListOfReplicators().size());
- rq2.removeAllQueues();
-
- assertEquals(0, rq2.getListOfReplicators().size());
+ assertEquals(6, rqs.getAllQueues(server2).size());
+ removeAllQueues(server2);
+ rqs.removeReplicatorIfQueueIsEmpty(server2);
+ assertEquals(0, rqs.getListOfReplicators().size());
}
@Test
@@ -197,7 +185,6 @@ public abstract class TestReplicationStateBasic {
@Test
public void testHfileRefsReplicationQueues() throws ReplicationException, KeeperException {
rp.init();
- rq1.init(server1.getServerName());
List<Pair<Path, Path>> files1 = new ArrayList<>(3);
files1.add(new Pair<>(null, new Path("file_1")));
@@ -206,8 +193,8 @@ public abstract class TestReplicationStateBasic {
assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
- rq1.addPeerToHFileRefs(ID_ONE);
- rq1.addHFileRefs(ID_ONE, files1);
+ rqs.addPeerToHFileRefs(ID_ONE);
+ rqs.addHFileRefs(ID_ONE, files1);
assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
assertEquals(3, rqs.getReplicableHFiles(ID_ONE).size());
List<String> hfiles2 = new ArrayList<>(files1.size());
@@ -215,43 +202,41 @@ public abstract class TestReplicationStateBasic {
hfiles2.add(p.getSecond().getName());
}
String removedString = hfiles2.remove(0);
- rq1.removeHFileRefs(ID_ONE, hfiles2);
+ rqs.removeHFileRefs(ID_ONE, hfiles2);
assertEquals(1, rqs.getReplicableHFiles(ID_ONE).size());
hfiles2 = new ArrayList<>(1);
hfiles2.add(removedString);
- rq1.removeHFileRefs(ID_ONE, hfiles2);
+ rqs.removeHFileRefs(ID_ONE, hfiles2);
assertEquals(0, rqs.getReplicableHFiles(ID_ONE).size());
rp.unregisterPeer(ID_ONE);
}
@Test
public void testRemovePeerForHFileRefs() throws ReplicationException, KeeperException {
- rq1.init(server1.getServerName());
-
rp.init();
rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
- rq1.addPeerToHFileRefs(ID_ONE);
+ rqs.addPeerToHFileRefs(ID_ONE);
rp.registerPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO));
- rq1.addPeerToHFileRefs(ID_TWO);
+ rqs.addPeerToHFileRefs(ID_TWO);
List<Pair<Path, Path>> files1 = new ArrayList<>(3);
files1.add(new Pair<>(null, new Path("file_1")));
files1.add(new Pair<>(null, new Path("file_2")));
files1.add(new Pair<>(null, new Path("file_3")));
- rq1.addHFileRefs(ID_ONE, files1);
- rq1.addHFileRefs(ID_TWO, files1);
+ rqs.addHFileRefs(ID_ONE, files1);
+ rqs.addHFileRefs(ID_TWO, files1);
assertEquals(2, rqs.getAllPeersFromHFileRefsQueue().size());
assertEquals(3, rqs.getReplicableHFiles(ID_ONE).size());
assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
rp.unregisterPeer(ID_ONE);
- rq1.removePeerFromHFileRefs(ID_ONE);
+ rqs.removePeerFromHFileRefs(ID_ONE);
assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
rp.unregisterPeer(ID_TWO);
- rq1.removePeerFromHFileRefs(ID_TWO);
+ rqs.removePeerFromHFileRefs(ID_TWO);
assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
assertTrue(rqs.getReplicableHFiles(ID_TWO).isEmpty());
}
@@ -363,15 +348,15 @@ public abstract class TestReplicationStateBasic {
* 3, 4, 5 log files respectively
*/
protected void populateQueues() throws ReplicationException {
- rq1.addLog("trash", "trash");
- rq1.removeQueue("trash");
+ rqs.addWAL(server1, "trash", "trash");
+ rqs.removeQueue(server1, "trash");
- rq2.addLog("qId1", "trash");
- rq2.removeLog("qId1", "trash");
+ rqs.addWAL(server2, "qId1", "trash");
+ rqs.removeWAL(server2, "qId1", "trash");
for (int i = 1; i < 6; i++) {
for (int j = 0; j < i; j++) {
- rq3.addLog("qId" + i, "filename" + j);
+ rqs.addWAL(server3, "qId" + i, "filename" + j);
}
// Add peers for the corresponding queues so they are not orphans
rp.registerPeer("qId" + i,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
index 5fe7c55..ac869d9 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
@@ -17,10 +17,6 @@
*/
package org.apache.hadoop.hbase.replication;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
@@ -40,7 +36,6 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +49,6 @@ public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
private static HBaseZKTestingUtility utility;
private static ZKWatcher zkw;
private static String replicationZNode;
- private ReplicationQueuesZKImpl rqZK;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -84,23 +78,9 @@ public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
@Before
public void setUp() {
zkTimeoutCount = 0;
- WarnOnlyAbortable abortable = new WarnOnlyAbortable();
- try {
- rq1 = ReplicationFactory
- .getReplicationQueues(new ReplicationQueuesArguments(conf, abortable, zkw));
- rq2 = ReplicationFactory
- .getReplicationQueues(new ReplicationQueuesArguments(conf, abortable, zkw));
- rq3 = ReplicationFactory
- .getReplicationQueues(new ReplicationQueuesArguments(conf, abortable, zkw));
- rqs = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
- } catch (Exception e) {
- // This should not occur, because getReplicationQueues() only throws for
- // TableBasedReplicationQueuesImpl
- fail("ReplicationFactory.getReplicationQueues() threw an IO Exception");
- }
- rp = ReplicationFactory.getReplicationPeers(zkw, conf, zkw);
+ rqs = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
+ rp = ReplicationFactory.getReplicationPeers(zkw, conf, new WarnOnlyAbortable());
OUR_KEY = ZKConfig.getZooKeeperClusterKey(conf);
- rqZK = new ReplicationQueuesZKImpl(zkw, conf, abortable);
}
@After
@@ -113,23 +93,6 @@ public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
utility.shutdownMiniZKCluster();
}
- @Test
- public void testIsPeerPath_PathToParentOfPeerNode() {
- assertFalse(rqZK.isPeerPath(rqZK.peersZNode));
- }
-
- @Test
- public void testIsPeerPath_PathToChildOfPeerNode() {
- String peerChild = ZNodePaths.joinZNode(ZNodePaths.joinZNode(rqZK.peersZNode, "1"), "child");
- assertFalse(rqZK.isPeerPath(peerChild));
- }
-
- @Test
- public void testIsPeerPath_ActualPeerPath() {
- String peerPath = ZNodePaths.joinZNode(rqZK.peersZNode, "1");
- assertTrue(rqZK.isPeerPath(peerPath));
- }
-
private static class WarnOnlyAbortable implements Abortable {
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index d8f9625..73e600e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -49,8 +49,6 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
@@ -307,14 +305,10 @@ public class DumpReplicationQueues extends Configured implements Tool {
boolean hdfs) throws Exception {
ReplicationQueueStorage queueStorage;
ReplicationPeers replicationPeers;
- ReplicationQueues replicationQueues;
ReplicationTracker replicationTracker;
- ReplicationQueuesArguments replicationArgs =
- new ReplicationQueuesArguments(getConf(), new WarnOnlyAbortable(), zkw);
StringBuilder sb = new StringBuilder();
queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
- replicationQueues = ReplicationFactory.getReplicationQueues(replicationArgs);
replicationPeers =
ReplicationFactory.getReplicationPeers(zkw, getConf(), queueStorage, connection);
replicationTracker = ReplicationFactory.getReplicationTracker(zkw, replicationPeers, getConf(),
@@ -328,7 +322,6 @@ public class DumpReplicationQueues extends Configured implements Tool {
}
for (ServerName regionserver : regionservers) {
List<String> queueIds = queueStorage.getAllQueues(regionserver);
- replicationQueues.init(regionserver.getServerName());
if (!liveRegionServers.contains(regionserver.getServerName())) {
deadRegionServers.add(regionserver.getServerName());
}
@@ -338,17 +331,17 @@ public class DumpReplicationQueues extends Configured implements Tool {
if (!peerIds.contains(queueInfo.getPeerId())) {
deletedQueues.add(regionserver + "/" + queueId);
sb.append(
- formatQueue(regionserver, replicationQueues, queueInfo, queueId, wals, true, hdfs));
+ formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs));
} else {
sb.append(
- formatQueue(regionserver, replicationQueues, queueInfo, queueId, wals, false, hdfs));
+ formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, false, hdfs));
}
}
}
return sb.toString();
}
- private String formatQueue(ServerName regionserver, ReplicationQueues replicationQueues,
+ private String formatQueue(ServerName regionserver, ReplicationQueueStorage queueStorage,
ReplicationQueueInfo queueInfo, String queueId, List<String> wals, boolean isDeleted,
boolean hdfs) throws Exception {
StringBuilder sb = new StringBuilder();
@@ -370,7 +363,7 @@ public class DumpReplicationQueues extends Configured implements Tool {
peersQueueSize.addAndGet(queueInfo.getPeerId(), wals.size());
for (String wal : wals) {
- long position = replicationQueues.getLogPosition(queueInfo.getPeerId(), wal);
+ long position = queueStorage.getWALPosition(regionserver, queueInfo.getPeerId(), wal);
sb.append(" Replication position for " + wal + ": " + (position > 0 ? position : "0"
+ " (not started or nothing to replicate)") + "\n");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index bd191e3..e0c45d5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -29,15 +28,15 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Class that handles the recovered source of a replication stream, which is transfered from
@@ -52,10 +51,10 @@ public class RecoveredReplicationSource extends ReplicationSource {
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueues replicationQueues, ReplicationPeers replicationPeers, Server server,
+ ReplicationQueueStorage queueStorage, ReplicationPeers replicationPeers, Server server,
String peerClusterZnode, UUID clusterId, ReplicationEndpoint replicationEndpoint,
WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException {
- super.init(conf, fs, manager, replicationQueues, replicationPeers, server, peerClusterZnode,
+ super.init(conf, fs, manager, queueStorage, replicationPeers, server, peerClusterZnode,
clusterId, replicationEndpoint, walFileLengthProvider, metrics);
this.actualPeerId = this.replicationQueueInfo.getPeerId();
}
@@ -64,7 +63,7 @@ public class RecoveredReplicationSource extends ReplicationSource {
protected void tryStartNewShipper(String walGroupId, PriorityBlockingQueue<Path> queue) {
final RecoveredReplicationSourceShipper worker =
new RecoveredReplicationSourceShipper(conf, walGroupId, queue, this,
- this.replicationQueues);
+ this.queueStorage);
ReplicationSourceShipper extant = workerThreads.putIfAbsent(walGroupId, worker);
if (extant != null) {
LOG.debug("Someone has beat us to start a worker thread for wal group " + walGroupId);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
index 630b90b..fb365bc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
@@ -23,13 +23,13 @@ import java.util.concurrent.PriorityBlockingQueue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.WALEntryBatch;
import org.apache.hadoop.hbase.util.Threads;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Used by a {@link RecoveredReplicationSource}.
@@ -40,14 +40,14 @@ public class RecoveredReplicationSourceShipper extends ReplicationSourceShipper
LoggerFactory.getLogger(RecoveredReplicationSourceShipper.class);
protected final RecoveredReplicationSource source;
- private final ReplicationQueues replicationQueues;
+ private final ReplicationQueueStorage replicationQueues;
public RecoveredReplicationSourceShipper(Configuration conf, String walGroupId,
PriorityBlockingQueue<Path> queue, RecoveredReplicationSource source,
- ReplicationQueues replicationQueues) {
+ ReplicationQueueStorage queueStorage) {
super(conf, walGroupId, queue, source);
this.source = source;
- this.replicationQueues = replicationQueues;
+ this.replicationQueues = queueStorage;
}
@Override
@@ -116,11 +116,11 @@ public class RecoveredReplicationSourceShipper extends ReplicationSourceShipper
long startPosition = 0;
String peerClusterZnode = source.getPeerClusterZnode();
try {
- startPosition = this.replicationQueues.getLogPosition(peerClusterZnode,
- this.queue.peek().getName());
+ startPosition = this.replicationQueues.getWALPosition(source.getServerWALsBelongTo(),
+ peerClusterZnode, this.queue.peek().getName());
if (LOG.isTraceEnabled()) {
- LOG.trace("Recovered queue started with log " + this.queue.peek() + " at position "
- + startPosition);
+ LOG.trace("Recovered queue started with log " + this.queue.peek() + " at position " +
+ startPosition);
}
} catch (ReplicationException e) {
terminate("Couldn't get the position of this recovered queue " + peerClusterZnode, e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 47f08f9..d555c6e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,12 +27,6 @@ import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.hbase.wal.WALKey;
-import org.apache.hadoop.hbase.wal.WALKeyImpl;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -43,27 +36,33 @@ import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
-import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.regionserver.ReplicationSinkService;
import org.apache.hadoop.hbase.regionserver.ReplicationSourceService;
+import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
-import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
-import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.wal.WALEdit;
+import org.apache.hadoop.hbase.wal.WALKey;
+import org.apache.hadoop.hbase.wal.WALKeyImpl;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
+
+import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* Gateway to Replication. Used by {@link org.apache.hadoop.hbase.regionserver.HRegionServer}.
*/
@@ -74,7 +73,7 @@ public class Replication implements
LoggerFactory.getLogger(Replication.class);
private boolean replicationForBulkLoadData;
private ReplicationSourceManager replicationManager;
- private ReplicationQueues replicationQueues;
+ private ReplicationQueueStorage queueStorage;
private ReplicationPeers replicationPeers;
private ReplicationTracker replicationTracker;
private Configuration conf;
@@ -128,10 +127,8 @@ public class Replication implements
}
try {
- this.replicationQueues =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, this.server,
- server.getZooKeeper()));
- this.replicationQueues.init(this.server.getServerName().toString());
+ this.queueStorage =
+ ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
this.replicationPeers =
ReplicationFactory.getReplicationPeers(server.getZooKeeper(), this.conf, this.server);
this.replicationPeers.init();
@@ -148,7 +145,7 @@ public class Replication implements
throw new IOException("Could not read cluster id", ke);
}
this.replicationManager =
- new ReplicationSourceManager(replicationQueues, replicationPeers, replicationTracker, conf,
+ new ReplicationSourceManager(queueStorage, replicationPeers, replicationTracker, conf,
this.server, fs, logDir, oldLogDir, clusterId, walFileLengthProvider);
this.statsThreadPeriod =
this.conf.getInt("replication.stats.thread.period.seconds", 5 * 60);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index a2eb202..58ea6ee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -31,7 +30,6 @@ import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -49,7 +47,7 @@ import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.SystemTableWALEntryFilter;
import org.apache.hadoop.hbase.replication.WALEntryFilter;
import org.apache.hadoop.hbase.util.Bytes;
@@ -83,7 +81,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
private Map<String, PriorityBlockingQueue<Path>> queues = new HashMap<>();
// per group queue size, keep no more than this number of logs in each wal group
protected int queueSizePerGroup;
- protected ReplicationQueues replicationQueues;
+ protected ReplicationQueueStorage queueStorage;
private ReplicationPeers replicationPeers;
protected Configuration conf;
@@ -148,7 +146,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
*/
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueues replicationQueues, ReplicationPeers replicationPeers, Server server,
+ ReplicationQueueStorage queueStorage, ReplicationPeers replicationPeers, Server server,
String peerClusterZnode, UUID clusterId, ReplicationEndpoint replicationEndpoint,
WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException {
this.server = server;
@@ -161,7 +159,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
this.maxRetriesMultiplier =
this.conf.getInt("replication.source.maxretriesmultiplier", 300); // 5 minutes @ 1 sec per
this.queueSizePerGroup = this.conf.getInt("hbase.regionserver.maxlogs", 32);
- this.replicationQueues = replicationQueues;
+ this.queueStorage = queueStorage;
this.replicationPeers = replicationPeers;
this.manager = manager;
this.fs = fs;
@@ -230,7 +228,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
List<String> tableCfs = tableCFMap.get(tableName);
if (tableCFMap.containsKey(tableName)
&& (tableCfs == null || tableCfs.contains(Bytes.toString(family)))) {
- this.replicationQueues.addHFileRefs(peerId, pairs);
+ this.queueStorage.addHFileRefs(peerId, pairs);
metrics.incrSizeOfHFileRefsQueue(pairs.size());
} else {
LOG.debug("HFiles will not be replicated belonging to the table " + tableName + " family "
@@ -239,7 +237,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
} else {
// user has explicitly not defined any table cfs for replication, means replicate all the
// data
- this.replicationQueues.addHFileRefs(peerId, pairs);
+ this.queueStorage.addHFileRefs(peerId, pairs);
metrics.incrSizeOfHFileRefsQueue(pairs.size());
}
}
[25/50] [abbrv] hbase git commit: HBASE-19599 Remove
ReplicationQueuesClient, use ReplicationQueueStorage directly
Posted by zh...@apache.org.
HBASE-19599 Remove ReplicationQueuesClient, use ReplicationQueueStorage directly
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0a754e99
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0a754e99
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0a754e99
Branch: refs/heads/HBASE-19397-branch-2
Commit: 0a754e998b2c5c43485c848cf0340254e0924179
Parents: 7213a0d
Author: zhangduo <zh...@apache.org>
Authored: Mon Dec 25 18:49:56 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../hbase/replication/ReplicationFactory.java | 19 +-
.../replication/ReplicationPeersZKImpl.java | 21 +-
.../replication/ReplicationQueueStorage.java | 26 +-
.../replication/ReplicationQueuesClient.java | 93 -----
.../ReplicationQueuesClientArguments.java | 40 --
.../ReplicationQueuesClientZKImpl.java | 176 ---------
.../replication/ZKReplicationQueueStorage.java | 90 ++++-
.../replication/TestReplicationStateBasic.java | 378 +++++++++++++++++++
.../replication/TestReplicationStateZKImpl.java | 148 ++++++++
.../TestZKReplicationQueueStorage.java | 74 ++++
.../cleaner/ReplicationZKNodeCleaner.java | 71 ++--
.../cleaner/ReplicationZKNodeCleanerChore.java | 5 +-
.../replication/ReplicationPeerManager.java | 31 +-
.../master/ReplicationHFileCleaner.java | 109 ++----
.../master/ReplicationLogCleaner.java | 35 +-
.../regionserver/DumpReplicationQueues.java | 78 ++--
.../hbase/util/hbck/ReplicationChecker.java | 14 +-
.../client/TestAsyncReplicationAdminApi.java | 31 +-
.../replication/TestReplicationAdmin.java | 2 +
.../hbase/master/cleaner/TestLogsCleaner.java | 30 +-
.../cleaner/TestReplicationHFileCleaner.java | 59 +--
.../cleaner/TestReplicationZKNodeCleaner.java | 12 +-
.../replication/TestReplicationStateBasic.java | 378 -------------------
.../replication/TestReplicationStateZKImpl.java | 227 -----------
.../TestReplicationSourceManagerZkImpl.java | 84 ++---
25 files changed, 905 insertions(+), 1326 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 9f4ad18..6c1c213 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -37,20 +36,14 @@ public class ReplicationFactory {
args);
}
- public static ReplicationQueuesClient
- getReplicationQueuesClient(ReplicationQueuesClientArguments args) throws Exception {
- return (ReplicationQueuesClient) ConstructorUtils
- .invokeConstructor(ReplicationQueuesClientZKImpl.class, args);
- }
-
- public static ReplicationPeers getReplicationPeers(final ZKWatcher zk, Configuration conf,
- Abortable abortable) {
+ public static ReplicationPeers getReplicationPeers(ZKWatcher zk, Configuration conf,
+ Abortable abortable) {
return getReplicationPeers(zk, conf, null, abortable);
}
- public static ReplicationPeers getReplicationPeers(final ZKWatcher zk, Configuration conf,
- final ReplicationQueuesClient queuesClient, Abortable abortable) {
- return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);
+ public static ReplicationPeers getReplicationPeers(ZKWatcher zk, Configuration conf,
+ ReplicationQueueStorage queueStorage, Abortable abortable) {
+ return new ReplicationPeersZKImpl(zk, conf, queueStorage, abortable);
}
public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 8ed0150..7de4619 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.CompoundConfiguration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -80,17 +81,17 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
// Map of peer clusters keyed by their id
private Map<String, ReplicationPeerZKImpl> peerClusters;
- private final ReplicationQueuesClient queuesClient;
+ private final ReplicationQueueStorage queueStorage;
private Abortable abortable;
private static final Logger LOG = LoggerFactory.getLogger(ReplicationPeersZKImpl.class);
- public ReplicationPeersZKImpl(final ZKWatcher zk, final Configuration conf,
- final ReplicationQueuesClient queuesClient, Abortable abortable) {
+ public ReplicationPeersZKImpl(ZKWatcher zk, Configuration conf,
+ ReplicationQueueStorage queueStorage, Abortable abortable) {
super(zk, conf, abortable);
this.abortable = abortable;
this.peerClusters = new ConcurrentHashMap<>();
- this.queuesClient = queuesClient;
+ this.queueStorage = queueStorage;
}
@Override
@@ -510,14 +511,16 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
}
private void checkQueuesDeleted(String peerId) throws ReplicationException {
- if (queuesClient == null) return;
+ if (queueStorage == null) {
+ return;
+ }
try {
- List<String> replicators = queuesClient.getListOfReplicators();
+ List<ServerName> replicators = queueStorage.getListOfReplicators();
if (replicators == null || replicators.isEmpty()) {
return;
}
- for (String replicator : replicators) {
- List<String> queueIds = queuesClient.getAllQueues(replicator);
+ for (ServerName replicator : replicators) {
+ List<String> queueIds = queueStorage.getAllQueues(replicator);
for (String queueId : queueIds) {
ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
if (queueInfo.getPeerId().equals(peerId)) {
@@ -528,7 +531,7 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
}
// Check for hfile-refs queue
if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
- && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
+ && queueStorage.getAllPeersFromHFileRefsQueue().contains(peerId)) {
throw new IllegalArgumentException("Undeleted queue for peerId: " + peerId
+ ", found in hfile-refs node path " + hfileRefsZNode);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
index 7210d9a..e774148 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
@@ -78,6 +78,14 @@ public interface ReplicationQueueStorage {
throws ReplicationException;
/**
+ * Get a list of all WALs in the given queue on the given region server.
+ * @param serverName the server name of the region server that owns the queue
+ * @param queueId a String that identifies the queue
+ * @return a list of WALs
+ */
+ List<String> getWALsInQueue(ServerName serverName, String queueId) throws ReplicationException;
+
+ /**
* Get a list of all queues for the specified region server.
* @param serverName the server name of the region server that owns the set of queues
* @return a list of queueIds
@@ -108,8 +116,8 @@ public interface ReplicationQueueStorage {
/**
* Load all wals in all replication queues. This method guarantees to return a snapshot which
- * contains all WALs in the zookeeper at the start of this call even there is concurrent queue
- * failover. However, some newly created WALs during the call may not be included.
+ * contains all WALs at the start of this call even there is concurrent queue failover. However,
+ * some newly created WALs during the call may not be included.
*/
Set<String> getAllWALs() throws ReplicationException;
@@ -143,13 +151,6 @@ public interface ReplicationQueueStorage {
void removeHFileRefs(String peerId, List<String> files) throws ReplicationException;
/**
- * Get the change version number of replication hfile references node. This can be used as
- * optimistic locking to get a consistent snapshot of the replication queues of hfile references.
- * @return change version number of hfile references node
- */
- int getHFileRefsNodeChangeVersion() throws ReplicationException;
-
- /**
* Get list of all peers from hfile reference queue.
* @return a list of peer ids
*/
@@ -161,4 +162,11 @@ public interface ReplicationQueueStorage {
* @return a list of hfile references
*/
List<String> getReplicableHFiles(String peerId) throws ReplicationException;
+
+ /**
+ * Load all hfile references in all replication queues. This method guarantees to return a
+ * snapshot which contains all hfile references at the start of this call. However, some newly
+ * created hfile references during the call may not be included.
+ */
+ Set<String> getAllHFileRefs() throws ReplicationException;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java
deleted file mode 100644
index 2c513fa..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.List;
-import java.util.Set;
-
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * This provides an interface for clients of replication to view replication queues. These queues
- * keep track of the sources(WALs/HFile references) that still need to be replicated to remote
- * clusters.
- */
-@InterfaceAudience.Private
-public interface ReplicationQueuesClient {
-
- /**
- * Initialize the replication queue client interface.
- */
- public void init() throws ReplicationException;
-
- /**
- * Get a list of all region servers that have outstanding replication queues. These servers could
- * be alive, dead or from a previous run of the cluster.
- * @return a list of server names
- * @throws KeeperException zookeeper exception
- */
- List<String> getListOfReplicators() throws KeeperException;
-
- /**
- * Get a list of all WALs in the given queue on the given region server.
- * @param serverName the server name of the region server that owns the queue
- * @param queueId a String that identifies the queue
- * @return a list of WALs, null if this region server is dead and has no outstanding queues
- * @throws KeeperException zookeeper exception
- */
- List<String> getLogsInQueue(String serverName, String queueId) throws KeeperException;
-
- /**
- * Get a list of all queues for the specified region server.
- * @param serverName the server name of the region server that owns the set of queues
- * @return a list of queueIds, null if this region server is not a replicator.
- */
- List<String> getAllQueues(String serverName) throws KeeperException;
-
- /**
- * Load all wals in all replication queues from ZK. This method guarantees to return a
- * snapshot which contains all WALs in the zookeeper at the start of this call even there
- * is concurrent queue failover. However, some newly created WALs during the call may
- * not be included.
- */
- Set<String> getAllWALs() throws KeeperException;
-
- /**
- * Get the change version number of replication hfile references node. This can be used as
- * optimistic locking to get a consistent snapshot of the replication queues of hfile references.
- * @return change version number of hfile references node
- */
- int getHFileRefsNodeChangeVersion() throws KeeperException;
-
- /**
- * Get list of all peers from hfile reference queue.
- * @return a list of peer ids
- * @throws KeeperException zookeeper exception
- */
- List<String> getAllPeersFromHFileRefsQueue() throws KeeperException;
-
- /**
- * Get a list of all hfile references in the given peer.
- * @param peerId a String that identifies the peer
- * @return a list of hfile references, null if not found any
- * @throws KeeperException zookeeper exception
- */
- List<String> getReplicableHFiles(String peerId) throws KeeperException;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
deleted file mode 100644
index 9b79294..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
-
-/**
- * Wrapper around common arguments used to construct ReplicationQueuesClient. Used to construct
- * various ReplicationQueuesClient Implementations with different constructor arguments by
- * reflection.
- */
-@InterfaceAudience.Private
-public class ReplicationQueuesClientArguments extends ReplicationQueuesArguments {
- public ReplicationQueuesClientArguments(Configuration conf, Abortable abort,
- ZKWatcher zk) {
- super(conf, abort, zk);
- }
- public ReplicationQueuesClientArguments(Configuration conf, Abortable abort) {
- super(conf, abort);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java
deleted file mode 100644
index e00a7a2..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@InterfaceAudience.Private
-public class ReplicationQueuesClientZKImpl extends ReplicationStateZKBase implements
- ReplicationQueuesClient {
-
- Logger LOG = LoggerFactory.getLogger(ReplicationQueuesClientZKImpl.class);
-
- public ReplicationQueuesClientZKImpl(ReplicationQueuesClientArguments args) {
- this(args.getZk(), args.getConf(), args.getAbortable());
- }
-
- public ReplicationQueuesClientZKImpl(final ZKWatcher zk, Configuration conf,
- Abortable abortable) {
- super(zk, conf, abortable);
- }
-
- @Override
- public void init() throws ReplicationException {
- try {
- if (ZKUtil.checkExists(this.zookeeper, this.queuesZNode) < 0) {
- ZKUtil.createWithParents(this.zookeeper, this.queuesZNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Internal error while initializing a queues client", e);
- }
- }
-
- @Override
- public List<String> getLogsInQueue(String serverName, String queueId) throws KeeperException {
- String znode = ZNodePaths.joinZNode(this.queuesZNode, serverName);
- znode = ZNodePaths.joinZNode(znode, queueId);
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of wals for queueId=" + queueId
- + " and serverName=" + serverName, e);
- throw e;
- }
- return result;
- }
-
- @Override
- public List<String> getAllQueues(String serverName) throws KeeperException {
- String znode = ZNodePaths.joinZNode(this.queuesZNode, serverName);
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of queues for serverName=" + serverName, e);
- throw e;
- }
- return result;
- }
-
- @Override
- public Set<String> getAllWALs() throws KeeperException {
- /**
- * Load all wals in all replication queues from ZK. This method guarantees to return a
- * snapshot which contains all WALs in the zookeeper at the start of this call even there
- * is concurrent queue failover. However, some newly created WALs during the call may
- * not be included.
- */
- for (int retry = 0; ; retry++) {
- int v0 = getQueuesZNodeCversion();
- List<String> rss = getListOfReplicators();
- if (rss == null || rss.isEmpty()) {
- LOG.debug("Didn't find any region server that replicates, won't prevent any deletions.");
- return ImmutableSet.of();
- }
- Set<String> wals = Sets.newHashSet();
- for (String rs : rss) {
- List<String> listOfPeers = getAllQueues(rs);
- // if rs just died, this will be null
- if (listOfPeers == null) {
- continue;
- }
- for (String id : listOfPeers) {
- List<String> peersWals = getLogsInQueue(rs, id);
- if (peersWals != null) {
- wals.addAll(peersWals);
- }
- }
- }
- int v1 = getQueuesZNodeCversion();
- if (v0 == v1) {
- return wals;
- }
- LOG.info(String.format("Replication queue node cversion changed from %d to %d, retry = %d",
- v0, v1, retry));
- }
- }
-
- public int getQueuesZNodeCversion() throws KeeperException {
- try {
- Stat stat = new Stat();
- ZKUtil.getDataNoWatch(this.zookeeper, this.queuesZNode, stat);
- return stat.getCversion();
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get stat of replication rs node", e);
- throw e;
- }
- }
-
- @Override
- public int getHFileRefsNodeChangeVersion() throws KeeperException {
- Stat stat = new Stat();
- try {
- ZKUtil.getDataNoWatch(this.zookeeper, this.hfileRefsZNode, stat);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get stat of replication hfile references node.", e);
- throw e;
- }
- return stat.getCversion();
- }
-
- @Override
- public List<String> getAllPeersFromHFileRefsQueue() throws KeeperException {
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.hfileRefsZNode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of all peers in hfile references node.", e);
- throw e;
- }
- return result;
- }
-
- @Override
- public List<String> getReplicableHFiles(String peerId) throws KeeperException {
- String znode = ZNodePaths.joinZNode(this.hfileRefsZNode, peerId);
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of hfile references for peerId=" + peerId, e);
- throw e;
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
index 7015d7f..0275d52 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.util.CollectionUtils.nullToEmpty;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
@@ -49,7 +50,7 @@ import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Sets;
+import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
/**
* ZK based replication queue storage.
@@ -61,7 +62,7 @@ class ZKReplicationQueueStorage extends ZKReplicationStorageBase
private static final Logger LOG = LoggerFactory.getLogger(ZKReplicationQueueStorage.class);
public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY =
- "zookeeper.znode.replication.hfile.refs";
+ "zookeeper.znode.replication.hfile.refs";
public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT = "hfile-refs";
/**
@@ -256,11 +257,23 @@ class ZKReplicationQueueStorage extends ZKReplicationStorageBase
}
}
- private List<String> getLogsInQueue0(ServerName serverName, String queueId)
+ private List<String> getWALsInQueue0(ServerName serverName, String queueId)
throws KeeperException {
return nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, getQueueNode(serverName, queueId)));
}
+ @Override
+ public List<String> getWALsInQueue(ServerName serverName, String queueId)
+ throws ReplicationException {
+ try {
+ return getWALsInQueue0(serverName, queueId);
+ } catch (KeeperException e) {
+ throw new ReplicationException(
+ "Failed to get wals in queue (serverName=" + serverName + ", queueId=" + queueId + ")",
+ e);
+ }
+ }
+
private List<String> getAllQueues0(ServerName serverName) throws KeeperException {
return nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, getRsNode(serverName)));
}
@@ -274,7 +287,9 @@ class ZKReplicationQueueStorage extends ZKReplicationStorageBase
}
}
- private int getQueuesZNodeCversion() throws KeeperException {
+ // will be overridden in UTs
+ @VisibleForTesting
+ protected int getQueuesZNodeCversion() throws KeeperException {
Stat stat = new Stat();
ZKUtil.getDataNoWatch(this.zookeeper, this.queuesZNode, stat);
return stat.getCversion();
@@ -290,10 +305,10 @@ class ZKReplicationQueueStorage extends ZKReplicationStorageBase
LOG.debug("Didn't find any region server that replicates, won't prevent any deletions.");
return Collections.emptySet();
}
- Set<String> wals = Sets.newHashSet();
+ Set<String> wals = new HashSet<>();
for (ServerName rs : rss) {
for (String queueId : getAllQueues0(rs)) {
- wals.addAll(getLogsInQueue0(rs, queueId));
+ wals.addAll(getWALsInQueue0(rs, queueId));
}
}
int v1 = getQueuesZNodeCversion();
@@ -356,9 +371,9 @@ class ZKReplicationQueueStorage extends ZKReplicationStorageBase
if (debugEnabled) {
LOG.debug("Adding hfile references " + pairs + " in queue " + peerNode);
}
- List<ZKUtilOp> listOfOps =
- pairs.stream().map(p -> p.getSecond().getName()).map(n -> getHFileNode(peerNode, n))
- .map(f -> ZKUtilOp.createAndFailSilent(f, HConstants.EMPTY_BYTE_ARRAY)).collect(toList());
+ List<ZKUtilOp> listOfOps = pairs.stream().map(p -> p.getSecond().getName())
+ .map(n -> getHFileNode(peerNode, n))
+ .map(f -> ZKUtilOp.createAndFailSilent(f, HConstants.EMPTY_BYTE_ARRAY)).collect(toList());
if (debugEnabled) {
LOG.debug("The multi list size for adding hfile references in zk for node " + peerNode +
" is " + listOfOps.size());
@@ -391,35 +406,70 @@ class ZKReplicationQueueStorage extends ZKReplicationStorageBase
}
}
- @Override
- public int getHFileRefsNodeChangeVersion() throws ReplicationException {
- Stat stat = new Stat();
- try {
- ZKUtil.getDataNoWatch(zookeeper, hfileRefsZNode, stat);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get stat of replication hfile references node.", e);
- }
- return stat.getCversion();
+ private List<String> getAllPeersFromHFileRefsQueue0() throws KeeperException {
+ return nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, hfileRefsZNode));
}
@Override
public List<String> getAllPeersFromHFileRefsQueue() throws ReplicationException {
try {
- return nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, hfileRefsZNode));
+ return getAllPeersFromHFileRefsQueue0();
} catch (KeeperException e) {
throw new ReplicationException("Failed to get list of all peers in hfile references node.",
e);
}
}
+ private List<String> getReplicableHFiles0(String peerId) throws KeeperException {
+ return nullToEmpty(ZKUtil.listChildrenNoWatch(this.zookeeper, getHFileRefsPeerNode(peerId)));
+ }
+
@Override
public List<String> getReplicableHFiles(String peerId) throws ReplicationException {
try {
- return nullToEmpty(ZKUtil.listChildrenNoWatch(this.zookeeper, getHFileRefsPeerNode(peerId)));
+ return getReplicableHFiles0(peerId);
} catch (KeeperException e) {
throw new ReplicationException("Failed to get list of hfile references for peer " + peerId,
e);
}
}
+ // will be overridden in UTs
+ @VisibleForTesting
+ protected int getHFileRefsZNodeCversion() throws ReplicationException {
+ Stat stat = new Stat();
+ try {
+ ZKUtil.getDataNoWatch(zookeeper, hfileRefsZNode, stat);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to get stat of replication hfile references node.", e);
+ }
+ return stat.getCversion();
+ }
+
+ @Override
+ public Set<String> getAllHFileRefs() throws ReplicationException {
+ try {
+ for (int retry = 0;; retry++) {
+ int v0 = getHFileRefsZNodeCversion();
+ List<String> peers = getAllPeersFromHFileRefsQueue();
+ if (peers.isEmpty()) {
+ LOG.debug("Didn't find any peers with hfile references, won't prevent any deletions.");
+ return Collections.emptySet();
+ }
+ Set<String> hfileRefs = new HashSet<>();
+ for (String peer : peers) {
+ hfileRefs.addAll(getReplicableHFiles0(peer));
+ }
+ int v1 = getHFileRefsZNodeCversion();
+ if (v0 == v1) {
+ return hfileRefs;
+ }
+ LOG.debug(String.format(
+ "Replication hfile references node cversion changed from " + "%d to %d, retry = %d", v0,
+ v1, retry));
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to get all hfile refs", e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
new file mode 100644
index 0000000..6fe869c
--- /dev/null
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
@@ -0,0 +1,378 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
+import org.apache.zookeeper.KeeperException;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * White box testing for replication state interfaces. Implementations should extend this class, and
+ * initialize the interfaces properly.
+ */
+public abstract class TestReplicationStateBasic {
+
+ protected ReplicationQueues rq1;
+ protected ReplicationQueues rq2;
+ protected ReplicationQueues rq3;
+ protected ReplicationQueueStorage rqs;
+ protected ServerName server1 = ServerName.valueOf("hostname1.example.org", 1234, 12345);
+ protected ServerName server2 = ServerName.valueOf("hostname2.example.org", 1234, 12345);
+ protected ServerName server3 = ServerName.valueOf("hostname3.example.org", 1234, 12345);
+ protected ReplicationPeers rp;
+ protected static final String ID_ONE = "1";
+ protected static final String ID_TWO = "2";
+ protected static String KEY_ONE;
+ protected static String KEY_TWO;
+
+ // For testing when we try to replicate to ourself
+ protected String OUR_ID = "3";
+ protected String OUR_KEY;
+
+ protected static int zkTimeoutCount;
+ protected static final int ZK_MAX_COUNT = 300;
+ protected static final int ZK_SLEEP_INTERVAL = 100; // millis
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestReplicationStateBasic.class);
+
+ @Test
+ public void testReplicationQueueStorage() throws ReplicationException {
+ // Test methods with empty state
+ assertEquals(0, rqs.getListOfReplicators().size());
+ assertTrue(rqs.getWALsInQueue(server1, "qId1").isEmpty());
+ assertTrue(rqs.getAllQueues(server1).isEmpty());
+
+ /*
+ * Set up data Two replicators: -- server1: three queues with 0, 1 and 2 log files each --
+ * server2: zero queues
+ */
+ rq1.init(server1.getServerName());
+ rq2.init(server2.getServerName());
+ rq1.addLog("qId1", "trash");
+ rq1.removeLog("qId1", "trash");
+ rq1.addLog("qId2", "filename1");
+ rq1.addLog("qId3", "filename2");
+ rq1.addLog("qId3", "filename3");
+ rq2.addLog("trash", "trash");
+ rq2.removeQueue("trash");
+
+ List<ServerName> reps = rqs.getListOfReplicators();
+ assertEquals(2, reps.size());
+ assertTrue(server1.getServerName(), reps.contains(server1));
+ assertTrue(server2.getServerName(), reps.contains(server2));
+
+ assertTrue(rqs.getWALsInQueue(ServerName.valueOf("bogus", 12345, 12345), "bogus").isEmpty());
+ assertTrue(rqs.getWALsInQueue(server1, "bogus").isEmpty());
+ assertEquals(0, rqs.getWALsInQueue(server1, "qId1").size());
+ assertEquals(1, rqs.getWALsInQueue(server1, "qId2").size());
+ assertEquals("filename1", rqs.getWALsInQueue(server1, "qId2").get(0));
+
+ assertTrue(rqs.getAllQueues(ServerName.valueOf("bogus", 12345, -1L)).isEmpty());
+ assertEquals(0, rqs.getAllQueues(server2).size());
+ List<String> list = rqs.getAllQueues(server1);
+ assertEquals(3, list.size());
+ assertTrue(list.contains("qId2"));
+ assertTrue(list.contains("qId3"));
+ }
+
+ @Test
+ public void testReplicationQueues() throws ReplicationException {
+ rq1.init(server1.getServerName());
+ rq2.init(server2.getServerName());
+ rq3.init(server3.getServerName());
+ // Initialize ReplicationPeer so we can add peers (we don't transfer lone queues)
+ rp.init();
+
+ // 3 replicators should exist
+ assertEquals(3, rq1.getListOfReplicators().size());
+ rq1.removeQueue("bogus");
+ rq1.removeLog("bogus", "bogus");
+ rq1.removeAllQueues();
+ assertEquals(0, rq1.getAllQueues().size());
+ assertEquals(0, rq1.getLogPosition("bogus", "bogus"));
+ assertNull(rq1.getLogsInQueue("bogus"));
+ assertNull(rq1.getUnClaimedQueueIds(ServerName.valueOf("bogus", 1234, -1L).toString()));
+
+ rq1.setLogPosition("bogus", "bogus", 5L);
+
+ populateQueues();
+
+ assertEquals(3, rq1.getListOfReplicators().size());
+ assertEquals(0, rq2.getLogsInQueue("qId1").size());
+ assertEquals(5, rq3.getLogsInQueue("qId5").size());
+ assertEquals(0, rq3.getLogPosition("qId1", "filename0"));
+ rq3.setLogPosition("qId5", "filename4", 354L);
+ assertEquals(354L, rq3.getLogPosition("qId5", "filename4"));
+
+ assertEquals(5, rq3.getLogsInQueue("qId5").size());
+ assertEquals(0, rq2.getLogsInQueue("qId1").size());
+ assertEquals(0, rq1.getAllQueues().size());
+ assertEquals(1, rq2.getAllQueues().size());
+ assertEquals(5, rq3.getAllQueues().size());
+
+ assertEquals(0, rq3.getUnClaimedQueueIds(server1.getServerName()).size());
+ rq3.removeReplicatorIfQueueIsEmpty(server1.getServerName());
+ assertEquals(2, rq3.getListOfReplicators().size());
+
+ List<String> queues = rq2.getUnClaimedQueueIds(server3.getServerName());
+ assertEquals(5, queues.size());
+ for (String queue : queues) {
+ rq2.claimQueue(server3.getServerName(), queue);
+ }
+ rq2.removeReplicatorIfQueueIsEmpty(server3.getServerName());
+ assertEquals(1, rq2.getListOfReplicators().size());
+
+ // Try to claim our own queues
+ assertNull(rq2.getUnClaimedQueueIds(server2.getServerName()));
+ rq2.removeReplicatorIfQueueIsEmpty(server2.getServerName());
+
+ assertEquals(6, rq2.getAllQueues().size());
+
+ rq2.removeAllQueues();
+
+ assertEquals(0, rq2.getListOfReplicators().size());
+ }
+
+ @Test
+ public void testInvalidClusterKeys() throws ReplicationException, KeeperException {
+ rp.init();
+
+ try {
+ rp.registerPeer(ID_ONE,
+ new ReplicationPeerConfig().setClusterKey("hostname1.example.org:1234:hbase"));
+ fail("Should throw an IllegalArgumentException because " +
+ "zookeeper.znode.parent is missing leading '/'.");
+ } catch (IllegalArgumentException e) {
+ // Expected.
+ }
+
+ try {
+ rp.registerPeer(ID_ONE,
+ new ReplicationPeerConfig().setClusterKey("hostname1.example.org:1234:/"));
+ fail("Should throw an IllegalArgumentException because zookeeper.znode.parent is missing.");
+ } catch (IllegalArgumentException e) {
+ // Expected.
+ }
+
+ try {
+ rp.registerPeer(ID_ONE,
+ new ReplicationPeerConfig().setClusterKey("hostname1.example.org::/hbase"));
+ fail("Should throw an IllegalArgumentException because " +
+ "hbase.zookeeper.property.clientPort is missing.");
+ } catch (IllegalArgumentException e) {
+ // Expected.
+ }
+ }
+
+ @Test
+ public void testHfileRefsReplicationQueues() throws ReplicationException, KeeperException {
+ rp.init();
+ rq1.init(server1.getServerName());
+
+ List<Pair<Path, Path>> files1 = new ArrayList<>(3);
+ files1.add(new Pair<>(null, new Path("file_1")));
+ files1.add(new Pair<>(null, new Path("file_2")));
+ files1.add(new Pair<>(null, new Path("file_3")));
+ assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
+ assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
+ rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
+ rq1.addPeerToHFileRefs(ID_ONE);
+ rq1.addHFileRefs(ID_ONE, files1);
+ assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
+ assertEquals(3, rqs.getReplicableHFiles(ID_ONE).size());
+ List<String> hfiles2 = new ArrayList<>(files1.size());
+ for (Pair<Path, Path> p : files1) {
+ hfiles2.add(p.getSecond().getName());
+ }
+ String removedString = hfiles2.remove(0);
+ rq1.removeHFileRefs(ID_ONE, hfiles2);
+ assertEquals(1, rqs.getReplicableHFiles(ID_ONE).size());
+ hfiles2 = new ArrayList<>(1);
+ hfiles2.add(removedString);
+ rq1.removeHFileRefs(ID_ONE, hfiles2);
+ assertEquals(0, rqs.getReplicableHFiles(ID_ONE).size());
+ rp.unregisterPeer(ID_ONE);
+ }
+
+ @Test
+ public void testRemovePeerForHFileRefs() throws ReplicationException, KeeperException {
+ rq1.init(server1.getServerName());
+
+ rp.init();
+ rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
+ rq1.addPeerToHFileRefs(ID_ONE);
+ rp.registerPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO));
+ rq1.addPeerToHFileRefs(ID_TWO);
+
+ List<Pair<Path, Path>> files1 = new ArrayList<>(3);
+ files1.add(new Pair<>(null, new Path("file_1")));
+ files1.add(new Pair<>(null, new Path("file_2")));
+ files1.add(new Pair<>(null, new Path("file_3")));
+ rq1.addHFileRefs(ID_ONE, files1);
+ rq1.addHFileRefs(ID_TWO, files1);
+ assertEquals(2, rqs.getAllPeersFromHFileRefsQueue().size());
+ assertEquals(3, rqs.getReplicableHFiles(ID_ONE).size());
+ assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
+
+ rp.unregisterPeer(ID_ONE);
+ rq1.removePeerFromHFileRefs(ID_ONE);
+ assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
+ assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
+ assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
+
+ rp.unregisterPeer(ID_TWO);
+ rq1.removePeerFromHFileRefs(ID_TWO);
+ assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
+ assertTrue(rqs.getReplicableHFiles(ID_TWO).isEmpty());
+ }
+
+ @Test
+ public void testReplicationPeers() throws Exception {
+ rp.init();
+
+ // Test methods with non-existent peer ids
+ try {
+ rp.unregisterPeer("bogus");
+ fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
+ } catch (IllegalArgumentException e) {
+ }
+ try {
+ rp.enablePeer("bogus");
+ fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
+ } catch (IllegalArgumentException e) {
+ }
+ try {
+ rp.disablePeer("bogus");
+ fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
+ } catch (IllegalArgumentException e) {
+ }
+ try {
+ rp.getStatusOfPeer("bogus");
+ fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
+ } catch (IllegalArgumentException e) {
+ }
+ assertFalse(rp.peerConnected("bogus"));
+ rp.peerDisconnected("bogus");
+
+ assertNull(rp.getPeerConf("bogus"));
+ assertNumberOfPeers(0);
+
+ // Add some peers
+ rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
+ assertNumberOfPeers(1);
+ rp.registerPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO));
+ assertNumberOfPeers(2);
+
+ // Test methods with a peer that is added but not connected
+ try {
+ rp.getStatusOfPeer(ID_ONE);
+ fail("There are no connected peers, should have thrown an IllegalArgumentException");
+ } catch (IllegalArgumentException e) {
+ }
+ assertEquals(KEY_ONE, ZKConfig.getZooKeeperClusterKey(rp.getPeerConf(ID_ONE).getSecond()));
+ rp.unregisterPeer(ID_ONE);
+ rp.peerDisconnected(ID_ONE);
+ assertNumberOfPeers(1);
+
+ // Add one peer
+ rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
+ rp.peerConnected(ID_ONE);
+ assertNumberOfPeers(2);
+ assertTrue(rp.getStatusOfPeer(ID_ONE));
+ rp.disablePeer(ID_ONE);
+ // now we do not rely on zk watcher to trigger the state change so we need to trigger it
+ // manually...
+ assertEquals(PeerState.DISABLED, rp.getConnectedPeer(ID_ONE).getPeerState(true));
+ assertConnectedPeerStatus(false, ID_ONE);
+ rp.enablePeer(ID_ONE);
+ // now we do not rely on zk watcher to trigger the state change so we need to trigger it
+ // manually...
+ assertEquals(PeerState.ENABLED, rp.getConnectedPeer(ID_ONE).getPeerState(true));
+ assertConnectedPeerStatus(true, ID_ONE);
+
+ // Disconnect peer
+ rp.peerDisconnected(ID_ONE);
+ assertNumberOfPeers(2);
+ try {
+ rp.getStatusOfPeer(ID_ONE);
+ fail("There are no connected peers, should have thrown an IllegalArgumentException");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+
+ protected void assertConnectedPeerStatus(boolean status, String peerId) throws Exception {
+ // we can first check if the value was changed in the store, if it wasn't then fail right away
+ if (status != rp.getStatusOfPeerFromBackingStore(peerId)) {
+ fail("ConnectedPeerStatus was " + !status + " but expected " + status + " in ZK");
+ }
+ while (true) {
+ if (status == rp.getStatusOfPeer(peerId)) {
+ return;
+ }
+ if (zkTimeoutCount < ZK_MAX_COUNT) {
+ LOG.debug("ConnectedPeerStatus was " + !status + " but expected " + status +
+ ", sleeping and trying again.");
+ Thread.sleep(ZK_SLEEP_INTERVAL);
+ } else {
+ fail("Timed out waiting for ConnectedPeerStatus to be " + status);
+ }
+ }
+ }
+
+ protected void assertNumberOfPeers(int total) {
+ assertEquals(total, rp.getAllPeerConfigs().size());
+ assertEquals(total, rp.getAllPeerIds().size());
+ assertEquals(total, rp.getAllPeerIds().size());
+ }
+
+ /*
+ * three replicators: rq1 has 0 queues, rq2 has 1 queue with no logs, rq3 has 5 queues with 1, 2,
+ * 3, 4, 5 log files respectively
+ */
+ protected void populateQueues() throws ReplicationException {
+ rq1.addLog("trash", "trash");
+ rq1.removeQueue("trash");
+
+ rq2.addLog("qId1", "trash");
+ rq2.removeLog("qId1", "trash");
+
+ for (int i = 1; i < 6; i++) {
+ for (int j = 0; j < i; j++) {
+ rq3.addLog("qId" + i, "filename" + j);
+ }
+ // Add peers for the corresponding queues so they are not orphans
+ rp.registerPeer("qId" + i,
+ new ReplicationPeerConfig().setClusterKey("localhost:2818:/bogus" + i));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
new file mode 100644
index 0000000..5fe7c55
--- /dev/null
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.HBaseZKTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category({ ReplicationTests.class, MediumTests.class })
+public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestReplicationStateZKImpl.class);
+
+ private static Configuration conf;
+ private static HBaseZKTestingUtility utility;
+ private static ZKWatcher zkw;
+ private static String replicationZNode;
+ private ReplicationQueuesZKImpl rqZK;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ utility = new HBaseZKTestingUtility();
+ utility.startMiniZKCluster();
+ conf = utility.getConfiguration();
+ conf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
+ zkw = utility.getZooKeeperWatcher();
+ String replicationZNodeName = conf.get("zookeeper.znode.replication", "replication");
+ replicationZNode = ZNodePaths.joinZNode(zkw.znodePaths.baseZNode, replicationZNodeName);
+ KEY_ONE = initPeerClusterState("/hbase1");
+ KEY_TWO = initPeerClusterState("/hbase2");
+ }
+
+ private static String initPeerClusterState(String baseZKNode)
+ throws IOException, KeeperException {
+ // Add a dummy region server and set up the cluster id
+ Configuration testConf = new Configuration(conf);
+ testConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, baseZKNode);
+ ZKWatcher zkw1 = new ZKWatcher(testConf, "test1", null);
+ String fakeRs = ZNodePaths.joinZNode(zkw1.znodePaths.rsZNode, "hostname1.example.org:1234");
+ ZKUtil.createWithParents(zkw1, fakeRs);
+ ZKClusterId.setClusterId(zkw1, new ClusterId());
+ return ZKConfig.getZooKeeperClusterKey(testConf);
+ }
+
+ @Before
+ public void setUp() {
+ zkTimeoutCount = 0;
+ WarnOnlyAbortable abortable = new WarnOnlyAbortable();
+ try {
+ rq1 = ReplicationFactory
+ .getReplicationQueues(new ReplicationQueuesArguments(conf, abortable, zkw));
+ rq2 = ReplicationFactory
+ .getReplicationQueues(new ReplicationQueuesArguments(conf, abortable, zkw));
+ rq3 = ReplicationFactory
+ .getReplicationQueues(new ReplicationQueuesArguments(conf, abortable, zkw));
+ rqs = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
+ } catch (Exception e) {
+ // This should not occur, because getReplicationQueues() only throws for
+ // TableBasedReplicationQueuesImpl
+ fail("ReplicationFactory.getReplicationQueues() threw an IO Exception");
+ }
+ rp = ReplicationFactory.getReplicationPeers(zkw, conf, zkw);
+ OUR_KEY = ZKConfig.getZooKeeperClusterKey(conf);
+ rqZK = new ReplicationQueuesZKImpl(zkw, conf, abortable);
+ }
+
+ @After
+ public void tearDown() throws KeeperException, IOException {
+ ZKUtil.deleteNodeRecursively(zkw, replicationZNode);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ utility.shutdownMiniZKCluster();
+ }
+
+ @Test
+ public void testIsPeerPath_PathToParentOfPeerNode() {
+ assertFalse(rqZK.isPeerPath(rqZK.peersZNode));
+ }
+
+ @Test
+ public void testIsPeerPath_PathToChildOfPeerNode() {
+ String peerChild = ZNodePaths.joinZNode(ZNodePaths.joinZNode(rqZK.peersZNode, "1"), "child");
+ assertFalse(rqZK.isPeerPath(peerChild));
+ }
+
+ @Test
+ public void testIsPeerPath_ActualPeerPath() {
+ String peerPath = ZNodePaths.joinZNode(rqZK.peersZNode, "1");
+ assertTrue(rqZK.isPeerPath(peerPath));
+ }
+
+ private static class WarnOnlyAbortable implements Abortable {
+
+ @Override
+ public void abort(String why, Throwable e) {
+ LOG.warn("TestReplicationStateZKImpl received abort, ignoring. Reason: " + why);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(e.toString(), e);
+ }
+ }
+
+ @Override
+ public boolean isAborted() {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
index d5bba0d..786730f 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
@@ -23,15 +23,18 @@ import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseZKTestingUtility;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -115,6 +118,15 @@ public class TestZKReplicationQueueStorage {
assertEquals(2, queueIds.size());
assertThat(queueIds, hasItems("1", "2"));
+ List<String> wals1 = STORAGE.getWALsInQueue(serverName1, queue1);
+ List<String> wals2 = STORAGE.getWALsInQueue(serverName1, queue2);
+ assertEquals(10, wals1.size());
+ assertEquals(10, wals1.size());
+ for (int i = 0; i < 10; i++) {
+ assertThat(wals1, hasItems(getFileName("file1", i)));
+ assertThat(wals2, hasItems(getFileName("file2", i)));
+ }
+
for (int i = 0; i < 10; i++) {
assertEquals(0, STORAGE.getWALPosition(serverName1, queue1, getFileName("file1", i)));
assertEquals(0, STORAGE.getWALPosition(serverName1, queue2, getFileName("file2", i)));
@@ -157,10 +169,20 @@ public class TestZKReplicationQueueStorage {
queueIds = STORAGE.getAllQueues(serverName1);
assertEquals(1, queueIds.size());
assertThat(queueIds, hasItems("2"));
+ wals2 = STORAGE.getWALsInQueue(serverName1, queue2);
+ assertEquals(5, wals2.size());
+ for (i = 0; i < 10; i += 2) {
+ assertThat(wals2, hasItems(getFileName("file2", i)));
+ }
queueIds = STORAGE.getAllQueues(serverName2);
assertEquals(1, queueIds.size());
assertThat(queueIds, hasItems(peer1.getFirst()));
+ wals1 = STORAGE.getWALsInQueue(serverName2, peer1.getFirst());
+ assertEquals(5, wals1.size());
+ for (i = 1; i < 10; i += 2) {
+ assertThat(wals1, hasItems(getFileName("file1", i)));
+ }
Set<String> allWals = STORAGE.getAllWALs();
assertEquals(10, allWals.size());
@@ -168,4 +190,56 @@ public class TestZKReplicationQueueStorage {
assertThat(allWals, hasItems(i % 2 == 0 ? getFileName("file2", i) : getFileName("file1", i)));
}
}
+
+ // For HBASE-12865
+ @Test
+ public void testClaimQueueChangeCversion() throws ReplicationException, KeeperException {
+ ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
+ STORAGE.addWAL(serverName1, "1", "file");
+
+ int v0 = STORAGE.getQueuesZNodeCversion();
+ ServerName serverName2 = ServerName.valueOf("127.0.0.1", 8001, 10001);
+ STORAGE.claimQueue(serverName1, "1", serverName2);
+ int v1 = STORAGE.getQueuesZNodeCversion();
+ // cversion should increase by 1 since a child node is deleted
+ assertEquals(1, v1 - v0);
+ }
+
+ private ZKReplicationQueueStorage createWithUnstableCversion() throws IOException {
+ return new ZKReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration()) {
+
+ private int called = 0;
+
+ @Override
+ protected int getQueuesZNodeCversion() throws KeeperException {
+ if (called < 4) {
+ called++;
+ }
+ return called;
+ }
+ };
+ }
+
+ @Test
+ public void testGetAllWALsCversionChange() throws IOException, ReplicationException {
+ ZKReplicationQueueStorage storage = createWithUnstableCversion();
+ storage.addWAL(getServerName(0), "1", "file");
+ // This should return eventually when cversion stabilizes
+ Set<String> allWals = storage.getAllWALs();
+ assertEquals(1, allWals.size());
+ assertThat(allWals, hasItems("file"));
+ }
+
+ // For HBASE-14621
+ @Test
+ public void testGetAllHFileRefsCversionChange() throws IOException, ReplicationException {
+ ZKReplicationQueueStorage storage = createWithUnstableCversion();
+ storage.addPeerToHFileRefs("1");
+ Path p = new Path("/test");
+ storage.addHFileRefs("1", Arrays.asList(Pair.newPair(p, p)));
+ // This should return eventually when cversion stabilizes
+ Set<String> allHFileRefs = storage.getAllHFileRefs();
+ assertEquals(1, allHFileRefs.size());
+ assertThat(allHFileRefs, hasItems("test"));
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
index 97deab5..af41399 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
@@ -23,21 +23,23 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,23 +50,19 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class ReplicationZKNodeCleaner {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationZKNodeCleaner.class);
- private final ZKWatcher zkw;
- private final ReplicationQueuesClient queuesClient;
+ private final ReplicationQueueStorage queueStorage;
private final ReplicationPeers replicationPeers;
private final ReplicationQueueDeletor queueDeletor;
public ReplicationZKNodeCleaner(Configuration conf, ZKWatcher zkw, Abortable abortable)
throws IOException {
try {
- this.zkw = zkw;
- this.queuesClient = ReplicationFactory
- .getReplicationQueuesClient(new ReplicationQueuesClientArguments(conf, abortable, zkw));
- this.queuesClient.init();
- this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf, this.queuesClient,
- abortable);
+ this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
+ this.replicationPeers =
+ ReplicationFactory.getReplicationPeers(zkw, conf, this.queueStorage, abortable);
this.replicationPeers.init();
this.queueDeletor = new ReplicationQueueDeletor(zkw, conf, abortable);
- } catch (Exception e) {
+ } catch (ReplicationException e) {
throw new IOException("failed to construct ReplicationZKNodeCleaner", e);
}
}
@@ -73,16 +71,16 @@ public class ReplicationZKNodeCleaner {
* @return undeletedQueues replicator with its queueIds for removed peers
* @throws IOException
*/
- public Map<String, List<String>> getUnDeletedQueues() throws IOException {
- Map<String, List<String>> undeletedQueues = new HashMap<>();
+ public Map<ServerName, List<String>> getUnDeletedQueues() throws IOException {
+ Map<ServerName, List<String>> undeletedQueues = new HashMap<>();
Set<String> peerIds = new HashSet<>(this.replicationPeers.getAllPeerIds());
try {
- List<String> replicators = this.queuesClient.getListOfReplicators();
+ List<ServerName> replicators = this.queueStorage.getListOfReplicators();
if (replicators == null || replicators.isEmpty()) {
return undeletedQueues;
}
- for (String replicator : replicators) {
- List<String> queueIds = this.queuesClient.getAllQueues(replicator);
+ for (ServerName replicator : replicators) {
+ List<String> queueIds = this.queueStorage.getAllQueues(replicator);
for (String queueId : queueIds) {
ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
if (!peerIds.contains(queueInfo.getPeerId())) {
@@ -96,7 +94,7 @@ public class ReplicationZKNodeCleaner {
}
}
}
- } catch (KeeperException ke) {
+ } catch (ReplicationException ke) {
throw new IOException("Failed to get the replication queues of all replicators", ke);
}
return undeletedQueues;
@@ -105,25 +103,21 @@ public class ReplicationZKNodeCleaner {
/**
* @return undeletedHFileRefsQueue replicator with its undeleted queueIds for removed peers in
* hfile-refs queue
- * @throws IOException
*/
public Set<String> getUnDeletedHFileRefsQueues() throws IOException {
Set<String> undeletedHFileRefsQueue = new HashSet<>();
Set<String> peerIds = new HashSet<>(this.replicationPeers.getAllPeerIds());
String hfileRefsZNode = queueDeletor.getHfileRefsZNode();
try {
- if (-1 == ZKUtil.checkExists(zkw, hfileRefsZNode)) {
- return null;
- }
- List<String> listOfPeers = this.queuesClient.getAllPeersFromHFileRefsQueue();
+ List<String> listOfPeers = this.queueStorage.getAllPeersFromHFileRefsQueue();
Set<String> peers = new HashSet<>(listOfPeers);
peers.removeAll(peerIds);
if (!peers.isEmpty()) {
undeletedHFileRefsQueue.addAll(peers);
}
- } catch (KeeperException e) {
- throw new IOException("Failed to get list of all peers from hfile-refs znode "
- + hfileRefsZNode, e);
+ } catch (ReplicationException e) {
+ throw new IOException(
+ "Failed to get list of all peers from hfile-refs znode " + hfileRefsZNode, e);
}
return undeletedHFileRefsQueue;
}
@@ -137,21 +131,20 @@ public class ReplicationZKNodeCleaner {
/**
* @param replicator The regionserver which has undeleted queue
* @param queueId The undeleted queue id
- * @throws IOException
*/
- public void removeQueue(final String replicator, final String queueId) throws IOException {
- String queueZnodePath =
- ZNodePaths.joinZNode(ZNodePaths.joinZNode(this.queuesZNode, replicator), queueId);
+ public void removeQueue(final ServerName replicator, final String queueId) throws IOException {
+ String queueZnodePath = ZNodePaths
+ .joinZNode(ZNodePaths.joinZNode(this.queuesZNode, replicator.getServerName()), queueId);
try {
ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
if (!replicationPeers.getAllPeerIds().contains(queueInfo.getPeerId())) {
ZKUtil.deleteNodeRecursively(this.zookeeper, queueZnodePath);
- LOG.info("Successfully removed replication queue, replicator: " + replicator
- + ", queueId: " + queueId);
+ LOG.info("Successfully removed replication queue, replicator: " + replicator +
+ ", queueId: " + queueId);
}
} catch (KeeperException e) {
- throw new IOException("Failed to delete queue, replicator: " + replicator + ", queueId: "
- + queueId);
+ throw new IOException(
+ "Failed to delete queue, replicator: " + replicator + ", queueId: " + queueId);
}
}
@@ -183,9 +176,9 @@ public class ReplicationZKNodeCleaner {
* @param undeletedQueues replicator with its queueIds for removed peers
* @throws IOException
*/
- public void removeQueues(final Map<String, List<String>> undeletedQueues) throws IOException {
- for (Entry<String, List<String>> replicatorAndQueueIds : undeletedQueues.entrySet()) {
- String replicator = replicatorAndQueueIds.getKey();
+ public void removeQueues(final Map<ServerName, List<String>> undeletedQueues) throws IOException {
+ for (Entry<ServerName, List<String>> replicatorAndQueueIds : undeletedQueues.entrySet()) {
+ ServerName replicator = replicatorAndQueueIds.getKey();
for (String queueId : replicatorAndQueueIds.getValue()) {
queueDeletor.removeQueue(replicator, queueId);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleanerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleanerChore.java
index 8d5df9b..19ca804 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleanerChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleanerChore.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.master.cleaner;
import java.io.IOException;
@@ -23,6 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hbase.ScheduledChore;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -45,11 +45,10 @@ public class ReplicationZKNodeCleanerChore extends ScheduledChore {
@Override
protected void chore() {
try {
- Map<String, List<String>> undeletedQueues = cleaner.getUnDeletedQueues();
+ Map<ServerName, List<String>> undeletedQueues = cleaner.getUnDeletedQueues();
cleaner.removeQueues(undeletedQueues);
} catch (IOException e) {
LOG.warn("Failed to clean replication zk node", e);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 5abd874..84abfeb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
@@ -147,28 +148,13 @@ public final class ReplicationPeerManager {
}
}
- private ReplicationPeerConfig copy(ReplicationPeerConfig peerConfig) {
- ReplicationPeerConfig copiedPeerConfig = new ReplicationPeerConfig();
- copiedPeerConfig.getConfiguration().putAll(peerConfig.getConfiguration());
- copiedPeerConfig.getPeerData().putAll(peerConfig.getPeerData());
- copiedPeerConfig.setTableCFsMap(peerConfig.getTableCFsMap());
- copiedPeerConfig.setNamespaces(peerConfig.getNamespaces());
- copiedPeerConfig.setExcludeTableCFsMap(peerConfig.getExcludeTableCFsMap());
- copiedPeerConfig.setExcludeNamespaces(peerConfig.getExcludeNamespaces());
- copiedPeerConfig.setBandwidth(peerConfig.getBandwidth());
- copiedPeerConfig.setReplicateAllUserTables(peerConfig.replicateAllUserTables());
- copiedPeerConfig.setClusterKey(peerConfig.getClusterKey());
- copiedPeerConfig.setReplicationEndpointImpl(peerConfig.getReplicationEndpointImpl());
- return copiedPeerConfig;
- }
-
public void addPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
throws ReplicationException {
if (peers.containsKey(peerId)) {
// this should be a retry, just return
return;
}
- ReplicationPeerConfig copiedPeerConfig = copy(peerConfig);
+ ReplicationPeerConfig copiedPeerConfig = ReplicationPeerConfig.newBuilder(peerConfig).build();
peerStorage.addPeer(peerId, copiedPeerConfig, enabled);
peers.put(peerId, new ReplicationPeerDescription(peerId, enabled, copiedPeerConfig));
}
@@ -205,13 +191,14 @@ public final class ReplicationPeerManager {
// the checking rules are too complicated here so we give up checking whether this is a retry.
ReplicationPeerDescription desc = peers.get(peerId);
ReplicationPeerConfig oldPeerConfig = desc.getPeerConfig();
- ReplicationPeerConfig newPeerConfig = copy(peerConfig);
+ ReplicationPeerConfigBuilder newPeerConfigBuilder =
+ ReplicationPeerConfig.newBuilder(peerConfig);
// we need to use the new conf to overwrite the old one.
- newPeerConfig.getConfiguration().putAll(oldPeerConfig.getConfiguration());
- newPeerConfig.getConfiguration().putAll(peerConfig.getConfiguration());
- newPeerConfig.getPeerData().putAll(oldPeerConfig.getPeerData());
- newPeerConfig.getPeerData().putAll(peerConfig.getPeerData());
-
+ newPeerConfigBuilder.putAllConfiguration(oldPeerConfig.getConfiguration());
+ newPeerConfigBuilder.putAllConfiguration(peerConfig.getConfiguration());
+ newPeerConfigBuilder.putAllConfiguration(oldPeerConfig.getConfiguration());
+ newPeerConfigBuilder.putAllConfiguration(peerConfig.getConfiguration());
+ ReplicationPeerConfig newPeerConfig = newPeerConfigBuilder.build();
peerStorage.updatePeerConfig(peerId, newPeerConfig);
peers.put(peerId, new ReplicationPeerDescription(peerId, desc.isEnabled(), newPeerConfig));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
index 5f1df44..7b62169 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
@@ -1,42 +1,43 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
- * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
- * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
- * for the specific language governing permissions and limitations under the License.
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.hadoop.hbase.replication.master;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hbase.thirdparty.com.google.common.base.Predicate;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
-import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
-
import java.io.IOException;
import java.util.Collections;
-import java.util.List;
import java.util.Set;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.zookeeper.KeeperException;
+import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hbase.thirdparty.com.google.common.base.Predicate;
+import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+
/**
* Implementation of a file cleaner that checks if a hfile is still scheduled for replication before
* deleting it from hfile archive directory.
@@ -45,7 +46,7 @@ import org.slf4j.LoggerFactory;
public class ReplicationHFileCleaner extends BaseHFileCleanerDelegate {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationHFileCleaner.class);
private ZKWatcher zkw;
- private ReplicationQueuesClient rqc;
+ private ReplicationQueueStorage rqs;
private boolean stopped = false;
@Override
@@ -60,8 +61,8 @@ public class ReplicationHFileCleaner extends BaseHFileCleanerDelegate {
try {
// The concurrently created new hfile entries in ZK may not be included in the return list,
// but they won't be deleted because they're not in the checking set.
- hfileRefs = loadHFileRefsFromPeers();
- } catch (KeeperException e) {
+ hfileRefs = rqs.getAllHFileRefs();
+ } catch (ReplicationException e) {
LOG.warn("Failed to read hfile references from zookeeper, skipping checking deletable files");
return Collections.emptyList();
}
@@ -82,37 +83,6 @@ public class ReplicationHFileCleaner extends BaseHFileCleanerDelegate {
});
}
- /**
- * Load all hfile references in all replication queues from ZK. This method guarantees to return a
- * snapshot which contains all hfile references in the zookeeper at the start of this call.
- * However, some newly created hfile references during the call may not be included.
- */
- private Set<String> loadHFileRefsFromPeers() throws KeeperException {
- Set<String> hfileRefs = Sets.newHashSet();
- List<String> listOfPeers;
- for (int retry = 0;; retry++) {
- int v0 = rqc.getHFileRefsNodeChangeVersion();
- hfileRefs.clear();
- listOfPeers = rqc.getAllPeersFromHFileRefsQueue();
- if (listOfPeers == null) {
- LOG.debug("Didn't find any peers with hfile references, won't prevent any deletions.");
- return ImmutableSet.of();
- }
- for (String id : listOfPeers) {
- List<String> peerHFileRefs = rqc.getReplicableHFiles(id);
- if (peerHFileRefs != null) {
- hfileRefs.addAll(peerHFileRefs);
- }
- }
- int v1 = rqc.getHFileRefsNodeChangeVersion();
- if (v0 == v1) {
- return hfileRefs;
- }
- LOG.debug(String.format("Replication hfile references node cversion changed from "
- + "%d to %d, retry = %d", v0, v1, retry));
- }
- }
-
@Override
public void setConf(Configuration config) {
// If either replication or replication of bulk load hfiles is disabled, keep all members null
@@ -139,17 +109,15 @@ public class ReplicationHFileCleaner extends BaseHFileCleanerDelegate {
public void setConf(Configuration conf, ZKWatcher zk) {
super.setConf(conf);
try {
- initReplicationQueuesClient(conf, zk);
+ initReplicationQueueStorage(conf, zk);
} catch (Exception e) {
LOG.error("Error while configuring " + this.getClass().getName(), e);
}
}
- private void initReplicationQueuesClient(Configuration conf, ZKWatcher zk)
- throws Exception {
+ private void initReplicationQueueStorage(Configuration conf, ZKWatcher zk) {
this.zkw = zk;
- this.rqc = ReplicationFactory.getReplicationQueuesClient(new ReplicationQueuesClientArguments(
- conf, new WarnOnlyAbortable(), zkw));
+ this.rqs = ReplicationStorageFactory.getReplicationQueueStorage(zk, conf);
}
@Override
@@ -179,25 +147,12 @@ public class ReplicationHFileCleaner extends BaseHFileCleanerDelegate {
}
try {
- hfileRefsFromQueue = loadHFileRefsFromPeers();
- } catch (KeeperException e) {
+ hfileRefsFromQueue = rqs.getAllHFileRefs();
+ } catch (ReplicationException e) {
LOG.warn("Failed to read hfile references from zookeeper, skipping checking deletable "
+ "file for " + fStat.getPath());
return false;
}
return !hfileRefsFromQueue.contains(fStat.getPath().getName());
}
-
- private static class WarnOnlyAbortable implements Abortable {
- @Override
- public void abort(String why, Throwable e) {
- LOG.warn("ReplicationHFileCleaner received abort, ignoring. Reason: " + why);
- LOG.debug(e.toString(), e);
- }
-
- @Override
- public boolean isAborted() {
- return false;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 86f98da..15aa21a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -24,16 +23,14 @@ import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,7 +46,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationLogCleaner.class);
private ZKWatcher zkw;
- private ReplicationQueuesClient replicationQueues;
+ private ReplicationQueueStorage queueStorage;
private boolean stopped = false;
private Set<String> wals;
private long readZKTimestamp = 0;
@@ -60,8 +57,8 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
try {
// The concurrently created new WALs may not be included in the return list,
// but they won't be deleted because they're not in the checking set.
- wals = replicationQueues.getAllWALs();
- } catch (KeeperException e) {
+ wals = queueStorage.getAllWALs();
+ } catch (ReplicationException e) {
LOG.warn("Failed to read zookeeper, skipping checking deletable files");
wals = null;
}
@@ -110,9 +107,7 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
super.setConf(conf);
try {
this.zkw = zk;
- this.replicationQueues = ReplicationFactory.getReplicationQueuesClient(
- new ReplicationQueuesClientArguments(conf, new WarnOnlyAbortable(), zkw));
- this.replicationQueues.init();
+ this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zk, conf);
} catch (Exception e) {
LOG.error("Error while configuring " + this.getClass().getName(), e);
}
@@ -132,18 +127,4 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
public boolean isStopped() {
return this.stopped;
}
-
- private static class WarnOnlyAbortable implements Abortable {
-
- @Override
- public void abort(String why, Throwable e) {
- LOG.warn("ReplicationLogCleaner received abort, ignoring. Reason: " + why);
- LOG.debug(e.toString(), e);
- }
-
- @Override
- public boolean isAborted() {
- return false;
- }
- }
}
[05/50] [abbrv] hbase git commit: HBASE-19811 Fix findbugs and
error-prone warnings in hbase-server (branch-2)
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java
index 189b37f..2adcf9e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDefaultVisLabelService.java
@@ -82,6 +82,7 @@ public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibili
public void testAddLabels() throws Throwable {
PrivilegedExceptionAction<VisibilityLabelsResponse> action =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
String[] labels = { "L1", SECRET, "L2", "invalid~", "L3" };
VisibilityLabelsResponse response = null;
@@ -122,6 +123,7 @@ public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibili
do {
PrivilegedExceptionAction<VisibilityLabelsResponse> action =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, "ABC", "XYZ" };
try (Connection conn = ConnectionFactory.createConnection(conf)) {
@@ -170,6 +172,7 @@ public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibili
public void testListLabels() throws Throwable {
PrivilegedExceptionAction<ListLabelsResponse> action =
new PrivilegedExceptionAction<ListLabelsResponse>() {
+ @Override
public ListLabelsResponse run() throws Exception {
ListLabelsResponse response = null;
try (Connection conn = ConnectionFactory.createConnection(conf)) {
@@ -200,6 +203,7 @@ public class TestVisibilityLabelsWithDefaultVisLabelService extends TestVisibili
public void testListLabelsWithRegEx() throws Throwable {
PrivilegedExceptionAction<ListLabelsResponse> action =
new PrivilegedExceptionAction<ListLabelsResponse>() {
+ @Override
public ListLabelsResponse run() throws Exception {
ListLabelsResponse response = null;
try (Connection conn = ConnectionFactory.createConnection(conf)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
index 0a7d918..9d536fe 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
@@ -17,6 +17,17 @@
*/
package org.apache.hadoop.hbase.security.visibility;
+import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
@@ -42,14 +53,10 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.SecurityTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.DefaultEnvironmentEdge;
-import org.apache.hadoop.hbase.util.EnvironmentEdge;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Threads;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -57,17 +64,6 @@ import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
/**
* Tests visibility labels with deletes
*/
@@ -205,7 +201,7 @@ public class TestVisibilityLabelsWithDeletes {
public void testVisibilityLabelsWithDeleteFamilyVersion() throws Exception {
setAuths();
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
- long[] ts = new long[] { 123l, 125l };
+ long[] ts = new long[] { 123L, 125L };
try (Table table = createTableAndWriteDataWithLabels(ts,
CONFIDENTIAL + "|" + TOPSECRET, SECRET)) {
PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
@@ -215,7 +211,7 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL));
- d.addFamilyVersion(fam, 123l);
+ d.addFamilyVersion(fam, 123L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -243,7 +239,7 @@ public class TestVisibilityLabelsWithDeletes {
public void testVisibilityLabelsWithDeleteColumnExactVersion() throws Exception {
setAuths();
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
- long[] ts = new long[] { 123l, 125l };
+ long[] ts = new long[] { 123L, 125L };
try (Table table = createTableAndWriteDataWithLabels(ts,
CONFIDENTIAL + "|" + TOPSECRET, SECRET);) {
PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
@@ -253,7 +249,7 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(TOPSECRET + "|" + CONFIDENTIAL));
- d.addColumn(fam, qual, 123l);
+ d.addColumn(fam, qual, 123L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -291,7 +287,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" +
SECRET + "&" + TOPSECRET+")"));
- d.addColumns(fam, qual, 125l);
+ d.addColumns(fam, qual, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -313,17 +309,17 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -508,7 +504,7 @@ public class TestVisibilityLabelsWithDeletes {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 1);
+ assertEquals(1, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -529,7 +525,7 @@ public class TestVisibilityLabelsWithDeletes {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 0);
+ assertEquals(0, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -566,7 +562,7 @@ public class TestVisibilityLabelsWithDeletes {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 0);
+ assertEquals(0, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -586,7 +582,7 @@ public class TestVisibilityLabelsWithDeletes {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 0);
+ assertEquals(0, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -622,7 +618,7 @@ public class TestVisibilityLabelsWithDeletes {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 1);
+ assertEquals(1, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -643,7 +639,7 @@ public class TestVisibilityLabelsWithDeletes {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 0);
+ assertEquals(0, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -680,7 +676,7 @@ public class TestVisibilityLabelsWithDeletes {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 0);
+ assertEquals(0, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -700,7 +696,7 @@ public class TestVisibilityLabelsWithDeletes {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 0);
+ assertEquals(0, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -737,7 +733,7 @@ public class TestVisibilityLabelsWithDeletes {
ResultScanner scanner = table.getScanner(s);
// The delete would not be able to apply it because of visibility mismatch
Result[] next = scanner.next(3);
- assertEquals(next.length, 1);
+ assertEquals(1, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -759,7 +755,7 @@ public class TestVisibilityLabelsWithDeletes {
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
// this will alone match
- assertEquals(next.length, 0);
+ assertEquals(0, next.length);
} catch (Throwable t) {
throw new IOException(t);
}
@@ -810,7 +806,7 @@ public class TestVisibilityLabelsWithDeletes {
s.setAuthorizations(new Authorizations(SECRET));
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 1);
+ assertEquals(1, next.length);
put = new Put(Bytes.toBytes("row1"));
put.addColumn(fam, qual, value1);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
@@ -836,13 +832,13 @@ public class TestVisibilityLabelsWithDeletes {
s.setAuthorizations(new Authorizations(CONFIDENTIAL));
scanner = table.getScanner(s);
next = scanner.next(3);
- assertEquals(next.length, 1);
+ assertEquals(1, next.length);
s = new Scan();
s.setMaxVersions(5);
s.setAuthorizations(new Authorizations(SECRET));
scanner = table.getScanner(s);
Result[] next1 = scanner.next(3);
- assertEquals(next1.length, 0);
+ assertEquals(0, next1.length);
}
}
@@ -886,7 +882,7 @@ public class TestVisibilityLabelsWithDeletes {
s.setAuthorizations(new Authorizations(SECRET));
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 1);
+ assertEquals(1, next.length);
put = new Put(Bytes.toBytes("row1"));
put.addColumn(fam, qual, value1);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
@@ -912,13 +908,13 @@ public class TestVisibilityLabelsWithDeletes {
s.setAuthorizations(new Authorizations(CONFIDENTIAL));
scanner = table.getScanner(s);
next = scanner.next(3);
- assertEquals(next.length, 1);
+ assertEquals(1, next.length);
s = new Scan();
s.setMaxVersions(5);
s.setAuthorizations(new Authorizations(SECRET));
scanner = table.getScanner(s);
Result[] next1 = scanner.next(3);
- assertEquals(next1.length, 0);
+ assertEquals(0, next1.length);
}
}
@@ -933,11 +929,11 @@ public class TestVisibilityLabelsWithDeletes {
hBaseAdmin.createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 123l, value);
+ put.addColumn(fam, qual, 123L, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
table.put(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 124l, value1);
+ put.addColumn(fam, qual, 124L, value1);
put.setCellVisibility(new CellVisibility(SECRET));
table.put(put);
PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
@@ -947,7 +943,7 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(SECRET));
- d.addColumns(fam, qual, 126l);
+ d.addColumns(fam, qual, 126L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -957,7 +953,7 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(CONFIDENTIAL));
- d.addColumn(fam, qual, 123l);
+ d.addColumn(fam, qual, 123L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -971,7 +967,7 @@ public class TestVisibilityLabelsWithDeletes {
s.setAuthorizations(new Authorizations(CONFIDENTIAL, SECRET));
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 0);
+ assertEquals(0, next.length);
}
}
@Test
@@ -987,11 +983,11 @@ public class TestVisibilityLabelsWithDeletes {
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put1 = new Put(Bytes.toBytes("row1"));
- put1.addColumn(fam, qual, 123l, value);
+ put1.addColumn(fam, qual, 123L, value);
put1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
Put put2 = new Put(Bytes.toBytes("row1"));
- put2.addColumn(fam, qual, 123l, value1);
+ put2.addColumn(fam, qual, 123L, value1);
put2.setCellVisibility(new CellVisibility(SECRET));
table.put(createList(put1, put2));
@@ -1000,7 +996,7 @@ public class TestVisibilityLabelsWithDeletes {
s.setAuthorizations(new Authorizations(CONFIDENTIAL, SECRET));
ResultScanner scanner = table.getScanner(s);
- assertEquals(scanner.next(3).length, 1);
+ assertEquals(1, scanner.next(3).length);
scanner.close();
PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
@@ -1010,7 +1006,7 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(CONFIDENTIAL));
- d.addColumn(fam, qual, 123l);
+ d.addColumn(fam, qual, 123L);
table.delete(d);
}
@@ -1018,7 +1014,7 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(SECRET));
- d.addColumn(fam, qual, 123l);
+ d.addColumn(fam, qual, 123L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -1031,7 +1027,7 @@ public class TestVisibilityLabelsWithDeletes {
s.setMaxVersions(5);
s.setAuthorizations(new Authorizations(CONFIDENTIAL));
scanner = table.getScanner(s);
- assertEquals(scanner.next(3).length, 0);
+ assertEquals(0, scanner.next(3).length);
scanner.close();
}
}
@@ -1126,22 +1122,22 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -1162,36 +1158,36 @@ public class TestVisibilityLabelsWithDeletes {
List<Put> puts = new ArrayList<>(5);
Put put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 123l, value);
+ put.addColumn(fam, qual, 123L, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 124l, value);
+ put.addColumn(fam, qual, 124L, value);
put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 125l, value);
+ put.addColumn(fam, qual, 125L, value);
put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 126l, value);
+ put.addColumn(fam, qual, 126L, value);
put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 127l, value);
+ put.addColumn(fam, qual, 127L, value);
put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
puts.add(put);
TEST_UTIL.getAdmin().flush(tableName);
put = new Put(Bytes.toBytes("row2"));
- put.addColumn(fam, qual, 127l, value);
+ put.addColumn(fam, qual, 127L, value);
put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|(" + TOPSECRET
+ "&" + SECRET + ")"));
puts.add(put);
@@ -1212,28 +1208,28 @@ public class TestVisibilityLabelsWithDeletes {
List<Put> puts = new ArrayList<>(5);
Put put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 123l, value);
+ put.addColumn(fam, qual, 123L, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 124l, value);
+ put.addColumn(fam, qual, 124L, value);
put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 125l, value);
+ put.addColumn(fam, qual, 125L, value);
put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual1, 126l, value);
+ put.addColumn(fam, qual1, 126L, value);
put.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual2, 127l, value);
+ put.addColumn(fam, qual2, 127L, value);
put.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
puts.add(put);
@@ -1253,23 +1249,23 @@ public class TestVisibilityLabelsWithDeletes {
hBaseAdmin.createTable(desc);
List<Put> puts = new ArrayList<>(5);
Put put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 123l, value);
+ put.addColumn(fam, qual, 123L, value);
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 124l, value);
+ put.addColumn(fam, qual, 124L, value);
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 125l, value);
+ put.addColumn(fam, qual, 125L, value);
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 126l, value);
+ put.addColumn(fam, qual, 126L, value);
puts.add(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 127l, value);
+ put.addColumn(fam, qual, 127L, value);
puts.add(put);
Table table = TEST_UTIL.getConnection().getTable(tableName);
@@ -1278,7 +1274,7 @@ public class TestVisibilityLabelsWithDeletes {
TEST_UTIL.getAdmin().flush(tableName);
put = new Put(Bytes.toBytes("row2"));
- put.addColumn(fam, qual, 127l, value);
+ put.addColumn(fam, qual, 127L, value);
table.put(put);
return table;
@@ -1300,7 +1296,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" +
SECRET + "&" + TOPSECRET+")"));
- d.addColumn(fam, qual, 125l);
+ d.addColumn(fam, qual, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -1322,27 +1318,27 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -1386,22 +1382,22 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -1417,7 +1413,7 @@ public class TestVisibilityLabelsWithDeletes {
try (Table table = doPuts(tableName)) {
TEST_UTIL.getAdmin().flush(tableName);
Put put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 128l, value);
+ put.addColumn(fam, qual, 128L, value);
put.setCellVisibility(new CellVisibility(TOPSECRET));
table.put(put);
PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
@@ -1449,27 +1445,27 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 128l);
+ assertEquals(128L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -1477,7 +1473,7 @@ public class TestVisibilityLabelsWithDeletes {
current.getRowLength(), row2, 0, row2.length));
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 129l, value);
+ put.addColumn(fam, qual, 129L, value);
put.setCellVisibility(new CellVisibility(SECRET));
table.put(put);
@@ -1493,7 +1489,7 @@ public class TestVisibilityLabelsWithDeletes {
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 129l);
+ assertEquals(129L, current.getTimestamp());
}
}
@Test
@@ -1521,7 +1517,7 @@ public class TestVisibilityLabelsWithDeletes {
SUPERUSER.runAs(actiona);
TEST_UTIL.getAdmin().flush(tableName);
Put put = new Put(Bytes.toBytes("row3"));
- put.addColumn(fam, qual, 127l, value);
+ put.addColumn(fam, qual, 127L, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE));
table.put(put);
TEST_UTIL.getAdmin().flush(tableName);
@@ -1539,22 +1535,22 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -1598,12 +1594,12 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -1623,7 +1619,7 @@ public class TestVisibilityLabelsWithDeletes {
public Void run() throws Exception {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
- d.addColumns(fam, qual, 125l);
+ d.addColumns(fam, qual, 125L);
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
table.delete(d);
@@ -1647,24 +1643,24 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(),
current.getQualifierLength(), qual1, 0, qual1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(),
current.getQualifierLength(), qual2, 0, qual2.length));
}
@@ -1681,11 +1677,11 @@ public class TestVisibilityLabelsWithDeletes {
hBaseAdmin.createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual1, 125l, value);
+ put.addColumn(fam, qual1, 125L, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
table.put(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual1, 126l, value);
+ put.addColumn(fam, qual1, 126L, value);
put.setCellVisibility(new CellVisibility(SECRET));
table.put(put);
TEST_UTIL.getAdmin().flush(tableName);
@@ -1694,11 +1690,11 @@ public class TestVisibilityLabelsWithDeletes {
public Void run() throws Exception {
Delete d1 = new Delete(row1);
d1.setCellVisibility(new CellVisibility(SECRET));
- d1.addColumns(fam, qual, 126l);
+ d1.addColumns(fam, qual, 126L);
Delete d2 = new Delete(row1);
d2.setCellVisibility(new CellVisibility(CONFIDENTIAL));
- d2.addColumns(fam, qual1, 125l);
+ d2.addColumns(fam, qual1, 125L);
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
@@ -1715,7 +1711,7 @@ public class TestVisibilityLabelsWithDeletes {
s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 1);
+ assertEquals(1, next.length);
}
}
@Test
@@ -1729,11 +1725,11 @@ public class TestVisibilityLabelsWithDeletes {
hBaseAdmin.createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual1, 125l, value);
+ put.addColumn(fam, qual1, 125L, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
table.put(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual1, 126l, value);
+ put.addColumn(fam, qual1, 126L, value);
put.setCellVisibility(new CellVisibility(SECRET));
table.put(put);
TEST_UTIL.getAdmin().flush(tableName);
@@ -1742,11 +1738,11 @@ public class TestVisibilityLabelsWithDeletes {
public Void run() throws Exception {
Delete d1 = new Delete(row1);
d1.setCellVisibility(new CellVisibility(SECRET));
- d1.addColumns(fam, qual, 126l);
+ d1.addColumns(fam, qual, 126L);
Delete d2 = new Delete(row1);
d2.setCellVisibility(new CellVisibility(CONFIDENTIAL));
- d2.addColumns(fam, qual1, 126l);
+ d2.addColumns(fam, qual1, 126L);
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
@@ -1763,7 +1759,7 @@ public class TestVisibilityLabelsWithDeletes {
s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
- assertEquals(next.length, 1);
+ assertEquals(1, next.length);
}
}
@Test
@@ -1838,27 +1834,27 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -1881,7 +1877,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|("
+ SECRET + "&" + TOPSECRET + ")"));
- d.addFamily(fam, 126l);
+ d.addFamily(fam, 126L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -1903,17 +1899,17 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -1936,7 +1932,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" +
SECRET + "&" + TOPSECRET+")"));
- d.addFamily(fam, 126l);
+ d.addFamily(fam, 126L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -1948,7 +1944,7 @@ public class TestVisibilityLabelsWithDeletes {
TEST_UTIL.getAdmin().flush(tableName);
Put put = new Put(Bytes.toBytes("row3"));
- put.addColumn(fam, qual, 127l, value);
+ put.addColumn(fam, qual, 127L, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE));
table.put(put);
TEST_UTIL.getAdmin().flush(tableName);
@@ -1966,7 +1962,7 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -1989,7 +1985,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|("
+ TOPSECRET + "&" + SECRET+")"));
- d.addFamily(fam, 125l);
+ d.addFamily(fam, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2010,22 +2006,22 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -2041,7 +2037,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
- d.addFamily(fam, 127l);
+ d.addFamily(fam, 127L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2061,18 +2057,18 @@ public class TestVisibilityLabelsWithDeletes {
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row2, 0, row2.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
}
}
@@ -2100,11 +2096,11 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(CONFIDENTIAL));
- d.addFamilyVersion(fam, 123l);
+ d.addFamilyVersion(fam, 123L);
table.delete(d);
d = new Delete(row1);
d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
- d.addFamilyVersion(fam, 125l);
+ d.addFamilyVersion(fam, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2126,17 +2122,17 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
}
}
@@ -2153,11 +2149,11 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET + ")"));
- d.addColumn(fam, qual, 126l);
+ d.addColumn(fam, qual, 126L);
table.delete(d);
d = new Delete(row1);
d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
- d.addFamilyVersion(fam, 125l);
+ d.addFamilyVersion(fam, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2179,17 +2175,17 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
// Issue 2nd delete
actiona = new PrivilegedExceptionAction<Void>() {
@Override
@@ -2218,12 +2214,12 @@ public class TestVisibilityLabelsWithDeletes {
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
}
}
@@ -2258,7 +2254,7 @@ public class TestVisibilityLabelsWithDeletes {
d = new Delete(row1);
d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
- d.addFamilyVersion(fam, 125l);
+ d.addFamilyVersion(fam, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2280,17 +2276,17 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
// Issue 2nd delete
actiona = new PrivilegedExceptionAction<Void>() {
@Override
@@ -2319,12 +2315,12 @@ public class TestVisibilityLabelsWithDeletes {
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
}
}
@@ -2340,7 +2336,7 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
- d.addColumn(fam, qual, 125l);
+ d.addColumn(fam, qual, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2361,22 +2357,22 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -2392,7 +2388,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
- d.addColumn(fam, qual, 127l);
+ d.addColumn(fam, qual, 127L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2412,23 +2408,23 @@ public class TestVisibilityLabelsWithDeletes {
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row2, 0, row2.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
}
}
@@ -2446,7 +2442,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")" +
"|(" + TOPSECRET + "&" + SECRET + ")"));
- d.addColumn(fam, qual, 127l);
+ d.addColumn(fam, qual, 127L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2467,22 +2463,22 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -2497,7 +2493,7 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
- d.addColumn(fam, qual, 127l);
+ d.addColumn(fam, qual, 127L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2517,28 +2513,28 @@ public class TestVisibilityLabelsWithDeletes {
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row2, 0, row2.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
}
}
@Test
@@ -2556,7 +2552,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|("
+ TOPSECRET + "&" + SECRET+")"));
- d.addColumn(fam, qual, 125l);
+ d.addColumn(fam, qual, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2577,27 +2573,27 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -2613,7 +2609,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
- d.addColumn(fam, qual, 127l);
+ d.addColumn(fam, qual, 127L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2633,28 +2629,28 @@ public class TestVisibilityLabelsWithDeletes {
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row2, 0, row2.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
}
}
@Test
@@ -2671,7 +2667,7 @@ public class TestVisibilityLabelsWithDeletes {
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
- d.addColumn(fam, qual, 125l);
+ d.addColumn(fam, qual, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2692,22 +2688,22 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -2723,7 +2719,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
- d.addFamily(fam, 124l);
+ d.addFamily(fam, 124L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2743,18 +2739,18 @@ public class TestVisibilityLabelsWithDeletes {
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row2, 0, row2.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
}
}
@@ -2789,7 +2785,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|("
+ TOPSECRET + "&" + SECRET+")"));
- d.addColumns(fam, qual, 125l);
+ d.addColumns(fam, qual, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2810,22 +2806,22 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -2841,7 +2837,7 @@ public class TestVisibilityLabelsWithDeletes {
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility("(" + CONFIDENTIAL + "&" + PRIVATE + ")|("
+ TOPSECRET + "&" + SECRET+")"));
- d.addColumn(fam, qual, 127l);
+ d.addColumn(fam, qual, 127L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2861,17 +2857,17 @@ public class TestVisibilityLabelsWithDeletes {
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -2892,7 +2888,7 @@ public class TestVisibilityLabelsWithDeletes {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
- d.addColumn(fam, qual, 125l);
+ d.addColumn(fam, qual, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2916,7 +2912,7 @@ public class TestVisibilityLabelsWithDeletes {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
- d.addColumns(fam, qual, 125l);
+ d.addColumns(fam, qual, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -2941,7 +2937,7 @@ public class TestVisibilityLabelsWithDeletes {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
- d.addFamily(fam, 125l);
+ d.addFamily(fam, 125L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -3016,7 +3012,7 @@ public class TestVisibilityLabelsWithDeletes {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
- d.addFamilyVersion(fam, 126l);
+ d.addFamilyVersion(fam, 126L);
table.delete(d);
} catch (Throwable t) {
throw new IOException(t);
@@ -3043,27 +3039,27 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 127l);
+ assertEquals(127L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 126l);
+ assertEquals(126L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 125l);
+ assertEquals(125L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
cellScanner = next[1].cellScanner();
cellScanner.advance();
current = cellScanner.current();
@@ -3083,11 +3079,11 @@ public class TestVisibilityLabelsWithDeletes {
hBaseAdmin.createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 123l, value);
+ put.addColumn(fam, qual, 123L, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
table.put(put);
put = new Put(Bytes.toBytes("row1"));
- put.addColumn(fam, qual, 124l, value);
+ put.addColumn(fam, qual, 124L, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "|" + PRIVATE));
table.put(put);
TEST_UTIL.getAdmin().flush(tableName);
@@ -3097,7 +3093,7 @@ public class TestVisibilityLabelsWithDeletes {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Delete d = new Delete(row1);
- d.addColumn(fam, qual, 124l);
+ d.addColumn(fam, qual, 124L);
d.setCellVisibility(new CellVisibility(PRIVATE ));
table.delete(d);
} catch (Throwable t) {
@@ -3120,12 +3116,12 @@ public class TestVisibilityLabelsWithDeletes {
Cell current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 124l);
+ assertEquals(124L, current.getTimestamp());
cellScanner.advance();
current = cellScanner.current();
assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
current.getRowLength(), row1, 0, row1.length));
- assertEquals(current.getTimestamp(), 123l);
+ assertEquals(123L, current.getTimestamp());
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithSLGStack.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithSLGStack.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithSLGStack.java
index b6a1c6d..b82d503 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithSLGStack.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithSLGStack.java
@@ -107,6 +107,7 @@ public class TestVisibilityLabelsWithSLGStack {
private static void addLabels() throws Exception {
PrivilegedExceptionAction<VisibilityLabelsResponse> action =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
String[] labels = { SECRET, CONFIDENTIAL };
try (Connection conn = ConnectionFactory.createConnection(conf)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLablesWithGroups.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLablesWithGroups.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLablesWithGroups.java
index 9f24f6c..906b9d5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLablesWithGroups.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLablesWithGroups.java
@@ -96,6 +96,7 @@ public class TestVisibilityLablesWithGroups {
// Set up for the test
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
VisibilityClient.addLabels(conn, new String[] { SECRET, CONFIDENTIAL });
@@ -116,6 +117,7 @@ public class TestVisibilityLablesWithGroups {
TEST_UTIL.createTable(tableName, CF);
// put the data.
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
@@ -137,6 +139,7 @@ public class TestVisibilityLablesWithGroups {
// 'admin' user is part of 'supergroup', thus can see all the cells.
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
@@ -178,6 +181,7 @@ public class TestVisibilityLablesWithGroups {
// Get testgroup's labels.
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
GetAuthsResponse authsResponse = null;
try (Connection conn = ConnectionFactory.createConnection(conf)) {
@@ -197,6 +201,7 @@ public class TestVisibilityLablesWithGroups {
// Test that test user can see what 'testgroup' has been authorized to.
TESTUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
@@ -281,6 +286,7 @@ public class TestVisibilityLablesWithGroups {
// Clear 'testgroup' of CONFIDENTIAL label.
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
VisibilityLabelsResponse response = null;
try (Connection conn = ConnectionFactory.createConnection(conf)) {
@@ -295,6 +301,7 @@ public class TestVisibilityLablesWithGroups {
// Get testgroup's labels. No label is returned.
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
GetAuthsResponse authsResponse = null;
try (Connection conn = ConnectionFactory.createConnection(conf)) {
@@ -313,6 +320,7 @@ public class TestVisibilityLablesWithGroups {
// Test that test user cannot see the cells with the labels anymore.
TESTUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityWithCheckAuths.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityWithCheckAuths.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityWithCheckAuths.java
index fa88a53..ebf38a5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityWithCheckAuths.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityWithCheckAuths.java
@@ -136,7 +136,7 @@ public class TestVisibilityWithCheckAuths {
Table table = connection.getTable(tableName)) {
Put p = new Put(row1);
p.setCellVisibility(new CellVisibility(PUBLIC + "&" + TOPSECRET));
- p.addColumn(fam, qual, 125l, value);
+ p.addColumn(fam, qual, 125L, value);
table.put(p);
Assert.fail("Testcase should fail with AccesDeniedException");
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestWithDisabledAuthorization.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestWithDisabledAuthorization.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestWithDisabledAuthorization.java
index 43ba304..7a55584 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestWithDisabledAuthorization.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestWithDisabledAuthorization.java
@@ -209,25 +209,25 @@ public class TestWithDisabledAuthorization {
s.setAuthorizations(new Authorizations());
try (ResultScanner scanner = t.getScanner(s)) {
Result[] next = scanner.next(10);
- assertEquals(next.length, 4);
+ assertEquals(4, next.length);
}
s = new Scan();
s.setAuthorizations(new Authorizations(SECRET));
try (ResultScanner scanner = t.getScanner(s)) {
Result[] next = scanner.next(10);
- assertEquals(next.length, 4);
+ assertEquals(4, next.length);
}
s = new Scan();
s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
try (ResultScanner scanner = t.getScanner(s)) {
Result[] next = scanner.next(10);
- assertEquals(next.length, 4);
+ assertEquals(4, next.length);
}
s = new Scan();
s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
try (ResultScanner scanner = t.getScanner(s)) {
Result[] next = scanner.next(10);
- assertEquals(next.length, 4);
+ assertEquals(4, next.length);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index a2c015c..9b02096 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -313,6 +313,7 @@ public class TestFlushSnapshotFromClient {
// Merge two regions
List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
Collections.sort(regions, new Comparator<HRegionInfo>() {
+ @Override
public int compare(HRegionInfo r1, HRegionInfo r2) {
return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
}
@@ -354,6 +355,7 @@ public class TestFlushSnapshotFromClient {
// Merge two regions
List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
Collections.sort(regions, new Comparator<HRegionInfo>() {
+ @Override
public int compare(HRegionInfo r1, HRegionInfo r2) {
return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/MapreduceTestingShim.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/MapreduceTestingShim.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/MapreduceTestingShim.java
index 3f7d441..07fdcd2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/MapreduceTestingShim.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/MapreduceTestingShim.java
@@ -77,6 +77,7 @@ abstract public class MapreduceTestingShim {
}
private static class MapreduceV1Shim extends MapreduceTestingShim {
+ @Override
public JobContext newJobContext(Configuration jobConf) throws IOException {
// Implementing:
// return new JobContext(jobConf, new JobID());
@@ -105,6 +106,7 @@ abstract public class MapreduceTestingShim {
}
}
+ @Override
public JobConf obtainJobConf(MiniMRCluster cluster) {
if (cluster == null) return null;
try {
@@ -129,6 +131,7 @@ abstract public class MapreduceTestingShim {
};
private static class MapreduceV2Shim extends MapreduceTestingShim {
+ @Override
public JobContext newJobContext(Configuration jobConf) {
return newJob(jobConf);
}
@@ -147,6 +150,7 @@ abstract public class MapreduceTestingShim {
}
}
+ @Override
public JobConf obtainJobConf(MiniMRCluster cluster) {
try {
Method meth = MiniMRCluster.class.getMethod("getJobTrackerConf", emptyParam);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
index 8b1c96e..a3ca323 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFiles.java
@@ -681,7 +681,7 @@ public class TestLoadIncrementalHFiles {
compare[1] = "r".getBytes();
compare[2] = "u".getBytes();
- assertEquals(keysArray.length, 3);
+ assertEquals(3, keysArray.length);
for (int row = 0; row < keysArray.length; row++) {
assertArrayEquals(keysArray[row], compare[row]);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFilesSplitRecovery.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFilesSplitRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFilesSplitRecovery.java
index 56c9eca..2d32a3c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFilesSplitRecovery.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestLoadIncrementalHFilesSplitRecovery.java
@@ -420,7 +420,7 @@ public class TestLoadIncrementalHFilesSplitRecovery {
// check that data was loaded
// The three expected attempts are 1) failure because need to split, 2)
// load of split top 3) load of split bottom
- assertEquals(attemptedCalls.get(), 3);
+ assertEquals(3, attemptedCalls.get());
assertExpectedTable(table, ROWCOUNT, 2);
}
}
[23/50] [abbrv] hbase git commit: HBASE-19525 RS side changes for
moving peer modification from zk watcher to procedure
Posted by zh...@apache.org.
HBASE-19525 RS side changes for moving peer modification from zk watcher to procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8fa2fc5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8fa2fc5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8fa2fc5
Branch: refs/heads/HBASE-19397-branch-2
Commit: f8fa2fc5160ab60ffaba1cdf2fd6d5f3e35ee88a
Parents: bcb79d5
Author: huzheng <op...@gmail.com>
Authored: Wed Dec 20 10:47:18 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../hadoop/hbase/protobuf/ProtobufUtil.java | 11 +-
.../hbase/shaded/protobuf/ProtobufUtil.java | 13 +-
.../hbase/replication/ReplicationListener.java | 14 --
.../hbase/replication/ReplicationPeer.java | 28 ++-
.../replication/ReplicationPeerZKImpl.java | 186 ++++-----------
.../replication/ReplicationPeersZKImpl.java | 19 +-
.../replication/ReplicationTrackerZKImpl.java | 73 +-----
.../regionserver/ReplicationSourceService.java | 9 +-
.../handler/RSProcedureHandler.java | 3 +
.../replication/BaseReplicationEndpoint.java | 2 +-
.../regionserver/PeerProcedureHandler.java | 38 ++++
.../regionserver/PeerProcedureHandlerImpl.java | 81 +++++++
.../regionserver/RefreshPeerCallable.java | 39 +++-
.../replication/regionserver/Replication.java | 10 +
.../regionserver/ReplicationSource.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 37 ++-
.../TestReplicationAdminUsingProcedure.java | 226 +++++++++++++++++++
.../replication/DummyModifyPeerProcedure.java | 48 ----
.../TestDummyModifyPeerProcedure.java | 80 -------
.../TestReplicationTrackerZKImpl.java | 51 -----
.../TestReplicationSourceManager.java | 32 ++-
21 files changed, 535 insertions(+), 473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 9739254..f500088 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.protobuf;
+import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC;
+
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -203,7 +205,7 @@ public final class ProtobufUtil {
* byte array that is <code>bytes.length</code> plus {@link ProtobufMagic#PB_MAGIC}.length.
*/
public static byte [] prependPBMagic(final byte [] bytes) {
- return Bytes.add(ProtobufMagic.PB_MAGIC, bytes);
+ return Bytes.add(PB_MAGIC, bytes);
}
/**
@@ -228,10 +230,11 @@ public final class ProtobufUtil {
* @param bytes bytes to check
* @throws DeserializationException if we are missing the pb magic prefix
*/
- public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
+ public static void expectPBMagicPrefix(final byte[] bytes) throws DeserializationException {
if (!isPBMagicPrefix(bytes)) {
- throw new DeserializationException("Missing pb magic " +
- Bytes.toString(ProtobufMagic.PB_MAGIC) + " prefix");
+ String bytesPrefix = bytes == null ? "null" : Bytes.toStringBinary(bytes, 0, PB_MAGIC.length);
+ throw new DeserializationException(
+ "Missing pb magic " + Bytes.toString(PB_MAGIC) + " prefix, bytes: " + bytesPrefix);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index b26802f..5e6b3db 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.shaded.protobuf;
+import static org.apache.hadoop.hbase.protobuf.ProtobufMagic.PB_MAGIC;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -280,7 +282,7 @@ public final class ProtobufUtil {
* byte array that is <code>bytes.length</code> plus {@link ProtobufMagic#PB_MAGIC}.length.
*/
public static byte [] prependPBMagic(final byte [] bytes) {
- return Bytes.add(ProtobufMagic.PB_MAGIC, bytes);
+ return Bytes.add(PB_MAGIC, bytes);
}
/**
@@ -305,10 +307,11 @@ public final class ProtobufUtil {
* @param bytes bytes to check
* @throws DeserializationException if we are missing the pb magic prefix
*/
- public static void expectPBMagicPrefix(final byte [] bytes) throws DeserializationException {
+ public static void expectPBMagicPrefix(final byte[] bytes) throws DeserializationException {
if (!isPBMagicPrefix(bytes)) {
- throw new DeserializationException("Missing pb magic " +
- Bytes.toString(ProtobufMagic.PB_MAGIC) + " prefix");
+ String bytesPrefix = bytes == null ? "null" : Bytes.toStringBinary(bytes, 0, PB_MAGIC.length);
+ throw new DeserializationException(
+ "Missing pb magic " + Bytes.toString(PB_MAGIC) + " prefix" + ", bytes: " + bytesPrefix);
}
}
@@ -1946,7 +1949,7 @@ public final class ProtobufUtil {
public static byte [] toDelimitedByteArray(final Message m) throws IOException {
// Allocate arbitrary big size so we avoid resizing.
ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
- baos.write(ProtobufMagic.PB_MAGIC);
+ baos.write(PB_MAGIC);
m.writeDelimitedTo(baos);
return baos.toByteArray();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
index 3edfcf9..f040bf9 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
@@ -18,8 +18,6 @@
*/
package org.apache.hadoop.hbase.replication;
-import java.util.List;
-
import org.apache.yetus.audience.InterfaceAudience;
/**
@@ -36,16 +34,4 @@ public interface ReplicationListener {
* @param regionServer the removed region server
*/
public void regionServerRemoved(String regionServer);
-
- /**
- * A peer cluster has been removed (i.e. unregistered) from replication.
- * @param peerId The peer id of the cluster that has been removed
- */
- public void peerRemoved(String peerId);
-
- /**
- * The list of registered peer clusters has changed.
- * @param peerIds A list of all currently registered peer clusters
- */
- public void peerListChanged(List<String> peerIds);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index 97e2ddb..b66d76d 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -55,12 +55,27 @@ public interface ReplicationPeer {
public ReplicationPeerConfig getPeerConfig();
/**
- * Returns the state of the peer
+ * Get the peer config object. if loadFromBackingStore is true, it will load from backing store
+ * directly and update its load peer config. otherwise, just return the local cached peer config.
+ * @return the ReplicationPeerConfig for this peer
+ */
+ public ReplicationPeerConfig getPeerConfig(boolean loadFromBackingStore)
+ throws ReplicationException;
+
+ /**
+ * Returns the state of the peer by reading local cache.
* @return the enabled state
*/
PeerState getPeerState();
/**
+ * Returns the state of peer, if loadFromBackingStore is true, it will load from backing store
+ * directly and update its local peer state. otherwise, just return the local cached peer state.
+ * @return the enabled state
+ */
+ PeerState getPeerState(boolean loadFromBackingStore) throws ReplicationException;
+
+ /**
* Get the configuration object required to communicate with this peer
* @return configuration object
*/
@@ -84,6 +99,15 @@ public interface ReplicationPeer {
*/
public long getPeerBandwidth();
- void trackPeerConfigChanges(ReplicationPeerConfigListener listener);
+ /**
+ * Register a peer config listener to catch the peer config change event.
+ * @param listener listener to catch the peer config change event.
+ */
+ public void registerPeerConfigListener(ReplicationPeerConfigListener listener);
+ /**
+ * Notify all the registered ReplicationPeerConfigListener to update their peer config.
+ * @param newPeerConfig the new peer config.
+ */
+ public void triggerPeerConfigChange(ReplicationPeerConfig newPeerConfig);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
index 454d09c..49b9460 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
@@ -20,41 +20,39 @@ package org.apache.hadoop.hbase.replication;
import java.io.Closeable;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.log.HBaseMarkers;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-import org.apache.hadoop.hbase.zookeeper.ZKNodeTracker;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+
@InterfaceAudience.Private
public class ReplicationPeerZKImpl extends ReplicationStateZKBase
implements ReplicationPeer, Abortable, Closeable {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationPeerZKImpl.class);
- private ReplicationPeerConfig peerConfig;
+ private volatile ReplicationPeerConfig peerConfig;
private final String id;
private volatile PeerState peerState;
private volatile Map<TableName, List<String>> tableCFs = new HashMap<>();
private final Configuration conf;
- private PeerStateTracker peerStateTracker;
- private PeerConfigTracker peerConfigTracker;
+ private final List<ReplicationPeerConfigListener> peerConfigListeners;
/**
* Constructor that takes all the objects required to communicate with the specified peer, except
@@ -63,62 +61,35 @@ public class ReplicationPeerZKImpl extends ReplicationStateZKBase
* @param id string representation of this peer's identifier
* @param peerConfig configuration for the replication peer
*/
- public ReplicationPeerZKImpl(ZKWatcher zkWatcher, Configuration conf,
- String id, ReplicationPeerConfig peerConfig,
- Abortable abortable)
- throws ReplicationException {
+ public ReplicationPeerZKImpl(ZKWatcher zkWatcher, Configuration conf, String id,
+ ReplicationPeerConfig peerConfig, Abortable abortable) throws ReplicationException {
super(zkWatcher, conf, abortable);
this.conf = conf;
this.peerConfig = peerConfig;
this.id = id;
+ this.peerConfigListeners = new ArrayList<>();
}
- /**
- * start a state tracker to check whether this peer is enabled or not
- *
- * @param peerStateNode path to zk node which stores peer state
- * @throws KeeperException
- */
- public void startStateTracker(String peerStateNode)
- throws KeeperException {
- ensurePeerEnabled(peerStateNode);
- this.peerStateTracker = new PeerStateTracker(peerStateNode, zookeeper, this);
- this.peerStateTracker.start();
+ private PeerState readPeerState() throws ReplicationException {
try {
- this.readPeerStateZnode();
- } catch (DeserializationException e) {
- throw ZKUtil.convert(e);
+ byte[] data = ZKUtil.getData(zookeeper, this.getPeerStateNode(id));
+ this.peerState = isStateEnabled(data) ? PeerState.ENABLED : PeerState.DISABLED;
+ } catch (DeserializationException | KeeperException | InterruptedException e) {
+ throw new ReplicationException("Get and deserialize peer state data from zookeeper failed: ",
+ e);
}
+ return this.peerState;
}
- private void readPeerStateZnode() throws DeserializationException {
- this.peerState =
- isStateEnabled(this.peerStateTracker.getData(false))
- ? PeerState.ENABLED
- : PeerState.DISABLED;
- }
-
- /**
- * start a table-cfs tracker to listen the (table, cf-list) map change
- * @param peerConfigNode path to zk node which stores table-cfs
- * @throws KeeperException
- */
- public void startPeerConfigTracker(String peerConfigNode)
- throws KeeperException {
- this.peerConfigTracker = new PeerConfigTracker(peerConfigNode, zookeeper,
- this);
- this.peerConfigTracker.start();
- this.readPeerConfig();
- }
-
- private ReplicationPeerConfig readPeerConfig() {
+ private ReplicationPeerConfig readPeerConfig() throws ReplicationException {
try {
- byte[] data = peerConfigTracker.getData(false);
+ byte[] data = ZKUtil.getData(zookeeper, this.getPeerNode(id));
if (data != null) {
this.peerConfig = ReplicationPeerConfigUtil.parsePeerFrom(data);
}
- } catch (DeserializationException e) {
- LOG.error("", e);
+ } catch (DeserializationException | KeeperException | InterruptedException e) {
+ throw new ReplicationException("Get and deserialize peer config date from zookeeper failed: ",
+ e);
}
return this.peerConfig;
}
@@ -128,6 +99,15 @@ public class ReplicationPeerZKImpl extends ReplicationStateZKBase
return peerState;
}
+ @Override
+ public PeerState getPeerState(boolean loadFromBackingStore) throws ReplicationException {
+ if (loadFromBackingStore) {
+ return readPeerState();
+ } else {
+ return peerState;
+ }
+ }
+
/**
* Get the identifier of this peer
* @return string representation of the id (short)
@@ -146,6 +126,16 @@ public class ReplicationPeerZKImpl extends ReplicationStateZKBase
return peerConfig;
}
+ @Override
+ public ReplicationPeerConfig getPeerConfig(boolean loadFromBackingStore)
+ throws ReplicationException {
+ if (loadFromBackingStore) {
+ return readPeerConfig();
+ } else {
+ return peerConfig;
+ }
+ }
+
/**
* Get the configuration object required to communicate with this peer
* @return configuration object
@@ -180,9 +170,14 @@ public class ReplicationPeerZKImpl extends ReplicationStateZKBase
}
@Override
- public void trackPeerConfigChanges(ReplicationPeerConfigListener listener) {
- if (this.peerConfigTracker != null){
- this.peerConfigTracker.setListener(listener);
+ public void registerPeerConfigListener(ReplicationPeerConfigListener listener) {
+ this.peerConfigListeners.add(listener);
+ }
+
+ @Override
+ public void triggerPeerConfigChange(ReplicationPeerConfig newPeerConfig) {
+ for (ReplicationPeerConfigListener listener : this.peerConfigListeners) {
+ listener.peerConfigUpdated(newPeerConfig);
}
}
@@ -223,97 +218,16 @@ public class ReplicationPeerZKImpl extends ReplicationStateZKBase
private static ReplicationProtos.ReplicationState.State parseStateFrom(final byte[] bytes)
throws DeserializationException {
ProtobufUtil.expectPBMagicPrefix(bytes);
- int pblen = ProtobufUtil.lengthOfPBMagic();
+ int pbLen = ProtobufUtil.lengthOfPBMagic();
ReplicationProtos.ReplicationState.Builder builder =
ReplicationProtos.ReplicationState.newBuilder();
ReplicationProtos.ReplicationState state;
try {
- ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
+ ProtobufUtil.mergeFrom(builder, bytes, pbLen, bytes.length - pbLen);
state = builder.build();
return state.getState();
} catch (IOException e) {
throw new DeserializationException(e);
}
}
-
- /**
- * Utility method to ensure an ENABLED znode is in place; if not present, we create it.
- * @param path Path to znode to check
- * @return True if we created the znode.
- * @throws NodeExistsException
- * @throws KeeperException
- */
- private boolean ensurePeerEnabled(final String path)
- throws NodeExistsException, KeeperException {
- if (ZKUtil.checkExists(zookeeper, path) == -1) {
- // There is a race b/w PeerWatcher and ReplicationZookeeper#add method to create the
- // peer-state znode. This happens while adding a peer.
- // The peer state data is set as "ENABLED" by default.
- ZKUtil.createNodeIfNotExistsAndWatch(zookeeper, path,
- ReplicationStateZKBase.ENABLED_ZNODE_BYTES);
- return true;
- }
- return false;
- }
-
- /**
- * Tracker for state of this peer
- */
- public class PeerStateTracker extends ZKNodeTracker {
-
- public PeerStateTracker(String peerStateZNode, ZKWatcher watcher,
- Abortable abortable) {
- super(watcher, peerStateZNode, abortable);
- }
-
- @Override
- public synchronized void nodeDataChanged(String path) {
- if (path.equals(node)) {
- super.nodeDataChanged(path);
- try {
- readPeerStateZnode();
- } catch (DeserializationException e) {
- LOG.warn("Failed deserializing the content of " + path, e);
- }
- }
- }
- }
-
- /**
- * Tracker for PeerConfigNode of this peer
- */
- public class PeerConfigTracker extends ZKNodeTracker {
-
- ReplicationPeerConfigListener listener;
-
- public PeerConfigTracker(String peerConfigNode, ZKWatcher watcher,
- Abortable abortable) {
- super(watcher, peerConfigNode, abortable);
- }
-
- public synchronized void setListener(ReplicationPeerConfigListener listener){
- this.listener = listener;
- }
-
- @Override
- public synchronized void nodeCreated(String path) {
- if (path.equals(node)) {
- super.nodeCreated(path);
- ReplicationPeerConfig config = readPeerConfig();
- if (listener != null){
- listener.peerConfigUpdated(config);
- }
- }
- }
-
- @Override
- public synchronized void nodeDataChanged(String path) {
- //superclass calls nodeCreated
- if (path.equals(node)) {
- super.nodeDataChanged(path);
- }
-
- }
-
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index e4b1875..8ed0150 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -499,21 +499,12 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
}
Configuration peerConf = pair.getSecond();
- ReplicationPeerZKImpl peer = new ReplicationPeerZKImpl(zookeeper,
- peerConf, peerId, pair.getFirst(), abortable);
- try {
- peer.startStateTracker(this.getPeerStateNode(peerId));
- } catch (KeeperException e) {
- throw new ReplicationException("Error starting the peer state tracker for peerId=" +
- peerId, e);
- }
+ ReplicationPeerZKImpl peer =
+ new ReplicationPeerZKImpl(zookeeper, peerConf, peerId, pair.getFirst(), abortable);
- try {
- peer.startPeerConfigTracker(this.getPeerNode(peerId));
- } catch (KeeperException e) {
- throw new ReplicationException("Error starting the peer tableCFs tracker for peerId=" +
- peerId, e);
- }
+ // Load peer state and peer config by reading zookeeper directly.
+ peer.getPeerState(true);
+ peer.getPeerConfig(true);
return peer;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
index 9a1d9aa..2c522f6 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
@@ -48,16 +48,12 @@ public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements
private final List<ReplicationListener> listeners = new CopyOnWriteArrayList<>();
// List of all the other region servers in this cluster
private final ArrayList<String> otherRegionServers = new ArrayList<>();
- private final ReplicationPeers replicationPeers;
- public ReplicationTrackerZKImpl(ZKWatcher zookeeper,
- final ReplicationPeers replicationPeers, Configuration conf, Abortable abortable,
- Stoppable stopper) {
+ public ReplicationTrackerZKImpl(ZKWatcher zookeeper, final ReplicationPeers replicationPeers,
+ Configuration conf, Abortable abortable, Stoppable stopper) {
super(zookeeper, conf, abortable);
- this.replicationPeers = replicationPeers;
this.stopper = stopper;
this.zookeeper.registerListener(new OtherRegionServerWatcher(this.zookeeper));
- this.zookeeper.registerListener(new PeersWatcher(this.zookeeper));
}
@Override
@@ -146,71 +142,6 @@ public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements
}
/**
- * Watcher used to follow the creation and deletion of peer clusters.
- */
- public class PeersWatcher extends ZKListener {
-
- /**
- * Construct a ZooKeeper event listener.
- */
- public PeersWatcher(ZKWatcher watcher) {
- super(watcher);
- }
-
- /**
- * Called when a node has been deleted
- * @param path full path of the deleted node
- */
- @Override
- public void nodeDeleted(String path) {
- List<String> peers = refreshPeersList(path);
- if (peers == null) {
- return;
- }
- if (isPeerPath(path)) {
- String id = getZNodeName(path);
- LOG.info(path + " znode expired, triggering peerRemoved event");
- for (ReplicationListener rl : listeners) {
- rl.peerRemoved(id);
- }
- }
- }
-
- /**
- * Called when an existing node has a child node added or removed.
- * @param path full path of the node whose children have changed
- */
- @Override
- public void nodeChildrenChanged(String path) {
- List<String> peers = refreshPeersList(path);
- if (peers == null) {
- return;
- }
- LOG.info(path + " znode expired, triggering peerListChanged event");
- for (ReplicationListener rl : listeners) {
- rl.peerListChanged(peers);
- }
- }
- }
-
- /**
- * Verify if this event is meant for us, and if so then get the latest peers' list from ZK. Also
- * reset the watches.
- * @param path path to check against
- * @return A list of peers' identifiers if the event concerns this watcher, else null.
- */
- private List<String> refreshPeersList(String path) {
- if (!path.startsWith(getPeersZNode())) {
- return null;
- }
- return this.replicationPeers.getAllPeerIds();
- }
-
- private String getPeersZNode() {
- return this.peersZNode;
- }
-
- /**
* Extracts the znode name of a peer cluster from a ZK path
* @param fullPath Path to extract the id from
* @return the id or an empty string if path is invalid
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
index 835ffbf..a82fa3d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import org.apache.hadoop.hbase.replication.regionserver.PeerProcedureHandler;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
@@ -29,8 +30,14 @@ import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
@InterfaceAudience.Private
public interface ReplicationSourceService extends ReplicationService {
/**
- * Returns a WALObserver for the service. This is needed to
+ * Returns a WALObserver for the service. This is needed to
* observe log rolls and log archival events.
*/
WALActionsListener getWALActionsListener();
+
+
+ /**
+ * Returns a Handler to handle peer procedures.
+ */
+ PeerProcedureHandler getPeerProcedureHandler();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
index 94bcfec..240b0a7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver.handler;
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.procedure2.RSProcedureCallable;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.log4j.Logger;
import org.apache.yetus.audience.InterfaceAudience;
/**
@@ -28,6 +29,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class RSProcedureHandler extends EventHandler {
+ private static final Logger LOG = Logger.getLogger(RSProcedureHandler.class);
private final long procId;
private final RSProcedureCallable callable;
@@ -44,6 +46,7 @@ public class RSProcedureHandler extends EventHandler {
try {
callable.call();
} catch (Exception e) {
+ LOG.error("Catch exception when call RSProcedureCallable: ", e);
error = e;
}
((HRegionServer) server).reportProcedureDone(procId, error);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
index 42d0299..56576a6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
@@ -49,7 +49,7 @@ public abstract class BaseReplicationEndpoint extends AbstractService
if (this.ctx != null){
ReplicationPeer peer = this.ctx.getReplicationPeer();
if (peer != null){
- peer.trackPeerConfigChanges(this);
+ peer.registerPeerConfigListener(this);
} else {
LOG.warn("Not tracking replication peer config changes for Peer Id " + this.ctx.getPeerId() +
" because there's no such peer");
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
new file mode 100644
index 0000000..b392985
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public interface PeerProcedureHandler {
+
+ public void addPeer(String peerId) throws ReplicationException, IOException;
+
+ public void removePeer(String peerId) throws ReplicationException, IOException;
+
+ public void disablePeer(String peerId) throws ReplicationException, IOException;
+
+ public void enablePeer(String peerId) throws ReplicationException, IOException;
+
+ public void updatePeerConfig(String peerId) throws ReplicationException, IOException;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
new file mode 100644
index 0000000..9b493d9
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationPeer;
+import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.log4j.Logger;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
+ private static final Logger LOG = Logger.getLogger(PeerProcedureHandlerImpl.class);
+
+ private ReplicationSourceManager replicationSourceManager;
+
+ public PeerProcedureHandlerImpl(ReplicationSourceManager replicationSourceManager) {
+ this.replicationSourceManager = replicationSourceManager;
+ }
+
+ @Override
+ public void addPeer(String peerId) throws ReplicationException, IOException {
+ replicationSourceManager.addPeer(peerId);
+ }
+
+ @Override
+ public void removePeer(String peerId) throws ReplicationException, IOException {
+ replicationSourceManager.removePeer(peerId);
+ }
+
+ @Override
+ public void disablePeer(String peerId) throws ReplicationException, IOException {
+ ReplicationPeer peer = replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
+ if (peer != null) {
+ PeerState peerState = peer.getPeerState(true);
+ LOG.info("disablePeer state, peer id: " + peerId + ", state: " + peerState);
+ } else {
+ throw new ReplicationException("No connected peer found, peerId=" + peerId);
+ }
+ }
+
+ @Override
+ public void enablePeer(String peerId) throws ReplicationException, IOException {
+ ReplicationPeer peer = replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
+ if (peer != null) {
+ PeerState peerState = peer.getPeerState(true);
+ LOG.info("enablePeer state, peer id: " + peerId + ", state: " + peerState);
+ } else {
+ throw new ReplicationException("No connected peer found, peerId=" + peerId);
+ }
+ }
+
+ @Override
+ public void updatePeerConfig(String peerId) throws ReplicationException, IOException {
+ ReplicationPeer peer = replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
+ if (peer == null) {
+ throw new ReplicationException("No connected peer found, peerId=" + peerId);
+ }
+ ReplicationPeerConfig rpc = peer.getPeerConfig(true);
+ peer.triggerPeerConfigChange(rpc);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
index a47a483..c3f33aa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
@@ -17,27 +17,29 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.executor.EventType;
import org.apache.hadoop.hbase.procedure2.RSProcedureCallable;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationType;
+import org.apache.log4j.Logger;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshPeerParameter;
/**
- * The callable executed at RS side to refresh the peer config/state.
- * <p>
- * TODO: only a dummy implementation for verifying the framework, will add implementation later.
+ * The callable executed at RS side to refresh the peer config/state. <br/>
*/
@InterfaceAudience.Private
public class RefreshPeerCallable implements RSProcedureCallable {
+ private static final Logger LOG = Logger.getLogger(RefreshPeerCallable.class);
private HRegionServer rs;
private String peerId;
+ private PeerModificationType type;
+
private Exception initError;
@Override
@@ -45,9 +47,27 @@ public class RefreshPeerCallable implements RSProcedureCallable {
if (initError != null) {
throw initError;
}
- Path dir = new Path("/" + peerId);
- if (rs.getFileSystem().exists(dir)) {
- rs.getFileSystem().create(new Path(dir, rs.getServerName().toString())).close();
+
+ LOG.info("Received a peer change event, peerId=" + peerId + ", type=" + type);
+ PeerProcedureHandler handler = rs.getReplicationSourceService().getPeerProcedureHandler();
+ switch (type) {
+ case ADD_PEER:
+ handler.addPeer(this.peerId);
+ break;
+ case REMOVE_PEER:
+ handler.removePeer(this.peerId);
+ break;
+ case ENABLE_PEER:
+ handler.enablePeer(this.peerId);
+ break;
+ case DISABLE_PEER:
+ handler.disablePeer(this.peerId);
+ break;
+ case UPDATE_PEER_CONFIG:
+ handler.updatePeerConfig(this.peerId);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown peer modification type: " + type);
}
return null;
}
@@ -56,10 +76,11 @@ public class RefreshPeerCallable implements RSProcedureCallable {
public void init(byte[] parameter, HRegionServer rs) {
this.rs = rs;
try {
- this.peerId = RefreshPeerParameter.parseFrom(parameter).getPeerId();
+ RefreshPeerParameter param = RefreshPeerParameter.parseFrom(parameter);
+ this.peerId = param.getPeerId();
+ this.type = param.getType();
} catch (InvalidProtocolBufferException e) {
initError = e;
- return;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 28494e6..47f08f9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -87,6 +87,8 @@ public class Replication implements
// ReplicationLoad to access replication metrics
private ReplicationLoad replicationLoad;
+ private PeerProcedureHandler peerProcedureHandler;
+
/**
* Instantiate the replication management (if rep is enabled).
* @param server Hosting server
@@ -152,6 +154,8 @@ public class Replication implements
this.conf.getInt("replication.stats.thread.period.seconds", 5 * 60);
LOG.debug("ReplicationStatisticsThread " + this.statsThreadPeriod);
this.replicationLoad = new ReplicationLoad();
+
+ this.peerProcedureHandler = new PeerProcedureHandlerImpl(replicationManager);
}
/**
@@ -170,6 +174,12 @@ public class Replication implements
public WALActionsListener getWALActionsListener() {
return this;
}
+
+ @Override
+ public PeerProcedureHandler getPeerProcedureHandler() {
+ return peerProcedureHandler;
+ }
+
/**
* Stops replication service.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 9db3cc2..a2eb202 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -446,12 +446,10 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
public void terminate(String reason, Exception cause, boolean join) {
if (cause == null) {
- LOG.info("Closing source "
- + this.peerClusterZnode + " because: " + reason);
-
+ LOG.info("Closing source " + this.peerClusterZnode + " because: " + reason);
} else {
- LOG.error("Closing source " + this.peerClusterZnode
- + " because an error occurred: " + reason, cause);
+ LOG.error("Closing source " + this.peerClusterZnode + " because an error occurred: " + reason,
+ cause);
}
this.sourceRunning = false;
Collection<ReplicationSourceShipper> workers = workerThreads.values();
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 55ebdc1..2f202fb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -564,6 +564,18 @@ public class ReplicationSourceManager implements ReplicationListener {
this.walsById.remove(src.getPeerClusterZnode());
}
+ public void addPeer(String id) throws ReplicationException, IOException {
+ LOG.info("Trying to add peer, peerId: " + id);
+ boolean added = this.replicationPeers.peerConnected(id);
+ if (added) {
+ LOG.info("Peer " + id + " connected success, trying to start the replication source thread.");
+ addSource(id);
+ if (replicationForBulkLoadDataEnabled) {
+ this.replicationQueues.addPeerToHFileRefs(id);
+ }
+ }
+ }
+
/**
* Thie method first deletes all the recovered sources for the specified
* id, then deletes the normal source (deleting all related data in ZK).
@@ -611,6 +623,8 @@ public class ReplicationSourceManager implements ReplicationListener {
}
deleteSource(id, true);
}
+ // Remove HFile Refs znode from zookeeper
+ this.replicationQueues.removePeerFromHFileRefs(id);
}
@Override
@@ -618,29 +632,6 @@ public class ReplicationSourceManager implements ReplicationListener {
transferQueues(regionserver);
}
- @Override
- public void peerRemoved(String peerId) {
- removePeer(peerId);
- this.replicationQueues.removePeerFromHFileRefs(peerId);
- }
-
- @Override
- public void peerListChanged(List<String> peerIds) {
- for (String id : peerIds) {
- try {
- boolean added = this.replicationPeers.peerConnected(id);
- if (added) {
- addSource(id);
- if (replicationForBulkLoadDataEnabled) {
- this.replicationQueues.addPeerToHFileRefs(id);
- }
- }
- } catch (Exception e) {
- LOG.error("Error while adding a new peer", e);
- }
- }
- }
-
/**
* Class responsible to setup new ReplicationSources to take care of the
* queues from dead region servers.
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
new file mode 100644
index 0000000..b09a8a7
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client.replication;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.TestReplicationBase;
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.log4j.Logger;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MediumTests.class, ClientTests.class })
+public class TestReplicationAdminUsingProcedure extends TestReplicationBase {
+
+ private static final String PEER_ID = "2";
+ private static final Logger LOG = Logger.getLogger(TestReplicationAdminUsingProcedure.class);
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ conf1.setInt("hbase.multihconnection.threads.max", 10);
+
+ // Start the master & slave mini cluster.
+ TestReplicationBase.setUpBeforeClass();
+
+ // Remove the replication peer
+ hbaseAdmin.removeReplicationPeer(PEER_ID);
+ }
+
+ private void loadData(int startRowKey, int endRowKey) throws IOException {
+ for (int i = startRowKey; i < endRowKey; i++) {
+ byte[] rowKey = Bytes.add(row, Bytes.toBytes(i));
+ Put put = new Put(rowKey);
+ put.addColumn(famName, null, Bytes.toBytes(i));
+ htable1.put(put);
+ }
+ }
+
+ private void waitForReplication(int expectedRows, int retries)
+ throws IOException, InterruptedException {
+ Scan scan;
+ for (int i = 0; i < retries; i++) {
+ scan = new Scan();
+ if (i == retries - 1) {
+ throw new IOException("Waited too much time for normal batch replication");
+ }
+ try (ResultScanner scanner = htable2.getScanner(scan)) {
+ int count = 0;
+ for (Result res : scanner) {
+ count++;
+ }
+ if (count != expectedRows) {
+ LOG.info("Only got " + count + " rows, expected rows: " + expectedRows);
+ Thread.sleep(SLEEP_TIME);
+ } else {
+ return;
+ }
+ }
+ }
+ }
+
+ @Before
+ public void setUp() throws IOException {
+ ReplicationPeerConfig rpc = new ReplicationPeerConfig();
+ rpc.setClusterKey(utility2.getClusterKey());
+ hbaseAdmin.addReplicationPeer(PEER_ID, rpc);
+
+ utility1.waitUntilAllRegionsAssigned(tableName);
+ utility2.waitUntilAllRegionsAssigned(tableName);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ hbaseAdmin.removeReplicationPeer(PEER_ID);
+ truncateBoth();
+ }
+
+ private void truncateBoth() throws IOException {
+ utility1.deleteTableData(tableName);
+ utility2.deleteTableData(tableName);
+ }
+
+ @Test
+ public void testAddPeer() throws Exception {
+ // Load data
+ loadData(0, NB_ROWS_IN_BATCH);
+
+ // Wait the replication finished
+ waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
+ }
+
+ @Test
+ public void testRemovePeer() throws Exception {
+ // prev-check
+ waitForReplication(0, NB_RETRIES);
+
+ // Load data
+ loadData(0, NB_ROWS_IN_BATCH);
+
+ // Wait the replication finished
+ waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
+
+ // Remove the peer id
+ hbaseAdmin.removeReplicationPeer(PEER_ID);
+
+ // Load data again
+ loadData(NB_ROWS_IN_BATCH, 2 * NB_ROWS_IN_BATCH);
+
+ // Wait the replication again
+ boolean foundException = false;
+ try {
+ waitForReplication(NB_ROWS_IN_BATCH * 2, NB_RETRIES);
+ } catch (IOException e) {
+ foundException = true;
+ }
+ Assert.assertTrue(foundException);
+
+ // Truncate the table in source cluster
+ truncateBoth();
+
+ // Add peer again
+ ReplicationPeerConfig rpc = new ReplicationPeerConfig();
+ rpc.setClusterKey(utility2.getClusterKey());
+ hbaseAdmin.addReplicationPeer(PEER_ID, rpc);
+
+ // Load data again
+ loadData(0, NB_ROWS_IN_BATCH);
+
+ // Wait the replication finished
+ waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
+ }
+
+ @Test
+ public void testDisableAndEnablePeer() throws Exception {
+ // disable peer
+ hbaseAdmin.disableReplicationPeer(PEER_ID);
+
+ // Load data
+ loadData(0, NB_ROWS_IN_BATCH);
+
+ // Will failed to wait the replication.
+ boolean foundException = false;
+ try {
+ waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
+ } catch (IOException e) {
+ foundException = true;
+ }
+ Assert.assertTrue(foundException);
+
+ // Enable the peer
+ hbaseAdmin.enableReplicationPeer(PEER_ID);
+ waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
+
+ // Load more data
+ loadData(NB_ROWS_IN_BATCH, NB_ROWS_IN_BATCH * 2);
+
+ // Wait replication again.
+ waitForReplication(NB_ROWS_IN_BATCH * 2, NB_RETRIES);
+ }
+
+ @Test
+ public void testUpdatePeerConfig() throws Exception {
+ ReplicationPeerConfig rpc = new ReplicationPeerConfig();
+ rpc.setClusterKey(utility2.getClusterKey());
+ rpc.setExcludeTableCFsMap(
+ ImmutableMap.of(tableName, ImmutableList.of(Bytes.toString(famName))));
+
+ // Update the peer config to exclude the test table name.
+ hbaseAdmin.updateReplicationPeerConfig(PEER_ID, rpc);
+
+ // Load data
+ loadData(0, NB_ROWS_IN_BATCH);
+
+ // Will failed to wait the replication
+ boolean foundException = false;
+ try {
+ waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
+ } catch (IOException e) {
+ foundException = true;
+ }
+ Assert.assertTrue(foundException);
+
+ // Truncate the table in source cluster
+ truncateBoth();
+
+ // Update the peer config to include the test table name.
+ ReplicationPeerConfig rpc2 = new ReplicationPeerConfig();
+ rpc2.setClusterKey(utility2.getClusterKey());
+ hbaseAdmin.updateReplicationPeerConfig(PEER_ID, rpc2);
+
+ // Load data again
+ loadData(0, NB_ROWS_IN_BATCH);
+
+ // Wait the replication finished
+ waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
deleted file mode 100644
index ed7c6fa..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.replication;
-
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-
-public class DummyModifyPeerProcedure extends ModifyPeerProcedure {
-
- public DummyModifyPeerProcedure() {
- }
-
- public DummyModifyPeerProcedure(String peerId) {
- super(peerId);
- }
-
- @Override
- public PeerOperationType getPeerOperationType() {
- return PeerOperationType.ADD;
- }
-
- @Override
- protected void prePeerModification(MasterProcedureEnv env) {
- }
-
- @Override
- protected void updatePeerStorage(MasterProcedureEnv env) {
- }
-
- @Override
- protected void postPeerModification(MasterProcedureEnv env) {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestDummyModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestDummyModifyPeerProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestDummyModifyPeerProcedure.java
deleted file mode 100644
index ec06306..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestDummyModifyPeerProcedure.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.replication;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.MasterTests;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category({ MasterTests.class, LargeTests.class })
-public class TestDummyModifyPeerProcedure {
-
- private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
- private static String PEER_ID;
-
- private static Path DIR;
-
- @BeforeClass
- public static void setUp() throws Exception {
- UTIL.startMiniCluster(3);
- PEER_ID = "testPeer";
- DIR = new Path("/" + PEER_ID);
- UTIL.getTestFileSystem().mkdirs(DIR);
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- UTIL.shutdownMiniCluster();
- }
-
- @Test
- public void test() throws Exception {
- ProcedureExecutor<?> executor =
- UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
- long procId = executor.submitProcedure(new DummyModifyPeerProcedure(PEER_ID));
- UTIL.waitFor(30000, new Waiter.Predicate<Exception>() {
-
- @Override
- public boolean evaluate() throws Exception {
- return executor.isFinished(procId);
- }
- });
- Set<String> serverNames = UTIL.getHBaseCluster().getRegionServerThreads().stream()
- .map(t -> t.getRegionServer().getServerName().toString())
- .collect(Collectors.toCollection(HashSet::new));
- for (FileStatus s : UTIL.getTestFileSystem().listStatus(DIR)) {
- assertTrue(serverNames.remove(s.getPath().getName()));
- }
- assertTrue(serverNames.isEmpty());
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
index a04d524..f118ca3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.replication;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@@ -156,41 +155,6 @@ public class TestReplicationTrackerZKImpl {
}
@Test(timeout = 30000)
- public void testPeerRemovedEvent() throws Exception {
- rp.registerPeer("5", new ReplicationPeerConfig().setClusterKey(utility.getClusterKey()));
- rt.registerListener(new DummyReplicationListener());
- rp.unregisterPeer("5");
- // wait for event
- while (peerRemovedCount.get() < 1) {
- Thread.sleep(5);
- }
- assertEquals("5", peerRemovedData);
- }
-
- @Test(timeout = 30000)
- public void testPeerListChangedEvent() throws Exception {
- // add a peer
- rp.registerPeer("5", new ReplicationPeerConfig().setClusterKey(utility.getClusterKey()));
- zkw.getRecoverableZooKeeper().getZooKeeper().getChildren("/hbase/replication/peers/5", true);
- rt.registerListener(new DummyReplicationListener());
- rp.disablePeer("5");
- int tmp = plChangedCount.get();
- LOG.info("Peer count=" + tmp);
- ZKUtil.deleteNode(zkw, "/hbase/replication/peers/5/peer-state");
- // wait for event
- while (plChangedCount.get() <= tmp) {
- Thread.sleep(100);
- LOG.info("Peer count=" + tmp);
- }
- assertEquals(1, plChangedData.size());
- assertTrue(plChangedData.contains("5"));
-
- // clean up
- //ZKUtil.deleteNode(zkw, "/hbase/replication/peers/5");
- rp.unregisterPeer("5");
- }
-
- @Test(timeout = 30000)
public void testPeerNameControl() throws Exception {
int exists = 0;
int hyphen = 0;
@@ -222,21 +186,6 @@ public class TestReplicationTrackerZKImpl {
rsRemovedCount.getAndIncrement();
LOG.debug("Received regionServerRemoved event: " + regionServer);
}
-
- @Override
- public void peerRemoved(String peerId) {
- peerRemovedData = peerId;
- peerRemovedCount.getAndIncrement();
- LOG.debug("Received peerDisconnected event: " + peerId);
- }
-
- @Override
- public void peerListChanged(List<String> peerIds) {
- plChangedData.clear();
- plChangedData.addAll(peerIds);
- int count = plChangedCount.getAndIncrement();
- LOG.debug("Received peerListChanged event " + count);
- }
}
private class DummyServer implements Server {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8fa2fc5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 057c64d..b5a7461 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -176,6 +176,12 @@ public abstract class TestReplicationSourceManager {
replication = new Replication(new DummyServer(), fs, logDir, oldLogDir);
managerOfCluster = getManagerFromCluster();
+ if (managerOfCluster != null) {
+ // After replication procedure, we need to add peer by hand (other than by receiving
+ // notification from zk)
+ managerOfCluster.addPeer(slaveId);
+ }
+
manager = replication.getReplicationManager();
manager.addSource(slaveId);
if (managerOfCluster != null) {
@@ -535,18 +541,16 @@ public abstract class TestReplicationSourceManager {
final int globalLogQueueSizeInitial = globalSource.getSizeOfLogQueue();
final long sizeOfLatestPath = getSizeOfLatestPath();
addPeerAndWait(peerId, peerConfig, true);
- assertEquals(sizeOfLatestPath + globalLogQueueSizeInitial,
- globalSource.getSizeOfLogQueue());
+ assertEquals(sizeOfLatestPath + globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
ReplicationSourceInterface source = manager.getSource(peerId);
// Sanity check
assertNotNull(source);
final int sizeOfSingleLogQueue = source.getSourceMetrics().getSizeOfLogQueue();
// Enqueue log and check if metrics updated
source.enqueueLog(new Path("abc"));
- assertEquals(1 + sizeOfSingleLogQueue,
- source.getSourceMetrics().getSizeOfLogQueue());
- assertEquals(source.getSourceMetrics().getSizeOfLogQueue()
- + globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
+ assertEquals(1 + sizeOfSingleLogQueue, source.getSourceMetrics().getSizeOfLogQueue());
+ assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
+ globalSource.getSizeOfLogQueue());
// Removing the peer should reset the global metrics
removePeerAndWait(peerId);
@@ -556,9 +560,8 @@ public abstract class TestReplicationSourceManager {
addPeerAndWait(peerId, peerConfig, true);
source = manager.getSource(peerId);
assertNotNull(source);
- assertEquals(sizeOfLatestPath, source.getSourceMetrics().getSizeOfLogQueue());
- assertEquals(source.getSourceMetrics().getSizeOfLogQueue()
- + globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
+ assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
+ globalSource.getSizeOfLogQueue());
} finally {
removePeerAndWait(peerId);
}
@@ -575,8 +578,14 @@ public abstract class TestReplicationSourceManager {
final boolean waitForSource) throws Exception {
final ReplicationPeers rp = manager.getReplicationPeers();
rp.registerPeer(peerId, peerConfig);
+ try {
+ manager.addPeer(peerId);
+ } catch (Exception e) {
+ // ignore the failed exception, because we'll test both success & failed case.
+ }
waitPeer(peerId, manager, waitForSource);
if (managerOfCluster != null) {
+ managerOfCluster.addPeer(peerId);
waitPeer(peerId, managerOfCluster, waitForSource);
}
}
@@ -609,6 +618,11 @@ public abstract class TestReplicationSourceManager {
final ReplicationPeers rp = manager.getReplicationPeers();
if (rp.getAllPeerIds().contains(peerId)) {
rp.unregisterPeer(peerId);
+ try {
+ manager.removePeer(peerId);
+ } catch (Exception e) {
+ // ignore the failed exception and continue.
+ }
}
Waiter.waitFor(conf, 20000, new Waiter.Predicate<Exception>() {
@Override public boolean evaluate() throws Exception {
[35/50] [abbrv] hbase git commit: HBASE-19622 Reimplement
ReplicationPeers with the new replication storage interface
Posted by zh...@apache.org.
HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a791c238
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a791c238
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a791c238
Branch: refs/heads/HBASE-19397-branch-2
Commit: a791c23828731c91feac831afba97943543368cf
Parents: 5245829
Author: huzheng <op...@gmail.com>
Authored: Tue Dec 26 16:46:10 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../replication/ReplicationPeerConfigUtil.java | 10 +-
.../replication/VerifyReplication.java | 9 +-
.../hbase/replication/ReplicationFactory.java | 10 +-
.../hbase/replication/ReplicationPeerImpl.java | 60 +-
.../replication/ReplicationPeerStorage.java | 3 +-
.../hbase/replication/ReplicationPeers.java | 238 ++++----
.../replication/ReplicationPeersZKImpl.java | 552 -------------------
.../replication/ZKReplicationPeerStorage.java | 12 +-
.../replication/ZKReplicationStorageBase.java | 3 +-
.../replication/TestReplicationStateBasic.java | 125 ++---
.../replication/TestReplicationStateZKImpl.java | 2 +-
.../TestZKReplicationPeerStorage.java | 12 +-
.../cleaner/ReplicationZKNodeCleaner.java | 57 +-
.../replication/ReplicationPeerManager.java | 6 +-
.../regionserver/DumpReplicationQueues.java | 2 +-
.../regionserver/PeerProcedureHandlerImpl.java | 49 +-
.../replication/regionserver/Replication.java | 2 +-
.../regionserver/ReplicationSource.java | 7 +-
.../regionserver/ReplicationSourceManager.java | 45 +-
.../cleaner/TestReplicationHFileCleaner.java | 7 +-
.../replication/TestMultiSlaveReplication.java | 2 -
.../TestReplicationTrackerZKImpl.java | 36 +-
.../TestReplicationSourceManager.java | 17 +-
.../hadoop/hbase/HBaseZKTestingUtility.java | 3 +-
24 files changed, 308 insertions(+), 961 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
index 022bf64..a234a9b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationPeerConfigUtil.java
@@ -247,22 +247,22 @@ public final class ReplicationPeerConfigUtil {
public static ReplicationPeerConfig parsePeerFrom(final byte[] bytes)
throws DeserializationException {
if (ProtobufUtil.isPBMagicPrefix(bytes)) {
- int pblen = ProtobufUtil.lengthOfPBMagic();
+ int pbLen = ProtobufUtil.lengthOfPBMagic();
ReplicationProtos.ReplicationPeer.Builder builder =
ReplicationProtos.ReplicationPeer.newBuilder();
ReplicationProtos.ReplicationPeer peer;
try {
- ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
+ ProtobufUtil.mergeFrom(builder, bytes, pbLen, bytes.length - pbLen);
peer = builder.build();
} catch (IOException e) {
throw new DeserializationException(e);
}
return convert(peer);
} else {
- if (bytes.length > 0) {
- return ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build();
+ if (bytes == null || bytes.length <= 0) {
+ throw new DeserializationException("Bytes to deserialize should not be empty.");
}
- return ReplicationPeerConfig.newBuilder().setClusterKey("").build();
+ return ReplicationPeerConfig.newBuilder().setClusterKey(Bytes.toString(bytes)).build();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 09d4b4b..f0070f0 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -339,15 +339,10 @@ public class VerifyReplication extends Configured implements Tool {
@Override public boolean isAborted() {return false;}
});
- ReplicationPeers rp = ReplicationFactory.getReplicationPeers(localZKW, conf, localZKW);
+ ReplicationPeers rp = ReplicationFactory.getReplicationPeers(localZKW, conf);
rp.init();
- Pair<ReplicationPeerConfig, Configuration> pair = rp.getPeerConf(peerId);
- if (pair == null) {
- throw new IOException("Couldn't get peer conf!");
- }
-
- return pair;
+ return Pair.newPair(rp.getPeerConfig(peerId), rp.getPeerClusterConfiguration(peerId));
} catch (ReplicationException e) {
throw new IOException(
"An error occurred while trying to connect to the remove peer cluster", e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 5e70e57..6c66aff 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -29,14 +29,8 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class ReplicationFactory {
- public static ReplicationPeers getReplicationPeers(ZKWatcher zk, Configuration conf,
- Abortable abortable) {
- return getReplicationPeers(zk, conf, null, abortable);
- }
-
- public static ReplicationPeers getReplicationPeers(ZKWatcher zk, Configuration conf,
- ReplicationQueueStorage queueStorage, Abortable abortable) {
- return new ReplicationPeersZKImpl(zk, conf, queueStorage, abortable);
+ public static ReplicationPeers getReplicationPeers(ZKWatcher zk, Configuration conf) {
+ return new ReplicationPeers(zk, conf);
}
public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index 2c7ea9b..3e17025 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -18,28 +18,16 @@
*/
package org.apache.hadoop.hbase.replication;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
@InterfaceAudience.Private
public class ReplicationPeerImpl implements ReplicationPeer {
- private static final Logger LOG = LoggerFactory.getLogger(ReplicationPeerImpl.class);
-
- private final ReplicationPeerStorage peerStorage;
-
private final Configuration conf;
private final String id;
@@ -57,21 +45,21 @@ public class ReplicationPeerImpl implements ReplicationPeer {
* @param id string representation of this peer's identifier
* @param peerConfig configuration for the replication peer
*/
- public ReplicationPeerImpl(ZKWatcher zkWatcher, Configuration conf, String id,
+ public ReplicationPeerImpl(Configuration conf, String id, boolean peerState,
ReplicationPeerConfig peerConfig) {
- this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zkWatcher, conf);
this.conf = conf;
- this.peerConfig = peerConfig;
this.id = id;
+ this.peerState = peerState ? PeerState.ENABLED : PeerState.DISABLED;
+ this.peerConfig = peerConfig;
this.peerConfigListeners = new ArrayList<>();
}
- public void refreshPeerState() throws ReplicationException {
- this.peerState = peerStorage.isPeerEnabled(id) ? PeerState.ENABLED : PeerState.DISABLED;
+ void setPeerState(boolean enabled) {
+ this.peerState = enabled ? PeerState.ENABLED : PeerState.DISABLED;
}
- public void refreshPeerConfig() throws ReplicationException {
- this.peerConfig = peerStorage.getPeerConfig(id).orElse(peerConfig);
+ void setPeerConfig(ReplicationPeerConfig peerConfig) {
+ this.peerConfig = peerConfig;
peerConfigListeners.forEach(listener -> listener.peerConfigUpdated(peerConfig));
}
@@ -134,36 +122,4 @@ public class ReplicationPeerImpl implements ReplicationPeer {
public void registerPeerConfigListener(ReplicationPeerConfigListener listener) {
this.peerConfigListeners.add(listener);
}
-
- /**
- * Parse the raw data from ZK to get a peer's state
- * @param bytes raw ZK data
- * @return True if the passed in <code>bytes</code> are those of a pb serialized ENABLED state.
- * @throws DeserializationException
- */
- public static boolean isStateEnabled(final byte[] bytes) throws DeserializationException {
- ReplicationProtos.ReplicationState.State state = parseStateFrom(bytes);
- return ReplicationProtos.ReplicationState.State.ENABLED == state;
- }
-
- /**
- * @param bytes Content of a state znode.
- * @return State parsed from the passed bytes.
- * @throws DeserializationException
- */
- private static ReplicationProtos.ReplicationState.State parseStateFrom(final byte[] bytes)
- throws DeserializationException {
- ProtobufUtil.expectPBMagicPrefix(bytes);
- int pbLen = ProtobufUtil.lengthOfPBMagic();
- ReplicationProtos.ReplicationState.Builder builder =
- ReplicationProtos.ReplicationState.newBuilder();
- ReplicationProtos.ReplicationState state;
- try {
- ProtobufUtil.mergeFrom(builder, bytes, pbLen, bytes.length - pbLen);
- state = builder.build();
- return state.getState();
- } catch (IOException e) {
- throw new DeserializationException(e);
- }
- }
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
index e00cd0d..1adda02 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerStorage.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.replication;
import java.util.List;
-import java.util.Optional;
import org.apache.yetus.audience.InterfaceAudience;
@@ -70,5 +69,5 @@ public interface ReplicationPeerStorage {
* Get the peer config of a replication peer.
* @throws ReplicationException if there are errors accessing the storage service.
*/
- Optional<ReplicationPeerConfig> getPeerConfig(String peerId) throws ReplicationException;
+ ReplicationPeerConfig getPeerConfig(String peerId) throws ReplicationException;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index afc19bd..e58482e 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,58 +17,53 @@
*/
package org.apache.hadoop.hbase.replication;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
+import java.io.IOException;
import java.util.Set;
-
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.CompoundConfiguration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
- * This provides an interface for maintaining a set of peer clusters. These peers are remote slave
- * clusters that data is replicated to. A peer cluster can be in three different states:
- *
- * 1. Not-Registered - There is no notion of the peer cluster.
- * 2. Registered - The peer has an id and is being tracked but there is no connection.
- * 3. Connected - There is an active connection to the remote peer.
- *
- * In the registered or connected state, a peer cluster can either be enabled or disabled.
+ * This provides an class for maintaining a set of peer clusters. These peers are remote slave
+ * clusters that data is replicated to.
*/
@InterfaceAudience.Private
-public interface ReplicationPeers {
+public class ReplicationPeers {
- /**
- * Initialize the ReplicationPeers interface.
- */
- void init() throws ReplicationException;
+ private static final Logger LOG = LoggerFactory.getLogger(ReplicationPeers.class);
- /**
- * Add a new remote slave cluster for replication.
- * @param peerId a short that identifies the cluster
- * @param peerConfig configuration for the replication slave cluster
- */
- default void registerPeer(String peerId, ReplicationPeerConfig peerConfig)
- throws ReplicationException {
- registerPeer(peerId, peerConfig, true);
+ private final Configuration conf;
+
+ // Map of peer clusters keyed by their id
+ private final ConcurrentMap<String, ReplicationPeerImpl> peerCache;
+ private final ReplicationPeerStorage peerStorage;
+
+ protected ReplicationPeers(ZKWatcher zookeeper, Configuration conf) {
+ this.conf = conf;
+ this.peerCache = new ConcurrentHashMap<>();
+ this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zookeeper, conf);
}
- /**
- * Add a new remote slave cluster for replication.
- * @param peerId a short that identifies the cluster
- * @param peerConfig configuration for the replication slave cluster
- * @param enabled peer state, true if ENABLED and false if DISABLED
- */
- void registerPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
- throws ReplicationException;
+ public void init() throws ReplicationException {
+ // Loading all existing peerIds into peer cache.
+ for (String peerId : this.peerStorage.listPeerIds()) {
+ addPeer(peerId);
+ }
+ }
- /**
- * Removes a remote slave cluster and stops the replication to it.
- * @param peerId a short that identifies the cluster
- */
- void unregisterPeer(String peerId) throws ReplicationException;
+ @VisibleForTesting
+ public ReplicationPeerStorage getPeerStorage() {
+ return this.peerStorage;
+ }
/**
* Method called after a peer has been connected. It will create a ReplicationPeer to track the
@@ -78,111 +72,115 @@ public interface ReplicationPeers {
* @return whether a ReplicationPeer was successfully created
* @throws ReplicationException
*/
- boolean peerConnected(String peerId) throws ReplicationException;
-
- /**
- * Method called after a peer has been disconnected. It will remove the ReplicationPeer that
- * tracked the disconnected cluster.
- * @param peerId a short that identifies the cluster
- */
- void peerDisconnected(String peerId);
+ public boolean addPeer(String peerId) throws ReplicationException {
+ if (this.peerCache.containsKey(peerId)) {
+ return false;
+ }
- /**
- * Restart the replication to the specified remote slave cluster.
- * @param peerId a short that identifies the cluster
- */
- void enablePeer(String peerId) throws ReplicationException;
-
- /**
- * Stop the replication to the specified remote slave cluster.
- * @param peerId a short that identifies the cluster
- */
- void disablePeer(String peerId) throws ReplicationException;
+ peerCache.put(peerId, createPeer(peerId));
+ return true;
+ }
- /**
- * Get the table and column-family list string of the peer from the underlying storage.
- * @param peerId a short that identifies the cluster
- */
- public Map<TableName, List<String>> getPeerTableCFsConfig(String peerId)
- throws ReplicationException;
+ public void removePeer(String peerId) {
+ peerCache.remove(peerId);
+ }
/**
- * Set the table and column-family list string of the peer to the underlying storage.
+ * Get the peer state for the specified connected remote slave cluster. The value might be read
+ * from cache, so it is recommended to use {@link #peerStorage } to read storage directly if
+ * reading the state after enabling or disabling it.
* @param peerId a short that identifies the cluster
- * @param tableCFs the table and column-family list which will be replicated for this peer
+ * @return true if replication is enabled, false otherwise.
*/
- public void setPeerTableCFsConfig(String peerId,
- Map<TableName, ? extends Collection<String>> tableCFs)
- throws ReplicationException;
+ public boolean isPeerEnabled(String peerId) {
+ ReplicationPeer replicationPeer = this.peerCache.get(peerId);
+ if (replicationPeer == null) {
+ throw new IllegalArgumentException("Peer with id= " + peerId + " is not cached");
+ }
+ return replicationPeer.getPeerState() == PeerState.ENABLED;
+ }
/**
- * Returns the ReplicationPeerImpl for the specified connected peer. This ReplicationPeer will
- * continue to track changes to the Peer's state and config. This method returns null if no
- * peer has been connected with the given peerId.
+ * Returns the ReplicationPeerImpl for the specified cached peer. This ReplicationPeer will
+ * continue to track changes to the Peer's state and config. This method returns null if no peer
+ * has been cached with the given peerId.
* @param peerId id for the peer
* @return ReplicationPeer object
*/
- ReplicationPeerImpl getConnectedPeer(String peerId);
+ public ReplicationPeerImpl getPeer(String peerId) {
+ return peerCache.get(peerId);
+ }
/**
* Returns the set of peerIds of the clusters that have been connected and have an underlying
* ReplicationPeer.
* @return a Set of Strings for peerIds
*/
- public Set<String> getConnectedPeerIds();
+ public Set<String> getAllPeerIds() {
+ return peerCache.keySet();
+ }
- /**
- * Get the replication status for the specified connected remote slave cluster.
- * The value might be read from cache, so it is recommended to
- * use {@link #getStatusOfPeerFromBackingStore(String)}
- * if reading the state after enabling or disabling it.
- * @param peerId a short that identifies the cluster
- * @return true if replication is enabled, false otherwise.
- */
- boolean getStatusOfPeer(String peerId);
+ public ReplicationPeerConfig getPeerConfig(String peerId) {
+ ReplicationPeer replicationPeer = this.peerCache.get(peerId);
+ if (replicationPeer == null) {
+ throw new IllegalArgumentException("Peer with id= " + peerId + " is not cached");
+ }
+ return replicationPeer.getPeerConfig();
+ }
- /**
- * Get the replication status for the specified remote slave cluster, which doesn't
- * have to be connected. The state is read directly from the backing store.
- * @param peerId a short that identifies the cluster
- * @return true if replication is enabled, false otherwise.
- * @throws ReplicationException thrown if there's an error contacting the store
- */
- boolean getStatusOfPeerFromBackingStore(String peerId) throws ReplicationException;
+ public Configuration getPeerClusterConfiguration(String peerId) throws ReplicationException {
+ ReplicationPeerConfig peerConfig = peerStorage.getPeerConfig(peerId);
- /**
- * List the cluster replication configs of all remote slave clusters (whether they are
- * enabled/disabled or connected/disconnected).
- * @return A map of peer ids to peer cluster keys
- */
- Map<String, ReplicationPeerConfig> getAllPeerConfigs();
+ Configuration otherConf;
+ try {
+ otherConf = HBaseConfiguration.createClusterConf(this.conf, peerConfig.getClusterKey());
+ } catch (IOException e) {
+ throw new ReplicationException("Can't get peer configuration for peerId=" + peerId, e);
+ }
- /**
- * List the peer ids of all remote slave clusters (whether they are enabled/disabled or
- * connected/disconnected).
- * @return A list of peer ids
- */
- List<String> getAllPeerIds();
+ if (!peerConfig.getConfiguration().isEmpty()) {
+ CompoundConfiguration compound = new CompoundConfiguration();
+ compound.add(otherConf);
+ compound.addStringMap(peerConfig.getConfiguration());
+ return compound;
+ }
- /**
- * Returns the configured ReplicationPeerConfig for this peerId
- * @param peerId a short name that identifies the cluster
- * @return ReplicationPeerConfig for the peer
- */
- ReplicationPeerConfig getReplicationPeerConfig(String peerId) throws ReplicationException;
+ return otherConf;
+ }
- /**
- * Returns the configuration needed to talk to the remote slave cluster.
- * @param peerId a short that identifies the cluster
- * @return the configuration for the peer cluster, null if it was unable to get the configuration
- */
- Pair<ReplicationPeerConfig, Configuration> getPeerConf(String peerId) throws ReplicationException;
+ public PeerState refreshPeerState(String peerId) throws ReplicationException {
+ ReplicationPeerImpl peer = peerCache.get(peerId);
+ if (peer == null) {
+ throw new ReplicationException("Peer with id=" + peerId + " is not cached.");
+ }
+ peer.setPeerState(peerStorage.isPeerEnabled(peerId));
+ return peer.getPeerState();
+ }
+
+ public ReplicationPeerConfig refreshPeerConfig(String peerId) throws ReplicationException {
+ ReplicationPeerImpl peer = peerCache.get(peerId);
+ if (peer == null) {
+ throw new ReplicationException("Peer with id=" + peerId + " is not cached.");
+ }
+ peer.setPeerConfig(peerStorage.getPeerConfig(peerId));
+ return peer.getPeerConfig();
+ }
/**
+<<<<<<< 2bb2fd611d4b88c724a2b561f10433b56c6fd3dd
* Update the peerConfig for the a given peer cluster
* @param id a short that identifies the cluster
* @param peerConfig new config for the peer cluster
* @throws ReplicationException
- */
- void updatePeerConfig(String id, ReplicationPeerConfig peerConfig) throws ReplicationException;
+=======
+ * Helper method to connect to a peer
+ * @param peerId peer's identifier
+ * @return object representing the peer
+>>>>>>> HBASE-19622 Reimplement ReplicationPeers with the new replication storage interface
+ */
+ private ReplicationPeerImpl createPeer(String peerId) throws ReplicationException {
+ ReplicationPeerConfig peerConf = peerStorage.getPeerConfig(peerId);
+ boolean enabled = peerStorage.isPeerEnabled(peerId);
+ return new ReplicationPeerImpl(getPeerClusterConfiguration(peerId), peerId, enabled, peerConf);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
deleted file mode 100644
index 7f6498d..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ /dev/null
@@ -1,552 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.CompoundConfiguration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class provides an implementation of the ReplicationPeers interface using ZooKeeper. The
- * peers znode contains a list of all peer replication clusters and the current replication state of
- * those clusters. It has one child peer znode for each peer cluster. The peer znode is named with
- * the cluster id provided by the user in the HBase shell. The value of the peer znode contains the
- * peers cluster key provided by the user in the HBase Shell. The cluster key contains a list of
- * zookeeper quorum peers, the client port for the zookeeper quorum, and the base znode for HBase.
- * For example:
- *
- * /hbase/replication/peers/1 [Value: zk1.host.com,zk2.host.com,zk3.host.com:2181:/hbase]
- * /hbase/replication/peers/2 [Value: zk5.host.com,zk6.host.com,zk7.host.com:2181:/hbase]
- *
- * Each of these peer znodes has a child znode that indicates whether or not replication is enabled
- * on that peer cluster. These peer-state znodes do not have child znodes and simply contain a
- * boolean value (i.e. ENABLED or DISABLED). This value is read/maintained by the
- * ReplicationPeer.PeerStateTracker class. For example:
- *
- * /hbase/replication/peers/1/peer-state [Value: ENABLED]
- *
- * Each of these peer znodes has a child znode that indicates which data will be replicated
- * to the peer cluster. These peer-tableCFs znodes do not have child znodes and only have a
- * table/cf list config. This value is read/maintained by the ReplicationPeer.TableCFsTracker
- * class. For example:
- *
- * /hbase/replication/peers/1/tableCFs [Value: "table1; table2:cf1,cf3; table3:cfx,cfy"]
- */
-@InterfaceAudience.Private
-public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements ReplicationPeers {
-
- // Map of peer clusters keyed by their id
- private ConcurrentMap<String, ReplicationPeerImpl> peerClusters;
- private final ReplicationQueueStorage queueStorage;
- private Abortable abortable;
-
- private static final Logger LOG = LoggerFactory.getLogger(ReplicationPeersZKImpl.class);
-
- public ReplicationPeersZKImpl(ZKWatcher zk, Configuration conf,
- ReplicationQueueStorage queueStorage, Abortable abortable) {
- super(zk, conf, abortable);
- this.abortable = abortable;
- this.peerClusters = new ConcurrentHashMap<>();
- this.queueStorage = queueStorage;
- }
-
- @Override
- public void init() throws ReplicationException {
- try {
- if (ZKUtil.checkExists(this.zookeeper, this.peersZNode) < 0) {
- ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Could not initialize replication peers", e);
- }
- addExistingPeers();
- }
-
- @Override
- public void registerPeer(String id, ReplicationPeerConfig peerConfig, boolean enabled)
- throws ReplicationException {
- try {
- if (peerExists(id)) {
- throw new IllegalArgumentException("Cannot add a peer with id=" + id
- + " because that id already exists.");
- }
-
- if(id.contains("-")){
- throw new IllegalArgumentException("Found invalid peer name:" + id);
- }
-
- if (peerConfig.getClusterKey() != null) {
- try {
- ZKConfig.validateClusterKey(peerConfig.getClusterKey());
- } catch (IOException ioe) {
- throw new IllegalArgumentException(ioe.getMessage());
- }
- }
-
- checkQueuesDeleted(id);
-
- ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
-
- List<ZKUtilOp> listOfOps = new ArrayList<>(2);
- ZKUtilOp op1 =
- ZKUtilOp.createAndFailSilent(getPeerNode(id),
- ReplicationPeerConfigUtil.toByteArray(peerConfig));
- ZKUtilOp op2 =
- ZKUtilOp.createAndFailSilent(getPeerStateNode(id), enabled ? ENABLED_ZNODE_BYTES
- : DISABLED_ZNODE_BYTES);
- listOfOps.add(op1);
- listOfOps.add(op2);
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
- } catch (KeeperException e) {
- throw new ReplicationException("Could not add peer with id=" + id + ", peerConfif=>"
- + peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"), e);
- }
- }
-
- @Override
- public void unregisterPeer(String id) throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("Cannot remove peer with id=" + id
- + " because that id does not exist.");
- }
- ZKUtil.deleteNodeRecursively(this.zookeeper, ZNodePaths.joinZNode(this.peersZNode, id));
- } catch (KeeperException e) {
- throw new ReplicationException("Could not remove peer with id=" + id, e);
- }
- }
-
- @Override
- public void enablePeer(String id) throws ReplicationException {
- changePeerState(id, ReplicationProtos.ReplicationState.State.ENABLED);
- LOG.info("peer " + id + " is enabled");
- }
-
- @Override
- public void disablePeer(String id) throws ReplicationException {
- changePeerState(id, ReplicationProtos.ReplicationState.State.DISABLED);
- LOG.info("peer " + id + " is disabled");
- }
-
- @Override
- public Map<TableName, List<String>> getPeerTableCFsConfig(String id) throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("peer " + id + " doesn't exist");
- }
- try {
- ReplicationPeerConfig rpc = getReplicationPeerConfig(id);
- if (rpc == null) {
- throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id);
- }
- return rpc.getTableCFsMap();
- } catch (Exception e) {
- throw new ReplicationException(e);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id, e);
- }
- }
-
- @Override
- public void setPeerTableCFsConfig(String id,
- Map<TableName, ? extends Collection<String>> tableCFs)
- throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("Cannot set peer tableCFs because id=" + id
- + " does not exist.");
- }
- ReplicationPeerConfig rpc = getReplicationPeerConfig(id);
- if (rpc == null) {
- throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id);
- }
- rpc.setTableCFsMap(tableCFs);
- ZKUtil.setData(this.zookeeper, getPeerNode(id),
- ReplicationPeerConfigUtil.toByteArray(rpc));
- LOG.info("Peer tableCFs with id= " + id + " is now " +
- ReplicationPeerConfigUtil.convertToString(tableCFs));
- } catch (KeeperException e) {
- throw new ReplicationException("Unable to change tableCFs of the peer with id=" + id, e);
- }
- }
-
- @Override
- public boolean getStatusOfPeer(String id) {
- ReplicationPeer replicationPeer = this.peerClusters.get(id);
- if (replicationPeer == null) {
- throw new IllegalArgumentException("Peer with id= " + id + " is not cached");
- }
- return replicationPeer.getPeerState() == PeerState.ENABLED;
- }
-
- @Override
- public boolean getStatusOfPeerFromBackingStore(String id) throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("peer " + id + " doesn't exist");
- }
- String peerStateZNode = getPeerStateNode(id);
- try {
- return ReplicationPeerImpl.isStateEnabled(ZKUtil.getData(this.zookeeper, peerStateZNode));
- } catch (KeeperException e) {
- throw new ReplicationException(e);
- } catch (DeserializationException e) {
- throw new ReplicationException(e);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Unable to get status of the peer with id=" + id +
- " from backing store", e);
- } catch (InterruptedException e) {
- throw new ReplicationException(e);
- }
- }
-
- @Override
- public Map<String, ReplicationPeerConfig> getAllPeerConfigs() {
- Map<String, ReplicationPeerConfig> peers = new TreeMap<>();
- List<String> ids = null;
- try {
- ids = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
- for (String id : ids) {
- ReplicationPeerConfig peerConfig = getReplicationPeerConfig(id);
- if (peerConfig == null) {
- LOG.warn("Failed to get replication peer configuration of clusterid=" + id
- + " znode content, continuing.");
- continue;
- }
- peers.put(id, peerConfig);
- }
- } catch (KeeperException e) {
- this.abortable.abort("Cannot get the list of peers ", e);
- } catch (ReplicationException e) {
- this.abortable.abort("Cannot get the list of peers ", e);
- }
- return peers;
- }
-
- @Override
- public ReplicationPeerImpl getConnectedPeer(String peerId) {
- return peerClusters.get(peerId);
- }
-
- @Override
- public Set<String> getConnectedPeerIds() {
- return peerClusters.keySet(); // this is not thread-safe
- }
-
- /**
- * Returns a ReplicationPeerConfig from the znode or null for the given peerId.
- */
- @Override
- public ReplicationPeerConfig getReplicationPeerConfig(String peerId)
- throws ReplicationException {
- String znode = getPeerNode(peerId);
- byte[] data = null;
- try {
- data = ZKUtil.getData(this.zookeeper, znode);
- } catch (InterruptedException e) {
- LOG.warn("Could not get configuration for peer because the thread " +
- "was interrupted. peerId=" + peerId);
- Thread.currentThread().interrupt();
- return null;
- } catch (KeeperException e) {
- throw new ReplicationException("Error getting configuration for peer with id="
- + peerId, e);
- }
- if (data == null) {
- LOG.error("Could not get configuration for peer because it doesn't exist. peerId=" + peerId);
- return null;
- }
-
- try {
- return ReplicationPeerConfigUtil.parsePeerFrom(data);
- } catch (DeserializationException e) {
- LOG.warn("Failed to parse cluster key from peerId=" + peerId
- + ", specifically the content from the following znode: " + znode);
- return null;
- }
- }
-
- @Override
- public Pair<ReplicationPeerConfig, Configuration> getPeerConf(String peerId)
- throws ReplicationException {
- ReplicationPeerConfig peerConfig = getReplicationPeerConfig(peerId);
-
- if (peerConfig == null) {
- return null;
- }
-
- Configuration otherConf;
- try {
- otherConf = HBaseConfiguration.createClusterConf(this.conf, peerConfig.getClusterKey());
- } catch (IOException e) {
- LOG.error("Can't get peer configuration for peerId=" + peerId + " because:", e);
- return null;
- }
-
- if (!peerConfig.getConfiguration().isEmpty()) {
- CompoundConfiguration compound = new CompoundConfiguration();
- compound.add(otherConf);
- compound.addStringMap(peerConfig.getConfiguration());
- return new Pair<>(peerConfig, compound);
- }
-
- return new Pair<>(peerConfig, otherConf);
- }
-
- @Override
- public void updatePeerConfig(String id, ReplicationPeerConfig newConfig)
- throws ReplicationException {
- ReplicationPeer peer = getConnectedPeer(id);
- if (peer == null){
- throw new ReplicationException("Could not find peer Id " + id + " in connected peers");
- }
- ReplicationPeerConfig existingConfig = peer.getPeerConfig();
- if (!isStringEquals(newConfig.getClusterKey(), existingConfig.getClusterKey())) {
- throw new ReplicationException(
- "Changing the cluster key on an existing peer is not allowed." + " Existing key '" +
- existingConfig.getClusterKey() + "' does not match new key '" +
- newConfig.getClusterKey() + "'");
- }
- if (!isStringEquals(newConfig.getReplicationEndpointImpl(),
- existingConfig.getReplicationEndpointImpl())) {
- throw new ReplicationException("Changing the replication endpoint implementation class " +
- "on an existing peer is not allowed. Existing class '" +
- existingConfig.getReplicationEndpointImpl() + "' does not match new class '" +
- newConfig.getReplicationEndpointImpl() + "'");
- }
-
- // Update existingConfig's peer config and peer data with the new values, but don't touch config
- // or data that weren't explicitly changed
- ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder(existingConfig);
- builder.putAllConfiguration(newConfig.getConfiguration())
- .putAllPeerData(newConfig.getPeerData())
- .setReplicateAllUserTables(newConfig.replicateAllUserTables())
- .setNamespaces(newConfig.getNamespaces()).setTableCFsMap(newConfig.getTableCFsMap())
- .setExcludeNamespaces(newConfig.getExcludeNamespaces())
- .setExcludeTableCFsMap(newConfig.getExcludeTableCFsMap())
- .setBandwidth(newConfig.getBandwidth());
-
- try {
- ZKUtil.setData(this.zookeeper, getPeerNode(id),
- ReplicationPeerConfigUtil.toByteArray(builder.build()));
- }
- catch(KeeperException ke){
- throw new ReplicationException("There was a problem trying to save changes to the " +
- "replication peer " + id, ke);
- }
- }
-
- /**
- * List all registered peer clusters and set a watch on their znodes.
- */
- @Override
- public List<String> getAllPeerIds() {
- List<String> ids = null;
- try {
- ids = ZKUtil.listChildrenAndWatchThem(this.zookeeper, this.peersZNode);
- } catch (KeeperException e) {
- this.abortable.abort("Cannot get the list of peers ", e);
- }
- return ids;
- }
-
- /**
- * A private method used during initialization. This method attempts to add all registered
- * peer clusters. This method does not set a watch on the peer cluster znodes.
- */
- private void addExistingPeers() throws ReplicationException {
- List<String> znodes = null;
- try {
- znodes = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
- } catch (KeeperException e) {
- throw new ReplicationException("Error getting the list of peer clusters.", e);
- }
- if (znodes != null) {
- for (String z : znodes) {
- createAndAddPeer(z);
- }
- }
- }
-
- @Override
- public boolean peerConnected(String peerId) throws ReplicationException {
- return createAndAddPeer(peerId);
- }
-
- @Override
- public void peerDisconnected(String peerId) {
- ReplicationPeer rp = this.peerClusters.get(peerId);
- if (rp != null) {
- peerClusters.remove(peerId, rp);
- }
- }
-
- /**
- * Attempt to connect to a new remote slave cluster.
- * @param peerId a short that identifies the cluster
- * @return true if a new connection was made, false if no new connection was made.
- */
- public boolean createAndAddPeer(String peerId) throws ReplicationException {
- if (peerClusters == null) {
- return false;
- }
- if (this.peerClusters.containsKey(peerId)) {
- return false;
- }
-
- ReplicationPeerImpl peer = null;
- try {
- peer = createPeer(peerId);
- } catch (Exception e) {
- throw new ReplicationException("Error adding peer with id=" + peerId, e);
- }
- if (peer == null) {
- return false;
- }
- ReplicationPeerImpl previous = peerClusters.putIfAbsent(peerId, peer);
- if (previous == null) {
- LOG.info("Added new peer cluster=" + peer.getPeerConfig().getClusterKey());
- } else {
- LOG.info("Peer already present, " + previous.getPeerConfig().getClusterKey() +
- ", new cluster=" + peer.getPeerConfig().getClusterKey());
- }
- return true;
- }
-
- /**
- * Update the state znode of a peer cluster.
- * @param id
- * @param state
- */
- private void changePeerState(String id, ReplicationProtos.ReplicationState.State state)
- throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("Cannot enable/disable peer because id=" + id
- + " does not exist.");
- }
- String peerStateZNode = getPeerStateNode(id);
- byte[] stateBytes =
- (state == ReplicationProtos.ReplicationState.State.ENABLED) ? ENABLED_ZNODE_BYTES
- : DISABLED_ZNODE_BYTES;
- if (ZKUtil.checkExists(this.zookeeper, peerStateZNode) != -1) {
- ZKUtil.setData(this.zookeeper, peerStateZNode, stateBytes);
- } else {
- ZKUtil.createAndWatch(this.zookeeper, peerStateZNode, stateBytes);
- }
- LOG.info("Peer with id= " + id + " is now " + state.name());
- } catch (KeeperException e) {
- throw new ReplicationException("Unable to change state of the peer with id=" + id, e);
- }
- }
-
- /**
- * Helper method to connect to a peer
- * @param peerId peer's identifier
- * @return object representing the peer
- * @throws ReplicationException
- */
- private ReplicationPeerImpl createPeer(String peerId) throws ReplicationException {
- Pair<ReplicationPeerConfig, Configuration> pair = getPeerConf(peerId);
- if (pair == null) {
- return null;
- }
- Configuration peerConf = pair.getSecond();
-
- ReplicationPeerImpl peer =
- new ReplicationPeerImpl(zookeeper, peerConf, peerId, pair.getFirst());
-
- // Load peer state and peer config by reading zookeeper directly.
- peer.refreshPeerState();
- peer.refreshPeerConfig();
-
- return peer;
- }
-
- private void checkQueuesDeleted(String peerId) throws ReplicationException {
- if (queueStorage == null) {
- return;
- }
- try {
- List<ServerName> replicators = queueStorage.getListOfReplicators();
- if (replicators == null || replicators.isEmpty()) {
- return;
- }
- for (ServerName replicator : replicators) {
- List<String> queueIds = queueStorage.getAllQueues(replicator);
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (queueInfo.getPeerId().equals(peerId)) {
- throw new IllegalArgumentException("undeleted queue for peerId: " + peerId
- + ", replicator: " + replicator + ", queueId: " + queueId);
- }
- }
- }
- // Check for hfile-refs queue
- if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
- && queueStorage.getAllPeersFromHFileRefsQueue().contains(peerId)) {
- throw new IllegalArgumentException("Undeleted queue for peerId: " + peerId
- + ", found in hfile-refs node path " + hfileRefsZNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Could not check queues deleted with id=" + peerId, e);
- }
- }
-
- /**
- * For replication peer cluster key or endpoint class, null and empty string is same. So here
- * don't use {@link StringUtils#equals(CharSequence, CharSequence)} directly.
- */
- private boolean isStringEquals(String s1, String s2) {
- if (StringUtils.isBlank(s1)) {
- return StringUtils.isBlank(s2);
- }
- return s1.equals(s2);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
index 49af4c3..bf448e8 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.replication;
import java.util.Arrays;
import java.util.List;
-import java.util.Optional;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
@@ -144,7 +143,7 @@ class ZKReplicationPeerStorage extends ZKReplicationStorageBase implements Repli
}
@Override
- public Optional<ReplicationPeerConfig> getPeerConfig(String peerId) throws ReplicationException {
+ public ReplicationPeerConfig getPeerConfig(String peerId) throws ReplicationException {
byte[] data;
try {
data = ZKUtil.getData(zookeeper, getPeerNode(peerId));
@@ -152,13 +151,14 @@ class ZKReplicationPeerStorage extends ZKReplicationStorageBase implements Repli
throw new ReplicationException("Error getting configuration for peer with id=" + peerId, e);
}
if (data == null || data.length == 0) {
- return Optional.empty();
+ throw new ReplicationException(
+ "Replication peer config data shouldn't be empty, peerId=" + peerId);
}
try {
- return Optional.of(ReplicationPeerConfigUtil.parsePeerFrom(data));
+ return ReplicationPeerConfigUtil.parsePeerFrom(data);
} catch (DeserializationException e) {
- LOG.warn("Failed to parse replication peer config for peer with id=" + peerId, e);
- return Optional.empty();
+ throw new ReplicationException(
+ "Failed to parse replication peer config for peer with id=" + peerId, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
index b8a2044..d09a56b 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
@@ -48,8 +48,7 @@ class ZKReplicationStorageBase {
String replicationZNodeName = conf.get("zookeeper.znode.replication", "replication");
this.replicationZNode =
- ZNodePaths.joinZNode(this.zookeeper.znodePaths.baseZNode, replicationZNodeName);
-
+ ZNodePaths.joinZNode(this.zookeeper.znodePaths.baseZNode, replicationZNodeName);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
index 4afda5d..2589199 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
@@ -55,7 +55,6 @@ public abstract class TestReplicationStateBasic {
protected static String KEY_TWO;
// For testing when we try to replicate to ourself
- protected String OUR_ID = "3";
protected String OUR_KEY;
protected static int zkTimeoutCount;
@@ -152,37 +151,6 @@ public abstract class TestReplicationStateBasic {
}
@Test
- public void testInvalidClusterKeys() throws ReplicationException, KeeperException {
- rp.init();
-
- try {
- rp.registerPeer(ID_ONE,
- new ReplicationPeerConfig().setClusterKey("hostname1.example.org:1234:hbase"));
- fail("Should throw an IllegalArgumentException because " +
- "zookeeper.znode.parent is missing leading '/'.");
- } catch (IllegalArgumentException e) {
- // Expected.
- }
-
- try {
- rp.registerPeer(ID_ONE,
- new ReplicationPeerConfig().setClusterKey("hostname1.example.org:1234:/"));
- fail("Should throw an IllegalArgumentException because zookeeper.znode.parent is missing.");
- } catch (IllegalArgumentException e) {
- // Expected.
- }
-
- try {
- rp.registerPeer(ID_ONE,
- new ReplicationPeerConfig().setClusterKey("hostname1.example.org::/hbase"));
- fail("Should throw an IllegalArgumentException because " +
- "hbase.zookeeper.property.clientPort is missing.");
- } catch (IllegalArgumentException e) {
- // Expected.
- }
- }
-
- @Test
public void testHfileRefsReplicationQueues() throws ReplicationException, KeeperException {
rp.init();
@@ -192,7 +160,8 @@ public abstract class TestReplicationStateBasic {
files1.add(new Pair<>(null, new Path("file_3")));
assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
- rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
+ rp.getPeerStorage().addPeer(ID_ONE,
+ ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build(), true);
rqs.addPeerToHFileRefs(ID_ONE);
rqs.addHFileRefs(ID_ONE, files1);
assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
@@ -208,15 +177,17 @@ public abstract class TestReplicationStateBasic {
hfiles2.add(removedString);
rqs.removeHFileRefs(ID_ONE, hfiles2);
assertEquals(0, rqs.getReplicableHFiles(ID_ONE).size());
- rp.unregisterPeer(ID_ONE);
+ rp.getPeerStorage().removePeer(ID_ONE);
}
@Test
public void testRemovePeerForHFileRefs() throws ReplicationException, KeeperException {
rp.init();
- rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
+ rp.getPeerStorage().addPeer(ID_ONE,
+ ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build(), true);
rqs.addPeerToHFileRefs(ID_ONE);
- rp.registerPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO));
+ rp.getPeerStorage().addPeer(ID_TWO,
+ ReplicationPeerConfig.newBuilder().setClusterKey(KEY_TWO).build(), true);
rqs.addPeerToHFileRefs(ID_TWO);
List<Pair<Path, Path>> files1 = new ArrayList<>(3);
@@ -229,13 +200,13 @@ public abstract class TestReplicationStateBasic {
assertEquals(3, rqs.getReplicableHFiles(ID_ONE).size());
assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
- rp.unregisterPeer(ID_ONE);
+ rp.getPeerStorage().removePeer(ID_ONE);
rqs.removePeerFromHFileRefs(ID_ONE);
assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
- rp.unregisterPeer(ID_TWO);
+ rp.getPeerStorage().removePeer(ID_TWO);
rqs.removePeerFromHFileRefs(ID_TWO);
assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
assertTrue(rqs.getReplicableHFiles(ID_TWO).isEmpty());
@@ -245,74 +216,77 @@ public abstract class TestReplicationStateBasic {
public void testReplicationPeers() throws Exception {
rp.init();
- // Test methods with non-existent peer ids
try {
- rp.unregisterPeer("bogus");
+ rp.getPeerStorage().setPeerState("bogus", true);
fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
- } catch (IllegalArgumentException e) {
+ } catch (ReplicationException e) {
}
try {
- rp.enablePeer("bogus");
+ rp.getPeerStorage().setPeerState("bogus", false);
fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
- } catch (IllegalArgumentException e) {
+ } catch (ReplicationException e) {
}
try {
- rp.disablePeer("bogus");
+ rp.isPeerEnabled("bogus");
fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
} catch (IllegalArgumentException e) {
}
+
try {
- rp.getStatusOfPeer("bogus");
- fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
- } catch (IllegalArgumentException e) {
+ assertFalse(rp.addPeer("bogus"));
+ fail("Should have thrown an ReplicationException when passed a bogus peerId");
+ } catch (ReplicationException e) {
+ }
+
+ try {
+ assertNull(rp.getPeerClusterConfiguration("bogus"));
+ fail("Should have thrown an ReplicationException when passed a bogus peerId");
+ } catch (ReplicationException e) {
}
- assertFalse(rp.peerConnected("bogus"));
- rp.peerDisconnected("bogus");
- assertNull(rp.getPeerConf("bogus"));
assertNumberOfPeers(0);
// Add some peers
- rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
+ rp.getPeerStorage().addPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE), true);
assertNumberOfPeers(1);
- rp.registerPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO));
+ rp.getPeerStorage().addPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO), true);
assertNumberOfPeers(2);
// Test methods with a peer that is added but not connected
try {
- rp.getStatusOfPeer(ID_ONE);
+ rp.isPeerEnabled(ID_ONE);
fail("There are no connected peers, should have thrown an IllegalArgumentException");
} catch (IllegalArgumentException e) {
}
- assertEquals(KEY_ONE, ZKConfig.getZooKeeperClusterKey(rp.getPeerConf(ID_ONE).getSecond()));
- rp.unregisterPeer(ID_ONE);
- rp.peerDisconnected(ID_ONE);
+ assertEquals(KEY_ONE, ZKConfig.getZooKeeperClusterKey(rp.getPeerClusterConfiguration(ID_ONE)));
+ rp.getPeerStorage().removePeer(ID_ONE);
+ rp.removePeer(ID_ONE);
assertNumberOfPeers(1);
// Add one peer
- rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
- rp.peerConnected(ID_ONE);
+ rp.getPeerStorage().addPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE), true);
+ rp.addPeer(ID_ONE);
assertNumberOfPeers(2);
- assertTrue(rp.getStatusOfPeer(ID_ONE));
- rp.disablePeer(ID_ONE);
+ assertTrue(rp.isPeerEnabled(ID_ONE));
+ rp.getPeerStorage().setPeerState(ID_ONE, false);
// now we do not rely on zk watcher to trigger the state change so we need to trigger it
// manually...
- ReplicationPeerImpl peer = rp.getConnectedPeer(ID_ONE);
- peer.refreshPeerState();
+ ReplicationPeerImpl peer = rp.getPeer(ID_ONE);
+ rp.refreshPeerState(peer.getId());
assertEquals(PeerState.DISABLED, peer.getPeerState());
assertConnectedPeerStatus(false, ID_ONE);
- rp.enablePeer(ID_ONE);
+ rp.getPeerStorage().setPeerState(ID_ONE, true);
// now we do not rely on zk watcher to trigger the state change so we need to trigger it
// manually...
- peer.refreshPeerState();
+ rp.refreshPeerState(peer.getId());
assertEquals(PeerState.ENABLED, peer.getPeerState());
assertConnectedPeerStatus(true, ID_ONE);
// Disconnect peer
- rp.peerDisconnected(ID_ONE);
+ rp.removePeer(ID_ONE);
assertNumberOfPeers(2);
try {
- rp.getStatusOfPeer(ID_ONE);
+ rp.isPeerEnabled(ID_ONE);
fail("There are no connected peers, should have thrown an IllegalArgumentException");
} catch (IllegalArgumentException e) {
}
@@ -320,16 +294,16 @@ public abstract class TestReplicationStateBasic {
protected void assertConnectedPeerStatus(boolean status, String peerId) throws Exception {
// we can first check if the value was changed in the store, if it wasn't then fail right away
- if (status != rp.getStatusOfPeerFromBackingStore(peerId)) {
+ if (status != rp.getPeerStorage().isPeerEnabled(peerId)) {
fail("ConnectedPeerStatus was " + !status + " but expected " + status + " in ZK");
}
while (true) {
- if (status == rp.getStatusOfPeer(peerId)) {
+ if (status == rp.isPeerEnabled(peerId)) {
return;
}
if (zkTimeoutCount < ZK_MAX_COUNT) {
- LOG.debug("ConnectedPeerStatus was " + !status + " but expected " + status +
- ", sleeping and trying again.");
+ LOG.debug("ConnectedPeerStatus was " + !status + " but expected " + status
+ + ", sleeping and trying again.");
Thread.sleep(ZK_SLEEP_INTERVAL);
} else {
fail("Timed out waiting for ConnectedPeerStatus to be " + status);
@@ -337,10 +311,8 @@ public abstract class TestReplicationStateBasic {
}
}
- protected void assertNumberOfPeers(int total) {
- assertEquals(total, rp.getAllPeerConfigs().size());
- assertEquals(total, rp.getAllPeerIds().size());
- assertEquals(total, rp.getAllPeerIds().size());
+ protected void assertNumberOfPeers(int total) throws ReplicationException {
+ assertEquals(total, rp.getPeerStorage().listPeerIds().size());
}
/*
@@ -359,8 +331,9 @@ public abstract class TestReplicationStateBasic {
rqs.addWAL(server3, "qId" + i, "filename" + j);
}
// Add peers for the corresponding queues so they are not orphans
- rp.registerPeer("qId" + i,
- new ReplicationPeerConfig().setClusterKey("localhost:2818:/bogus" + i));
+ rp.getPeerStorage().addPeer("qId" + i,
+ ReplicationPeerConfig.newBuilder().setClusterKey("localhost:2818:/bogus" + i).build(),
+ true);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
index ac869d9..6825c36 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
@@ -79,7 +79,7 @@ public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
public void setUp() {
zkTimeoutCount = 0;
rqs = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
- rp = ReplicationFactory.getReplicationPeers(zkw, conf, new WarnOnlyAbortable());
+ rp = ReplicationFactory.getReplicationPeers(zkw, conf);
OUR_KEY = ZKConfig.getZooKeeperClusterKey(conf);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
index a3be1e6..e8098c8 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.HashMap;
@@ -143,14 +144,14 @@ public class TestZKReplicationPeerStorage {
assertEquals(peerCount, peerIds.size());
for (String peerId : peerIds) {
int seed = Integer.parseInt(peerId);
- assertConfigEquals(getConfig(seed), STORAGE.getPeerConfig(peerId).get());
+ assertConfigEquals(getConfig(seed), STORAGE.getPeerConfig(peerId));
}
for (int i = 0; i < peerCount; i++) {
STORAGE.updatePeerConfig(Integer.toString(i), getConfig(i + 1));
}
for (String peerId : peerIds) {
int seed = Integer.parseInt(peerId);
- assertConfigEquals(getConfig(seed + 1), STORAGE.getPeerConfig(peerId).get());
+ assertConfigEquals(getConfig(seed + 1), STORAGE.getPeerConfig(peerId));
}
for (int i = 0; i < peerCount; i++) {
assertEquals(i % 2 == 0, STORAGE.isPeerEnabled(Integer.toString(i)));
@@ -166,6 +167,11 @@ public class TestZKReplicationPeerStorage {
peerIds = STORAGE.listPeerIds();
assertEquals(peerCount - 1, peerIds.size());
assertFalse(peerIds.contains(toRemove));
- assertFalse(STORAGE.getPeerConfig(toRemove).isPresent());
+
+ try {
+ STORAGE.getPeerConfig(toRemove);
+ fail("Should throw a ReplicationException when get peer config of a peerId");
+ } catch (ReplicationException e) {
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
index af41399..f2c3ec9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/ReplicationZKNodeCleaner.java
@@ -30,8 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
+import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
@@ -51,20 +50,14 @@ import org.slf4j.LoggerFactory;
public class ReplicationZKNodeCleaner {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationZKNodeCleaner.class);
private final ReplicationQueueStorage queueStorage;
- private final ReplicationPeers replicationPeers;
+ private final ReplicationPeerStorage peerStorage;
private final ReplicationQueueDeletor queueDeletor;
public ReplicationZKNodeCleaner(Configuration conf, ZKWatcher zkw, Abortable abortable)
throws IOException {
- try {
- this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
- this.replicationPeers =
- ReplicationFactory.getReplicationPeers(zkw, conf, this.queueStorage, abortable);
- this.replicationPeers.init();
- this.queueDeletor = new ReplicationQueueDeletor(zkw, conf, abortable);
- } catch (ReplicationException e) {
- throw new IOException("failed to construct ReplicationZKNodeCleaner", e);
- }
+ this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
+ this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zkw, conf);
+ this.queueDeletor = new ReplicationQueueDeletor(zkw, conf, abortable);
}
/**
@@ -73,8 +66,8 @@ public class ReplicationZKNodeCleaner {
*/
public Map<ServerName, List<String>> getUnDeletedQueues() throws IOException {
Map<ServerName, List<String>> undeletedQueues = new HashMap<>();
- Set<String> peerIds = new HashSet<>(this.replicationPeers.getAllPeerIds());
try {
+ Set<String> peerIds = new HashSet<>(peerStorage.listPeerIds());
List<ServerName> replicators = this.queueStorage.getListOfReplicators();
if (replicators == null || replicators.isEmpty()) {
return undeletedQueues;
@@ -84,8 +77,7 @@ public class ReplicationZKNodeCleaner {
for (String queueId : queueIds) {
ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
if (!peerIds.contains(queueInfo.getPeerId())) {
- undeletedQueues.computeIfAbsent(replicator, (key) -> new ArrayList<>()).add(
- queueId);
+ undeletedQueues.computeIfAbsent(replicator, (key) -> new ArrayList<>()).add(queueId);
if (LOG.isDebugEnabled()) {
LOG.debug("Undeleted replication queue for removed peer found: "
+ String.format("[removedPeerId=%s, replicator=%s, queueId=%s]",
@@ -106,9 +98,9 @@ public class ReplicationZKNodeCleaner {
*/
public Set<String> getUnDeletedHFileRefsQueues() throws IOException {
Set<String> undeletedHFileRefsQueue = new HashSet<>();
- Set<String> peerIds = new HashSet<>(this.replicationPeers.getAllPeerIds());
String hfileRefsZNode = queueDeletor.getHfileRefsZNode();
try {
+ Set<String> peerIds = new HashSet<>(peerStorage.listPeerIds());
List<String> listOfPeers = this.queueStorage.getAllPeersFromHFileRefsQueue();
Set<String> peers = new HashSet<>(listOfPeers);
peers.removeAll(peerIds);
@@ -116,15 +108,15 @@ public class ReplicationZKNodeCleaner {
undeletedHFileRefsQueue.addAll(peers);
}
} catch (ReplicationException e) {
- throw new IOException(
- "Failed to get list of all peers from hfile-refs znode " + hfileRefsZNode, e);
+ throw new IOException("Failed to get list of all peers from hfile-refs znode "
+ + hfileRefsZNode, e);
}
return undeletedHFileRefsQueue;
}
private class ReplicationQueueDeletor extends ReplicationStateZKBase {
- public ReplicationQueueDeletor(ZKWatcher zk, Configuration conf, Abortable abortable) {
+ ReplicationQueueDeletor(ZKWatcher zk, Configuration conf, Abortable abortable) {
super(zk, conf, abortable);
}
@@ -132,19 +124,20 @@ public class ReplicationZKNodeCleaner {
* @param replicator The regionserver which has undeleted queue
* @param queueId The undeleted queue id
*/
- public void removeQueue(final ServerName replicator, final String queueId) throws IOException {
- String queueZnodePath = ZNodePaths
- .joinZNode(ZNodePaths.joinZNode(this.queuesZNode, replicator.getServerName()), queueId);
+ void removeQueue(final ServerName replicator, final String queueId) throws IOException {
+ String queueZnodePath =
+ ZNodePaths.joinZNode(ZNodePaths.joinZNode(this.queuesZNode, replicator.getServerName()),
+ queueId);
try {
ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (!replicationPeers.getAllPeerIds().contains(queueInfo.getPeerId())) {
+ if (!peerStorage.listPeerIds().contains(queueInfo.getPeerId())) {
ZKUtil.deleteNodeRecursively(this.zookeeper, queueZnodePath);
- LOG.info("Successfully removed replication queue, replicator: " + replicator +
- ", queueId: " + queueId);
+ LOG.info("Successfully removed replication queue, replicator: " + replicator
+ + ", queueId: " + queueId);
}
- } catch (KeeperException e) {
- throw new IOException(
- "Failed to delete queue, replicator: " + replicator + ", queueId: " + queueId);
+ } catch (ReplicationException | KeeperException e) {
+ throw new IOException("Failed to delete queue, replicator: " + replicator + ", queueId: "
+ + queueId);
}
}
@@ -152,17 +145,17 @@ public class ReplicationZKNodeCleaner {
* @param hfileRefsQueueId The undeleted hfile-refs queue id
* @throws IOException
*/
- public void removeHFileRefsQueue(final String hfileRefsQueueId) throws IOException {
+ void removeHFileRefsQueue(final String hfileRefsQueueId) throws IOException {
String node = ZNodePaths.joinZNode(this.hfileRefsZNode, hfileRefsQueueId);
try {
- if (!replicationPeers.getAllPeerIds().contains(hfileRefsQueueId)) {
+ if (!peerStorage.listPeerIds().contains(hfileRefsQueueId)) {
ZKUtil.deleteNodeRecursively(this.zookeeper, node);
LOG.info("Successfully removed hfile-refs queue " + hfileRefsQueueId + " from path "
+ hfileRefsZNode);
}
- } catch (KeeperException e) {
+ } catch (ReplicationException | KeeperException e) {
throw new IOException("Failed to delete hfile-refs queue " + hfileRefsQueueId
- + " from path " + hfileRefsZNode);
+ + " from path " + hfileRefsZNode, e);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index f4ccce8..b6732d7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -314,12 +314,12 @@ public class ReplicationPeerManager {
public static ReplicationPeerManager create(ZKWatcher zk, Configuration conf)
throws ReplicationException {
ReplicationPeerStorage peerStorage =
- ReplicationStorageFactory.getReplicationPeerStorage(zk, conf);
+ ReplicationStorageFactory.getReplicationPeerStorage(zk, conf);
ConcurrentMap<String, ReplicationPeerDescription> peers = new ConcurrentHashMap<>();
for (String peerId : peerStorage.listPeerIds()) {
- Optional<ReplicationPeerConfig> peerConfig = peerStorage.getPeerConfig(peerId);
+ ReplicationPeerConfig peerConfig = peerStorage.getPeerConfig(peerId);
boolean enabled = peerStorage.isPeerEnabled(peerId);
- peers.put(peerId, new ReplicationPeerDescription(peerId, enabled, peerConfig.get()));
+ peers.put(peerId, new ReplicationPeerDescription(peerId, enabled, peerConfig));
}
return new ReplicationPeerManager(peerStorage,
ReplicationStorageFactory.getReplicationQueueStorage(zk, conf), peers);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 73e600e..27bda2d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -310,7 +310,7 @@ public class DumpReplicationQueues extends Configured implements Tool {
queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
replicationPeers =
- ReplicationFactory.getReplicationPeers(zkw, getConf(), queueStorage, connection);
+ ReplicationFactory.getReplicationPeers(zkw, getConf());
replicationTracker = ReplicationFactory.getReplicationTracker(zkw, replicationPeers, getConf(),
new WarnOnlyAbortable(), new WarnOnlyStoppable());
Set<String> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index 598357c..1efe180 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -19,9 +19,10 @@
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationPeerImpl;
+import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,7 +31,8 @@ import org.slf4j.LoggerFactory;
public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
private static final Logger LOG = LoggerFactory.getLogger(PeerProcedureHandlerImpl.class);
- private ReplicationSourceManager replicationSourceManager;
+ private final ReplicationSourceManager replicationSourceManager;
+ private final ReentrantLock peersLock = new ReentrantLock();
public PeerProcedureHandlerImpl(ReplicationSourceManager replicationSourceManager) {
this.replicationSourceManager = replicationSourceManager;
@@ -38,45 +40,40 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
@Override
public void addPeer(String peerId) throws ReplicationException, IOException {
- replicationSourceManager.addPeer(peerId);
+ peersLock.lock();
+ try {
+ replicationSourceManager.addPeer(peerId);
+ } finally {
+ peersLock.unlock();
+ }
}
@Override
public void removePeer(String peerId) throws ReplicationException, IOException {
- replicationSourceManager.removePeer(peerId);
+ peersLock.lock();
+ try {
+ if (replicationSourceManager.getReplicationPeers().getPeer(peerId) != null) {
+ replicationSourceManager.removePeer(peerId);
+ }
+ } finally {
+ peersLock.unlock();
+ }
}
@Override
public void disablePeer(String peerId) throws ReplicationException, IOException {
- ReplicationPeerImpl peer =
- replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
- if (peer != null) {
- peer.refreshPeerState();
- LOG.info("disable replication peer, id: " + peerId + ", new state: " + peer.getPeerState());
- } else {
- throw new ReplicationException("No connected peer found, peerId=" + peerId);
- }
+ PeerState newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+ LOG.info("disable replication peer, id: " + peerId + ", new state: " + newState);
}
@Override
public void enablePeer(String peerId) throws ReplicationException, IOException {
- ReplicationPeerImpl peer =
- replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
- if (peer != null) {
- peer.refreshPeerState();
- LOG.info("enable replication peer, id: " + peerId + ", new state: " + peer.getPeerState());
- } else {
- throw new ReplicationException("No connected peer found, peerId=" + peerId);
- }
+ PeerState newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+ LOG.info("enable replication peer, id: " + peerId + ", new state: " + newState);
}
@Override
public void updatePeerConfig(String peerId) throws ReplicationException, IOException {
- ReplicationPeerImpl peer =
- replicationSourceManager.getReplicationPeers().getConnectedPeer(peerId);
- if (peer == null) {
- throw new ReplicationException("No connected peer found, peerId=" + peerId);
- }
- peer.refreshPeerConfig();
+ replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index d555c6e..a8991a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -130,7 +130,7 @@ public class Replication implements
this.queueStorage =
ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
this.replicationPeers =
- ReplicationFactory.getReplicationPeers(server.getZooKeeper(), this.conf, this.server);
+ ReplicationFactory.getReplicationPeers(server.getZooKeeper(), this.conf);
this.replicationPeers.init();
this.replicationTracker =
ReplicationFactory.getReplicationTracker(server.getZooKeeper(), this.replicationPeers,
http://git-wip-us.apache.org/repos/asf/hbase/blob/a791c238/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 58ea6ee..8250992 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -222,8 +222,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
// A peerId will not have "-" in its name, see HBASE-11394
peerId = peerClusterZnode.split("-")[0];
}
- Map<TableName, List<String>> tableCFMap =
- replicationPeers.getConnectedPeer(peerId).getTableCFs();
+ Map<TableName, List<String>> tableCFMap = replicationPeers.getPeer(peerId).getTableCFs();
if (tableCFMap != null) {
List<String> tableCfs = tableCFMap.get(tableName);
if (tableCFMap.containsKey(tableName)
@@ -371,7 +370,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
}
private long getCurrentBandwidth() {
- ReplicationPeer replicationPeer = this.replicationPeers.getConnectedPeer(peerId);
+ ReplicationPeer replicationPeer = this.replicationPeers.getPeer(peerId);
long peerBandwidth = replicationPeer != null ? replicationPeer.getPeerBandwidth() : 0;
// user can set peer bandwidth to 0 to use default bandwidth
return peerBandwidth != 0 ? peerBandwidth : defaultBandwidth;
@@ -416,7 +415,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
*/
@Override
public boolean isPeerEnabled() {
- return this.replicationPeers.getStatusOfPeer(this.peerId);
+ return this.replicationPeers.isPeerEnabled(this.peerId);
}
@Override
[46/50] [abbrv] hbase git commit: HBASE-19719 Fix checkstyle issues
Posted by zh...@apache.org.
HBASE-19719 Fix checkstyle issues
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2aff1edf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2aff1edf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2aff1edf
Branch: refs/heads/HBASE-19397-branch-2
Commit: 2aff1edf39d161bcbf3a3d332dedb1ad809ccf4e
Parents: e4687c9
Author: zhangduo <zh...@apache.org>
Authored: Sat Jan 6 08:30:55 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:20:38 2018 +0800
----------------------------------------------------------------------
.../hbase/replication/ReplicationStorageFactory.java | 2 +-
.../master/assignment/RegionTransitionProcedure.java | 4 ++--
.../hbase/master/procedure/RSProcedureDispatcher.java | 13 ++++++-------
.../master/ReplicationPeerConfigUpgrader.java | 8 ++++----
4 files changed, 13 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aff1edf/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
index 60d0749..462cfed 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
@@ -27,7 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* For now we only have zk based implementation.
*/
@InterfaceAudience.Private
-public class ReplicationStorageFactory {
+public final class ReplicationStorageFactory {
private ReplicationStorageFactory() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aff1edf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index 1724a38..8277dbe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -36,11 +36,11 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionTransitionState;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-
/**
* Base class for the Assign and Unassign Procedure.
*
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aff1edf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index eb4680c..bfdb682 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.master.procedure;
import java.io.IOException;
@@ -36,6 +35,12 @@ import org.apache.hadoop.ipc.RemoteException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
@@ -47,12 +52,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionR
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.RemoteProcedureRequest;
-import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
-import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
-import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
-import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
-import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
-
/**
* A remote procecdure dispatcher for regionservers.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aff1edf/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java
index b6e8862..f524fd7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -24,7 +23,6 @@ import static org.apache.hadoop.hbase.replication.ZKReplicationStorageBase.REPLI
import static org.apache.hadoop.hbase.replication.ZKReplicationStorageBase.REPLICATION_ZNODE_DEFAULT;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Admin;
@@ -43,9 +41,11 @@ import org.apache.yetus.audience.InterfaceStability;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+
/**
* This class is used to upgrade TableCFs from HBase 1.0, 1.1, 1.2, 1.3 to HBase 1.4 or 2.x. It will
* be removed in HBase 3.x. See HBASE-11393
[48/50] [abbrv] hbase git commit: HBASE-19707 Race in start and
terminate of a replication source after we async start replicatione endpoint
Posted by zh...@apache.org.
HBASE-19707 Race in start and terminate of a replication source after we async start replicatione endpoint
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b0e88ed1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b0e88ed1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b0e88ed1
Branch: refs/heads/HBASE-19397-branch-2
Commit: b0e88ed14aa6b40a1d26d9d4cd9175c769dc97ed
Parents: aab18b4
Author: zhangduo <zh...@apache.org>
Authored: Fri Jan 5 18:28:44 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:20:38 2018 +0800
----------------------------------------------------------------------
.../RecoveredReplicationSource.java | 16 +-
.../regionserver/ReplicationSource.java | 203 ++++++++++---------
2 files changed, 116 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b0e88ed1/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index 1be9a88..3cae0f2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -68,7 +68,7 @@ public class RecoveredReplicationSource extends ReplicationSource {
LOG.debug("Someone has beat us to start a worker thread for wal group " + walGroupId);
} else {
LOG.debug("Starting up worker for wal group " + walGroupId);
- worker.startup(getUncaughtExceptionHandler());
+ worker.startup(this::uncaughtException);
worker.setWALReader(
startNewWALReader(worker.getName(), walGroupId, queue, worker.getStartPosition()));
workerThreads.put(walGroupId, worker);
@@ -76,13 +76,13 @@ public class RecoveredReplicationSource extends ReplicationSource {
}
@Override
- protected ReplicationSourceWALReader startNewWALReader(String threadName,
- String walGroupId, PriorityBlockingQueue<Path> queue, long startPosition) {
- ReplicationSourceWALReader walReader = new RecoveredReplicationSourceWALReader(fs,
- conf, queue, startPosition, walEntryFilter, this);
- Threads.setDaemonThreadRunning(walReader, threadName
- + ".replicationSource.replicationWALReaderThread." + walGroupId + "," + queueId,
- getUncaughtExceptionHandler());
+ protected ReplicationSourceWALReader startNewWALReader(String threadName, String walGroupId,
+ PriorityBlockingQueue<Path> queue, long startPosition) {
+ ReplicationSourceWALReader walReader =
+ new RecoveredReplicationSourceWALReader(fs, conf, queue, startPosition, walEntryFilter, this);
+ Threads.setDaemonThreadRunning(walReader,
+ threadName + ".replicationSource.replicationWALReaderThread." + walGroupId + "," + queueId,
+ this::uncaughtException);
return walReader;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b0e88ed1/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 0092251..09b6cc1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -75,7 +75,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
* </p>
*/
@InterfaceAudience.Private
-public class ReplicationSource extends Thread implements ReplicationSourceInterface {
+public class ReplicationSource implements ReplicationSourceInterface {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationSource.class);
// Queues of logs to process, entry in format of walGroupId->queue,
@@ -114,10 +114,8 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
private MetricsSource metrics;
// WARN threshold for the number of queued logs, defaults to 2
private int logQueueWarnThreshold;
- // whether the replication endpoint has been initialized
- private volatile boolean endpointInitialized = false;
// ReplicationEndpoint which will handle the actual replication
- private ReplicationEndpoint replicationEndpoint;
+ private volatile ReplicationEndpoint replicationEndpoint;
// A filter (or a chain of filters) for the WAL entries.
protected WALEntryFilter walEntryFilter;
// throttler
@@ -135,6 +133,8 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
public static final int DEFAULT_WAIT_ON_ENDPOINT_SECONDS = 30;
private int waitOnEndpointSeconds = -1;
+ private Thread initThread;
+
/**
* Instantiation method used by region servers
* @param conf configuration to use
@@ -196,7 +196,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
if (queue == null) {
queue = new PriorityBlockingQueue<>(queueSizePerGroup, new LogsComparator());
queues.put(logPrefix, queue);
- if (this.isSourceActive() && this.endpointInitialized) {
+ if (this.isSourceActive() && this.replicationEndpoint != null) {
// new wal group observed after source startup, start a new worker thread to track it
// notice: it's possible that log enqueued when this.running is set but worker thread
// still not launched, so it's necessary to check workerThreads before start the worker
@@ -235,28 +235,36 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
}
}
- private void initAndStartReplicationEndpoint() throws Exception {
+ private ReplicationEndpoint createReplicationEndpoint()
+ throws InstantiationException, IllegalAccessException, ClassNotFoundException, IOException {
RegionServerCoprocessorHost rsServerHost = null;
- TableDescriptors tableDescriptors = null;
if (server instanceof HRegionServer) {
rsServerHost = ((HRegionServer) server).getRegionServerCoprocessorHost();
- tableDescriptors = ((HRegionServer) server).getTableDescriptors();
}
String replicationEndpointImpl = replicationPeer.getPeerConfig().getReplicationEndpointImpl();
if (replicationEndpointImpl == null) {
// Default to HBase inter-cluster replication endpoint
replicationEndpointImpl = HBaseInterClusterReplicationEndpoint.class.getName();
}
- replicationEndpoint =
- Class.forName(replicationEndpointImpl).asSubclass(ReplicationEndpoint.class).newInstance();
+ ReplicationEndpoint replicationEndpoint =
+ Class.forName(replicationEndpointImpl).asSubclass(ReplicationEndpoint.class).newInstance();
if (rsServerHost != null) {
ReplicationEndpoint newReplicationEndPoint =
- rsServerHost.postCreateReplicationEndPoint(replicationEndpoint);
+ rsServerHost.postCreateReplicationEndPoint(replicationEndpoint);
if (newReplicationEndPoint != null) {
// Override the newly created endpoint from the hook with configured end point
replicationEndpoint = newReplicationEndPoint;
}
}
+ return replicationEndpoint;
+ }
+
+ private void initAndStartReplicationEndpoint(ReplicationEndpoint replicationEndpoint)
+ throws IOException, TimeoutException {
+ TableDescriptors tableDescriptors = null;
+ if (server instanceof HRegionServer) {
+ tableDescriptors = ((HRegionServer) server).getTableDescriptors();
+ }
replicationEndpoint
.init(new ReplicationEndpoint.Context(conf, replicationPeer.getConfiguration(), fs, peerId,
clusterId, replicationPeer, metrics, tableDescriptors, server));
@@ -264,60 +272,6 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
replicationEndpoint.awaitRunning(waitOnEndpointSeconds, TimeUnit.SECONDS);
}
- @Override
- public void run() {
- // mark we are running now
- this.sourceRunning = true;
-
- int sleepMultiplier = 1;
- while (this.isSourceActive()) {
- try {
- initAndStartReplicationEndpoint();
- break;
- } catch (Exception e) {
- LOG.warn("Error starting ReplicationEndpoint, retrying", e);
- if (replicationEndpoint != null) {
- replicationEndpoint.stop();
- replicationEndpoint = null;
- }
- if (sleepForRetries("Error starting ReplicationEndpoint", sleepMultiplier)) {
- sleepMultiplier++;
- }
- }
- }
- this.endpointInitialized = true;
-
- sleepMultiplier = 1;
- // delay this until we are in an asynchronous thread
- while (this.isSourceActive() && this.peerClusterId == null) {
- this.peerClusterId = replicationEndpoint.getPeerUUID();
- if (this.isSourceActive() && this.peerClusterId == null) {
- if (sleepForRetries("Cannot contact the peer's zk ensemble", sleepMultiplier)) {
- sleepMultiplier++;
- }
- }
- }
-
- // In rare case, zookeeper setting may be messed up. That leads to the incorrect
- // peerClusterId value, which is the same as the source clusterId
- if (clusterId.equals(peerClusterId) && !replicationEndpoint.canReplicateToSameCluster()) {
- this.terminate("ClusterId " + clusterId + " is replicating to itself: peerClusterId "
- + peerClusterId + " which is not allowed by ReplicationEndpoint:"
- + replicationEndpoint.getClass().getName(), null, false);
- this.manager.removeSource(this);
- return;
- }
- LOG.info("Replicating " + clusterId + " -> " + peerClusterId);
-
- initializeWALEntryFilter();
- // start workers
- for (Map.Entry<String, PriorityBlockingQueue<Path>> entry : queues.entrySet()) {
- String walGroupId = entry.getKey();
- PriorityBlockingQueue<Path> queue = entry.getValue();
- tryStartNewShipper(walGroupId, queue);
- }
- }
-
private void initializeWALEntryFilter() {
// get the WALEntryFilter from ReplicationEndpoint and add it to default filters
ArrayList<WALEntryFilter> filters =
@@ -331,37 +285,31 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
}
protected void tryStartNewShipper(String walGroupId, PriorityBlockingQueue<Path> queue) {
- final ReplicationSourceShipper worker = new ReplicationSourceShipper(conf,
- walGroupId, queue, this);
+ ReplicationSourceShipper worker = new ReplicationSourceShipper(conf, walGroupId, queue, this);
ReplicationSourceShipper extant = workerThreads.putIfAbsent(walGroupId, worker);
if (extant != null) {
LOG.debug("Someone has beat us to start a worker thread for wal group " + walGroupId);
} else {
LOG.debug("Starting up worker for wal group " + walGroupId);
- worker.startup(getUncaughtExceptionHandler());
- worker.setWALReader(startNewWALReader(worker.getName(), walGroupId, queue,
- worker.getStartPosition()));
+ worker.startup(this::uncaughtException);
+ worker.setWALReader(
+ startNewWALReader(worker.getName(), walGroupId, queue, worker.getStartPosition()));
}
}
protected ReplicationSourceWALReader startNewWALReader(String threadName, String walGroupId,
PriorityBlockingQueue<Path> queue, long startPosition) {
ReplicationSourceWALReader walReader =
- new ReplicationSourceWALReader(fs, conf, queue, startPosition, walEntryFilter, this);
+ new ReplicationSourceWALReader(fs, conf, queue, startPosition, walEntryFilter, this);
return (ReplicationSourceWALReader) Threads.setDaemonThreadRunning(walReader,
threadName + ".replicationSource.wal-reader." + walGroupId + "," + queueId,
- getUncaughtExceptionHandler());
+ this::uncaughtException);
}
- public Thread.UncaughtExceptionHandler getUncaughtExceptionHandler() {
- return new Thread.UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(final Thread t, final Throwable e) {
- RSRpcServices.exitIfOOME(e);
- LOG.error("Unexpected exception in " + t.getName() + " currentPath=" + getCurrentPath(), e);
- server.stop("Unexpected exception in " + t.getName());
- }
- };
+ protected final void uncaughtException(Thread t, Throwable e) {
+ RSRpcServices.exitIfOOME(e);
+ LOG.error("Unexpected exception in " + t.getName() + " currentPath=" + getCurrentPath(), e);
+ server.abort("Unexpected exception in " + t.getName(), e);
}
@Override
@@ -434,17 +382,76 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
return replicationPeer.isPeerEnabled();
}
+ private void initialize() {
+ int sleepMultiplier = 1;
+ while (this.isSourceActive()) {
+ ReplicationEndpoint replicationEndpoint;
+ try {
+ replicationEndpoint = createReplicationEndpoint();
+ } catch (Exception e) {
+ LOG.warn("error creating ReplicationEndpoint, retry", e);
+ if (sleepForRetries("Error creating ReplicationEndpoint", sleepMultiplier)) {
+ sleepMultiplier++;
+ }
+ continue;
+ }
+
+ try {
+ initAndStartReplicationEndpoint(replicationEndpoint);
+ this.replicationEndpoint = replicationEndpoint;
+ break;
+ } catch (Exception e) {
+ LOG.warn("Error starting ReplicationEndpoint, retry", e);
+ replicationEndpoint.stop();
+ if (sleepForRetries("Error starting ReplicationEndpoint", sleepMultiplier)) {
+ sleepMultiplier++;
+ }
+ }
+ }
+
+ if (!this.isSourceActive()) {
+ return;
+ }
+
+ sleepMultiplier = 1;
+ // delay this until we are in an asynchronous thread
+ while (this.isSourceActive() && this.peerClusterId == null) {
+ this.peerClusterId = replicationEndpoint.getPeerUUID();
+ if (this.isSourceActive() && this.peerClusterId == null) {
+ if (sleepForRetries("Cannot contact the peer's zk ensemble", sleepMultiplier)) {
+ sleepMultiplier++;
+ }
+ }
+ }
+
+ // In rare case, zookeeper setting may be messed up. That leads to the incorrect
+ // peerClusterId value, which is the same as the source clusterId
+ if (clusterId.equals(peerClusterId) && !replicationEndpoint.canReplicateToSameCluster()) {
+ this.terminate("ClusterId " + clusterId + " is replicating to itself: peerClusterId "
+ + peerClusterId + " which is not allowed by ReplicationEndpoint:"
+ + replicationEndpoint.getClass().getName(), null, false);
+ this.manager.removeSource(this);
+ return;
+ }
+ LOG.info("Replicating " + clusterId + " -> " + peerClusterId);
+
+ initializeWALEntryFilter();
+ // start workers
+ for (Map.Entry<String, PriorityBlockingQueue<Path>> entry : queues.entrySet()) {
+ String walGroupId = entry.getKey();
+ PriorityBlockingQueue<Path> queue = entry.getValue();
+ tryStartNewShipper(walGroupId, queue);
+ }
+ }
+
@Override
public void startup() {
- String n = Thread.currentThread().getName();
- Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(final Thread t, final Throwable e) {
- LOG.error("Unexpected exception in ReplicationSource", e);
- }
- };
- Threads.setDaemonThreadRunning(this, n + ".replicationSource," + this.queueId,
- handler);
+ // mark we are running now
+ this.sourceRunning = true;
+ initThread = new Thread(this::initialize);
+ Threads.setDaemonThreadRunning(initThread,
+ Thread.currentThread().getName() + ".replicationSource," + this.queueId,
+ this::uncaughtException);
}
@Override
@@ -465,6 +472,13 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
cause);
}
this.sourceRunning = false;
+ if (initThread != null && Thread.currentThread() != initThread) {
+ // This usually won't happen but anyway, let's wait until the initialization thread exits.
+ // And notice that we may call terminate directly from the initThread so here we need to
+ // avoid join on ourselves.
+ initThread.interrupt();
+ Threads.shutdown(initThread, this.sleepForRetries);
+ }
Collection<ReplicationSourceShipper> workers = workerThreads.values();
for (ReplicationSourceShipper worker : workers) {
worker.stopWorker();
@@ -481,12 +495,11 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
}
if (this.replicationEndpoint != null) {
try {
- this.replicationEndpoint
- .awaitTerminated(sleepForRetries * maxRetriesMultiplier, TimeUnit.MILLISECONDS);
+ this.replicationEndpoint.awaitTerminated(sleepForRetries * maxRetriesMultiplier,
+ TimeUnit.MILLISECONDS);
} catch (TimeoutException te) {
- LOG.warn("Got exception while waiting for endpoint to shutdown for replication source :"
- + this.queueId,
- te);
+ LOG.warn("Got exception while waiting for endpoint to shutdown for replication source :" +
+ this.queueId, te);
}
}
}
[24/50] [abbrv] hbase git commit: HBASE-19599 Remove
ReplicationQueuesClient, use ReplicationQueueStorage directly
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 6e27a21..d8f9625 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -21,13 +21,13 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
@@ -48,17 +48,18 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicLongMap;
/**
@@ -303,57 +304,53 @@ public class DumpReplicationQueues extends Configured implements Tool {
}
public String dumpQueues(ClusterConnection connection, ZKWatcher zkw, Set<String> peerIds,
- boolean hdfs) throws Exception {
- ReplicationQueuesClient queuesClient;
+ boolean hdfs) throws Exception {
+ ReplicationQueueStorage queueStorage;
ReplicationPeers replicationPeers;
ReplicationQueues replicationQueues;
ReplicationTracker replicationTracker;
- ReplicationQueuesClientArguments replicationArgs =
- new ReplicationQueuesClientArguments(getConf(), new WarnOnlyAbortable(), zkw);
+ ReplicationQueuesArguments replicationArgs =
+ new ReplicationQueuesArguments(getConf(), new WarnOnlyAbortable(), zkw);
StringBuilder sb = new StringBuilder();
- queuesClient = ReplicationFactory.getReplicationQueuesClient(replicationArgs);
- queuesClient.init();
+ queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
replicationQueues = ReplicationFactory.getReplicationQueues(replicationArgs);
- replicationPeers = ReplicationFactory.getReplicationPeers(zkw, getConf(), queuesClient, connection);
+ replicationPeers =
+ ReplicationFactory.getReplicationPeers(zkw, getConf(), queueStorage, connection);
replicationTracker = ReplicationFactory.getReplicationTracker(zkw, replicationPeers, getConf(),
new WarnOnlyAbortable(), new WarnOnlyStoppable());
- List<String> liveRegionServers = replicationTracker.getListOfRegionServers();
+ Set<String> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());
// Loops each peer on each RS and dumps the queues
- try {
- List<String> regionservers = queuesClient.getListOfReplicators();
- if (regionservers == null || regionservers.isEmpty()) {
- return sb.toString();
+ List<ServerName> regionservers = queueStorage.getListOfReplicators();
+ if (regionservers == null || regionservers.isEmpty()) {
+ return sb.toString();
+ }
+ for (ServerName regionserver : regionservers) {
+ List<String> queueIds = queueStorage.getAllQueues(regionserver);
+ replicationQueues.init(regionserver.getServerName());
+ if (!liveRegionServers.contains(regionserver.getServerName())) {
+ deadRegionServers.add(regionserver.getServerName());
}
- for (String regionserver : regionservers) {
- List<String> queueIds = queuesClient.getAllQueues(regionserver);
- replicationQueues.init(regionserver);
- if (!liveRegionServers.contains(regionserver)) {
- deadRegionServers.add(regionserver);
- }
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- List<String> wals = queuesClient.getLogsInQueue(regionserver, queueId);
- if (!peerIds.contains(queueInfo.getPeerId())) {
- deletedQueues.add(regionserver + "/" + queueId);
- sb.append(formatQueue(regionserver, replicationQueues, queueInfo, queueId, wals, true,
- hdfs));
- } else {
- sb.append(formatQueue(regionserver, replicationQueues, queueInfo, queueId, wals, false,
- hdfs));
- }
+ for (String queueId : queueIds) {
+ ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
+ List<String> wals = queueStorage.getWALsInQueue(regionserver, queueId);
+ if (!peerIds.contains(queueInfo.getPeerId())) {
+ deletedQueues.add(regionserver + "/" + queueId);
+ sb.append(
+ formatQueue(regionserver, replicationQueues, queueInfo, queueId, wals, true, hdfs));
+ } else {
+ sb.append(
+ formatQueue(regionserver, replicationQueues, queueInfo, queueId, wals, false, hdfs));
}
}
- } catch (KeeperException ke) {
- throw new IOException(ke);
}
return sb.toString();
}
- private String formatQueue(String regionserver, ReplicationQueues replicationQueues, ReplicationQueueInfo queueInfo,
- String queueId, List<String> wals, boolean isDeleted, boolean hdfs) throws Exception {
-
+ private String formatQueue(ServerName regionserver, ReplicationQueues replicationQueues,
+ ReplicationQueueInfo queueInfo, String queueId, List<String> wals, boolean isDeleted,
+ boolean hdfs) throws Exception {
StringBuilder sb = new StringBuilder();
List<ServerName> deadServers;
@@ -389,13 +386,14 @@ public class DumpReplicationQueues extends Configured implements Tool {
/**
* return total size in bytes from a list of WALs
*/
- private long getTotalWALSize(FileSystem fs, List<String> wals, String server) throws IOException {
+ private long getTotalWALSize(FileSystem fs, List<String> wals, ServerName server)
+ throws IOException {
long size = 0;
FileStatus fileStatus;
for (String wal : wals) {
try {
- fileStatus = (new WALLink(getConf(), server, wal)).getFileStatus(fs);
+ fileStatus = (new WALLink(getConf(), server.getServerName(), wal)).getFileStatus(fs);
} catch (IOException e) {
if (e instanceof FileNotFoundException) {
numWalsNotFound++;
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index 839b5ad..85fa729 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.util.hbck;
import java.io.IOException;
@@ -27,22 +26,23 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.master.cleaner.ReplicationZKNodeCleaner;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.util.HBaseFsck;
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.yetus.audience.InterfaceAudience;
-/*
+/**
* Check and fix undeleted replication queues for removed peerId.
*/
@InterfaceAudience.Private
public class ReplicationChecker {
private final ErrorReporter errorReporter;
// replicator with its queueIds for removed peers
- private Map<String, List<String>> undeletedQueueIds = new HashMap<>();
+ private Map<ServerName, List<String>> undeletedQueueIds = new HashMap<>();
// replicator with its undeleted queueIds for removed peers in hfile-refs queue
private Set<String> undeletedHFileRefsQueueIds = new HashSet<>();
private final ReplicationZKNodeCleaner cleaner;
@@ -60,8 +60,8 @@ public class ReplicationChecker {
public void checkUnDeletedQueues() throws IOException {
undeletedQueueIds = cleaner.getUnDeletedQueues();
- for (Entry<String, List<String>> replicatorAndQueueIds : undeletedQueueIds.entrySet()) {
- String replicator = replicatorAndQueueIds.getKey();
+ for (Entry<ServerName, List<String>> replicatorAndQueueIds : undeletedQueueIds.entrySet()) {
+ ServerName replicator = replicatorAndQueueIds.getKey();
for (String queueId : replicatorAndQueueIds.getValue()) {
ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
String msg = "Undeleted replication queue for removed peer found: "
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
index 28a7562..b28eaaf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -34,12 +35,16 @@ import java.util.Set;
import java.util.concurrent.CompletionException;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -56,8 +61,8 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
private final String ID_ONE = "1";
private final String KEY_ONE = "127.0.0.1:2181:/hbase";
- private final String ID_SECOND = "2";
- private final String KEY_SECOND = "127.0.0.1:2181:/hbase2";
+ private final String ID_TWO = "2";
+ private final String KEY_TWO = "127.0.0.1:2181:/hbase2";
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -65,21 +70,27 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 120000);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
TEST_UTIL.getConfiguration().setInt(START_LOG_ERRORS_AFTER_COUNT_KEY, 0);
+ TEST_UTIL.getConfiguration().setInt(ReadOnlyZKClient.RECOVERY_RETRY, 1);
TEST_UTIL.startMiniCluster();
ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get();
}
@After
- public void cleanupPeer() {
+ public void clearPeerAndQueues() throws IOException, ReplicationException {
try {
admin.removeReplicationPeer(ID_ONE).join();
} catch (Exception e) {
- LOG.debug("Replication peer " + ID_ONE + " may already be removed");
}
try {
- admin.removeReplicationPeer(ID_SECOND).join();
+ admin.removeReplicationPeer(ID_TWO).join();
} catch (Exception e) {
- LOG.debug("Replication peer " + ID_SECOND + " may already be removed");
+ }
+ ReplicationQueueStorage queueStorage = ReplicationStorageFactory
+ .getReplicationQueueStorage(TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConfiguration());
+ for (ServerName serverName : queueStorage.getListOfReplicators()) {
+ for (String queue : queueStorage.getAllQueues(serverName)) {
+ queueStorage.removeQueue(serverName, queue);
+ }
}
}
@@ -88,7 +99,7 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
ReplicationPeerConfig rpc1 = new ReplicationPeerConfig();
rpc1.setClusterKey(KEY_ONE);
ReplicationPeerConfig rpc2 = new ReplicationPeerConfig();
- rpc2.setClusterKey(KEY_SECOND);
+ rpc2.setClusterKey(KEY_TWO);
// Add a valid peer
admin.addReplicationPeer(ID_ONE, rpc1).join();
// try adding the same (fails)
@@ -101,19 +112,19 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
assertEquals(1, admin.listReplicationPeers().get().size());
// Try to remove an inexisting peer
try {
- admin.removeReplicationPeer(ID_SECOND).join();
+ admin.removeReplicationPeer(ID_TWO).join();
fail("Test case should fail as removing a inexisting peer.");
} catch (CompletionException e) {
// OK!
}
assertEquals(1, admin.listReplicationPeers().get().size());
// Add a second since multi-slave is supported
- admin.addReplicationPeer(ID_SECOND, rpc2).join();
+ admin.addReplicationPeer(ID_TWO, rpc2).join();
assertEquals(2, admin.listReplicationPeers().get().size());
// Remove the first peer we added
admin.removeReplicationPeer(ID_ONE).join();
assertEquals(1, admin.listReplicationPeers().get().size());
- admin.removeReplicationPeer(ID_SECOND).join();
+ admin.removeReplicationPeer(ID_TWO).join();
assertEquals(0, admin.listReplicationPeers().get().size());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index d612eab..56a6cfe 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.replication.TestReplicationEndpoint.InterClusterR
import org.apache.hadoop.hbase.replication.TestReplicationEndpoint.ReplicationEndpointForTest;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -83,6 +84,7 @@ public class TestReplicationAdmin {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
+ TEST_UTIL.getConfiguration().setInt(ReadOnlyZKClient.RECOVERY_RETRY, 1);
TEST_UTIL.startMiniCluster();
admin = new ReplicationAdmin(TEST_UTIL.getConfiguration());
hbaseAdmin = TEST_UTIL.getAdmin();
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index 644de6a..7e6de5f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -24,16 +24,12 @@ import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
import java.io.IOException;
-import java.lang.reflect.Field;
import java.net.URLEncoder;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Random;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -51,7 +47,6 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationQueues;
import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientZKImpl;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
import org.apache.hadoop.hbase.replication.regionserver.Replication;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -65,10 +60,11 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+
@Category({MasterTests.class, MediumTests.class})
public class TestLogsCleaner {
@@ -195,24 +191,6 @@ public class TestLogsCleaner {
}
}
- @Test(timeout=5000)
- public void testZnodeCversionChange() throws Exception {
- Configuration conf = TEST_UTIL.getConfiguration();
- ReplicationLogCleaner cleaner = new ReplicationLogCleaner();
- cleaner.setConf(conf);
-
- ReplicationQueuesClientZKImpl rqcMock = Mockito.mock(ReplicationQueuesClientZKImpl.class);
- Mockito.when(rqcMock.getQueuesZNodeCversion()).thenReturn(1, 2, 3, 4);
-
- Field rqc = ReplicationLogCleaner.class.getDeclaredField("replicationQueues");
- rqc.setAccessible(true);
-
- rqc.set(cleaner, rqcMock);
-
- // This should return eventually when cversion stabilizes
- cleaner.getDeletableFiles(new LinkedList<>());
- }
-
/**
* ReplicationLogCleaner should be able to ride over ZooKeeper errors without aborting.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
index 85f0d1f..ca59094 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
@@ -1,12 +1,19 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
- * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
- * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
- * for the specific language governing permissions and limitations under the License.
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.hadoop.hbase.master.cleaner;
@@ -17,14 +24,10 @@ import static org.junit.Assert.fail;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-
import java.io.IOException;
-import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -45,7 +48,6 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueues;
import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
import org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl;
import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
import org.apache.hadoop.hbase.replication.regionserver.Replication;
@@ -63,10 +65,11 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+
@Category({ MasterTests.class, SmallTests.class })
public class TestReplicationHFileCleaner {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationQueuesZKImpl.class);
@@ -188,32 +191,6 @@ public class TestReplicationHFileCleaner {
assertTrue(deletableFilesIterator.next().getPath().equals(deletablefile));
}
- /*
- * Test for HBASE-14621. This test will not assert directly anything. Without the fix the test
- * will end up in a infinite loop, so it will timeout.
- */
- @Test(timeout = 15000)
- public void testForDifferntHFileRefsZnodeVersion() throws Exception {
- // 1. Create a file
- Path file = new Path(root, "testForDifferntHFileRefsZnodeVersion");
- fs.createNewFile(file);
- // 2. Assert file is successfully created
- assertTrue("Test file not created!", fs.exists(file));
- ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner();
- cleaner.setConf(conf);
-
- ReplicationQueuesClient replicationQueuesClient = Mockito.mock(ReplicationQueuesClient.class);
- //Return different znode version for each call
- Mockito.when(replicationQueuesClient.getHFileRefsNodeChangeVersion()).thenReturn(1, 2);
-
- Class<? extends ReplicationHFileCleaner> cleanerClass = cleaner.getClass();
- Field rqc = cleanerClass.getDeclaredField("rqc");
- rqc.setAccessible(true);
- rqc.set(cleaner, replicationQueuesClient);
-
- cleaner.isFileDeletable(fs.getFileStatus(file));
- }
-
/**
* ReplicationHFileCleaner should be able to ride over ZooKeeper errors without aborting.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
index 6aa59cb..8178266 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.master.cleaner;
import static org.junit.Assert.assertEquals;
@@ -26,6 +25,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationQueues;
import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
@@ -43,9 +43,9 @@ public class TestReplicationZKNodeCleaner {
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final String ID_ONE = "1";
- private final String SERVER_ONE = "server1";
+ private final ServerName SERVER_ONE = ServerName.valueOf("server1", 8000, 1234);
private final String ID_TWO = "2";
- private final String SERVER_TWO = "server2";
+ private final ServerName SERVER_TWO = ServerName.valueOf("server2", 8000, 1234);
private final Configuration conf;
private final ZKWatcher zkw;
@@ -72,12 +72,12 @@ public class TestReplicationZKNodeCleaner {
@Test
public void testReplicationZKNodeCleaner() throws Exception {
- repQueues.init(SERVER_ONE);
+ repQueues.init(SERVER_ONE.getServerName());
// add queue for ID_ONE which isn't exist
repQueues.addLog(ID_ONE, "file1");
ReplicationZKNodeCleaner cleaner = new ReplicationZKNodeCleaner(conf, zkw, null);
- Map<String, List<String>> undeletedQueues = cleaner.getUnDeletedQueues();
+ Map<ServerName, List<String>> undeletedQueues = cleaner.getUnDeletedQueues();
assertEquals(1, undeletedQueues.size());
assertTrue(undeletedQueues.containsKey(SERVER_ONE));
assertEquals(1, undeletedQueues.get(SERVER_ONE).size());
@@ -100,7 +100,7 @@ public class TestReplicationZKNodeCleaner {
@Test
public void testReplicationZKNodeCleanerChore() throws Exception {
- repQueues.init(SERVER_ONE);
+ repQueues.init(SERVER_ONE.getServerName());
// add queue for ID_ONE which isn't exist
repQueues.addLog(ID_ONE, "file1");
// add a recovery queue for ID_TWO which isn't exist
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
deleted file mode 100644
index 29c0930..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
+++ /dev/null
@@ -1,378 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.replication;
-
-import static org.junit.Assert.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.zookeeper.KeeperException;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * White box testing for replication state interfaces. Implementations should extend this class, and
- * initialize the interfaces properly.
- */
-public abstract class TestReplicationStateBasic {
-
- protected ReplicationQueues rq1;
- protected ReplicationQueues rq2;
- protected ReplicationQueues rq3;
- protected ReplicationQueuesClient rqc;
- protected String server1 = ServerName.valueOf("hostname1.example.org", 1234, -1L).toString();
- protected String server2 = ServerName.valueOf("hostname2.example.org", 1234, -1L).toString();
- protected String server3 = ServerName.valueOf("hostname3.example.org", 1234, -1L).toString();
- protected ReplicationPeers rp;
- protected static final String ID_ONE = "1";
- protected static final String ID_TWO = "2";
- protected static String KEY_ONE;
- protected static String KEY_TWO;
-
- // For testing when we try to replicate to ourself
- protected String OUR_ID = "3";
- protected String OUR_KEY;
-
- protected static int zkTimeoutCount;
- protected static final int ZK_MAX_COUNT = 300;
- protected static final int ZK_SLEEP_INTERVAL = 100; // millis
-
- private static final Logger LOG = LoggerFactory.getLogger(TestReplicationStateBasic.class);
-
- @Before
- public void setUp() {
- zkTimeoutCount = 0;
- }
-
- @Test
- public void testReplicationQueuesClient() throws ReplicationException, KeeperException {
- rqc.init();
- // Test methods with empty state
- assertEquals(0, rqc.getListOfReplicators().size());
- assertNull(rqc.getLogsInQueue(server1, "qId1"));
- assertNull(rqc.getAllQueues(server1));
-
- /*
- * Set up data Two replicators: -- server1: three queues with 0, 1 and 2 log files each --
- * server2: zero queues
- */
- rq1.init(server1);
- rq2.init(server2);
- rq1.addLog("qId1", "trash");
- rq1.removeLog("qId1", "trash");
- rq1.addLog("qId2", "filename1");
- rq1.addLog("qId3", "filename2");
- rq1.addLog("qId3", "filename3");
- rq2.addLog("trash", "trash");
- rq2.removeQueue("trash");
-
- List<String> reps = rqc.getListOfReplicators();
- assertEquals(2, reps.size());
- assertTrue(server1, reps.contains(server1));
- assertTrue(server2, reps.contains(server2));
-
- assertNull(rqc.getLogsInQueue("bogus", "bogus"));
- assertNull(rqc.getLogsInQueue(server1, "bogus"));
- assertEquals(0, rqc.getLogsInQueue(server1, "qId1").size());
- assertEquals(1, rqc.getLogsInQueue(server1, "qId2").size());
- assertEquals("filename1", rqc.getLogsInQueue(server1, "qId2").get(0));
-
- assertNull(rqc.getAllQueues("bogus"));
- assertEquals(0, rqc.getAllQueues(server2).size());
- List<String> list = rqc.getAllQueues(server1);
- assertEquals(3, list.size());
- assertTrue(list.contains("qId2"));
- assertTrue(list.contains("qId3"));
- }
-
- @Test
- public void testReplicationQueues() throws ReplicationException {
- rq1.init(server1);
- rq2.init(server2);
- rq3.init(server3);
- //Initialize ReplicationPeer so we can add peers (we don't transfer lone queues)
- rp.init();
-
- // 3 replicators should exist
- assertEquals(3, rq1.getListOfReplicators().size());
- rq1.removeQueue("bogus");
- rq1.removeLog("bogus", "bogus");
- rq1.removeAllQueues();
- assertEquals(0, rq1.getAllQueues().size());
- assertEquals(0, rq1.getLogPosition("bogus", "bogus"));
- assertNull(rq1.getLogsInQueue("bogus"));
- assertNull(rq1.getUnClaimedQueueIds(
- ServerName.valueOf("bogus", 1234, -1L).toString()));
-
- rq1.setLogPosition("bogus", "bogus", 5L);
-
- populateQueues();
-
- assertEquals(3, rq1.getListOfReplicators().size());
- assertEquals(0, rq2.getLogsInQueue("qId1").size());
- assertEquals(5, rq3.getLogsInQueue("qId5").size());
- assertEquals(0, rq3.getLogPosition("qId1", "filename0"));
- rq3.setLogPosition("qId5", "filename4", 354L);
- assertEquals(354L, rq3.getLogPosition("qId5", "filename4"));
-
- assertEquals(5, rq3.getLogsInQueue("qId5").size());
- assertEquals(0, rq2.getLogsInQueue("qId1").size());
- assertEquals(0, rq1.getAllQueues().size());
- assertEquals(1, rq2.getAllQueues().size());
- assertEquals(5, rq3.getAllQueues().size());
-
- assertEquals(0, rq3.getUnClaimedQueueIds(server1).size());
- rq3.removeReplicatorIfQueueIsEmpty(server1);
- assertEquals(2, rq3.getListOfReplicators().size());
-
- List<String> queues = rq2.getUnClaimedQueueIds(server3);
- assertEquals(5, queues.size());
- for(String queue: queues) {
- rq2.claimQueue(server3, queue);
- }
- rq2.removeReplicatorIfQueueIsEmpty(server3);
- assertEquals(1, rq2.getListOfReplicators().size());
-
- // Try to claim our own queues
- assertNull(rq2.getUnClaimedQueueIds(server2));
- rq2.removeReplicatorIfQueueIsEmpty(server2);
-
- assertEquals(6, rq2.getAllQueues().size());
-
- rq2.removeAllQueues();
-
- assertEquals(0, rq2.getListOfReplicators().size());
- }
-
- @Test
- public void testInvalidClusterKeys() throws ReplicationException, KeeperException {
- rp.init();
-
- try {
- rp.registerPeer(ID_ONE,
- new ReplicationPeerConfig().setClusterKey("hostname1.example.org:1234:hbase"));
- fail("Should throw an IllegalArgumentException because "
- + "zookeeper.znode.parent is missing leading '/'.");
- } catch (IllegalArgumentException e) {
- // Expected.
- }
-
- try {
- rp.registerPeer(ID_ONE,
- new ReplicationPeerConfig().setClusterKey("hostname1.example.org:1234:/"));
- fail("Should throw an IllegalArgumentException because zookeeper.znode.parent is missing.");
- } catch (IllegalArgumentException e) {
- // Expected.
- }
-
- try {
- rp.registerPeer(ID_ONE,
- new ReplicationPeerConfig().setClusterKey("hostname1.example.org::/hbase"));
- fail("Should throw an IllegalArgumentException because "
- + "hbase.zookeeper.property.clientPort is missing.");
- } catch (IllegalArgumentException e) {
- // Expected.
- }
- }
-
- @Test
- public void testHfileRefsReplicationQueues() throws ReplicationException, KeeperException {
- rp.init();
- rq1.init(server1);
- rqc.init();
-
- List<Pair<Path, Path>> files1 = new ArrayList<>(3);
- files1.add(new Pair<>(null, new Path("file_1")));
- files1.add(new Pair<>(null, new Path("file_2")));
- files1.add(new Pair<>(null, new Path("file_3")));
- assertNull(rqc.getReplicableHFiles(ID_ONE));
- assertEquals(0, rqc.getAllPeersFromHFileRefsQueue().size());
- rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
- rq1.addPeerToHFileRefs(ID_ONE);
- rq1.addHFileRefs(ID_ONE, files1);
- assertEquals(1, rqc.getAllPeersFromHFileRefsQueue().size());
- assertEquals(3, rqc.getReplicableHFiles(ID_ONE).size());
- List<String> hfiles2 = new ArrayList<>(files1.size());
- for (Pair<Path, Path> p : files1) {
- hfiles2.add(p.getSecond().getName());
- }
- String removedString = hfiles2.remove(0);
- rq1.removeHFileRefs(ID_ONE, hfiles2);
- assertEquals(1, rqc.getReplicableHFiles(ID_ONE).size());
- hfiles2 = new ArrayList<>(1);
- hfiles2.add(removedString);
- rq1.removeHFileRefs(ID_ONE, hfiles2);
- assertEquals(0, rqc.getReplicableHFiles(ID_ONE).size());
- rp.unregisterPeer(ID_ONE);
- }
-
- @Test
- public void testRemovePeerForHFileRefs() throws ReplicationException, KeeperException {
- rq1.init(server1);
- rqc.init();
-
- rp.init();
- rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
- rq1.addPeerToHFileRefs(ID_ONE);
- rp.registerPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO));
- rq1.addPeerToHFileRefs(ID_TWO);
-
- List<Pair<Path, Path>> files1 = new ArrayList<>(3);
- files1.add(new Pair<>(null, new Path("file_1")));
- files1.add(new Pair<>(null, new Path("file_2")));
- files1.add(new Pair<>(null, new Path("file_3")));
- rq1.addHFileRefs(ID_ONE, files1);
- rq1.addHFileRefs(ID_TWO, files1);
- assertEquals(2, rqc.getAllPeersFromHFileRefsQueue().size());
- assertEquals(3, rqc.getReplicableHFiles(ID_ONE).size());
- assertEquals(3, rqc.getReplicableHFiles(ID_TWO).size());
-
- rp.unregisterPeer(ID_ONE);
- rq1.removePeerFromHFileRefs(ID_ONE);
- assertEquals(1, rqc.getAllPeersFromHFileRefsQueue().size());
- assertNull(rqc.getReplicableHFiles(ID_ONE));
- assertEquals(3, rqc.getReplicableHFiles(ID_TWO).size());
-
- rp.unregisterPeer(ID_TWO);
- rq1.removePeerFromHFileRefs(ID_TWO);
- assertEquals(0, rqc.getAllPeersFromHFileRefsQueue().size());
- assertNull(rqc.getReplicableHFiles(ID_TWO));
- }
-
- @Test
- public void testReplicationPeers() throws Exception {
- rp.init();
-
- // Test methods with non-existent peer ids
- try {
- rp.unregisterPeer("bogus");
- fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
- } catch (IllegalArgumentException e) {
- }
- try {
- rp.enablePeer("bogus");
- fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
- } catch (IllegalArgumentException e) {
- }
- try {
- rp.disablePeer("bogus");
- fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
- } catch (IllegalArgumentException e) {
- }
- try {
- rp.getStatusOfPeer("bogus");
- fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
- } catch (IllegalArgumentException e) {
- }
- assertFalse(rp.peerConnected("bogus"));
- rp.peerDisconnected("bogus");
-
- assertNull(rp.getPeerConf("bogus"));
- assertNumberOfPeers(0);
-
- // Add some peers
- rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
- assertNumberOfPeers(1);
- rp.registerPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO));
- assertNumberOfPeers(2);
-
- // Test methods with a peer that is added but not connected
- try {
- rp.getStatusOfPeer(ID_ONE);
- fail("There are no connected peers, should have thrown an IllegalArgumentException");
- } catch (IllegalArgumentException e) {
- }
- assertEquals(KEY_ONE, ZKConfig.getZooKeeperClusterKey(rp.getPeerConf(ID_ONE).getSecond()));
- rp.unregisterPeer(ID_ONE);
- rp.peerDisconnected(ID_ONE);
- assertNumberOfPeers(1);
-
- // Add one peer
- rp.registerPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE));
- rp.peerConnected(ID_ONE);
- assertNumberOfPeers(2);
- assertTrue(rp.getStatusOfPeer(ID_ONE));
- rp.disablePeer(ID_ONE);
- assertConnectedPeerStatus(false, ID_ONE);
- rp.enablePeer(ID_ONE);
- assertConnectedPeerStatus(true, ID_ONE);
-
- // Disconnect peer
- rp.peerDisconnected(ID_ONE);
- assertNumberOfPeers(2);
- try {
- rp.getStatusOfPeer(ID_ONE);
- fail("There are no connected peers, should have thrown an IllegalArgumentException");
- } catch (IllegalArgumentException e) {
- }
- }
-
- protected void assertConnectedPeerStatus(boolean status, String peerId) throws Exception {
- // we can first check if the value was changed in the store, if it wasn't then fail right away
- if (status != rp.getStatusOfPeerFromBackingStore(peerId)) {
- fail("ConnectedPeerStatus was " + !status + " but expected " + status + " in ZK");
- }
- while (true) {
- if (status == rp.getStatusOfPeer(peerId)) {
- return;
- }
- if (zkTimeoutCount < ZK_MAX_COUNT) {
- LOG.debug("ConnectedPeerStatus was " + !status + " but expected " + status
- + ", sleeping and trying again.");
- Thread.sleep(ZK_SLEEP_INTERVAL);
- } else {
- fail("Timed out waiting for ConnectedPeerStatus to be " + status);
- }
- }
- }
-
- protected void assertNumberOfPeers(int total) {
- assertEquals(total, rp.getAllPeerConfigs().size());
- assertEquals(total, rp.getAllPeerIds().size());
- assertEquals(total, rp.getAllPeerIds().size());
- }
-
- /*
- * three replicators: rq1 has 0 queues, rq2 has 1 queue with no logs, rq3 has 5 queues with 1, 2,
- * 3, 4, 5 log files respectively
- */
- protected void populateQueues() throws ReplicationException {
- rq1.addLog("trash", "trash");
- rq1.removeQueue("trash");
-
- rq2.addLog("qId1", "trash");
- rq2.removeLog("qId1", "trash");
-
- for (int i = 1; i < 6; i++) {
- for (int j = 0; j < i; j++) {
- rq3.addLog("qId" + i, "filename" + j);
- }
- //Add peers for the corresponding queues so they are not orphans
- rp.registerPeer("qId" + i, new ReplicationPeerConfig().setClusterKey("localhost:2818:/bogus" + i));
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
deleted file mode 100644
index 231d655..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hbase.ChoreService;
-import org.apache.hadoop.hbase.ClusterId;
-import org.apache.hadoop.hbase.CoordinatedStateManager;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.ClusterConnection;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
-import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.zookeeper.KeeperException;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Category({ReplicationTests.class, MediumTests.class})
-public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestReplicationStateZKImpl.class);
-
- private static Configuration conf;
- private static HBaseTestingUtility utility;
- private static ZKWatcher zkw;
- private static String replicationZNode;
- private ReplicationQueuesZKImpl rqZK;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- utility = new HBaseTestingUtility();
- utility.startMiniZKCluster();
- conf = utility.getConfiguration();
- conf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
- zkw = HBaseTestingUtility.getZooKeeperWatcher(utility);
- String replicationZNodeName = conf.get("zookeeper.znode.replication", "replication");
- replicationZNode = ZNodePaths.joinZNode(zkw.znodePaths.baseZNode, replicationZNodeName);
- KEY_ONE = initPeerClusterState("/hbase1");
- KEY_TWO = initPeerClusterState("/hbase2");
- }
-
- private static String initPeerClusterState(String baseZKNode)
- throws IOException, KeeperException {
- // Add a dummy region server and set up the cluster id
- Configuration testConf = new Configuration(conf);
- testConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, baseZKNode);
- ZKWatcher zkw1 = new ZKWatcher(testConf, "test1", null);
- String fakeRs = ZNodePaths.joinZNode(zkw1.znodePaths.rsZNode, "hostname1.example.org:1234");
- ZKUtil.createWithParents(zkw1, fakeRs);
- ZKClusterId.setClusterId(zkw1, new ClusterId());
- return ZKConfig.getZooKeeperClusterKey(testConf);
- }
-
- @Before
- @Override
- public void setUp() {
- super.setUp();
- DummyServer ds1 = new DummyServer(server1);
- DummyServer ds2 = new DummyServer(server2);
- DummyServer ds3 = new DummyServer(server3);
- try {
- rq1 = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, ds1, zkw));
- rq2 = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, ds2, zkw));
- rq3 = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, ds3, zkw));
- rqc = ReplicationFactory.getReplicationQueuesClient(
- new ReplicationQueuesClientArguments(conf, ds1, zkw));
- } catch (Exception e) {
- // This should not occur, because getReplicationQueues() only throws for
- // TableBasedReplicationQueuesImpl
- fail("ReplicationFactory.getReplicationQueues() threw an IO Exception");
- }
- rp = ReplicationFactory.getReplicationPeers(zkw, conf, zkw);
- OUR_KEY = ZKConfig.getZooKeeperClusterKey(conf);
- rqZK = new ReplicationQueuesZKImpl(zkw, conf, ds1);
- }
-
- @After
- public void tearDown() throws KeeperException, IOException {
- ZKUtil.deleteNodeRecursively(zkw, replicationZNode);
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- utility.shutdownMiniZKCluster();
- }
-
- @Test
- public void testIsPeerPath_PathToParentOfPeerNode() {
- assertFalse(rqZK.isPeerPath(rqZK.peersZNode));
- }
-
- @Test
- public void testIsPeerPath_PathToChildOfPeerNode() {
- String peerChild = ZNodePaths.joinZNode(ZNodePaths.joinZNode(rqZK.peersZNode, "1"), "child");
- assertFalse(rqZK.isPeerPath(peerChild));
- }
-
- @Test
- public void testIsPeerPath_ActualPeerPath() {
- String peerPath = ZNodePaths.joinZNode(rqZK.peersZNode, "1");
- assertTrue(rqZK.isPeerPath(peerPath));
- }
-
- static class DummyServer implements Server {
- private String serverName;
- private boolean isAborted = false;
- private boolean isStopped = false;
-
- public DummyServer(String serverName) {
- this.serverName = serverName;
- }
-
- @Override
- public Configuration getConfiguration() {
- return conf;
- }
-
- @Override
- public ZKWatcher getZooKeeper() {
- return zkw;
- }
-
- @Override
- public CoordinatedStateManager getCoordinatedStateManager() {
- return null;
- }
-
- @Override
- public ClusterConnection getConnection() {
- return null;
- }
-
- @Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
- public ServerName getServerName() {
- return ServerName.valueOf(this.serverName);
- }
-
- @Override
- public void abort(String why, Throwable e) {
- LOG.info("Aborting " + serverName);
- this.isAborted = true;
- }
-
- @Override
- public boolean isAborted() {
- return this.isAborted;
- }
-
- @Override
- public void stop(String why) {
- this.isStopped = true;
- }
-
- @Override
- public boolean isStopped() {
- return this.isStopped;
- }
-
- @Override
- public ChoreService getChoreService() {
- return null;
- }
-
- @Override
- public ClusterConnection getClusterConnection() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public FileSystem getFileSystem() {
- return null;
- }
-
- @Override
- public boolean isStopping() {
- return false;
- }
-
- @Override
- public Connection createConnection(Configuration conf) throws IOException {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a754e99/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
index b47a8d3..aeab8b0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
@@ -1,34 +1,34 @@
-/*
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hbase.replication.regionserver;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueues;
import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientZKImpl;
import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
@@ -36,11 +36,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
/**
* Tests the ReplicationSourceManager with ReplicationQueueZkImpl's and
* ReplicationQueuesClientZkImpl. Also includes extra tests outside of those in
@@ -114,41 +109,4 @@ public class TestReplicationSourceManagerZkImpl extends TestReplicationSourceMan
server.stop("");
}
-
- @Test
- public void testFailoverDeadServerCversionChange() throws Exception {
- final Server s0 = new DummyServer("cversion-change0.example.org");
- ReplicationQueues repQueues =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, s0,
- s0.getZooKeeper()));
- repQueues.init(s0.getServerName().toString());
- // populate some znodes in the peer znode
- files.add("log1");
- files.add("log2");
- for (String file : files) {
- repQueues.addLog("1", file);
- }
- // simulate queue transfer
- Server s1 = new DummyServer("cversion-change1.example.org");
- ReplicationQueues rq1 =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s1.getConfiguration(), s1,
- s1.getZooKeeper()));
- rq1.init(s1.getServerName().toString());
-
- ReplicationQueuesClientZKImpl client =
- (ReplicationQueuesClientZKImpl)ReplicationFactory.getReplicationQueuesClient(
- new ReplicationQueuesClientArguments(s1.getConfiguration(), s1, s1.getZooKeeper()));
-
- int v0 = client.getQueuesZNodeCversion();
- List<String> queues = rq1.getUnClaimedQueueIds(s0.getServerName().getServerName());
- for(String queue : queues) {
- rq1.claimQueue(s0.getServerName().getServerName(), queue);
- }
- rq1.removeReplicatorIfQueueIsEmpty(s0.getServerName().getServerName());
- int v1 = client.getQueuesZNodeCversion();
- // cversion should increase by 1 since a child node is deleted
- assertEquals(v0 + 1, v1);
-
- s0.stop("");
- }
}
[03/50] [abbrv] hbase git commit: HBASE-19840 Flakey
TestMetaWithReplicas
Posted by zh...@apache.org.
HBASE-19840 Flakey TestMetaWithReplicas
Adding debuging
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1e5fc1ed
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1e5fc1ed
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1e5fc1ed
Branch: refs/heads/HBASE-19397-branch-2
Commit: 1e5fc1ed63039390ecd5d3a45c96232338e7ae83
Parents: 0fa24dd
Author: Michael Stack <st...@apache.org>
Authored: Mon Jan 22 12:24:52 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Mon Jan 22 12:26:03 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/client/TestMetaWithReplicas.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e5fc1ed/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index e01c28f..7099408 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -30,6 +30,8 @@ import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.CategoryBasedTimeout;
@@ -93,14 +95,14 @@ public class TestMetaWithReplicas {
// disable the balancer
LoadBalancerTracker l = new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(),
new Abortable() {
- boolean aborted = false;
+ AtomicBoolean aborted = new AtomicBoolean(false);
@Override
public boolean isAborted() {
- return aborted;
+ return aborted.get();
}
@Override
public void abort(String why, Throwable e) {
- aborted = true;
+ aborted.set(true);
}
});
l.setBalancerOn(false);
@@ -174,6 +176,7 @@ public class TestMetaWithReplicas {
conf.get("zookeeper.znode.metaserver", "meta-region-server"));
byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
ServerName primary = ProtobufUtil.toServerName(data);
+ LOG.info("Primary=" + primary.toString());
TableName TABLE = TableName.valueOf("testShutdownHandling");
byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
@@ -208,14 +211,17 @@ public class TestMetaWithReplicas {
master = util.getHBaseClusterInterface().getClusterMetrics().getMasterName();
// kill the master so that regionserver recovery is not triggered at all
// for the meta server
+ LOG.info("Stopping master=" + master.toString());
util.getHBaseClusterInterface().stopMaster(master);
util.getHBaseClusterInterface().waitForMasterToStop(master, 60000);
+ LOG.info("Master stopped!");
if (!master.equals(primary)) {
util.getHBaseClusterInterface().killRegionServer(primary);
util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000);
}
((ClusterConnection)c).clearRegionCache();
}
+ LOG.info("Running GETs");
Get get = null;
Result r = null;
byte[] row = "test".getBytes();
@@ -231,12 +237,15 @@ public class TestMetaWithReplicas {
assertTrue(Arrays.equals(r.getRow(), row));
// now start back the killed servers and disable use of replicas. That would mean
// calls go to the primary
+ LOG.info("Starting Master");
util.getHBaseClusterInterface().startMaster(master.getHostname(), 0);
util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0);
util.getHBaseClusterInterface().waitForActiveAndReadyMaster();
+ LOG.info("Master active!");
((ClusterConnection)c).clearRegionCache();
}
conf.setBoolean(HConstants.USE_META_REPLICAS, false);
+ LOG.info("Running GETs no replicas");
try (Table htable = c.getTable(TABLE);) {
r = htable.get(get);
assertTrue(Arrays.equals(r.getRow(), row));
[13/50] [abbrv] hbase git commit: HBASE-4224 Need a flush by
regionserver rather than by table option
Posted by zh...@apache.org.
HBASE-4224 Need a flush by regionserver rather than by table option
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f952779b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f952779b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f952779b
Branch: refs/heads/HBASE-19397-branch-2
Commit: f952779ba2be34f5c566353ab9b3619e60f29c4c
Parents: b1269ec
Author: Chia-Ping Tsai <ch...@gmail.com>
Authored: Sun Dec 17 01:59:19 2017 +0800
Committer: Chia-Ping Tsai <ch...@gmail.com>
Committed: Tue Jan 23 09:47:15 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Admin.java | 7 +
.../apache/hadoop/hbase/client/AsyncAdmin.java | 6 +
.../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 +
.../apache/hadoop/hbase/client/HBaseAdmin.java | 35 ++--
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 49 ++++--
.../hbase/client/TestFlushFromClient.java | 176 +++++++++++++++++++
6 files changed, 254 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f952779b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index f61b32e..40dac2f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -826,6 +826,13 @@ public interface Admin extends Abortable, Closeable {
void flushRegion(byte[] regionName) throws IOException;
/**
+ * Flush all regions on the region server. Synchronous operation.
+ * @param serverName the region server name to flush
+ * @throws IOException if a remote or network exception occurs
+ */
+ void flushRegionServer(ServerName serverName) throws IOException;
+
+ /**
* Compact a table. Asynchronous operation in that this method requests that a
* Compaction run and then it returns. It does not wait on the completion of Compaction
* (it can take a while).
http://git-wip-us.apache.org/repos/asf/hbase/blob/f952779b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
index a375265..35cdd3f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -295,6 +295,12 @@ public interface AsyncAdmin {
CompletableFuture<Void> flushRegion(byte[] regionName);
/**
+ * Flush all region on the region server.
+ * @param serverName server to flush
+ */
+ CompletableFuture<Void> flushRegionServer(ServerName serverName);
+
+ /**
* Compact a table. When the returned CompletableFuture is done, it only means the compact request
* was sent to HBase and may need some time to finish the compact operation.
* @param tableName table to compact
http://git-wip-us.apache.org/repos/asf/hbase/blob/f952779b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
index d0d19c1..9b2390c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -243,6 +243,11 @@ class AsyncHBaseAdmin implements AsyncAdmin {
}
@Override
+ public CompletableFuture<Void> flushRegionServer(ServerName sn) {
+ return wrap(rawAdmin.flushRegionServer(sn));
+ }
+
+ @Override
public CompletableFuture<Void> compact(TableName tableName,
CompactType compactType) {
return wrap(rawAdmin.compact(tableName, compactType));
http://git-wip-us.apache.org/repos/asf/hbase/blob/f952779b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 4ac1c21..c137383 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -1182,21 +1182,28 @@ public class HBaseAdmin implements Admin {
if (regionServerPair.getSecond() == null) {
throw new NoServerForRegionException(Bytes.toStringBinary(regionName));
}
- final RegionInfo hRegionInfo = regionServerPair.getFirst();
+ final RegionInfo regionInfo = regionServerPair.getFirst();
ServerName serverName = regionServerPair.getSecond();
- final AdminService.BlockingInterface admin = this.connection.getAdmin(serverName);
- Callable<Void> callable = new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- // TODO: There is no timeout on this controller. Set one!
- HBaseRpcController controller = rpcControllerFactory.newController();
- FlushRegionRequest request =
- RequestConverter.buildFlushRegionRequest(hRegionInfo.getRegionName());
- admin.flushRegion(controller, request);
- return null;
- }
- };
- ProtobufUtil.call(callable);
+ flush(this.connection.getAdmin(serverName), regionInfo);
+ }
+
+ private void flush(AdminService.BlockingInterface admin, final RegionInfo info)
+ throws IOException {
+ ProtobufUtil.call(() -> {
+ // TODO: There is no timeout on this controller. Set one!
+ HBaseRpcController controller = rpcControllerFactory.newController();
+ FlushRegionRequest request =
+ RequestConverter.buildFlushRegionRequest(info.getRegionName());
+ admin.flushRegion(controller, request);
+ return null;
+ });
+ }
+
+ @Override
+ public void flushRegionServer(ServerName serverName) throws IOException {
+ for (RegionInfo region : getRegions(serverName)) {
+ flush(this.connection.getAdmin(serverName), region);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/f952779b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
index 5111bfc..a826f8c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -826,25 +826,54 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
.toStringBinary(regionName)));
return;
}
-
- RegionInfo regionInfo = location.getRegion();
- this.<Void> newAdminCaller()
- .serverName(serverName)
- .action(
- (controller, stub) -> this.<FlushRegionRequest, FlushRegionResponse, Void> adminCall(
- controller, stub, RequestConverter.buildFlushRegionRequest(regionInfo
- .getRegionName()), (s, c, req, done) -> s.flushRegion(c, req, done),
- resp -> null)).call().whenComplete((ret, err2) -> {
+ flush(serverName, location.getRegion())
+ .whenComplete((ret, err2) -> {
if (err2 != null) {
future.completeExceptionally(err2);
} else {
future.complete(ret);
}
- });
+ });
});
return future;
}
+ private CompletableFuture<Void> flush(final ServerName serverName, final RegionInfo regionInfo) {
+ return this.<Void> newAdminCaller()
+ .serverName(serverName)
+ .action(
+ (controller, stub) -> this.<FlushRegionRequest, FlushRegionResponse, Void> adminCall(
+ controller, stub, RequestConverter.buildFlushRegionRequest(regionInfo
+ .getRegionName()), (s, c, req, done) -> s.flushRegion(c, req, done),
+ resp -> null))
+ .call();
+ }
+
+ @Override
+ public CompletableFuture<Void> flushRegionServer(ServerName sn) {
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ getRegions(sn).whenComplete((hRegionInfos, err) -> {
+ if (err != null) {
+ future.completeExceptionally(err);
+ return;
+ }
+ List<CompletableFuture<Void>> compactFutures = new ArrayList<>();
+ if (hRegionInfos != null) {
+ hRegionInfos.forEach(region -> compactFutures.add(flush(sn, region)));
+ }
+ CompletableFuture
+ .allOf(compactFutures.toArray(new CompletableFuture<?>[compactFutures.size()]))
+ .whenComplete((ret, err2) -> {
+ if (err2 != null) {
+ future.completeExceptionally(err2);
+ } else {
+ future.complete(ret);
+ }
+ });
+ });
+ return future;
+ }
+
@Override
public CompletableFuture<Void> compact(TableName tableName, CompactType compactType) {
return compact(tableName, null, false, compactType);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f952779b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java
new file mode 100644
index 0000000..9085fa5
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFlushFromClient.java
@@ -0,0 +1,176 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({MediumTests.class, ClientTests.class})
+public class TestFlushFromClient {
+ private static final Log LOG = LogFactory.getLog(TestFlushFromClient.class);
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static AsyncConnection asyncConn;
+ private static final byte[][] SPLITS = new byte[][]{Bytes.toBytes("3"), Bytes.toBytes("7")};
+ private static final List<byte[]> ROWS = Arrays.asList(
+ Bytes.toBytes("1"),
+ Bytes.toBytes("4"),
+ Bytes.toBytes("8"));
+ private static final byte[] FAMILY = Bytes.toBytes("f1");
+
+ @Rule
+ public TestName name = new TestName();
+
+ public TableName tableName;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.startMiniCluster(ROWS.size());
+ asyncConn = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ IOUtils.cleanup(null, asyncConn);
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ tableName = TableName.valueOf(name.getMethodName());
+ try (Table t = TEST_UTIL.createTable(tableName, FAMILY, SPLITS)) {
+ List<Put> puts = ROWS.stream().map(r -> new Put(r)).collect(Collectors.toList());
+ for (int i = 0; i != 20; ++i) {
+ byte[] value = Bytes.toBytes(i);
+ puts.forEach(p -> p.addColumn(FAMILY, value, value));
+ }
+ t.put(puts);
+ }
+ assertFalse(getRegionInfo().isEmpty());
+ assertTrue(getRegionInfo().stream().allMatch(r -> r.getMemStoreSize() != 0));
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ for (TableDescriptor htd : TEST_UTIL.getAdmin().listTableDescriptors()) {
+ LOG.info("Tear down, remove table=" + htd.getTableName());
+ TEST_UTIL.deleteTable(htd.getTableName());
+ }
+ }
+
+ @Test
+ public void testFlushTable() throws Exception {
+ try (Admin admin = TEST_UTIL.getAdmin()) {
+ admin.flush(tableName);
+ assertFalse(getRegionInfo().stream().anyMatch(r -> r.getMemStoreSize() != 0));
+ }
+ }
+
+ @Test
+ public void testAsyncFlushTable() throws Exception {
+ AsyncAdmin admin = asyncConn.getAdmin();
+ admin.flush(tableName).get();
+ assertFalse(getRegionInfo().stream().anyMatch(r -> r.getMemStoreSize() != 0));
+ }
+
+ @Test
+ public void testFlushRegion() throws Exception {
+ try (Admin admin = TEST_UTIL.getAdmin()) {
+ for (HRegion r : getRegionInfo()) {
+ admin.flushRegion(r.getRegionInfo().getRegionName());
+ TimeUnit.SECONDS.sleep(1);
+ assertEquals(0, r.getMemStoreSize());
+ }
+ }
+ }
+
+ @Test
+ public void testAsyncFlushRegion() throws Exception {
+ AsyncAdmin admin = asyncConn.getAdmin();
+ for (HRegion r : getRegionInfo()) {
+ admin.flushRegion(r.getRegionInfo().getRegionName()).get();
+ TimeUnit.SECONDS.sleep(1);
+ assertEquals(0, r.getMemStoreSize());
+ }
+ }
+
+ @Test
+ public void testFlushRegionServer() throws Exception {
+ try (Admin admin = TEST_UTIL.getAdmin()) {
+ for (HRegionServer rs : TEST_UTIL.getHBaseCluster()
+ .getLiveRegionServerThreads()
+ .stream().map(JVMClusterUtil.RegionServerThread::getRegionServer)
+ .collect(Collectors.toList())) {
+ admin.flushRegionServer(rs.getServerName());
+ assertFalse(getRegionInfo(rs).stream().anyMatch(r -> r.getMemStoreSize() != 0));
+ }
+ }
+ }
+
+ @Test
+ public void testAsyncFlushRegionServer() throws Exception {
+ AsyncAdmin admin = asyncConn.getAdmin();
+ for (HRegionServer rs : TEST_UTIL.getHBaseCluster()
+ .getLiveRegionServerThreads()
+ .stream().map(JVMClusterUtil.RegionServerThread::getRegionServer)
+ .collect(Collectors.toList())) {
+ admin.flushRegionServer(rs.getServerName()).get();
+ assertFalse(getRegionInfo(rs).stream().anyMatch(r -> r.getMemStoreSize() != 0));
+ }
+ }
+
+ private List<HRegion> getRegionInfo() {
+ return TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().stream()
+ .map(JVMClusterUtil.RegionServerThread::getRegionServer)
+ .flatMap(r -> r.getRegions().stream())
+ .filter(r -> r.getTableDescriptor().getTableName().equals(tableName))
+ .collect(Collectors.toList());
+ }
+
+ private List<HRegion> getRegionInfo(HRegionServer rs) {
+ return rs.getRegions().stream()
+ .filter(v -> v.getTableDescriptor().getTableName().equals(tableName))
+ .collect(Collectors.toList());
+ }
+}
[19/50] [abbrv] hbase git commit: HBASE-19580 Use slf4j instead of
commons-logging in new, just-added Peer Procedure classes
Posted by zh...@apache.org.
HBASE-19580 Use slf4j instead of commons-logging in new, just-added Peer Procedure classes
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bcb79d53
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bcb79d53
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bcb79d53
Branch: refs/heads/HBASE-19397-branch-2
Commit: bcb79d53c38859dc1bd8772eb989cde1401fd711
Parents: 211e6f5
Author: zhangduo <zh...@apache.org>
Authored: Thu Dec 21 21:59:46 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../hadoop/hbase/master/replication/AddPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/DisablePeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/EnablePeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/ModifyPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/RefreshPeerProcedure.java | 6 +++---
.../hadoop/hbase/master/replication/RemovePeerProcedure.java | 6 +++---
.../hbase/master/replication/UpdatePeerConfigProcedure.java | 6 +++---
7 files changed, 21 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/bcb79d53/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
index c3862d8..066c3e7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
@@ -28,6 +26,8 @@ import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AddPeerStateData;
@@ -37,7 +37,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.A
@InterfaceAudience.Private
public class AddPeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(AddPeerProcedure.class);
+ private static final Logger LOG = LoggerFactory.getLogger(AddPeerProcedure.class);
private ReplicationPeerConfig peerConfig;
http://git-wip-us.apache.org/repos/asf/hbase/blob/bcb79d53/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
index 0b32db9..9a28de6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The procedure for disabling a replication peer.
@@ -31,7 +31,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class DisablePeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(DisablePeerProcedure.class);
+ private static final Logger LOG = LoggerFactory.getLogger(DisablePeerProcedure.class);
public DisablePeerProcedure() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bcb79d53/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
index 92ba000..4855901 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The procedure for enabling a replication peer.
@@ -31,7 +31,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class EnablePeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(EnablePeerProcedure.class);
+ private static final Logger LOG = LoggerFactory.getLogger(EnablePeerProcedure.class);
public EnablePeerProcedure() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bcb79d53/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 23f6f87..c4552ed 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
@@ -30,6 +28,8 @@ import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.ModifyPeerStateData;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationState;
@@ -42,7 +42,7 @@ public abstract class ModifyPeerProcedure
extends StateMachineProcedure<MasterProcedureEnv, PeerModificationState>
implements PeerProcedureInterface {
- private static final Log LOG = LogFactory.getLog(ModifyPeerProcedure.class);
+ private static final Logger LOG = LoggerFactory.getLogger(ModifyPeerProcedure.class);
protected String peerId;
http://git-wip-us.apache.org/repos/asf/hbase/blob/bcb79d53/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
index ddc2401..ba4285f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
@@ -35,6 +33,8 @@ import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProced
import org.apache.hadoop.hbase.procedure2.RemoteProcedureException;
import org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationType;
@@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.R
public class RefreshPeerProcedure extends Procedure<MasterProcedureEnv>
implements PeerProcedureInterface, RemoteProcedure<MasterProcedureEnv, ServerName> {
- private static final Log LOG = LogFactory.getLog(RefreshPeerProcedure.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RefreshPeerProcedure.class);
private String peerId;
http://git-wip-us.apache.org/repos/asf/hbase/blob/bcb79d53/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
index 3daad6d..d40df02 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The procedure for removing a replication peer.
@@ -31,7 +31,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class RemovePeerProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(RemovePeerProcedure.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RemovePeerProcedure.class);
public RemovePeerProcedure() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bcb79d53/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
index 435eefc..d8154dc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
@@ -19,14 +19,14 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.UpdatePeerConfigStateData;
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.U
@InterfaceAudience.Private
public class UpdatePeerConfigProcedure extends ModifyPeerProcedure {
- private static final Log LOG = LogFactory.getLog(UpdatePeerConfigProcedure.class);
+ private static final Logger LOG = LoggerFactory.getLogger(UpdatePeerConfigProcedure.class);
private ReplicationPeerConfig peerConfig;
[04/50] [abbrv] hbase git commit: HBASE-19811 Fix findbugs and
error-prone warnings in hbase-server (branch-2)
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
index 5098e0b..410dd0c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
@@ -101,7 +101,7 @@ public abstract class MultiThreadedAction {
@Override
public byte[] getDeterministicUniqueKey(long keyBase) {
- return LoadTestKVGenerator.md5PrefixedKey(keyBase).getBytes();
+ return Bytes.toBytes(LoadTestKVGenerator.md5PrefixedKey(keyBase));
}
@Override
@@ -114,7 +114,7 @@ public abstract class MultiThreadedAction {
int numColumns = minColumnsPerKey + random.nextInt(maxColumnsPerKey - minColumnsPerKey + 1);
byte[][] columns = new byte[numColumns][];
for (int i = 0; i < numColumns; ++i) {
- columns[i] = Integer.toString(i).getBytes();
+ columns[i] = Bytes.toBytes(Integer.toString(i));
}
return columns;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java
index 447cca8..6864366 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReader.java
@@ -201,8 +201,7 @@ public class MultiThreadedReader extends MultiThreadedAction
"to read " + k + " is out of range (startKey=" + startKey +
", endKey=" + endKey + ")");
}
- if (k % numThreads != readerId ||
- writer != null && writer.failedToWriteKey(k)) {
+ if (k % numThreads != readerId || (writer != null && writer.failedToWriteKey(k))) {
// Skip keys that this thread should not read, as well as the keys
// that we know the writer failed to write.
continue;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestBoundedPriorityBlockingQueue.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestBoundedPriorityBlockingQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestBoundedPriorityBlockingQueue.java
index 7112d50..7746bea 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestBoundedPriorityBlockingQueue.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestBoundedPriorityBlockingQueue.java
@@ -42,7 +42,7 @@ import org.junit.experimental.categories.Category;
public class TestBoundedPriorityBlockingQueue {
private final static int CAPACITY = 16;
- class TestObject {
+ static class TestObject {
private final int priority;
private final int seqId;
@@ -60,7 +60,7 @@ public class TestBoundedPriorityBlockingQueue {
}
}
- class TestObjectComparator implements Comparator<TestObject> {
+ static class TestObjectComparator implements Comparator<TestObject> {
public TestObjectComparator() {}
@Override
@@ -208,6 +208,7 @@ public class TestBoundedPriorityBlockingQueue {
final CyclicBarrier threadsStarted = new CyclicBarrier(2);
ExecutorService executor = Executors.newFixedThreadPool(2);
executor.execute(new Runnable() {
+ @Override
public void run() {
try {
assertNull(queue.poll(1000, TimeUnit.MILLISECONDS));
@@ -221,6 +222,7 @@ public class TestBoundedPriorityBlockingQueue {
});
executor.execute(new Runnable() {
+ @Override
public void run() {
try {
threadsStarted.await();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestByteBuffUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestByteBuffUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestByteBuffUtils.java
index 4c6990e..ecc6611 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestByteBuffUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestByteBuffUtils.java
@@ -36,7 +36,7 @@ public class TestByteBuffUtils {
ByteBuffer bb2 = ByteBuffer.allocate(50);
MultiByteBuff src = new MultiByteBuff(bb1, bb2);
for (int i = 0; i < 7; i++) {
- src.putLong(8l);
+ src.putLong(8L);
}
src.put((byte) 1);
src.put((byte) 1);
@@ -58,7 +58,7 @@ public class TestByteBuffUtils {
bb3 = ByteBuffer.allocate(100);
SingleByteBuff sbb = new SingleByteBuff(bb3);
for (int i = 0; i < 7; i++) {
- sbb.putLong(8l);
+ sbb.putLong(8L);
}
sbb.put((byte) 1);
sbb.put((byte) 1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
index 8f503e0..a554e99 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSHDFSUtils.java
@@ -161,11 +161,12 @@ public class TestFSHDFSUtils {
/**
* Version of DFS that has HDFS-4525 in it.
*/
- class IsFileClosedDistributedFileSystem extends DistributedFileSystem {
+ static class IsFileClosedDistributedFileSystem extends DistributedFileSystem {
/**
* Close status of a file. Copied over from HDFS-4525
* @return true if file is already closed
**/
+ @Override
public boolean isFileClosed(Path f) throws IOException{
return false;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
index cb23a0b..be302d5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSVisitor.java
@@ -77,6 +77,7 @@ public class TestFSVisitor {
final Set<String> families = new HashSet<>();
final Set<String> hfiles = new HashSet<>();
FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
+ @Override
public void storeFile(final String region, final String family, final String hfileName)
throws IOException {
regions.add(region);
@@ -84,9 +85,9 @@ public class TestFSVisitor {
hfiles.add(hfileName);
}
});
- assertEquals(tableRegions, regions);
- assertEquals(tableFamilies, families);
- assertEquals(tableHFiles, hfiles);
+ assertEquals(regions, tableRegions);
+ assertEquals(families, tableFamilies);
+ assertEquals(hfiles, tableHFiles);
}
/*
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
index 641f66e..31f1909 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
@@ -131,12 +131,12 @@ public class TestHBaseFsckEncryption {
// Insure HBck doesn't consider them corrupt
HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, htd.getTableName());
- assertEquals(res.getRetCode(), 0);
+ assertEquals(0, res.getRetCode());
HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
- assertEquals(hfcc.getCorrupted().size(), 0);
- assertEquals(hfcc.getFailures().size(), 0);
- assertEquals(hfcc.getQuarantined().size(), 0);
- assertEquals(hfcc.getMissing().size(), 0);
+ assertEquals(0, hfcc.getCorrupted().size());
+ assertEquals(0, hfcc.getFailures().size());
+ assertEquals(0, hfcc.getQuarantined().size());
+ assertEquals(0, hfcc.getMissing().size());
}
private List<Path> findStorefilePaths(TableName tableName) throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
index 3661207..2548ecf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
@@ -118,18 +118,18 @@ public class TestHBaseFsckMOB extends BaseTestHBaseFsck {
// A corrupt mob file doesn't abort the start of regions, so we can enable the table.
admin.enableTable(table);
HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
- assertEquals(res.getRetCode(), 0);
+ assertEquals(0, res.getRetCode());
HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
- assertEquals(hfcc.getHFilesChecked(), 4);
- assertEquals(hfcc.getCorrupted().size(), 0);
- assertEquals(hfcc.getFailures().size(), 0);
- assertEquals(hfcc.getQuarantined().size(), 0);
- assertEquals(hfcc.getMissing().size(), 0);
- assertEquals(hfcc.getMobFilesChecked(), 5);
- assertEquals(hfcc.getCorruptedMobFiles().size(), 1);
- assertEquals(hfcc.getFailureMobFiles().size(), 0);
- assertEquals(hfcc.getQuarantinedMobFiles().size(), 1);
- assertEquals(hfcc.getMissedMobFiles().size(), 0);
+ assertEquals(4, hfcc.getHFilesChecked());
+ assertEquals(0, hfcc.getCorrupted().size());
+ assertEquals(0, hfcc.getFailures().size());
+ assertEquals(0, hfcc.getQuarantined().size());
+ assertEquals(0, hfcc.getMissing().size());
+ assertEquals(5, hfcc.getMobFilesChecked());
+ assertEquals(1, hfcc.getCorruptedMobFiles().size());
+ assertEquals(0, hfcc.getFailureMobFiles().size());
+ assertEquals(1, hfcc.getQuarantinedMobFiles().size());
+ assertEquals(0, hfcc.getMissedMobFiles().size());
String quarantinedMobFile = hfcc.getQuarantinedMobFiles().iterator().next().getName();
assertEquals(corruptMobFile, quarantinedMobFile);
} finally {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestJSONMetricUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestJSONMetricUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestJSONMetricUtil.java
index 13c6df5..bb68898 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestJSONMetricUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestJSONMetricUtil.java
@@ -86,7 +86,7 @@ public class TestJSONMetricUtil {
Hashtable<String, String> properties = JSONMetricUtil.buldKeyValueTable(keys, values);
ObjectName testObject = JSONMetricUtil.buildObjectName(JSONMetricUtil.JAVA_LANG_DOMAIN,
properties);
- assertEquals(testObject.getDomain(), JSONMetricUtil.JAVA_LANG_DOMAIN);
+ assertEquals(JSONMetricUtil.JAVA_LANG_DOMAIN, testObject.getDomain());
assertEquals(testObject.getKeyPropertyList(), properties);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadParallel.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadParallel.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadParallel.java
index 7b1cd2d..e6b05e9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadParallel.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadParallel.java
@@ -40,6 +40,7 @@ public class TestMiniClusterLoadParallel
super(isMultiPut, encoding);
}
+ @Override
@Test(timeout=TIMEOUT_MS)
public void loadTest() throws Exception {
prepareForLoadTest();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
index fd86beb..eee3030 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
@@ -68,6 +68,7 @@ public class TestRegionSplitCalculator {
return end;
}
+ @Override
public String toString() {
return "[" + Bytes.toString(start) + ", " + Bytes.toString(end) + "]";
}
@@ -135,8 +136,7 @@ public class TestRegionSplitCalculator {
LOG.info("Standard");
String res = dump(sc.getSplits(), regions);
checkDepths(sc.getSplits(), regions, 1, 1, 1, 0);
- assertEquals(res, "A:\t[A, B]\t\n" + "B:\t[B, C]\t\n" + "C:\t[C, D]\t\n"
- + "D:\t\n");
+ assertEquals("A:\t[A, B]\t\n" + "B:\t[B, C]\t\n" + "C:\t[C, D]\t\nD:\t\n", res);
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java
index 3650706..1aab1f8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java
@@ -20,10 +20,11 @@ import java.io.IOException;
import java.util.Random;
import java.util.Set;
-import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.util.LoadTestKVGenerator;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* A generator of random data (keys/cfs/columns/values) for load testing.
@@ -35,11 +36,11 @@ public abstract class LoadTestDataGenerator {
// The mutate info column stores information
// about update done to this column family this row.
- public final static byte[] MUTATE_INFO = "mutate_info".getBytes();
+ public final static byte[] MUTATE_INFO = Bytes.toBytes("mutate_info");
// The increment column always has a long value,
// which can be incremented later on during updates.
- public final static byte[] INCREMENT = "increment".getBytes();
+ public final static byte[] INCREMENT = Bytes.toBytes("increment");
protected String[] args;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
index f0f35e7..ceb43d5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
@@ -476,7 +476,7 @@ public class TestWALFactory {
reader.close();
// Reset the lease period
- setLeasePeriod.invoke(cluster, new Object[]{new Long(60000), new Long(3600000)});
+ setLeasePeriod.invoke(cluster, new Object[]{ 60000L, 3600000L });
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
index a1206aa..dded506 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
@@ -131,9 +131,9 @@ public class TestWALSplit {
private static final String FILENAME_BEING_SPLIT = "testfile";
private static final TableName TABLE_NAME =
TableName.valueOf("t1");
- private static final byte[] FAMILY = "f1".getBytes();
- private static final byte[] QUALIFIER = "q1".getBytes();
- private static final byte[] VALUE = "v1".getBytes();
+ private static final byte[] FAMILY = Bytes.toBytes("f1");
+ private static final byte[] QUALIFIER = Bytes.toBytes("q1");
+ private static final byte[] VALUE = Bytes.toBytes("v1");
private static final String WAL_FILE_PREFIX = "wal.dat.";
private static List<String> REGIONS = new ArrayList<>();
private static final String HBASE_SKIP_ERRORS = "hbase.hlog.split.skip.errors";
@@ -341,7 +341,7 @@ public class TestWALSplit {
while (!stop.get()) {
try {
long seq = appendEntry(writer, TABLE_NAME, regionBytes,
- ("r" + editsCount.get()).getBytes(), regionBytes, QUALIFIER, VALUE, 0);
+ Bytes.toBytes("r" + editsCount.get()), regionBytes, QUALIFIER, VALUE, 0);
long count = editsCount.incrementAndGet();
LOG.info(getName() + " sync count=" + count + ", seq=" + seq);
try {
@@ -407,7 +407,7 @@ public class TestWALSplit {
TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),
new WALEdit());
Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
- assertEquals(parent.getName(), HConstants.RECOVERED_EDITS_DIR);
+ assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName());
fs.createNewFile(parent); // create a recovered.edits file
Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR,
@@ -588,8 +588,8 @@ public class TestWALSplit {
archivedLogs.add(log.getPath().getName());
}
LOG.debug(archived.toString());
- assertEquals(failureType.name() + ": expected to find all of our wals corrupt.",
- walDirContents, archivedLogs);
+ assertEquals(failureType.name() + ": expected to find all of our wals corrupt.", archivedLogs,
+ walDirContents);
}
}
@@ -687,7 +687,7 @@ public class TestWALSplit {
// should not have stored the EOF files as corrupt
FileStatus[] archivedLogs = fs.listStatus(CORRUPTDIR);
- assertEquals(archivedLogs.length, 0);
+ assertEquals(0, archivedLogs.length);
}
@@ -749,7 +749,7 @@ public class TestWALSplit {
InstrumentedLogWriter.activateFailure = false;
appendEntry(writer, TABLE_NAME, Bytes.toBytes(region),
- ("r" + 999).getBytes(), FAMILY, QUALIFIER, VALUE, 0);
+ Bytes.toBytes("r" + 999), FAMILY, QUALIFIER, VALUE, 0);
writer.close();
try {
@@ -1206,8 +1206,8 @@ public class TestWALSplit {
int prefix = 0;
for (String region : REGIONS) {
String row_key = region + prefix++ + i + j;
- appendEntry(ws[i], TABLE_NAME, region.getBytes(), row_key.getBytes(), FAMILY, QUALIFIER,
- VALUE, seq++);
+ appendEntry(ws[i], TABLE_NAME, Bytes.toBytes(region), Bytes.toBytes(row_key), FAMILY,
+ QUALIFIER, VALUE, seq++);
if (numRegionEventsAdded < regionEvents) {
numRegionEventsAdded ++;
@@ -1233,7 +1233,7 @@ public class TestWALSplit {
Path tdir = FSUtils.getTableDir(rootdir, table);
@SuppressWarnings("deprecation")
Path editsdir = WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir,
- Bytes.toString(region.getBytes())));
+ Bytes.toString(Bytes.toBytes(region))));
FileStatus[] files = fs.listStatus(editsdir, new PathFilter() {
@Override
public boolean accept(Path p) {
@@ -1260,46 +1260,46 @@ public class TestWALSplit {
in.close();
switch (corruption) {
- case APPEND_GARBAGE:
- fs.delete(path, false);
- out = fs.create(path);
- out.write(corrupted_bytes);
- out.write("-----".getBytes());
- closeOrFlush(close, out);
- break;
-
- case INSERT_GARBAGE_ON_FIRST_LINE:
- fs.delete(path, false);
- out = fs.create(path);
- out.write(0);
- out.write(corrupted_bytes);
- closeOrFlush(close, out);
- break;
-
- case INSERT_GARBAGE_IN_THE_MIDDLE:
- fs.delete(path, false);
- out = fs.create(path);
- int middle = (int) Math.floor(corrupted_bytes.length / 2);
- out.write(corrupted_bytes, 0, middle);
- out.write(0);
- out.write(corrupted_bytes, middle, corrupted_bytes.length - middle);
- closeOrFlush(close, out);
- break;
-
- case TRUNCATE:
- fs.delete(path, false);
- out = fs.create(path);
- out.write(corrupted_bytes, 0, fileSize
- - (32 + ProtobufLogReader.PB_WAL_COMPLETE_MAGIC.length + Bytes.SIZEOF_INT));
- closeOrFlush(close, out);
- break;
-
- case TRUNCATE_TRAILER:
- fs.delete(path, false);
- out = fs.create(path);
- out.write(corrupted_bytes, 0, fileSize - Bytes.SIZEOF_INT);// trailer is truncated.
- closeOrFlush(close, out);
- break;
+ case APPEND_GARBAGE:
+ fs.delete(path, false);
+ out = fs.create(path);
+ out.write(corrupted_bytes);
+ out.write(Bytes.toBytes("-----"));
+ closeOrFlush(close, out);
+ break;
+
+ case INSERT_GARBAGE_ON_FIRST_LINE:
+ fs.delete(path, false);
+ out = fs.create(path);
+ out.write(0);
+ out.write(corrupted_bytes);
+ closeOrFlush(close, out);
+ break;
+
+ case INSERT_GARBAGE_IN_THE_MIDDLE:
+ fs.delete(path, false);
+ out = fs.create(path);
+ int middle = (int) Math.floor(corrupted_bytes.length / 2);
+ out.write(corrupted_bytes, 0, middle);
+ out.write(0);
+ out.write(corrupted_bytes, middle, corrupted_bytes.length - middle);
+ closeOrFlush(close, out);
+ break;
+
+ case TRUNCATE:
+ fs.delete(path, false);
+ out = fs.create(path);
+ out.write(corrupted_bytes, 0, fileSize
+ - (32 + ProtobufLogReader.PB_WAL_COMPLETE_MAGIC.length + Bytes.SIZEOF_INT));
+ closeOrFlush(close, out);
+ break;
+
+ case TRUNCATE_TRAILER:
+ fs.delete(path, false);
+ out = fs.create(path);
+ out.write(corrupted_bytes, 0, fileSize - Bytes.SIZEOF_INT);// trailer is truncated.
+ closeOrFlush(close, out);
+ break;
}
}
@@ -1360,14 +1360,14 @@ public class TestWALSplit {
WALProtos.RegionEventDescriptor regionOpenDesc = ProtobufUtil.toRegionEventDescriptor(
WALProtos.RegionEventDescriptor.EventType.REGION_OPEN,
TABLE_NAME.toBytes(),
- region.getBytes(),
- String.valueOf(region.hashCode()).getBytes(),
+ Bytes.toBytes(region),
+ Bytes.toBytes(String.valueOf(region.hashCode())),
1,
ServerName.parseServerName("ServerName:9099"), ImmutableMap.<byte[], List<Path>>of());
final long time = EnvironmentEdgeManager.currentTime();
- KeyValue kv = new KeyValue(region.getBytes(), WALEdit.METAFAMILY, WALEdit.REGION_EVENT,
+ KeyValue kv = new KeyValue(Bytes.toBytes(region), WALEdit.METAFAMILY, WALEdit.REGION_EVENT,
time, regionOpenDesc.toByteArray());
- final WALKeyImpl walKey = new WALKeyImpl(region.getBytes(), TABLE_NAME, 1, time,
+ final WALKeyImpl walKey = new WALKeyImpl(Bytes.toBytes(region), TABLE_NAME, 1, time,
HConstants.DEFAULT_CLUSTER_ID);
w.append(
new Entry(walKey, new WALEdit().add(kv)));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitBoundedLogWriterCreation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitBoundedLogWriterCreation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitBoundedLogWriterCreation.java
index 844cb3a..400d12b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitBoundedLogWriterCreation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplitBoundedLogWriterCreation.java
@@ -35,6 +35,7 @@ public class TestWALSplitBoundedLogWriterCreation extends TestWALSplit{
/**
* The logic of this test has conflict with the limit writers split logic, skip this test
*/
+ @Override
@Test(timeout=300000)
@Ignore
public void testThreadingSlowWriterSmallBuffer() throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
index 4e67b91..3e9e650 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
@@ -129,10 +129,10 @@ public class TestZooKeeperACL {
List<ACL> acls = zkw.getRecoverableZooKeeper().getZooKeeper()
.getACL("/hbase", new Stat());
- assertEquals(acls.size(),1);
- assertEquals(acls.get(0).getId().getScheme(),"sasl");
- assertEquals(acls.get(0).getId().getId(),"hbase");
- assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.ALL);
+ assertEquals(1, acls.size());
+ assertEquals("sasl", acls.get(0).getId().getScheme());
+ assertEquals("hbase", acls.get(0).getId().getId());
+ assertEquals(ZooDefs.Perms.ALL, acls.get(0).getPerms());
}
/**
@@ -148,20 +148,20 @@ public class TestZooKeeperACL {
List<ACL> acls = zkw.getRecoverableZooKeeper().getZooKeeper()
.getACL("/hbase/root-region-server", new Stat());
- assertEquals(acls.size(),2);
+ assertEquals(2, acls.size());
boolean foundWorldReadableAcl = false;
boolean foundHBaseOwnerAcl = false;
for(int i = 0; i < 2; i++) {
if (acls.get(i).getId().getScheme().equals("world") == true) {
- assertEquals(acls.get(0).getId().getId(),"anyone");
- assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.READ);
+ assertEquals("anyone", acls.get(0).getId().getId());
+ assertEquals(ZooDefs.Perms.READ, acls.get(0).getPerms());
foundWorldReadableAcl = true;
}
else {
if (acls.get(i).getId().getScheme().equals("sasl") == true) {
- assertEquals(acls.get(1).getId().getId(),"hbase");
- assertEquals(acls.get(1).getId().getScheme(),"sasl");
+ assertEquals("hbase", acls.get(1).getId().getId());
+ assertEquals("sasl", acls.get(1).getId().getScheme());
foundHBaseOwnerAcl = true;
} else { // error: should not get here: test fails.
assertTrue(false);
@@ -185,19 +185,19 @@ public class TestZooKeeperACL {
List<ACL> acls = zkw.getRecoverableZooKeeper().getZooKeeper()
.getACL("/hbase/master", new Stat());
- assertEquals(acls.size(),2);
+ assertEquals(2, acls.size());
boolean foundWorldReadableAcl = false;
boolean foundHBaseOwnerAcl = false;
for(int i = 0; i < 2; i++) {
if (acls.get(i).getId().getScheme().equals("world") == true) {
- assertEquals(acls.get(0).getId().getId(),"anyone");
- assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.READ);
+ assertEquals("anyone", acls.get(0).getId().getId());
+ assertEquals(ZooDefs.Perms.READ, acls.get(0).getPerms());
foundWorldReadableAcl = true;
} else {
if (acls.get(i).getId().getScheme().equals("sasl") == true) {
- assertEquals(acls.get(1).getId().getId(),"hbase");
- assertEquals(acls.get(1).getId().getScheme(),"sasl");
+ assertEquals("hbase", acls.get(1).getId().getId());
+ assertEquals("sasl", acls.get(1).getId().getScheme());
foundHBaseOwnerAcl = true;
} else { // error: should not get here: test fails.
assertTrue(false);
@@ -221,19 +221,19 @@ public class TestZooKeeperACL {
List<ACL> acls = zkw.getRecoverableZooKeeper().getZooKeeper()
.getACL("/hbase/hbaseid", new Stat());
- assertEquals(acls.size(),2);
+ assertEquals(2, acls.size());
boolean foundWorldReadableAcl = false;
boolean foundHBaseOwnerAcl = false;
for(int i = 0; i < 2; i++) {
if (acls.get(i).getId().getScheme().equals("world") == true) {
- assertEquals(acls.get(0).getId().getId(),"anyone");
- assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.READ);
+ assertEquals("anyone", acls.get(0).getId().getId());
+ assertEquals(ZooDefs.Perms.READ, acls.get(0).getPerms());
foundWorldReadableAcl = true;
} else {
if (acls.get(i).getId().getScheme().equals("sasl") == true) {
- assertEquals(acls.get(1).getId().getId(),"hbase");
- assertEquals(acls.get(1).getId().getScheme(),"sasl");
+ assertEquals("hbase", acls.get(1).getId().getId());
+ assertEquals("sasl", acls.get(1).getId().getScheme());
foundHBaseOwnerAcl = true;
} else { // error: should not get here: test fails.
assertTrue(false);
@@ -257,10 +257,10 @@ public class TestZooKeeperACL {
ZKUtil.createWithParents(zkw, "/testACLNode");
List<ACL> acls = zkw.getRecoverableZooKeeper().getZooKeeper()
.getACL("/testACLNode", new Stat());
- assertEquals(acls.size(),1);
- assertEquals(acls.get(0).getId().getScheme(),"sasl");
- assertEquals(acls.get(0).getId().getId(),"hbase");
- assertEquals(acls.get(0).getPerms(), ZooDefs.Perms.ALL);
+ assertEquals(1, acls.size());
+ assertEquals("sasl", acls.get(0).getId().getScheme());
+ assertEquals("hbase", acls.get(0).getId().getId());
+ assertEquals(ZooDefs.Perms.ALL, acls.get(0).getPerms());
}
/**
@@ -281,7 +281,7 @@ public class TestZooKeeperACL {
saslConfFile.getAbsolutePath());
testJaasConfig = ZKUtil.isSecureZooKeeper(new Configuration(TEST_UTIL.getConfiguration()));
- assertEquals(testJaasConfig, false);
+ assertEquals(false, testJaasConfig);
saslConfFile.delete();
}
@@ -295,13 +295,13 @@ public class TestZooKeeperACL {
Configuration config = new Configuration(HBaseConfiguration.create());
boolean testJaasConfig = ZKUtil.isSecureZooKeeper(config);
- assertEquals(testJaasConfig, false);
+ assertEquals(false, testJaasConfig);
// Now set authentication scheme to Kerberos still it should return false
// because no configuration set
config.set("hbase.security.authentication", "kerberos");
testJaasConfig = ZKUtil.isSecureZooKeeper(config);
- assertEquals(testJaasConfig, false);
+ assertEquals(false, testJaasConfig);
// Now set programmatic options related to security
config.set(HConstants.ZK_CLIENT_KEYTAB_FILE, "/dummy/file");
[44/50] [abbrv] hbase git commit: HBASE-19636 All rs should already
start work with the new peer change when replication peer procedure is
finished
Posted by zh...@apache.org.
HBASE-19636 All rs should already start work with the new peer change when replication peer procedure is finished
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aab18b45
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aab18b45
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aab18b45
Branch: refs/heads/HBASE-19397-branch-2
Commit: aab18b45ba20cea51df861914c12b87755008f7d
Parents: 6bd7fd8
Author: Guanghao Zhang <zg...@apache.org>
Authored: Thu Jan 4 16:58:01 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../replication/ReplicationPeerConfig.java | 1 -
.../hbase/replication/ReplicationPeerImpl.java | 4 +-
.../hbase/replication/ReplicationQueueInfo.java | 23 +-
.../hbase/replication/ReplicationUtils.java | 56 ++
.../replication/TestReplicationStateZKImpl.java | 22 -
.../regionserver/ReplicationSourceService.java | 3 +-
.../regionserver/PeerProcedureHandler.java | 3 +
.../regionserver/PeerProcedureHandlerImpl.java | 50 +-
.../RecoveredReplicationSource.java | 6 +-
.../RecoveredReplicationSourceShipper.java | 8 +-
.../replication/regionserver/Replication.java | 15 +-
.../regionserver/ReplicationSource.java | 34 +-
.../regionserver/ReplicationSourceFactory.java | 4 +-
.../ReplicationSourceInterface.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 895 ++++++++++---------
.../regionserver/ReplicationSourceShipper.java | 6 +-
.../ReplicationSourceWALReader.java | 2 +-
.../replication/ReplicationSourceDummy.java | 2 +-
.../replication/TestNamespaceReplication.java | 57 +-
.../TestReplicationSourceManager.java | 11 +-
.../TestReplicationSourceManagerZkImpl.java | 1 -
21 files changed, 659 insertions(+), 552 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index fdae288..bf8d030 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
index 3e17025..604e0bb 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerImpl.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,6 +27,7 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class ReplicationPeerImpl implements ReplicationPeer {
+
private final Configuration conf;
private final String id;
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
index ecd888f..cd65f9b 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
@@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.ServerName;
/**
- * This class is responsible for the parsing logic for a znode representing a queue.
+ * This class is responsible for the parsing logic for a queue id representing a queue.
* It will extract the peerId if it's recovered as well as the dead region servers
* that were part of the queue's history.
*/
@@ -38,21 +38,20 @@ public class ReplicationQueueInfo {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationQueueInfo.class);
private final String peerId;
- private final String peerClusterZnode;
+ private final String queueId;
private boolean queueRecovered;
// List of all the dead region servers that had this queue (if recovered)
private List<ServerName> deadRegionServers = new ArrayList<>();
/**
- * The passed znode will be either the id of the peer cluster or
- * the handling story of that queue in the form of id-servername-*
+ * The passed queueId will be either the id of the peer or the handling story of that queue
+ * in the form of id-servername-*
*/
- public ReplicationQueueInfo(String znode) {
- this.peerClusterZnode = znode;
- String[] parts = znode.split("-", 2);
+ public ReplicationQueueInfo(String queueId) {
+ this.queueId = queueId;
+ String[] parts = queueId.split("-", 2);
this.queueRecovered = parts.length != 1;
- this.peerId = this.queueRecovered ?
- parts[0] : peerClusterZnode;
+ this.peerId = this.queueRecovered ? parts[0] : queueId;
if (parts.length >= 2) {
// extract dead servers
extractDeadServersFromZNodeString(parts[1], this.deadRegionServers);
@@ -60,7 +59,7 @@ public class ReplicationQueueInfo {
}
/**
- * Parse dead server names from znode string servername can contain "-" such as
+ * Parse dead server names from queue id. servername can contain "-" such as
* "ip-10-46-221-101.ec2.internal", so we need skip some "-" during parsing for the following
* cases: 2-ip-10-46-221-101.ec2.internal,52170,1364333181125-<server name>-...
*/
@@ -119,8 +118,8 @@ public class ReplicationQueueInfo {
return this.peerId;
}
- public String getPeerClusterZnode() {
- return this.peerClusterZnode;
+ public String getQueueId() {
+ return this.queueId;
}
public boolean isQueueRecovered() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index 7b676ca..ebe68a7 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -18,11 +18,15 @@
package org.apache.hadoop.hbase.replication;
import java.io.IOException;
+import java.util.Collection;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CompoundConfiguration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
/**
@@ -66,4 +70,56 @@ public final class ReplicationUtils {
queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
}
}
+
+ private static boolean isCollectionEqual(Collection<String> c1, Collection<String> c2) {
+ if (c1 == null) {
+ return c2 == null;
+ }
+ if (c2 == null) {
+ return false;
+ }
+ return c1.size() == c2.size() && c1.containsAll(c2);
+ }
+
+ private static boolean isNamespacesEqual(Set<String> ns1, Set<String> ns2) {
+ return isCollectionEqual(ns1, ns2);
+ }
+
+ private static boolean isTableCFsEqual(Map<TableName, List<String>> tableCFs1,
+ Map<TableName, List<String>> tableCFs2) {
+ if (tableCFs1 == null) {
+ return tableCFs2 == null;
+ }
+ if (tableCFs2 == null) {
+ return false;
+ }
+ if (tableCFs1.size() != tableCFs2.size()) {
+ return false;
+ }
+ for (Map.Entry<TableName, List<String>> entry1 : tableCFs1.entrySet()) {
+ TableName table = entry1.getKey();
+ if (!tableCFs2.containsKey(table)) {
+ return false;
+ }
+ List<String> cfs1 = entry1.getValue();
+ List<String> cfs2 = tableCFs2.get(table);
+ if (!isCollectionEqual(cfs1, cfs2)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static boolean isKeyConfigEqual(ReplicationPeerConfig rpc1, ReplicationPeerConfig rpc2) {
+ if (rpc1.replicateAllUserTables() != rpc2.replicateAllUserTables()) {
+ return false;
+ }
+ if (rpc1.replicateAllUserTables()) {
+ return isNamespacesEqual(rpc1.getExcludeNamespaces(), rpc2.getExcludeNamespaces()) &&
+ isTableCFsEqual(rpc1.getExcludeTableCFsMap(), rpc2.getExcludeTableCFsMap());
+ } else {
+ return isNamespacesEqual(rpc1.getNamespaces(), rpc2.getNamespaces()) &&
+ isTableCFsEqual(rpc1.getTableCFsMap(), rpc2.getTableCFsMap());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
index 6825c36..2790bd0 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
@@ -18,9 +18,7 @@
package org.apache.hadoop.hbase.replication;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.ClusterId;
import org.apache.hadoop.hbase.HBaseZKTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@@ -37,14 +35,10 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.experimental.categories.Category;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
@Category({ ReplicationTests.class, MediumTests.class })
public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
- private static final Logger LOG = LoggerFactory.getLogger(TestReplicationStateZKImpl.class);
-
private static Configuration conf;
private static HBaseZKTestingUtility utility;
private static ZKWatcher zkw;
@@ -92,20 +86,4 @@ public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
public static void tearDownAfterClass() throws Exception {
utility.shutdownMiniZKCluster();
}
-
- private static class WarnOnlyAbortable implements Abortable {
-
- @Override
- public void abort(String why, Throwable e) {
- LOG.warn("TestReplicationStateZKImpl received abort, ignoring. Reason: " + why);
- if (LOG.isDebugEnabled()) {
- LOG.debug(e.toString(), e);
- }
- }
-
- @Override
- public boolean isAborted() {
- return false;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
index a82fa3d..2aef0a8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReplicationSourceService.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
index b392985..65da9af 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
@@ -23,6 +23,9 @@ import java.io.IOException;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.yetus.audience.InterfaceAudience;
+/**
+ * A handler for modifying replication peer in peer procedures.
+ */
@InterfaceAudience.Private
public interface PeerProcedureHandler {
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index c09c6a0..ce8fdae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -15,21 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
import java.util.concurrent.locks.Lock;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerImpl;
+import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.util.KeyLocker;
import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
- private static final Logger LOG = LoggerFactory.getLogger(PeerProcedureHandlerImpl.class);
private final ReplicationSourceManager replicationSourceManager;
private final KeyLocker<String> peersLock = new KeyLocker<>();
@@ -39,7 +38,7 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
}
@Override
- public void addPeer(String peerId) throws ReplicationException, IOException {
+ public void addPeer(String peerId) throws IOException {
Lock peerLock = peersLock.acquireLock(peerId);
try {
replicationSourceManager.addPeer(peerId);
@@ -49,7 +48,7 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
}
@Override
- public void removePeer(String peerId) throws ReplicationException, IOException {
+ public void removePeer(String peerId) throws IOException {
Lock peerLock = peersLock.acquireLock(peerId);
try {
if (replicationSourceManager.getReplicationPeers().getPeer(peerId) != null) {
@@ -60,35 +59,50 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
}
}
- @Override
- public void disablePeer(String peerId) throws ReplicationException, IOException {
+ private void refreshPeerState(String peerId) throws ReplicationException, IOException {
PeerState newState;
Lock peerLock = peersLock.acquireLock(peerId);
try {
+ ReplicationPeerImpl peer = replicationSourceManager.getReplicationPeers().getPeer(peerId);
+ if (peer == null) {
+ throw new ReplicationException("Peer with id=" + peerId + " is not cached.");
+ }
+ PeerState oldState = peer.getPeerState();
newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+ // RS need to start work with the new replication state change
+ if (oldState.equals(PeerState.ENABLED) && newState.equals(PeerState.DISABLED)) {
+ replicationSourceManager.refreshSources(peerId);
+ }
} finally {
peerLock.unlock();
}
- LOG.info("disable replication peer, id: {}, new state: {}", peerId, newState);
}
@Override
public void enablePeer(String peerId) throws ReplicationException, IOException {
- PeerState newState;
- Lock peerLock = peersLock.acquireLock(peerId);
- try {
- newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
- } finally {
- peerLock.unlock();
- }
- LOG.info("enable replication peer, id: {}, new state: {}", peerId, newState);
+ refreshPeerState(peerId);
+ }
+
+ @Override
+ public void disablePeer(String peerId) throws ReplicationException, IOException {
+ refreshPeerState(peerId);
}
@Override
public void updatePeerConfig(String peerId) throws ReplicationException, IOException {
Lock peerLock = peersLock.acquireLock(peerId);
try {
- replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+ ReplicationPeerImpl peer = replicationSourceManager.getReplicationPeers().getPeer(peerId);
+ if (peer == null) {
+ throw new ReplicationException("Peer with id=" + peerId + " is not cached.");
+ }
+ ReplicationPeerConfig oldConfig = peer.getPeerConfig();
+ ReplicationPeerConfig newConfig =
+ replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+ // RS need to start work with the new replication config change
+ if (!ReplicationUtils.isKeyConfigEqual(oldConfig, newConfig)) {
+ replicationSourceManager.refreshSources(peerId);
+ }
} finally {
peerLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index 7bceb78..1be9a88 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -81,7 +81,7 @@ public class RecoveredReplicationSource extends ReplicationSource {
ReplicationSourceWALReader walReader = new RecoveredReplicationSourceWALReader(fs,
conf, queue, startPosition, walEntryFilter, this);
Threads.setDaemonThreadRunning(walReader, threadName
- + ".replicationSource.replicationWALReaderThread." + walGroupId + "," + peerClusterZnode,
+ + ".replicationSource.replicationWALReaderThread." + walGroupId + "," + queueId,
getUncaughtExceptionHandler());
return walReader;
}
@@ -178,8 +178,8 @@ public class RecoveredReplicationSource extends ReplicationSource {
}
}
if (allTasksDone) {
- manager.closeRecoveredQueue(this);
- LOG.info("Finished recovering queue " + peerClusterZnode + " with the following stats: "
+ manager.removeRecoveredSource(this);
+ LOG.info("Finished recovering queue " + queueId + " with the following stats: "
+ getStats());
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
index fb365bc..1e45496 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
@@ -77,7 +77,7 @@ public class RecoveredReplicationSourceShipper extends ReplicationSourceShipper
if (entryBatch.getWalEntries().isEmpty()
&& entryBatch.getLastSeqIds().isEmpty()) {
LOG.debug("Finished recovering queue for group " + walGroupId + " of peer "
- + source.getPeerClusterZnode());
+ + source.getQueueId());
source.getSourceMetrics().incrCompletedRecoveryQueue();
setWorkerState(WorkerState.FINISHED);
continue;
@@ -114,7 +114,7 @@ public class RecoveredReplicationSourceShipper extends ReplicationSourceShipper
// normally has a position (unless the RS failed between 2 logs)
private long getRecoveredQueueStartPos() {
long startPosition = 0;
- String peerClusterZnode = source.getPeerClusterZnode();
+ String peerClusterZnode = source.getQueueId();
try {
startPosition = this.replicationQueues.getWALPosition(source.getServerWALsBelongTo(),
peerClusterZnode, this.queue.peek().getName());
@@ -130,8 +130,8 @@ public class RecoveredReplicationSourceShipper extends ReplicationSourceShipper
@Override
protected void updateLogPosition(long lastReadPosition) {
- source.getSourceManager().logPositionAndCleanOldLogs(currentPath, source.getPeerClusterZnode(),
- lastReadPosition, true, false);
+ source.getSourceManager().logPositionAndCleanOldLogs(currentPath, source.getQueueId(),
+ lastReadPosition, true);
lastLoggedPosition = lastReadPosition;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index ce056a1..2fa5a9b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.regionserver.ReplicationSinkService;
import org.apache.hadoop.hbase.regionserver.ReplicationSourceService;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
-import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
@@ -59,10 +58,10 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
-
-import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* Gateway to Replication. Used by {@link org.apache.hadoop.hbase.regionserver.HRegionServer}.
*/
@@ -223,11 +222,7 @@ public class Replication implements
*/
@Override
public void startReplicationService() throws IOException {
- try {
- this.replicationManager.init();
- } catch (ReplicationException e) {
- throw new IOException(e);
- }
+ this.replicationManager.init();
this.replicationSink = new ReplicationSink(this.conf, this.server);
this.scheduleThreadPool.scheduleAtFixedRate(
new ReplicationStatisticsThread(this.replicationSink, this.replicationManager),
@@ -285,9 +280,9 @@ public class Replication implements
throws IOException {
try {
this.replicationManager.addHFileRefs(tableName, family, pairs);
- } catch (ReplicationException e) {
+ } catch (IOException e) {
LOG.error("Failed to add hfile references in the replication queue.", e);
- throw new IOException(e);
+ throw e;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index ffed88d..0092251 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -105,7 +105,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
// total number of edits we replicated
private AtomicLong totalReplicatedEdits = new AtomicLong(0);
// The znode we currently play with
- protected String peerClusterZnode;
+ protected String queueId;
// Maximum number of retries before taking bold actions
private int maxRetriesMultiplier;
// Indicates if this particular source is running
@@ -141,14 +141,14 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
* @param fs file system to use
* @param manager replication manager to ping to
* @param server the server for this region server
- * @param peerClusterZnode the name of our znode
+ * @param queueId the id of our replication queue
* @param clusterId unique UUID for the cluster
* @param metrics metrics for replication source
*/
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
- String peerClusterZnode, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
+ String queueId, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
MetricsSource metrics) throws IOException {
this.server = server;
this.conf = HBaseConfiguration.create(conf);
@@ -167,8 +167,8 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
this.metrics = metrics;
this.clusterId = clusterId;
- this.peerClusterZnode = peerClusterZnode;
- this.replicationQueueInfo = new ReplicationQueueInfo(peerClusterZnode);
+ this.queueId = queueId;
+ this.replicationQueueInfo = new ReplicationQueueInfo(queueId);
// ReplicationQueueInfo parses the peerId out of the znode for us
this.peerId = this.replicationQueueInfo.getPeerId();
this.logQueueWarnThreshold = this.conf.getInt("replication.source.log.queue.warn", 2);
@@ -178,7 +178,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
this.throttler = new ReplicationThrottler((double) currentBandwidth / 10.0);
this.totalBufferUsed = manager.getTotalBufferUsed();
this.walFileLengthProvider = walFileLengthProvider;
- LOG.info("peerClusterZnode=" + peerClusterZnode + ", ReplicationSource : " + peerId
+ LOG.info("queueId=" + queueId + ", ReplicationSource : " + peerId
+ ", currentBandwidth=" + this.currentBandwidth);
}
@@ -216,12 +216,6 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
@Override
public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)
throws ReplicationException {
- String peerId = peerClusterZnode;
- if (peerId.contains("-")) {
- // peerClusterZnode will be in the form peerId + "-" + rsZNode.
- // A peerId will not have "-" in its name, see HBASE-11394
- peerId = peerClusterZnode.split("-")[0];
- }
Map<TableName, List<String>> tableCFMap = replicationPeer.getTableCFs();
if (tableCFMap != null) {
List<String> tableCfs = tableCFMap.get(tableName);
@@ -310,7 +304,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
this.terminate("ClusterId " + clusterId + " is replicating to itself: peerClusterId "
+ peerClusterId + " which is not allowed by ReplicationEndpoint:"
+ replicationEndpoint.getClass().getName(), null, false);
- this.manager.closeQueue(this);
+ this.manager.removeSource(this);
return;
}
LOG.info("Replicating " + clusterId + " -> " + peerClusterId);
@@ -355,7 +349,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
ReplicationSourceWALReader walReader =
new ReplicationSourceWALReader(fs, conf, queue, startPosition, walEntryFilter, this);
return (ReplicationSourceWALReader) Threads.setDaemonThreadRunning(walReader,
- threadName + ".replicationSource.wal-reader." + walGroupId + "," + peerClusterZnode,
+ threadName + ".replicationSource.wal-reader." + walGroupId + "," + queueId,
getUncaughtExceptionHandler());
}
@@ -449,7 +443,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
LOG.error("Unexpected exception in ReplicationSource", e);
}
};
- Threads.setDaemonThreadRunning(this, n + ".replicationSource," + this.peerClusterZnode,
+ Threads.setDaemonThreadRunning(this, n + ".replicationSource," + this.queueId,
handler);
}
@@ -465,9 +459,9 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
public void terminate(String reason, Exception cause, boolean join) {
if (cause == null) {
- LOG.info("Closing source " + this.peerClusterZnode + " because: " + reason);
+ LOG.info("Closing source " + this.queueId + " because: " + reason);
} else {
- LOG.error("Closing source " + this.peerClusterZnode + " because an error occurred: " + reason,
+ LOG.error("Closing source " + this.queueId + " because an error occurred: " + reason,
cause);
}
this.sourceRunning = false;
@@ -491,7 +485,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
.awaitTerminated(sleepForRetries * maxRetriesMultiplier, TimeUnit.MILLISECONDS);
} catch (TimeoutException te) {
LOG.warn("Got exception while waiting for endpoint to shutdown for replication source :"
- + this.peerClusterZnode,
+ + this.queueId,
te);
}
}
@@ -499,8 +493,8 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
}
@Override
- public String getPeerClusterZnode() {
- return this.peerClusterZnode;
+ public String getQueueId() {
+ return this.queueId;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
index 865a202..93e8331 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
@@ -32,8 +32,8 @@ public class ReplicationSourceFactory {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationSourceFactory.class);
- static ReplicationSourceInterface create(Configuration conf, String peerId) {
- ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(peerId);
+ static ReplicationSourceInterface create(Configuration conf, String queueId) {
+ ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);
boolean isQueueRecovered = replicationQueueInfo.isQueueRecovered();
ReplicationSourceInterface src;
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
index 4f10c73..d7cf9a3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
@@ -51,7 +51,7 @@ public interface ReplicationSourceInterface {
*/
void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
- String peerClusterZnode, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
+ String queueId, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
MetricsSource metrics) throws IOException;
/**
@@ -96,11 +96,11 @@ public interface ReplicationSourceInterface {
Path getCurrentPath();
/**
- * Get the id that the source is replicating to
+ * Get the queue id that the source is replicating to
*
- * @return peer cluster id
+ * @return queue id
*/
- String getPeerClusterZnode();
+ String getQueueId();
/**
* Get the id that the source is replicating to.
[45/50] [abbrv] hbase git commit: HBASE-19661 Replace
ReplicationStateZKBase with ZKReplicationStorageBase
Posted by zh...@apache.org.
HBASE-19661 Replace ReplicationStateZKBase with ZKReplicationStorageBase
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/424fa6cf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/424fa6cf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/424fa6cf
Branch: refs/heads/HBASE-19397-branch-2
Commit: 424fa6cf29c9df9b80d2c5e1a5dd3ac197d6240f
Parents: ba414a7
Author: huzheng <op...@gmail.com>
Authored: Fri Dec 29 15:55:28 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../hbase/replication/ReplicationFactory.java | 5 +-
.../replication/ReplicationStateZKBase.java | 153 -------------------
.../replication/ReplicationTrackerZKImpl.java | 21 +--
.../replication/ZKReplicationPeerStorage.java | 24 ++-
.../replication/ZKReplicationStorageBase.java | 13 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 4 +-
.../master/ReplicationPeerConfigUpgrader.java | 128 ++++++++--------
.../regionserver/DumpReplicationQueues.java | 18 +--
.../replication/regionserver/Replication.java | 3 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 3 +-
.../TestReplicationTrackerZKImpl.java | 3 +-
.../replication/master/TestTableCFsUpdater.java | 41 ++---
.../TestReplicationSourceManager.java | 6 +-
13 files changed, 136 insertions(+), 286 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 6c66aff..2a970ba 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -33,9 +33,8 @@ public class ReplicationFactory {
return new ReplicationPeers(zk, conf);
}
- public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper,
- final ReplicationPeers replicationPeers, Configuration conf, Abortable abortable,
+ public static ReplicationTracker getReplicationTracker(ZKWatcher zookeeper, Abortable abortable,
Stoppable stopper) {
- return new ReplicationTrackerZKImpl(zookeeper, replicationPeers, conf, abortable, stopper);
+ return new ReplicationTrackerZKImpl(zookeeper, abortable, stopper);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
deleted file mode 100644
index f49537c..0000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hbase.thirdparty.com.google.protobuf.CodedOutputStream;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * This is a base class for maintaining replication state in zookeeper.
- */
-@InterfaceAudience.Private
-public abstract class ReplicationStateZKBase {
-
- /**
- * The name of the znode that contains the replication status of a remote slave (i.e. peer)
- * cluster.
- */
- protected final String peerStateNodeName;
- /** The name of the base znode that contains all replication state. */
- protected final String replicationZNode;
- /** The name of the znode that contains a list of all remote slave (i.e. peer) clusters. */
- protected final String peersZNode;
- /** The name of the znode that contains all replication queues */
- protected final String queuesZNode;
- /** The name of the znode that contains queues of hfile references to be replicated */
- protected final String hfileRefsZNode;
- /** The cluster key of the local cluster */
- protected final String ourClusterKey;
- /** The name of the znode that contains tableCFs */
- protected final String tableCFsNodeName;
-
- protected final ZKWatcher zookeeper;
- protected final Configuration conf;
- protected final Abortable abortable;
-
- public static final byte[] ENABLED_ZNODE_BYTES =
- toByteArray(ReplicationProtos.ReplicationState.State.ENABLED);
- public static final byte[] DISABLED_ZNODE_BYTES =
- toByteArray(ReplicationProtos.ReplicationState.State.DISABLED);
- public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY =
- "zookeeper.znode.replication.hfile.refs";
- public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT = "hfile-refs";
-
- public ReplicationStateZKBase(ZKWatcher zookeeper, Configuration conf,
- Abortable abortable) {
- this.zookeeper = zookeeper;
- this.conf = conf;
- this.abortable = abortable;
-
- String replicationZNodeName = conf.get("zookeeper.znode.replication", "replication");
- String peersZNodeName = conf.get("zookeeper.znode.replication.peers", "peers");
- String queuesZNodeName = conf.get("zookeeper.znode.replication.rs", "rs");
- String hfileRefsZNodeName = conf.get(ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
- ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
- this.peerStateNodeName = conf.get("zookeeper.znode.replication.peers.state", "peer-state");
- this.tableCFsNodeName = conf.get("zookeeper.znode.replication.peers.tableCFs", "tableCFs");
- this.ourClusterKey = ZKConfig.getZooKeeperClusterKey(this.conf);
- this.replicationZNode = ZNodePaths.joinZNode(this.zookeeper.znodePaths.baseZNode,
- replicationZNodeName);
- this.peersZNode = ZNodePaths.joinZNode(replicationZNode, peersZNodeName);
- this.queuesZNode = ZNodePaths.joinZNode(replicationZNode, queuesZNodeName);
- this.hfileRefsZNode = ZNodePaths.joinZNode(replicationZNode, hfileRefsZNodeName);
- }
-
- public List<String> getListOfReplicators() {
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.queuesZNode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of replicators", e);
- }
- return result;
- }
-
- /**
- * @param state
- * @return Serialized protobuf of <code>state</code> with pb magic prefix prepended suitable for
- * use as content of a peer-state znode under a peer cluster id as in
- * /hbase/replication/peers/PEER_ID/peer-state.
- */
- protected static byte[] toByteArray(final ReplicationProtos.ReplicationState.State state) {
- ReplicationProtos.ReplicationState msg =
- ReplicationProtos.ReplicationState.newBuilder().setState(state).build();
- // There is no toByteArray on this pb Message?
- // 32 bytes is default which seems fair enough here.
- try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
- CodedOutputStream cos = CodedOutputStream.newInstance(baos, 16);
- msg.writeTo(cos);
- cos.flush();
- baos.flush();
- return ProtobufUtil.prependPBMagic(baos.toByteArray());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- protected boolean peerExists(String id) throws KeeperException {
- return ZKUtil.checkExists(this.zookeeper, ZNodePaths.joinZNode(this.peersZNode, id)) >= 0;
- }
-
- /**
- * Determine if a ZK path points to a peer node.
- * @param path path to be checked
- * @return true if the path points to a peer node, otherwise false
- */
- protected boolean isPeerPath(String path) {
- return path.split("/").length == peersZNode.split("/").length + 1;
- }
-
- @VisibleForTesting
- protected String getTableCFsNode(String id) {
- return ZNodePaths.joinZNode(this.peersZNode, ZNodePaths.joinZNode(id, this.tableCFsNodeName));
- }
-
- @VisibleForTesting
- protected String getPeerStateNode(String id) {
- return ZNodePaths.joinZNode(this.peersZNode, ZNodePaths.joinZNode(id, this.peerStateNodeName));
- }
- @VisibleForTesting
- protected String getPeerNode(String id) {
- return ZNodePaths.joinZNode(this.peersZNode, id);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
index 5659e4b..16a1668 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
@@ -20,14 +20,12 @@ package org.apache.hadoop.hbase.replication;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
-
-import org.apache.hadoop.hbase.zookeeper.ZKListener;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.zookeeper.ZKListener;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,9 +36,14 @@ import org.slf4j.LoggerFactory;
* interface.
*/
@InterfaceAudience.Private
-public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements ReplicationTracker {
+public class ReplicationTrackerZKImpl implements ReplicationTracker {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationTrackerZKImpl.class);
+
+ // Zookeeper
+ private final ZKWatcher zookeeper;
+ // Server to abort.
+ private final Abortable abortable;
// All about stopping
private final Stoppable stopper;
// listeners to be notified
@@ -48,9 +51,9 @@ public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements
// List of all the other region servers in this cluster
private final ArrayList<String> otherRegionServers = new ArrayList<>();
- public ReplicationTrackerZKImpl(ZKWatcher zookeeper, final ReplicationPeers replicationPeers,
- Configuration conf, Abortable abortable, Stoppable stopper) {
- super(zookeeper, conf, abortable);
+ public ReplicationTrackerZKImpl(ZKWatcher zookeeper, Abortable abortable, Stoppable stopper) {
+ this.zookeeper = zookeeper;
+ this.abortable = abortable;
this.stopper = stopper;
this.zookeeper.registerListener(new OtherRegionServerWatcher(this.zookeeper));
// watch the changes
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
index 42d4b3f..a53500a 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
@@ -36,7 +37,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
* ZK based replication peer storage.
*/
@InterfaceAudience.Private
-class ZKReplicationPeerStorage extends ZKReplicationStorageBase implements ReplicationPeerStorage {
+public class ZKReplicationPeerStorage extends ZKReplicationStorageBase
+ implements ReplicationPeerStorage {
+
+ public static final String PEERS_ZNODE = "zookeeper.znode.replication.peers";
+ public static final String PEERS_ZNODE_DEFAULT = "peers";
+
+ public static final String PEERS_STATE_ZNODE = "zookeeper.znode.replication.peers.state";
+ public static final String PEERS_STATE_ZNODE_DEFAULT = "peer-state";
public static final byte[] ENABLED_ZNODE_BYTES =
toByteArray(ReplicationProtos.ReplicationState.State.ENABLED);
@@ -56,16 +64,18 @@ class ZKReplicationPeerStorage extends ZKReplicationStorageBase implements Repli
public ZKReplicationPeerStorage(ZKWatcher zookeeper, Configuration conf) {
super(zookeeper, conf);
- this.peerStateNodeName = conf.get("zookeeper.znode.replication.peers.state", "peer-state");
- String peersZNodeName = conf.get("zookeeper.znode.replication.peers", "peers");
+ this.peerStateNodeName = conf.get(PEERS_STATE_ZNODE, PEERS_STATE_ZNODE_DEFAULT);
+ String peersZNodeName = conf.get(PEERS_ZNODE, PEERS_ZNODE_DEFAULT);
this.peersZNode = ZNodePaths.joinZNode(replicationZNode, peersZNodeName);
}
- private String getPeerStateNode(String peerId) {
+ @VisibleForTesting
+ public String getPeerStateNode(String peerId) {
return ZNodePaths.joinZNode(getPeerNode(peerId), peerStateNodeName);
}
- private String getPeerNode(String peerId) {
+ @VisibleForTesting
+ public String getPeerNode(String peerId) {
return ZNodePaths.joinZNode(peersZNode, peerId);
}
@@ -82,8 +92,8 @@ class ZKReplicationPeerStorage extends ZKReplicationStorageBase implements Repli
enabled ? ENABLED_ZNODE_BYTES : DISABLED_ZNODE_BYTES)),
false);
} catch (KeeperException e) {
- throw new ReplicationException("Could not add peer with id=" + peerId + ", peerConfif=>" +
- peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"), e);
+ throw new ReplicationException("Could not add peer with id=" + peerId + ", peerConfif=>"
+ + peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"), e);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
index 2321e4f..7190aeb 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationStorageBase.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.replication;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
@@ -34,7 +35,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
* zookeeper.
*/
@InterfaceAudience.Private
-class ZKReplicationStorageBase {
+public class ZKReplicationStorageBase {
+
+ public static final String REPLICATION_ZNODE = "zookeeper.znode.replication";
+ public static final String REPLICATION_ZNODE_DEFAULT = "replication";
/** The name of the base znode that contains all replication state. */
protected final String replicationZNode;
@@ -45,10 +49,9 @@ class ZKReplicationStorageBase {
protected ZKReplicationStorageBase(ZKWatcher zookeeper, Configuration conf) {
this.zookeeper = zookeeper;
this.conf = conf;
- String replicationZNodeName = conf.get("zookeeper.znode.replication", "replication");
- this.replicationZNode =
- ZNodePaths.joinZNode(this.zookeeper.znodePaths.baseZNode, replicationZNodeName);
+ this.replicationZNode = ZNodePaths.joinZNode(this.zookeeper.znodePaths.baseZNode,
+ conf.get(REPLICATION_ZNODE, REPLICATION_ZNODE_DEFAULT));
}
/**
@@ -58,7 +61,7 @@ class ZKReplicationStorageBase {
*/
protected static byte[] toByteArray(final ReplicationProtos.ReplicationState.State state) {
ReplicationProtos.ReplicationState msg =
- ReplicationProtos.ReplicationState.newBuilder().setState(state).build();
+ ReplicationProtos.ReplicationState.newBuilder().setState(state).build();
// There is no toByteArray on this pb Message?
// 32 bytes is default which seems fair enough here.
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index bfbc6a2..a7ef275 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -829,8 +829,8 @@ public class HMaster extends HRegionServer implements MasterServices {
// This is for backwards compatibility
// See HBASE-11393
status.setStatus("Update TableCFs node in ZNode");
- ReplicationPeerConfigUpgrader tableCFsUpdater = new ReplicationPeerConfigUpgrader(zooKeeper,
- conf, this.clusterConnection);
+ ReplicationPeerConfigUpgrader tableCFsUpdater =
+ new ReplicationPeerConfigUpgrader(zooKeeper, conf);
tableCFsUpdater.copyTableCFs();
// Add the Observer to delete space quotas on table deletion before starting all CPs by
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java
index ea5509f..b6e8862 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationPeerConfigUpgrader.java
@@ -18,96 +18,107 @@
*/
package org.apache.hadoop.hbase.replication.master;
+import static org.apache.hadoop.hbase.replication.ZKReplicationPeerStorage.PEERS_ZNODE;
+import static org.apache.hadoop.hbase.replication.ZKReplicationPeerStorage.PEERS_ZNODE_DEFAULT;
+import static org.apache.hadoop.hbase.replication.ZKReplicationStorageBase.REPLICATION_ZNODE;
+import static org.apache.hadoop.hbase.replication.ZKReplicationStorageBase.REPLICATION_ZNODE_DEFAULT;
+
import java.io.IOException;
-import java.util.List;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
+import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
- * This class is used to upgrade TableCFs from HBase 1.0, 1.1, 1.2, 1.3 to HBase 1.4 or 2.x.
- * It will be removed in HBase 3.x. See HBASE-11393
+ * This class is used to upgrade TableCFs from HBase 1.0, 1.1, 1.2, 1.3 to HBase 1.4 or 2.x. It will
+ * be removed in HBase 3.x. See HBASE-11393
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class ReplicationPeerConfigUpgrader extends ReplicationStateZKBase {
+public class ReplicationPeerConfigUpgrader{
+
+ private static final String TABLE_CFS_ZNODE = "zookeeper.znode.replication.peers.tableCFs";
+ private static final String TABLE_CFS_ZNODE_DEFAULT = "tableCFs";
private static final Logger LOG = LoggerFactory.getLogger(ReplicationPeerConfigUpgrader.class);
+ private final Configuration conf;
+ private final ZKWatcher zookeeper;
+ private final ReplicationPeerStorage peerStorage;
- public ReplicationPeerConfigUpgrader(ZKWatcher zookeeper,
- Configuration conf, Abortable abortable) {
- super(zookeeper, conf, abortable);
+ public ReplicationPeerConfigUpgrader(ZKWatcher zookeeper, Configuration conf) {
+ this.zookeeper = zookeeper;
+ this.conf = conf;
+ this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zookeeper, conf);
}
public void upgrade() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
Admin admin = conn.getAdmin();
- admin.listReplicationPeers().forEach(
- (peerDesc) -> {
- String peerId = peerDesc.getPeerId();
- ReplicationPeerConfig peerConfig = peerDesc.getPeerConfig();
- if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty())
- || (peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) {
- peerConfig.setReplicateAllUserTables(false);
- try {
- admin.updateReplicationPeerConfig(peerId, peerConfig);
- } catch (Exception e) {
- LOG.error("Failed to upgrade replication peer config for peerId=" + peerId, e);
- }
+ admin.listReplicationPeers().forEach((peerDesc) -> {
+ String peerId = peerDesc.getPeerId();
+ ReplicationPeerConfig peerConfig = peerDesc.getPeerConfig();
+ if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty())
+ || (peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) {
+ peerConfig.setReplicateAllUserTables(false);
+ try {
+ admin.updateReplicationPeerConfig(peerId, peerConfig);
+ } catch (Exception e) {
+ LOG.error("Failed to upgrade replication peer config for peerId=" + peerId, e);
}
- });
+ }
+ });
}
}
- public void copyTableCFs() {
- List<String> znodes = null;
- try {
- znodes = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
- } catch (KeeperException e) {
- LOG.error("Failed to get peers znode", e);
- }
- if (znodes != null) {
- for (String peerId : znodes) {
- if (!copyTableCFs(peerId)) {
- LOG.error("upgrade tableCFs failed for peerId=" + peerId);
- }
+ public void copyTableCFs() throws ReplicationException {
+ for (String peerId : peerStorage.listPeerIds()) {
+ if (!copyTableCFs(peerId)) {
+ LOG.error("upgrade tableCFs failed for peerId=" + peerId);
}
}
}
- public boolean copyTableCFs(String peerId) {
+ @VisibleForTesting
+ protected String getTableCFsNode(String peerId) {
+ String replicationZNode = ZNodePaths.joinZNode(zookeeper.znodePaths.baseZNode,
+ conf.get(REPLICATION_ZNODE, REPLICATION_ZNODE_DEFAULT));
+ String peersZNode =
+ ZNodePaths.joinZNode(replicationZNode, conf.get(PEERS_ZNODE, PEERS_ZNODE_DEFAULT));
+ return ZNodePaths.joinZNode(peersZNode,
+ ZNodePaths.joinZNode(peerId, conf.get(TABLE_CFS_ZNODE, TABLE_CFS_ZNODE_DEFAULT)));
+ }
+
+ public boolean copyTableCFs(String peerId) throws ReplicationException {
String tableCFsNode = getTableCFsNode(peerId);
try {
if (ZKUtil.checkExists(zookeeper, tableCFsNode) != -1) {
- String peerNode = getPeerNode(peerId);
- ReplicationPeerConfig rpc = getReplicationPeerConig(peerNode);
+ ReplicationPeerConfig rpc = peerStorage.getPeerConfig(peerId);
// We only need to copy data from tableCFs node to rpc Node the first time hmaster start.
if (rpc.getTableCFsMap() == null || rpc.getTableCFsMap().isEmpty()) {
// we copy TableCFs node into PeerNode
LOG.info("copy tableCFs into peerNode:" + peerId);
ReplicationProtos.TableCF[] tableCFs =
- ReplicationPeerConfigUtil.parseTableCFs(
- ZKUtil.getData(this.zookeeper, tableCFsNode));
+ ReplicationPeerConfigUtil.parseTableCFs(ZKUtil.getData(this.zookeeper, tableCFsNode));
if (tableCFs != null && tableCFs.length > 0) {
rpc.setTableCFsMap(ReplicationPeerConfigUtil.convert2Map(tableCFs));
- ZKUtil.setData(this.zookeeper, peerNode,
- ReplicationPeerConfigUtil.toByteArray(rpc));
+ peerStorage.updatePeerConfig(peerId, rpc);
}
} else {
LOG.info("No tableCFs in peerNode:" + peerId);
@@ -126,23 +137,6 @@ public class ReplicationPeerConfigUpgrader extends ReplicationStateZKBase {
return true;
}
- private ReplicationPeerConfig getReplicationPeerConig(String peerNode)
- throws KeeperException, InterruptedException {
- byte[] data = null;
- data = ZKUtil.getData(this.zookeeper, peerNode);
- if (data == null) {
- LOG.error("Could not get configuration for " +
- "peer because it doesn't exist. peer=" + peerNode);
- return null;
- }
- try {
- return ReplicationPeerConfigUtil.parsePeerFrom(data);
- } catch (DeserializationException e) {
- LOG.warn("Failed to parse cluster key from peer=" + peerNode);
- return null;
- }
- }
-
private static void printUsageAndExit() {
System.err.printf(
"Usage: hbase org.apache.hadoop.hbase.replication.master.ReplicationPeerConfigUpgrader"
@@ -163,19 +157,17 @@ public class ReplicationPeerConfigUpgrader extends ReplicationStateZKBase {
printUsageAndExit();
} else if (args[0].equals("copyTableCFs")) {
Configuration conf = HBaseConfiguration.create();
- ZKWatcher zkw = new ZKWatcher(conf, "ReplicationPeerConfigUpgrader", null);
- try {
- ReplicationPeerConfigUpgrader tableCFsUpdater = new ReplicationPeerConfigUpgrader(zkw,
- conf, null);
+ try (ZKWatcher zkw = new ZKWatcher(conf, "ReplicationPeerConfigUpgrader", null)) {
+ ReplicationPeerConfigUpgrader tableCFsUpdater =
+ new ReplicationPeerConfigUpgrader(zkw, conf);
tableCFsUpdater.copyTableCFs();
- } finally {
- zkw.close();
}
} else if (args[0].equals("upgrade")) {
Configuration conf = HBaseConfiguration.create();
- ZKWatcher zkw = new ZKWatcher(conf, "ReplicationPeerConfigUpgrader", null);
- ReplicationPeerConfigUpgrader upgrader = new ReplicationPeerConfigUpgrader(zkw, conf, null);
- upgrader.upgrade();
+ try (ZKWatcher zkw = new ZKWatcher(conf, "ReplicationPeerConfigUpgrader", null)) {
+ ReplicationPeerConfigUpgrader upgrader = new ReplicationPeerConfigUpgrader(zkw, conf);
+ upgrader.upgrade();
+ }
} else {
printUsageAndExit();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 27bda2d..22e8628 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.procedure2.util.StringUtils;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
@@ -237,7 +236,7 @@ public class DumpReplicationQueues extends Configured implements Tool {
LOG.info("Found [--distributed], will poll each RegionServer.");
Set<String> peerIds = peers.stream().map((peer) -> peer.getPeerId())
.collect(Collectors.toSet());
- System.out.println(dumpQueues(connection, zkw, peerIds, opts.isHdfs()));
+ System.out.println(dumpQueues(zkw, peerIds, opts.isHdfs()));
System.out.println(dumpReplicationSummary());
} else {
// use ZK instead
@@ -301,18 +300,15 @@ public class DumpReplicationQueues extends Configured implements Tool {
return sb.toString();
}
- public String dumpQueues(ClusterConnection connection, ZKWatcher zkw, Set<String> peerIds,
+ public String dumpQueues(ZKWatcher zkw, Set<String> peerIds,
boolean hdfs) throws Exception {
ReplicationQueueStorage queueStorage;
- ReplicationPeers replicationPeers;
ReplicationTracker replicationTracker;
StringBuilder sb = new StringBuilder();
queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
- replicationPeers =
- ReplicationFactory.getReplicationPeers(zkw, getConf());
- replicationTracker = ReplicationFactory.getReplicationTracker(zkw, replicationPeers, getConf(),
- new WarnOnlyAbortable(), new WarnOnlyStoppable());
+ replicationTracker = ReplicationFactory.getReplicationTracker(zkw, new WarnOnlyAbortable(),
+ new WarnOnlyStoppable());
Set<String> liveRegionServers = new HashSet<>(replicationTracker.getListOfRegionServers());
// Loops each peer on each RS and dumps the queues
@@ -330,11 +326,9 @@ public class DumpReplicationQueues extends Configured implements Tool {
List<String> wals = queueStorage.getWALsInQueue(regionserver, queueId);
if (!peerIds.contains(queueInfo.getPeerId())) {
deletedQueues.add(regionserver + "/" + queueId);
- sb.append(
- formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs));
+ sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs));
} else {
- sb.append(
- formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, false, hdfs));
+ sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, false, hdfs));
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index a8991a0..ce056a1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -133,8 +133,7 @@ public class Replication implements
ReplicationFactory.getReplicationPeers(server.getZooKeeper(), this.conf);
this.replicationPeers.init();
this.replicationTracker =
- ReplicationFactory.getReplicationTracker(server.getZooKeeper(), this.replicationPeers,
- this.conf, this.server, this.server);
+ ReplicationFactory.getReplicationTracker(server.getZooKeeper(), this.server, this.server);
} catch (Exception e) {
throw new IOException("Failed replication handler create", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 4c0630d..f6b6501 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -330,8 +330,7 @@ public class HBaseFsck extends Configured implements Closeable {
* @throws MasterNotRunningException if the master is not running
* @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
*/
- public HBaseFsck(Configuration conf) throws MasterNotRunningException,
- ZooKeeperConnectionException, IOException, ClassNotFoundException {
+ public HBaseFsck(Configuration conf) throws IOException, ClassNotFoundException {
this(conf, createThreadPool(conf));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
index fdfa6b7..757d9a9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
@@ -90,7 +90,8 @@ public class TestReplicationTrackerZKImpl {
ZKClusterId.setClusterId(zkw, new ClusterId());
rp = ReplicationFactory.getReplicationPeers(zkw, conf);
rp.init();
- rt = ReplicationFactory.getReplicationTracker(zkw, rp, conf, zkw, new DummyServer(fakeRs1));
+ rt = ReplicationFactory.getReplicationTracker(zkw, new DummyServer(fakeRs1),
+ new DummyServer(fakeRs1));
} catch (Exception e) {
fail("Exception during test setup: " + e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestTableCFsUpdater.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestTableCFsUpdater.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestTableCFsUpdater.java
index 2993043..19acc75 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestTableCFsUpdater.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestTableCFsUpdater.java
@@ -25,14 +25,13 @@ import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ZKReplicationPeerStorage;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -57,12 +56,19 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
private static ZKWatcher zkw = null;
private static Abortable abortable = null;
+ private static ZKStorageUtil zkStorageUtil = null;
+
+ private static class ZKStorageUtil extends ZKReplicationPeerStorage {
+ public ZKStorageUtil(ZKWatcher zookeeper, Configuration conf) {
+ super(zookeeper, conf);
+ }
+ }
@Rule
public TestName name = new TestName();
public TestTableCFsUpdater() {
- super(zkw, TEST_UTIL.getConfiguration(), abortable);
+ super(zkw, TEST_UTIL.getConfiguration());
}
@BeforeClass
@@ -81,6 +87,7 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
}
};
zkw = new ZKWatcher(conf, "TableCFs", abortable, true);
+ zkStorageUtil = new ZKStorageUtil(zkw, conf);
}
@AfterClass
@@ -89,8 +96,7 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
}
@Test
- public void testUpgrade() throws KeeperException, InterruptedException,
- DeserializationException {
+ public void testUpgrade() throws Exception {
String peerId = "1";
final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
@@ -98,13 +104,13 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
ReplicationPeerConfig rpc = new ReplicationPeerConfig();
rpc.setClusterKey(zkw.getQuorum());
- String peerNode = getPeerNode(peerId);
+ String peerNode = zkStorageUtil.getPeerNode(peerId);
ZKUtil.createWithParents(zkw, peerNode, ReplicationPeerConfigUtil.toByteArray(rpc));
String tableCFs = tableName1 + ":cf1,cf2;" + tableName2 + ":cf3;" + tableName3;
String tableCFsNode = getTableCFsNode(peerId);
LOG.info("create tableCFs :" + tableCFsNode + " for peerId=" + peerId);
- ZKUtil.createWithParents(zkw, tableCFsNode , Bytes.toBytes(tableCFs));
+ ZKUtil.createWithParents(zkw, tableCFsNode, Bytes.toBytes(tableCFs));
ReplicationPeerConfig actualRpc =
ReplicationPeerConfigUtil.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
@@ -117,13 +123,13 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
peerId = "2";
rpc = new ReplicationPeerConfig();
rpc.setClusterKey(zkw.getQuorum());
- peerNode = getPeerNode(peerId);
+ peerNode = zkStorageUtil.getPeerNode(peerId);
ZKUtil.createWithParents(zkw, peerNode, ReplicationPeerConfigUtil.toByteArray(rpc));
tableCFs = tableName1 + ":cf1,cf3;" + tableName2 + ":cf2";
tableCFsNode = getTableCFsNode(peerId);
LOG.info("create tableCFs :" + tableCFsNode + " for peerId=" + peerId);
- ZKUtil.createWithParents(zkw, tableCFsNode , Bytes.toBytes(tableCFs));
+ ZKUtil.createWithParents(zkw, tableCFsNode, Bytes.toBytes(tableCFs));
actualRpc = ReplicationPeerConfigUtil.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
actualTableCfs = Bytes.toString(ZKUtil.getData(zkw, tableCFsNode));
@@ -135,13 +141,13 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
peerId = "3";
rpc = new ReplicationPeerConfig();
rpc.setClusterKey(zkw.getQuorum());
- peerNode = getPeerNode(peerId);
+ peerNode = zkStorageUtil.getPeerNode(peerId);
ZKUtil.createWithParents(zkw, peerNode, ReplicationPeerConfigUtil.toByteArray(rpc));
tableCFs = "";
tableCFsNode = getTableCFsNode(peerId);
LOG.info("create tableCFs :" + tableCFsNode + " for peerId=" + peerId);
- ZKUtil.createWithParents(zkw, tableCFsNode , Bytes.toBytes(tableCFs));
+ ZKUtil.createWithParents(zkw, tableCFsNode, Bytes.toBytes(tableCFs));
actualRpc = ReplicationPeerConfigUtil.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
actualTableCfs = Bytes.toString(ZKUtil.getData(zkw, tableCFsNode));
@@ -153,7 +159,7 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
peerId = "4";
rpc = new ReplicationPeerConfig();
rpc.setClusterKey(zkw.getQuorum());
- peerNode = getPeerNode(peerId);
+ peerNode = zkStorageUtil.getPeerNode(peerId);
ZKUtil.createWithParents(zkw, peerNode, ReplicationPeerConfigUtil.toByteArray(rpc));
tableCFsNode = getTableCFsNode(peerId);
@@ -167,7 +173,7 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
copyTableCFs();
peerId = "1";
- peerNode = getPeerNode(peerId);
+ peerNode = zkStorageUtil.getPeerNode(peerId);
actualRpc = ReplicationPeerConfigUtil.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
Map<TableName, List<String>> tableNameListMap = actualRpc.getTableCFsMap();
@@ -182,9 +188,8 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
assertEquals("cf3", tableNameListMap.get(tableName2).get(0));
assertNull(tableNameListMap.get(tableName3));
-
peerId = "2";
- peerNode = getPeerNode(peerId);
+ peerNode = zkStorageUtil.getPeerNode(peerId);
actualRpc = ReplicationPeerConfigUtil.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
tableNameListMap = actualRpc.getTableCFsMap();
@@ -198,19 +203,17 @@ public class TestTableCFsUpdater extends ReplicationPeerConfigUpgrader {
assertEquals("cf2", tableNameListMap.get(tableName2).get(0));
peerId = "3";
- peerNode = getPeerNode(peerId);
+ peerNode = zkStorageUtil.getPeerNode(peerId);
actualRpc = ReplicationPeerConfigUtil.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
tableNameListMap = actualRpc.getTableCFsMap();
assertNull(tableNameListMap);
peerId = "4";
- peerNode = getPeerNode(peerId);
+ peerNode = zkStorageUtil.getPeerNode(peerId);
actualRpc = ReplicationPeerConfigUtil.parsePeerFrom(ZKUtil.getData(zkw, peerNode));
assertEquals(rpc.getClusterKey(), actualRpc.getClusterKey());
tableNameListMap = actualRpc.getTableCFsMap();
assertNull(tableNameListMap);
}
-
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/424fa6cf/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index f4d3901..1001aa5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -68,8 +68,8 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
-import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.ZKReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.NodeFailoverWorker;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
@@ -163,9 +163,9 @@ public abstract class TestReplicationSourceManager {
+ conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":/1"));
ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/peer-state");
ZKUtil.setData(zkw, "/hbase/replication/peers/1/peer-state",
- ReplicationStateZKBase.ENABLED_ZNODE_BYTES);
+ ZKReplicationPeerStorage.ENABLED_ZNODE_BYTES);
ZKUtil.createWithParents(zkw, "/hbase/replication/state");
- ZKUtil.setData(zkw, "/hbase/replication/state", ReplicationStateZKBase.ENABLED_ZNODE_BYTES);
+ ZKUtil.setData(zkw, "/hbase/replication/state", ZKReplicationPeerStorage.ENABLED_ZNODE_BYTES);
ZKClusterId.setClusterId(zkw, new ClusterId());
FSUtils.setRootDir(utility.getConfiguration(), utility.getDataTestDir());
[09/50] [abbrv] hbase git commit: HBASE-19811 Fix findbugs and
error-prone warnings in hbase-server (branch-2)
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java
index dcccfd1..91f7971 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.apache.hadoop.hbase.util.Threads;
-import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -204,14 +203,14 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
rs -> {
ServerName serverName = rs.getServerName();
try {
- Assert.assertEquals(admin.getRegions(serverName).get().size(), rs
+ assertEquals(admin.getRegions(serverName).get().size(), rs
.getRegions().size());
} catch (Exception e) {
fail("admin.getOnlineRegions() method throws a exception: " + e.getMessage());
}
regionServerCount.incrementAndGet();
});
- Assert.assertEquals(regionServerCount.get(), 2);
+ assertEquals(2, regionServerCount.get());
}
@Test
@@ -229,7 +228,7 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
ASYNC_CONN.getTable(tableName)
.put(new Put(hri.getStartKey()).addColumn(FAMILY, FAMILY_0, Bytes.toBytes("value-1")))
.join();
- Assert.assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreSize() > 0);
+ assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreSize() > 0);
// flush region and wait flush operation finished.
LOG.info("flushing region: " + Bytes.toStringBinary(hri.getRegionName()));
admin.flushRegion(hri.getRegionName()).get();
@@ -239,20 +238,20 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
Threads.sleep(50);
}
// check the memstore.
- Assert.assertEquals(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreSize(), 0);
+ assertEquals(0, regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreSize());
// write another put into the specific region
ASYNC_CONN.getTable(tableName)
.put(new Put(hri.getStartKey()).addColumn(FAMILY, FAMILY_0, Bytes.toBytes("value-2")))
.join();
- Assert.assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreSize() > 0);
+ assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreSize() > 0);
admin.flush(tableName).get();
Threads.sleepWithoutInterrupt(500);
while (regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreSize() > 0) {
Threads.sleep(50);
}
// check the memstore.
- Assert.assertEquals(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreSize(), 0);
+ assertEquals(0, regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreSize());
}
@Test
@@ -421,7 +420,7 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
LOG.error(e.toString(), e);
}
}
- assertEquals(count, 2);
+ assertEquals(2, count);
}
private void waitUntilMobCompactionFinished(TableName tableName)
@@ -471,23 +470,23 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
.map(rsThread -> rsThread.getRegionServer()).collect(Collectors.toList());
List<Region> regions = new ArrayList<>();
rsList.forEach(rs -> regions.addAll(rs.getRegions(tableName)));
- Assert.assertEquals(regions.size(), 1);
+ assertEquals(1, regions.size());
int countBefore = countStoreFilesInFamilies(regions, families);
- Assert.assertTrue(countBefore > 0);
+ assertTrue(countBefore > 0);
// Minor compaction for all region servers.
for (HRegionServer rs : rsList)
admin.compactRegionServer(rs.getServerName()).get();
Thread.sleep(5000);
int countAfterMinorCompaction = countStoreFilesInFamilies(regions, families);
- Assert.assertTrue(countAfterMinorCompaction < countBefore);
+ assertTrue(countAfterMinorCompaction < countBefore);
// Major compaction for all region servers.
for (HRegionServer rs : rsList)
admin.majorCompactRegionServer(rs.getServerName()).get();
Thread.sleep(5000);
int countAfterMajorCompaction = countStoreFilesInFamilies(regions, families);
- Assert.assertEquals(countAfterMajorCompaction, 3);
+ assertEquals(3, countAfterMajorCompaction);
}
@Test
@@ -512,7 +511,7 @@ public class TestAsyncRegionAdminApi extends TestAsyncAdminBase {
.getHBaseCluster()
.getLiveRegionServerThreads()
.forEach(rsThread -> regions.addAll(rsThread.getRegionServer().getRegions(tableName)));
- Assert.assertEquals(regions.size(), 1);
+ assertEquals(1, regions.size());
int countBefore = countStoreFilesInFamilies(regions, families);
int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApiWithClusters.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApiWithClusters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApiWithClusters.java
index 9ceb172..efea20e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApiWithClusters.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApiWithClusters.java
@@ -81,6 +81,7 @@ public class TestAsyncReplicationAdminApiWithClusters extends TestAsyncAdminBase
ASYNC_CONN.getAdmin().addReplicationPeer(ID_SECOND, rpc).join();
}
+ @Override
@After
public void tearDown() throws Exception {
Pattern pattern = Pattern.compile(tableName.getNameAsString() + ".*");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSnapshotAdminApi.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSnapshotAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSnapshotAdminApi.java
index 5014e96..7501192 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSnapshotAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSnapshotAdminApi.java
@@ -18,22 +18,25 @@
package org.apache.hadoop.hbase.client;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Pattern;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.regex.Pattern;
-
@RunWith(Parameterized.class)
@Category({ LargeTests.class, ClientTests.class })
public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
@@ -65,19 +68,19 @@ public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
admin.snapshot(snapshotName2, tableName).get();
List<SnapshotDescription> snapshots = syncAdmin.listSnapshots();
Collections.sort(snapshots, (snap1, snap2) -> {
- Assert.assertNotNull(snap1);
- Assert.assertNotNull(snap1.getName());
- Assert.assertNotNull(snap2);
- Assert.assertNotNull(snap2.getName());
+ assertNotNull(snap1);
+ assertNotNull(snap1.getName());
+ assertNotNull(snap2);
+ assertNotNull(snap2.getName());
return snap1.getName().compareTo(snap2.getName());
});
- Assert.assertEquals(snapshotName1, snapshots.get(0).getName());
- Assert.assertEquals(tableName, snapshots.get(0).getTableName());
- Assert.assertEquals(SnapshotType.FLUSH, snapshots.get(0).getType());
- Assert.assertEquals(snapshotName2, snapshots.get(1).getName());
- Assert.assertEquals(tableName, snapshots.get(1).getTableName());
- Assert.assertEquals(SnapshotType.FLUSH, snapshots.get(1).getType());
+ assertEquals(snapshotName1, snapshots.get(0).getName());
+ assertEquals(tableName, snapshots.get(0).getTableName());
+ assertEquals(SnapshotType.FLUSH, snapshots.get(0).getType());
+ assertEquals(snapshotName2, snapshots.get(1).getName());
+ assertEquals(tableName, snapshots.get(1).getTableName());
+ assertEquals(SnapshotType.FLUSH, snapshots.get(1).getType());
}
@Test
@@ -93,10 +96,10 @@ public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
admin.snapshot(snapshotName1, tableName).get();
List<SnapshotDescription> snapshots = syncAdmin.listSnapshots();
- Assert.assertEquals(snapshots.size(), 1);
- Assert.assertEquals(snapshotName1, snapshots.get(0).getName());
- Assert.assertEquals(tableName, snapshots.get(0).getTableName());
- Assert.assertEquals(SnapshotType.FLUSH, snapshots.get(0).getType());
+ assertEquals(1, snapshots.size());
+ assertEquals(snapshotName1, snapshots.get(0).getName());
+ assertEquals(tableName, snapshots.get(0).getTableName());
+ assertEquals(SnapshotType.FLUSH, snapshots.get(0).getType());
// cloneSnapshot into a existed table.
boolean failed = false;
@@ -105,10 +108,10 @@ public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
} catch (Exception e) {
failed = true;
}
- Assert.assertTrue(failed);
+ assertTrue(failed);
// cloneSnapshot into a new table.
- Assert.assertTrue(!syncAdmin.tableExists(tableName2));
+ assertTrue(!syncAdmin.tableExists(tableName2));
admin.cloneSnapshot(snapshotName1, tableName2).get();
syncAdmin.tableExists(tableName2);
}
@@ -120,12 +123,12 @@ public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
Result result;
int rowCount = 0;
while ((result = scanner.next()) != null) {
- Assert.assertArrayEquals(result.getRow(), Bytes.toBytes(rowCount));
- Assert.assertArrayEquals(result.getValue(Bytes.toBytes("f1"), Bytes.toBytes("cq")),
+ assertArrayEquals(result.getRow(), Bytes.toBytes(rowCount));
+ assertArrayEquals(result.getValue(Bytes.toBytes("f1"), Bytes.toBytes("cq")),
Bytes.toBytes(rowCount));
rowCount += 1;
}
- Assert.assertEquals(rowCount, expectedRowCount);
+ assertEquals(rowCount, expectedRowCount);
}
}
}
@@ -137,11 +140,11 @@ public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
table.put(new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("f1"), Bytes.toBytes("cq"),
Bytes.toBytes(i)));
}
- Assert.assertEquals(admin.listSnapshots().get().size(), 0);
+ assertEquals(0, admin.listSnapshots().get().size());
admin.snapshot(snapshotName1, tableName).get();
admin.snapshot(snapshotName2, tableName).get();
- Assert.assertEquals(admin.listSnapshots().get().size(), 2);
+ assertEquals(2, admin.listSnapshots().get().size());
admin.disableTable(tableName).get();
admin.restoreSnapshot(snapshotName1, true).get();
@@ -161,29 +164,23 @@ public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
table.put(new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("f1"), Bytes.toBytes("cq"),
Bytes.toBytes(i)));
}
- Assert.assertEquals(admin.listSnapshots().get().size(), 0);
+ assertEquals(0, admin.listSnapshots().get().size());
admin.snapshot(snapshotName1, tableName).get();
admin.snapshot(snapshotName2, tableName).get();
admin.snapshot(snapshotName3, tableName).get();
- Assert.assertEquals(admin.listSnapshots().get().size(), 3);
-
- Assert.assertEquals(admin.listSnapshots(Pattern.compile("(.*)")).get().size(), 3);
- Assert.assertEquals(admin.listSnapshots(Pattern.compile("snapshotName(\\d+)")).get().size(), 3);
- Assert.assertEquals(admin.listSnapshots(Pattern.compile("snapshotName[1|3]")).get().size(), 2);
- Assert.assertEquals(admin.listSnapshots(Pattern.compile("snapshot(.*)")).get().size(), 3);
- Assert.assertEquals(
- admin.listTableSnapshots(Pattern.compile("testListSnapshots"), Pattern.compile("s(.*)")).get()
- .size(),
- 3);
- Assert.assertEquals(
- admin.listTableSnapshots(Pattern.compile("fakeTableName"), Pattern.compile("snap(.*)")).get()
- .size(),
- 0);
- Assert.assertEquals(
- admin.listTableSnapshots(Pattern.compile("test(.*)"), Pattern.compile("snap(.*)[1|3]")).get()
- .size(),
- 2);
+ assertEquals(3, admin.listSnapshots().get().size());
+
+ assertEquals(3, admin.listSnapshots(Pattern.compile("(.*)")).get().size());
+ assertEquals(3, admin.listSnapshots(Pattern.compile("snapshotName(\\d+)")).get().size());
+ assertEquals(2, admin.listSnapshots(Pattern.compile("snapshotName[1|3]")).get().size());
+ assertEquals(3, admin.listSnapshots(Pattern.compile("snapshot(.*)")).get().size());
+ assertEquals(3, admin.listTableSnapshots(Pattern.compile("testListSnapshots"),
+ Pattern.compile("s(.*)")).get().size());
+ assertEquals(0, admin.listTableSnapshots(Pattern.compile("fakeTableName"),
+ Pattern.compile("snap(.*)")).get().size());
+ assertEquals(2, admin.listTableSnapshots(Pattern.compile("test(.*)"),
+ Pattern.compile("snap(.*)[1|3]")).get().size());
}
@Test
@@ -193,29 +190,29 @@ public class TestAsyncSnapshotAdminApi extends TestAsyncAdminBase {
table.put(new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("f1"), Bytes.toBytes("cq"),
Bytes.toBytes(i)));
}
- Assert.assertEquals(admin.listSnapshots().get().size(), 0);
+ assertEquals(0, admin.listSnapshots().get().size());
admin.snapshot(snapshotName1, tableName).get();
admin.snapshot(snapshotName2, tableName).get();
admin.snapshot(snapshotName3, tableName).get();
- Assert.assertEquals(admin.listSnapshots().get().size(), 3);
+ assertEquals(3, admin.listSnapshots().get().size());
admin.deleteSnapshot(snapshotName1).get();
- Assert.assertEquals(admin.listSnapshots().get().size(), 2);
+ assertEquals(2, admin.listSnapshots().get().size());
admin.deleteSnapshots(Pattern.compile("(.*)abc")).get();
- Assert.assertEquals(admin.listSnapshots().get().size(), 2);
+ assertEquals(2, admin.listSnapshots().get().size());
admin.deleteSnapshots(Pattern.compile("(.*)1")).get();
- Assert.assertEquals(admin.listSnapshots().get().size(), 2);
+ assertEquals(2, admin.listSnapshots().get().size());
admin.deleteTableSnapshots(Pattern.compile("(.*)"), Pattern.compile("(.*)1")).get();
- Assert.assertEquals(admin.listSnapshots().get().size(), 2);
+ assertEquals(2, admin.listSnapshots().get().size());
admin.deleteTableSnapshots(Pattern.compile("(.*)"), Pattern.compile("(.*)2")).get();
- Assert.assertEquals(admin.listSnapshots().get().size(), 1);
+ assertEquals(1, admin.listSnapshots().get().size());
admin.deleteTableSnapshots(Pattern.compile("(.*)"), Pattern.compile("(.*)3")).get();
- Assert.assertEquals(admin.listSnapshots().get().size(), 0);
+ assertEquals(0, admin.listSnapshots().get().size());
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java
index 529346f..8b2dce3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java
@@ -135,7 +135,7 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
admin.createTable(desc).join();
ModifyableTableDescriptor modifyableDesc = ((ModifyableTableDescriptor) desc);
TableDescriptor confirmedHtd = admin.getDescriptor(tableName).get();
- assertEquals(modifyableDesc.compareTo((ModifyableTableDescriptor) confirmedHtd), 0);
+ assertEquals(0, modifyableDesc.compareTo((ModifyableTableDescriptor) confirmedHtd));
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableBatch.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableBatch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableBatch.java
index f47e6e9..7848251 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableBatch.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableBatch.java
@@ -212,13 +212,13 @@ public class TestAsyncTableBatch {
.collect(Collectors.toList())).get();
List<Row> actions = new ArrayList<>();
actions.add(new Get(Bytes.toBytes(0)));
- actions.add(new Put(Bytes.toBytes(1)).addColumn(FAMILY, CQ, Bytes.toBytes((long) 2)));
+ actions.add(new Put(Bytes.toBytes(1)).addColumn(FAMILY, CQ, Bytes.toBytes(2L)));
actions.add(new Delete(Bytes.toBytes(2)));
actions.add(new Increment(Bytes.toBytes(3)).addColumn(FAMILY, CQ, 1));
actions.add(new Append(Bytes.toBytes(4)).addColumn(FAMILY, CQ, Bytes.toBytes(4)));
RowMutations rm = new RowMutations(Bytes.toBytes(5));
- rm.add(new Put(Bytes.toBytes(5)).addColumn(FAMILY, CQ, Bytes.toBytes((long) 100)));
- rm.add(new Put(Bytes.toBytes(5)).addColumn(FAMILY, CQ1, Bytes.toBytes((long) 200)));
+ rm.add(new Put(Bytes.toBytes(5)).addColumn(FAMILY, CQ, Bytes.toBytes(100L)));
+ rm.add(new Put(Bytes.toBytes(5)).addColumn(FAMILY, CQ1, Bytes.toBytes(200L)));
actions.add(rm);
actions.add(new Get(Bytes.toBytes(6)));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
index 521d2f5..ba61ab4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
@@ -189,7 +189,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
try (ResultScanner scanner = table.getScanner(s)) {
count = Iterables.size(scanner);
}
- assertEquals("Count all the rows ", count, 6);
+ assertEquals("Count all the rows ", 6, count);
// all the cache is loaded
// trigger a major compaction
ScannerThread scannerThread = new ScannerThread(table, cache);
@@ -200,7 +200,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
try (ResultScanner scanner = table.getScanner(s)) {
count = Iterables.size(scanner);
}
- assertEquals("Count all the rows ", count, 6);
+ assertEquals("Count all the rows ", 6, count);
} finally {
table.close();
}
@@ -215,6 +215,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
this.cache = cache;
}
+ @Override
public void run() {
Scan s = new Scan().withStartRow(ROW4).withStopRow(ROW5).setCaching(1);
try {
@@ -346,7 +347,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
try (ResultScanner scanner = table.getScanner(s)) {
count = Iterables.size(scanner);
}
- assertEquals("Count all the rows ", count, 6);
+ assertEquals("Count all the rows ", 6, count);
// Scan from cache
s = new Scan();
@@ -379,7 +380,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
iterator.next();
refBlockCount++;
}
- assertEquals("One block should be there ", refBlockCount, 1);
+ assertEquals("One block should be there ", 1, refBlockCount);
// Rescan to prepopulate the data
// cache this row.
Scan s1 = new Scan();
@@ -392,7 +393,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
try {
scanner = table.getScanner(s1);
int count = Iterables.size(scanner);
- assertEquals("Count the rows", count, 2);
+ assertEquals("Count the rows", 2, count);
iterator = cache.iterator();
List<BlockCacheKey> newCacheList = new ArrayList<>();
while (iterator.hasNext()) {
@@ -407,7 +408,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
}
}
- assertEquals("old blocks should still be found ", newBlockRefCount, 6);
+ assertEquals("old blocks should still be found ", 6, newBlockRefCount);
latch.countDown();
} catch (IOException e) {
@@ -423,7 +424,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
}
}
}
- assertEquals("Count should give all rows ", count, 10);
+ assertEquals("Count should give all rows ", 10, count);
} finally {
table.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientPushback.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientPushback.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientPushback.java
index 62eb316..9103c51 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientPushback.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientPushback.java
@@ -127,7 +127,7 @@ public class TestClientPushback {
regionStats.getMemStoreLoadPercent());
// check that the load reported produces a nonzero delay
long backoffTime = backoffPolicy.getBackoffTime(server, regionName, serverStats);
- assertNotEquals("Reported load does not produce a backoff", backoffTime, 0);
+ assertNotEquals("Reported load does not produce a backoff", 0, backoffTime);
LOG.debug("Backoff calculated for " + region.getRegionInfo().getRegionNameAsString() + " @ " +
server + " is " + backoffTime);
@@ -166,13 +166,13 @@ public class TestClientPushback {
MetricsConnection.RunnerStats runnerStats = conn.getConnectionMetrics().runnerStats;
- assertEquals(runnerStats.delayRunners.getCount(), 1);
- assertEquals(runnerStats.normalRunners.getCount(), 1);
+ assertEquals(1, runnerStats.delayRunners.getCount());
+ assertEquals(1, runnerStats.normalRunners.getCount());
assertEquals("", runnerStats.delayIntevalHist.getSnapshot().getMean(),
(double)backoffTime, 0.1);
latch.await(backoffTime * 2, TimeUnit.MILLISECONDS);
- assertNotEquals("AsyncProcess did not submit the work time", endTime.get(), 0);
+ assertNotEquals("AsyncProcess did not submit the work time", 0, endTime.get());
assertTrue("AsyncProcess did not delay long enough", endTime.get() - startTime >= backoffTime);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestConnectionImplementation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestConnectionImplementation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestConnectionImplementation.java
index 2ad49f8..f061fc9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestConnectionImplementation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestConnectionImplementation.java
@@ -120,6 +120,7 @@ public class TestConnectionImplementation {
TEST_UTIL.shutdownMiniCluster();
}
+ @Test
public void testClusterConnection() throws IOException {
ThreadPoolExecutor otherPool = new ThreadPoolExecutor(1, 1,
5, TimeUnit.SECONDS,
@@ -636,7 +637,7 @@ public class TestConnectionImplementation {
LOG.info("Put done, exception caught: " + e.getClass());
Assert.assertEquals(1, e.getNumExceptions());
Assert.assertEquals(1, e.getCauses().size());
- Assert.assertArrayEquals(e.getRow(0).getRow(), ROW);
+ Assert.assertArrayEquals(ROW, e.getRow(0).getRow());
// Check that we unserialized the exception as expected
Throwable cause = ClientExceptionsUtil.findException(e.getCause(0));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFastFail.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFastFail.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFastFail.java
index 47516ec..b2eae85 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFastFail.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFastFail.java
@@ -171,6 +171,7 @@ public class TestFastFail {
* will follow the killing of a regionserver so that we make sure that
* some of threads go into PreemptiveFastFailExcception
*/
+ @Override
public Boolean call() throws Exception {
try (Table table = connection.getTable(TableName.valueOf(tableName))) {
Thread.sleep(Math.abs(random.nextInt()) % 250); // Add some jitter here
@@ -275,7 +276,7 @@ public class TestFastFail {
"All the failures should be coming from the secondput failure",
numFailedThreads.get(), numThreadsReturnedFalse);
assertEquals("Number of threads that threw execution exceptions "
- + "otherwise should be 0", numThreadsThrewExceptions, 0);
+ + "otherwise should be 0", 0, numThreadsThrewExceptions);
assertEquals("The regionservers that returned true should equal to the"
+ " number of successful threads", numThreadsReturnedTrue,
numSuccessfullThreads.get());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index 3af245f..952905a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -203,125 +203,123 @@ public class TestFromClientSide {
/**
* Basic client side validation of HBASE-4536
*/
- @Test
- public void testKeepDeletedCells() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- final byte[] FAMILY = Bytes.toBytes("family");
- final byte[] C0 = Bytes.toBytes("c0");
-
- final byte[] T1 = Bytes.toBytes("T1");
- final byte[] T2 = Bytes.toBytes("T2");
- final byte[] T3 = Bytes.toBytes("T3");
- HColumnDescriptor hcd = new HColumnDescriptor(FAMILY)
- .setKeepDeletedCells(KeepDeletedCells.TRUE)
- .setMaxVersions(3);
-
- HTableDescriptor desc = new HTableDescriptor(tableName);
- desc.addFamily(hcd);
- TEST_UTIL.getAdmin().createTable(desc);
- Table h = TEST_UTIL.getConnection().getTable(tableName);
-
- long ts = System.currentTimeMillis();
- Put p = new Put(T1, ts);
- p.addColumn(FAMILY, C0, T1);
- h.put(p);
- p = new Put(T1, ts+2);
- p.addColumn(FAMILY, C0, T2);
- h.put(p);
- p = new Put(T1, ts+4);
- p.addColumn(FAMILY, C0, T3);
- h.put(p);
-
- Delete d = new Delete(T1, ts+3);
- h.delete(d);
-
- d = new Delete(T1, ts+3);
- d.addColumns(FAMILY, C0, ts+3);
- h.delete(d);
-
- Get g = new Get(T1);
- // does *not* include the delete
- g.setTimeRange(0, ts+3);
- Result r = h.get(g);
- assertArrayEquals(T2, r.getValue(FAMILY, C0));
-
- Scan s = new Scan(T1);
- s.setTimeRange(0, ts+3);
- s.setMaxVersions();
- ResultScanner scanner = h.getScanner(s);
- Cell[] kvs = scanner.next().rawCells();
- assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
- assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
- scanner.close();
-
- s = new Scan(T1);
- s.setRaw(true);
- s.setMaxVersions();
- scanner = h.getScanner(s);
- kvs = scanner.next().rawCells();
- assertTrue(PrivateCellUtil.isDeleteFamily(kvs[0]));
- assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
- assertTrue(CellUtil.isDelete(kvs[2]));
- assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
- assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
- scanner.close();
- h.close();
- }
-
- /**
- * Basic client side validation of HBASE-10118
- */
- @Test
- public void testPurgeFutureDeletes() throws Exception {
- final TableName tableName = TableName.valueOf(name.getMethodName());
- final byte[] ROW = Bytes.toBytes("row");
- final byte[] FAMILY = Bytes.toBytes("family");
- final byte[] COLUMN = Bytes.toBytes("column");
- final byte[] VALUE = Bytes.toBytes("value");
-
- Table table = TEST_UTIL.createTable(tableName, FAMILY);
-
- // future timestamp
- long ts = System.currentTimeMillis() * 2;
- Put put = new Put(ROW, ts);
- put.addColumn(FAMILY, COLUMN, VALUE);
- table.put(put);
-
- Get get = new Get(ROW);
- Result result = table.get(get);
- assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
-
- Delete del = new Delete(ROW);
- del.addColumn(FAMILY, COLUMN, ts);
- table.delete(del);
-
- get = new Get(ROW);
- result = table.get(get);
- assertNull(result.getValue(FAMILY, COLUMN));
-
- // major compaction, purged future deletes
- TEST_UTIL.getAdmin().flush(tableName);
- TEST_UTIL.getAdmin().majorCompact(tableName);
-
- // waiting for the major compaction to complete
- TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
- @Override
- public boolean evaluate() throws IOException {
- return TEST_UTIL.getAdmin().getCompactionState(tableName) ==
- CompactionState.NONE;
- }
- });
-
- put = new Put(ROW, ts);
- put.addColumn(FAMILY, COLUMN, VALUE);
- table.put(put);
-
- get = new Get(ROW);
- result = table.get(get);
- assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
-
- table.close();
- }
+ @Test
+ public void testKeepDeletedCells() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final byte[] FAMILY = Bytes.toBytes("family");
+ final byte[] C0 = Bytes.toBytes("c0");
+
+ final byte[] T1 = Bytes.toBytes("T1");
+ final byte[] T2 = Bytes.toBytes("T2");
+ final byte[] T3 = Bytes.toBytes("T3");
+ HColumnDescriptor hcd =
+ new HColumnDescriptor(FAMILY).setKeepDeletedCells(KeepDeletedCells.TRUE).setMaxVersions(3);
+
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(hcd);
+ TEST_UTIL.getAdmin().createTable(desc);
+ Table h = TEST_UTIL.getConnection().getTable(tableName);
+
+ long ts = System.currentTimeMillis();
+ Put p = new Put(T1, ts);
+ p.addColumn(FAMILY, C0, T1);
+ h.put(p);
+ p = new Put(T1, ts + 2);
+ p.addColumn(FAMILY, C0, T2);
+ h.put(p);
+ p = new Put(T1, ts + 4);
+ p.addColumn(FAMILY, C0, T3);
+ h.put(p);
+
+ Delete d = new Delete(T1, ts + 3);
+ h.delete(d);
+
+ d = new Delete(T1, ts + 3);
+ d.addColumns(FAMILY, C0, ts + 3);
+ h.delete(d);
+
+ Get g = new Get(T1);
+ // does *not* include the delete
+ g.setTimeRange(0, ts + 3);
+ Result r = h.get(g);
+ assertArrayEquals(T2, r.getValue(FAMILY, C0));
+
+ Scan s = new Scan(T1);
+ s.setTimeRange(0, ts + 3);
+ s.setMaxVersions();
+ ResultScanner scanner = h.getScanner(s);
+ Cell[] kvs = scanner.next().rawCells();
+ assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
+ assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
+ scanner.close();
+
+ s = new Scan(T1);
+ s.setRaw(true);
+ s.setMaxVersions();
+ scanner = h.getScanner(s);
+ kvs = scanner.next().rawCells();
+ assertTrue(PrivateCellUtil.isDeleteFamily(kvs[0]));
+ assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
+ assertTrue(CellUtil.isDelete(kvs[2]));
+ assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
+ assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
+ scanner.close();
+ h.close();
+ }
+
+ /**
+ * Basic client side validation of HBASE-10118
+ */
+ @Test
+ public void testPurgeFutureDeletes() throws Exception {
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final byte[] ROW = Bytes.toBytes("row");
+ final byte[] FAMILY = Bytes.toBytes("family");
+ final byte[] COLUMN = Bytes.toBytes("column");
+ final byte[] VALUE = Bytes.toBytes("value");
+
+ Table table = TEST_UTIL.createTable(tableName, FAMILY);
+
+ // future timestamp
+ long ts = System.currentTimeMillis() * 2;
+ Put put = new Put(ROW, ts);
+ put.addColumn(FAMILY, COLUMN, VALUE);
+ table.put(put);
+
+ Get get = new Get(ROW);
+ Result result = table.get(get);
+ assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
+
+ Delete del = new Delete(ROW);
+ del.addColumn(FAMILY, COLUMN, ts);
+ table.delete(del);
+
+ get = new Get(ROW);
+ result = table.get(get);
+ assertNull(result.getValue(FAMILY, COLUMN));
+
+ // major compaction, purged future deletes
+ TEST_UTIL.getAdmin().flush(tableName);
+ TEST_UTIL.getAdmin().majorCompact(tableName);
+
+ // waiting for the major compaction to complete
+ TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
+ @Override
+ public boolean evaluate() throws IOException {
+ return TEST_UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE;
+ }
+ });
+
+ put = new Put(ROW, ts);
+ put.addColumn(FAMILY, COLUMN, VALUE);
+ table.put(put);
+
+ get = new Get(ROW);
+ result = table.get(get);
+ assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
+
+ table.close();
+ }
/**
* Verifies that getConfiguration returns the same Configuration object used
@@ -660,13 +658,13 @@ public class TestFromClientSide {
ResultScanner scanner = ht.getScanner(scan);
int expectedIndex = 1;
for(Result result : ht.getScanner(scan)) {
- assertEquals(result.size(), 1);
+ assertEquals(1, result.size());
assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]),
QUALIFIERS[expectedIndex]));
expectedIndex++;
}
- assertEquals(expectedIndex, 6);
+ assertEquals(6, expectedIndex);
scanner.close();
}
@@ -693,11 +691,11 @@ public class TestFromClientSide {
ResultScanner scanner = ht.getScanner(scan);
int expectedIndex = 0;
for(Result result : ht.getScanner(scan)) {
- assertEquals(result.size(), 1);
+ assertEquals(1, result.size());
assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500);
expectedIndex++;
}
- assertEquals(expectedIndex, 4);
+ assertEquals(4, expectedIndex);
scanner.close();
}
@@ -726,12 +724,12 @@ public class TestFromClientSide {
ResultScanner scanner = ht.getScanner(scan);
int count = 0;
for(Result result : ht.getScanner(scan)) {
- assertEquals(result.size(), 1);
- assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
- assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
+ assertEquals(1, result.size());
+ assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
+ assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
count++;
}
- assertEquals(count, 10);
+ assertEquals(10, count);
scanner.close();
}
@@ -3499,7 +3497,9 @@ public class TestFromClientSide {
private long [] makeStamps(int n) {
long [] stamps = new long[n];
- for(int i=0;i<n;i++) stamps[i] = i+1;
+ for (int i = 0; i < n; i++) {
+ stamps[i] = i+1L;
+ }
return stamps;
}
@@ -3969,8 +3969,8 @@ public class TestFromClientSide {
put = new Put(row2);
put.addColumn(CONTENTS_FAMILY, null, value);
- assertEquals(put.size(), 1);
- assertEquals(put.getFamilyCellMap().get(CONTENTS_FAMILY).size(), 1);
+ assertEquals(1, put.size());
+ assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
// KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO
KeyValue kv = (KeyValue)put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
@@ -4794,22 +4794,22 @@ public class TestFromClientSide {
// row doesn't exist, so using non-null value should be considered "not match".
boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifEquals(VALUE).thenPut(put1);
- assertEquals(ok, false);
+ assertFalse(ok);
// row doesn't exist, so using "ifNotExists" should be considered "match".
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
- assertEquals(ok, true);
+ assertTrue(ok);
// row now exists, so using "ifNotExists" should be considered "not match".
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
- assertEquals(ok, false);
+ assertFalse(ok);
Put put2 = new Put(ROW);
put2.addColumn(FAMILY, QUALIFIER, value2);
// row now exists, use the matching value to check
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(VALUE).thenPut(put2);
- assertEquals(ok, true);
+ assertTrue(ok);
Put put3 = new Put(anotherrow);
put3.addColumn(FAMILY, QUALIFIER, VALUE);
@@ -4839,70 +4839,70 @@ public class TestFromClientSide {
// row doesn't exist, so using "ifNotExists" should be considered "match".
boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put2);
- assertEquals(ok, true);
+ assertTrue(ok);
// cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
// turns out "match"
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER, value1).thenPut(put2);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.EQUAL, value1).thenPut(put2);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenPut(put2);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS, value1).thenPut(put2);
- assertEquals(ok, true);
+ assertTrue(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenPut(put2);
- assertEquals(ok, true);
+ assertTrue(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.NOT_EQUAL, value1).thenPut(put3);
- assertEquals(ok, true);
+ assertTrue(ok);
// cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
// turns out "match"
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS, value4).thenPut(put3);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenPut(put3);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.EQUAL, value4).thenPut(put3);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER, value4).thenPut(put3);
- assertEquals(ok, true);
+ assertTrue(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenPut(put3);
- assertEquals(ok, true);
+ assertTrue(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.NOT_EQUAL, value4).thenPut(put2);
- assertEquals(ok, true);
+ assertTrue(ok);
// cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
// turns out "match"
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER, value2).thenPut(put2);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.NOT_EQUAL, value2).thenPut(put2);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS, value2).thenPut(put2);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenPut(put2);
- assertEquals(ok, true);
+ assertTrue(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenPut(put2);
- assertEquals(ok, true);
+ assertTrue(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.EQUAL, value2).thenPut(put3);
- assertEquals(ok, true);
+ assertTrue(ok);
}
@Test
@@ -4921,7 +4921,7 @@ public class TestFromClientSide {
boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifEquals(value1).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
}
@Test
@@ -4948,72 +4948,72 @@ public class TestFromClientSide {
// turns out "match"
boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER, value1).thenDelete(delete);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.EQUAL, value1).thenDelete(delete);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenDelete(delete);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS, value1).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
table.put(put2);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
table.put(put2);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.NOT_EQUAL, value1).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
// cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
// turns out "match"
table.put(put3);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS, value4).thenDelete(delete);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenDelete(delete);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.EQUAL, value4).thenDelete(delete);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER, value4).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
table.put(put3);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
table.put(put3);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.NOT_EQUAL, value4).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
// cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
// turns out "match"
table.put(put2);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER, value2).thenDelete(delete);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.NOT_EQUAL, value2).thenDelete(delete);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS, value2).thenDelete(delete);
- assertEquals(ok, false);
+ assertFalse(ok);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
table.put(put2);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
table.put(put2);
ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
.ifMatches(CompareOperator.EQUAL, value2).thenDelete(delete);
- assertEquals(ok, true);
+ assertTrue(ok);
}
/**
@@ -5726,7 +5726,7 @@ public class TestFromClientSide {
ResultScanner scanner = ht.getScanner(scan);
int expectedIndex = 5;
for (Result result : scanner) {
- assertEquals(result.size(), 1);
+ assertEquals(1, result.size());
Cell c = result.rawCells()[0];
assertTrue(Bytes.equals(c.getRowArray(), c.getRowOffset(), c.getRowLength(),
ROWS[expectedIndex], 0, ROWS[expectedIndex].length));
@@ -5734,7 +5734,7 @@ public class TestFromClientSide {
c.getQualifierLength(), QUALIFIERS[expectedIndex], 0, QUALIFIERS[expectedIndex].length));
expectedIndex--;
}
- assertEquals(expectedIndex, 0);
+ assertEquals(0, expectedIndex);
scanner.close();
ht.close();
}
@@ -5767,12 +5767,12 @@ public class TestFromClientSide {
ResultScanner scanner = ht.getScanner(scan);
int count = 0;
for (Result result : ht.getScanner(scan)) {
- assertEquals(result.size(), 1);
- assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
- assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
+ assertEquals(1, result.size());
+ assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
+ assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
count++;
}
- assertEquals(count, 10);
+ assertEquals(10, count);
scanner.close();
ht.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
index 2d67b3e..6b031d8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
@@ -19,6 +19,12 @@
*/
package org.apache.hadoop.hbase.client;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -35,37 +41,30 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.coprocessor.RegionObserver;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.junit.After;
import org.junit.AfterClass;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -75,6 +74,9 @@ import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
+
@Category({LargeTests.class, ClientTests.class})
public class TestFromClientSide3 {
private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide3.class);
@@ -497,12 +499,12 @@ public class TestFromClientSide3 {
Get get = new Get(ROW);
boolean exist = table.exists(get);
- assertEquals(exist, false);
+ assertFalse(exist);
table.put(put);
exist = table.exists(get);
- assertEquals(exist, true);
+ assertTrue(exist);
}
@Test
@@ -589,12 +591,12 @@ public class TestFromClientSide3 {
Get get = new Get(ROW);
boolean exist = table.exists(get);
- assertEquals(exist, false);
+ assertFalse(exist);
table.put(put);
exist = table.exists(get);
- assertEquals(exist, true);
+ assertTrue(exist);
}
@Test
@@ -614,10 +616,10 @@ public class TestFromClientSide3 {
LOG.info("Calling exists");
boolean[] results = table.existsAll(gets);
- assertEquals(results[0], false);
- assertEquals(results[1], false);
- assertEquals(results[2], true);
- assertEquals(results[3], false);
+ assertFalse(results[0]);
+ assertFalse(results[1]);
+ assertTrue(results[2]);
+ assertFalse(results[3]);
// Test with the first region.
put = new Put(new byte[] { 0x00 });
@@ -628,8 +630,8 @@ public class TestFromClientSide3 {
gets.add(new Get(new byte[] { 0x00 }));
gets.add(new Get(new byte[] { 0x00, 0x00 }));
results = table.existsAll(gets);
- assertEquals(results[0], true);
- assertEquals(results[1], false);
+ assertTrue(results[0]);
+ assertFalse(results[1]);
// Test with the last region
put = new Put(new byte[] { (byte) 0xff, (byte) 0xff });
@@ -641,9 +643,9 @@ public class TestFromClientSide3 {
gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff }));
gets.add(new Get(new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff }));
results = table.existsAll(gets);
- assertEquals(results[0], false);
- assertEquals(results[1], true);
- assertEquals(results[2], false);
+ assertFalse(results[0]);
+ assertTrue(results[1]);
+ assertFalse(results[2]);
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
index 093400e..3f618a1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java
@@ -279,16 +279,19 @@ public class TestMetaCache {
private int expCount = -1;
private List<Throwable> metaCachePreservingExceptions = metaCachePreservingExceptions();
+ @Override
public void throwOnGet(FakeRSRpcServices rpcServices, ClientProtos.GetRequest request)
throws ServiceException {
throwSomeExceptions(rpcServices, request.getRegion());
}
+ @Override
public void throwOnMutate(FakeRSRpcServices rpcServices, ClientProtos.MutateRequest request)
throws ServiceException {
throwSomeExceptions(rpcServices, request.getRegion());
}
+ @Override
public void throwOnScan(FakeRSRpcServices rpcServices, ClientProtos.ScanRequest request)
throws ServiceException {
if (!request.hasScannerId()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java
index ee39a83..d73ebc0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultipleTimestamps.java
@@ -180,7 +180,7 @@ public class TestMultipleTimestamps {
Integer[] scanRows = new Integer[] {5, 7};
Integer[] scanColumns = new Integer[] {3, 4, 5};
- Long[] scanTimestamps = new Long[] {2l, 3L};
+ Long[] scanTimestamps = new Long[] { 2L, 3L};
int scanMaxVersions = 2;
put(ht, FAMILY, putRows, putColumns, putTimestamps);
@@ -241,7 +241,7 @@ public class TestMultipleTimestamps {
Integer[] scanRows = new Integer[] {3, 5, 7};
Integer[] scanColumns = new Integer[] {3, 4, 5};
- Long[] scanTimestamps = new Long[] {2l, 4L};
+ Long[] scanTimestamps = new Long[] { 2L, 4L};
int scanMaxVersions = 5;
put(ht, FAMILY, putRows1, putColumns1, putTimestamps1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicasClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicasClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicasClient.java
index a06055d..f5ea10e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicasClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicasClient.java
@@ -261,9 +261,9 @@ public class TestReplicasClient {
AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(
getRS().getServerName(), hri, null);
AdminProtos.OpenRegionResponse responseOpen = getRS().getRSRpcServices().openRegion(null, orr);
- Assert.assertEquals(responseOpen.getOpeningStateCount(), 1);
- Assert.assertEquals(responseOpen.getOpeningState(0),
- AdminProtos.OpenRegionResponse.RegionOpeningState.OPENED);
+ Assert.assertEquals(1, responseOpen.getOpeningStateCount());
+ Assert.assertEquals(AdminProtos.OpenRegionResponse.RegionOpeningState.OPENED,
+ responseOpen.getOpeningState(0));
checkRegionIsOpened(hri);
}
@@ -583,8 +583,8 @@ public class TestReplicasClient {
r = table.get(g);
Assert.assertFalse(r.isStale());
Assert.assertFalse(r.getColumnCells(f, b1).isEmpty());
- Assert.assertEquals(hedgedReadOps.getCount(), 1);
- Assert.assertEquals(hedgedReadWin.getCount(), 0);
+ Assert.assertEquals(1, hedgedReadOps.getCount());
+ Assert.assertEquals(0, hedgedReadWin.getCount());
SlowMeCopro.sleepTime.set(0);
SlowMeCopro.getSecondaryCdl().get().countDown();
LOG.info("hedged read occurred but not faster");
@@ -597,8 +597,8 @@ public class TestReplicasClient {
r = table.get(g);
Assert.assertTrue(r.isStale());
Assert.assertTrue(r.getColumnCells(f, b1).isEmpty());
- Assert.assertEquals(hedgedReadOps.getCount(), 2);
- Assert.assertEquals(hedgedReadWin.getCount(), 1);
+ Assert.assertEquals(2, hedgedReadOps.getCount());
+ Assert.assertEquals(1, hedgedReadWin.getCount());
SlowMeCopro.getPrimaryCdl().get().countDown();
LOG.info("hedged read occurred and faster");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
index 3190fb9..fe1cc9c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
@@ -205,7 +205,7 @@ public class TestRestoreSnapshotFromClient {
HTableDescriptor htd = admin.getTableDescriptor(tableName);
assertEquals(2, htd.getFamilies().size());
SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, TEST_FAMILY2);
- long snapshot2Rows = snapshot1Rows + 500;
+ long snapshot2Rows = snapshot1Rows + 500L;
assertEquals(snapshot2Rows, countRows(table));
assertEquals(500, countRows(table, TEST_FAMILY2));
Set<String> fsFamilies = getFamiliesFromFS(tableName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
index 3335298..b494895 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
@@ -194,7 +194,7 @@ public class TestResult extends TestCase {
loadValueBuffer.clear();
r.loadValue(family, qf, loadValueBuffer);
loadValueBuffer.flip();
- assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), loadValueBuffer);
+ assertEquals(loadValueBuffer, ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))));
assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))),
r.getValueAsByteBuffer(family, qf));
}
@@ -219,7 +219,7 @@ public class TestResult extends TestCase {
loadValueBuffer.clear();
r.loadValue(family, qf, loadValueBuffer);
loadValueBuffer.flip();
- assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), loadValueBuffer);
+ assertEquals(loadValueBuffer, ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))));
assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))),
r.getValueAsByteBuffer(family, qf));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestServerBusyException.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestServerBusyException.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestServerBusyException.java
index b1126e5..387253c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestServerBusyException.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestServerBusyException.java
@@ -142,7 +142,7 @@ public class TestServerBusyException {
TEST_UTIL.shutdownMiniCluster();
}
- private class TestPutThread extends Thread {
+ private static class TestPutThread extends Thread {
Table table;
int getServerBusyException = 0;
@@ -163,7 +163,7 @@ public class TestServerBusyException {
}
}
- private class TestGetThread extends Thread {
+ private static class TestGetThread extends Thread {
Table table;
int getServerBusyException = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java
index 84bfa76..41e3034 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java
@@ -160,8 +160,8 @@ public class TestSizeFailures {
* @return An entry where the first item is rows observed and the second is entries observed.
*/
private Entry<Long,Long> sumTable(ResultScanner scanner) {
- long rowsObserved = 0l;
- long entriesObserved = 0l;
+ long rowsObserved = 0L;
+ long entriesObserved = 0L;
// Read all the records in the table
for (Result result : scanner) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSmallReversedScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSmallReversedScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSmallReversedScanner.java
index b050397..4de6a76 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSmallReversedScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSmallReversedScanner.java
@@ -105,7 +105,7 @@ public class TestSmallReversedScanner {
Assert.assertArrayEquals(r.getRow(), Bytes.toBytes(inputRowKeys[value]));
}
- Assert.assertEquals(value, 0);
+ Assert.assertEquals(0, value);
}
/**
@@ -133,6 +133,6 @@ public class TestSmallReversedScanner {
Assert.assertArrayEquals(r.getRow(), new byte[] { (char) 0x00 });
Assert.assertTrue(--count >= 0);
}
- Assert.assertEquals(count, 0);
+ Assert.assertEquals(0, count);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
index 1127a5e..b077d64 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
@@ -185,7 +185,7 @@ public class TestSnapshotFromClient {
admin.deleteSnapshots(Pattern.compile("TableSnapshot.*"));
List<SnapshotDescription> snapshots = admin.listSnapshots();
assertEquals(1, snapshots.size());
- assertEquals(snapshots.get(0).getName(), snapshot3);
+ assertEquals(snapshot3, snapshots.get(0).getName());
admin.deleteSnapshot(snapshot3);
admin.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
index 49c6560..fa3715d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java
@@ -212,8 +212,8 @@ public class TestSnapshotMetadata {
cloneHtd.getValues().size());
assertEquals(originalTableDescriptor.getConfiguration().size(),
cloneHtd.getConfiguration().size());
- assertEquals(cloneHtd.getValue(TEST_CUSTOM_VALUE), TEST_CUSTOM_VALUE);
- assertEquals(cloneHtd.getConfigurationValue(TEST_CONF_CUSTOM_VALUE), TEST_CONF_CUSTOM_VALUE);
+ assertEquals(TEST_CUSTOM_VALUE, cloneHtd.getValue(TEST_CUSTOM_VALUE));
+ assertEquals(TEST_CONF_CUSTOM_VALUE, cloneHtd.getConfigurationValue(TEST_CONF_CUSTOM_VALUE));
assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
assertEquals(originalTableDescriptor.getConfiguration(), cloneHtd.getConfiguration());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java
index df2a6e8..9e543dc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java
@@ -177,7 +177,7 @@ public class TestSnapshotWithAcl extends SecureTestUtil {
byte[] value = result.getValue(TEST_FAMILY, TEST_QUALIFIER);
Assert.assertArrayEquals(value, Bytes.toBytes(rowCount++));
}
- Assert.assertEquals(rowCount, ROW_COUNT);
+ Assert.assertEquals(ROW_COUNT, rowCount);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
index 678134b..d239eb8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
@@ -80,13 +80,13 @@ public class TestSplitOrMergeStatus {
Admin admin = TEST_UTIL.getAdmin();
initSwitchStatus(admin);
boolean[] results = admin.setSplitOrMergeEnabled(false, false, MasterSwitchType.SPLIT);
- assertEquals(results.length, 1);
+ assertEquals(1, results.length);
assertTrue(results[0]);
admin.split(t.getName());
int count = admin.getTableRegions(tableName).size();
assertTrue(originalCount == count);
results = admin.setSplitOrMergeEnabled(true, false, MasterSwitchType.SPLIT);
- assertEquals(results.length, 1);
+ assertEquals(1, results.length);
assertFalse(results[0]);
admin.split(t.getName());
while ((count = admin.getTableRegions(tableName).size()) == originalCount) {
@@ -117,7 +117,7 @@ public class TestSplitOrMergeStatus {
// Merge switch is off so merge should NOT succeed.
boolean[] results = admin.setSplitOrMergeEnabled(false, false, MasterSwitchType.MERGE);
- assertEquals(results.length, 1);
+ assertEquals(1, results.length);
assertTrue(results[0]);
List<HRegionInfo> regions = admin.getTableRegions(t.getName());
assertTrue(regions.size() > 1);
@@ -134,7 +134,7 @@ public class TestSplitOrMergeStatus {
results = admin.setSplitOrMergeEnabled(true, false, MasterSwitchType.MERGE);
regions = admin.getTableRegions(t.getName());
- assertEquals(results.length, 1);
+ assertEquals(1, results.length);
assertFalse(results[0]);
f = admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(),
regions.get(1).getEncodedNameAsBytes(), true);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java
index 89af5de..ea60ec2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTimestampsFilter.java
@@ -181,15 +181,15 @@ public class TestTimestampsFilter {
Table ht = TEST_UTIL.createTable(TableName.valueOf(TABLE), FAMILIES, Integer.MAX_VALUE);
Put p = new Put(Bytes.toBytes("row"));
- p.addColumn(FAMILY, Bytes.toBytes("column0"), (long) 3, Bytes.toBytes("value0-3"));
- p.addColumn(FAMILY, Bytes.toBytes("column1"), (long) 3, Bytes.toBytes("value1-3"));
- p.addColumn(FAMILY, Bytes.toBytes("column2"), (long) 1, Bytes.toBytes("value2-1"));
- p.addColumn(FAMILY, Bytes.toBytes("column2"), (long) 2, Bytes.toBytes("value2-2"));
- p.addColumn(FAMILY, Bytes.toBytes("column2"), (long) 3, Bytes.toBytes("value2-3"));
- p.addColumn(FAMILY, Bytes.toBytes("column3"), (long) 2, Bytes.toBytes("value3-2"));
- p.addColumn(FAMILY, Bytes.toBytes("column4"), (long) 1, Bytes.toBytes("value4-1"));
- p.addColumn(FAMILY, Bytes.toBytes("column4"), (long) 2, Bytes.toBytes("value4-2"));
- p.addColumn(FAMILY, Bytes.toBytes("column4"), (long) 3, Bytes.toBytes("value4-3"));
+ p.addColumn(FAMILY, Bytes.toBytes("column0"), 3L, Bytes.toBytes("value0-3"));
+ p.addColumn(FAMILY, Bytes.toBytes("column1"), 3L, Bytes.toBytes("value1-3"));
+ p.addColumn(FAMILY, Bytes.toBytes("column2"), 1L, Bytes.toBytes("value2-1"));
+ p.addColumn(FAMILY, Bytes.toBytes("column2"), 2L, Bytes.toBytes("value2-2"));
+ p.addColumn(FAMILY, Bytes.toBytes("column2"), 3L, Bytes.toBytes("value2-3"));
+ p.addColumn(FAMILY, Bytes.toBytes("column3"), 2L, Bytes.toBytes("value3-2"));
+ p.addColumn(FAMILY, Bytes.toBytes("column4"), 1L, Bytes.toBytes("value4-1"));
+ p.addColumn(FAMILY, Bytes.toBytes("column4"), 2L, Bytes.toBytes("value4-2"));
+ p.addColumn(FAMILY, Bytes.toBytes("column4"), 3L, Bytes.toBytes("value4-3"));
ht.put(p);
ArrayList<Long> timestamps = new ArrayList<>();
@@ -209,7 +209,7 @@ public class TestTimestampsFilter {
+ Bytes.toString(CellUtil.cloneValue(kv)));
}
- assertEquals(result.listCells().size(), 2);
+ assertEquals(2, result.listCells().size());
assertTrue(CellUtil.matchingValue(result.listCells().get(0), Bytes.toBytes("value2-3")));
assertTrue(CellUtil.matchingValue(result.listCells().get(1), Bytes.toBytes("value4-3")));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
index 6511a42..3f4029a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
@@ -70,7 +70,7 @@ public class TestUpdateConfiguration {
admin.updateConfiguration(server);
Configuration conf = TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
int custom = conf.getInt("hbase.custom.config", 0);
- assertEquals(custom, 1000);
+ assertEquals(1000, custom);
// restore hbase-site.xml
Files.copy(cnf3Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
}
@@ -97,17 +97,20 @@ public class TestUpdateConfiguration {
admin.updateConfiguration();
// Check the configuration of the Masters
- Configuration masterConfiguration = TEST_UTIL.getMiniHBaseCluster().getMaster(0).getConfiguration();
+ Configuration masterConfiguration =
+ TEST_UTIL.getMiniHBaseCluster().getMaster(0).getConfiguration();
int custom = masterConfiguration.getInt("hbase.custom.config", 0);
- assertEquals(custom, 1000);
- Configuration backupMasterConfiguration = TEST_UTIL.getMiniHBaseCluster().getMaster(1).getConfiguration();
+ assertEquals(1000, custom);
+ Configuration backupMasterConfiguration =
+ TEST_UTIL.getMiniHBaseCluster().getMaster(1).getConfiguration();
custom = backupMasterConfiguration.getInt("hbase.custom.config", 0);
- assertEquals(custom, 1000);
+ assertEquals(1000, custom);
// Check the configuration of the RegionServer
- Configuration regionServerConfiguration = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getConfiguration();
+ Configuration regionServerConfiguration =
+ TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getConfiguration();
custom = regionServerConfiguration.getInt("hbase.custom.config", 0);
- assertEquals(custom, 1000);
+ assertEquals(1000, custom);
// restore hbase-site.xml
Files.copy(cnf3Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
index 3b7fd84..82eda2a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminWithClusters.java
@@ -88,7 +88,7 @@ public class TestReplicationAdminWithClusters extends TestReplicationBase {
admin1.disableTableReplication(tableName);
table = admin1.getTableDescriptor(tableName);
for (HColumnDescriptor fam : table.getColumnFamilies()) {
- assertEquals(fam.getScope(), HConstants.REPLICATION_SCOPE_LOCAL);
+ assertEquals(HConstants.REPLICATION_SCOPE_LOCAL, fam.getScope());
}
}
@@ -119,7 +119,7 @@ public class TestReplicationAdminWithClusters extends TestReplicationBase {
admin1.enableTableReplication(tableName);
table = admin1.getTableDescriptor(tableName);
for (HColumnDescriptor fam : table.getColumnFamilies()) {
- assertEquals(fam.getScope(), HConstants.REPLICATION_SCOPE_GLOBAL);
+ assertEquals(HConstants.REPLICATION_SCOPE_GLOBAL, fam.getScope());
}
}
@@ -144,7 +144,7 @@ public class TestReplicationAdminWithClusters extends TestReplicationBase {
admin1.enableTableReplication(tableName);
table = admin1.getTableDescriptor(tableName);
for (HColumnDescriptor fam : table.getColumnFamilies()) {
- assertEquals(fam.getScope(), HConstants.REPLICATION_SCOPE_GLOBAL);
+ assertEquals(HConstants.REPLICATION_SCOPE_GLOBAL, fam.getScope());
}
}
@@ -153,12 +153,12 @@ public class TestReplicationAdminWithClusters extends TestReplicationBase {
admin1.disableTableReplication(tableName);
HTableDescriptor table = admin1.getTableDescriptor(tableName);
for (HColumnDescriptor fam : table.getColumnFamilies()) {
- assertEquals(fam.getScope(), HConstants.REPLICATION_SCOPE_LOCAL);
+ assertEquals(HConstants.REPLICATION_SCOPE_LOCAL, fam.getScope());
}
admin1.enableTableReplication(tableName);
table = admin1.getTableDescriptor(tableName);
for (HColumnDescriptor fam : table.getColumnFamilies()) {
- assertEquals(fam.getScope(), HConstants.REPLICATION_SCOPE_GLOBAL);
+ assertEquals(HConstants.REPLICATION_SCOPE_GLOBAL, fam.getScope());
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/conf/TestConfigurationManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/conf/TestConfigurationManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/conf/TestConfigurationManager.java
index 1f8dbc4..06e1a35 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/conf/TestConfigurationManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/conf/TestConfigurationManager.java
@@ -42,6 +42,7 @@ public class TestConfigurationManager {
register();
}
+ @Override
public void onConfigurationChange(Configuration conf) {
notifiedOnChange = true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java
index 35bcd77..d7c70da 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/constraint/TestConstraint.java
@@ -85,7 +85,7 @@ public class TestConstraint {
try {
// test that we don't fail on a valid put
Put put = new Put(row1);
- byte[] value = Integer.toString(10).getBytes();
+ byte[] value = Bytes.toBytes(Integer.toString(10));
byte[] qualifier = new byte[0];
put.addColumn(dummy, qualifier, value);
table.put(put);
@@ -119,7 +119,7 @@ public class TestConstraint {
// test that we do fail on violation
Put put = new Put(row1);
byte[] qualifier = new byte[0];
- put.addColumn(dummy, qualifier, "fail".getBytes());
+ put.addColumn(dummy, qualifier, Bytes.toBytes("fail"));
LOG.warn("Doing put in table");
try {
table.put(put);
@@ -158,7 +158,7 @@ public class TestConstraint {
// test that we don't fail because its disabled
Put put = new Put(row1);
byte[] qualifier = new byte[0];
- put.addColumn(dummy, qualifier, "pass".getBytes());
+ put.addColumn(dummy, qualifier, Bytes.toBytes("pass"));
table.put(put);
} finally {
table.close();
@@ -191,7 +191,7 @@ public class TestConstraint {
// test that we do fail on violation
Put put = new Put(row1);
byte[] qualifier = new byte[0];
- put.addColumn(dummy, qualifier, "pass".getBytes());
+ put.addColumn(dummy, qualifier, Bytes.toBytes("pass"));
LOG.warn("Doing put in table");
table.put(put);
} finally {
@@ -224,7 +224,7 @@ public class TestConstraint {
// test that we do fail on violation
Put put = new Put(row1);
byte[] qualifier = new byte[0];
- put.addColumn(dummy, qualifier, "pass".getBytes());
+ put.addColumn(dummy, qualifier, Bytes.toBytes("pass"));
try{
table.put(put);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALCoprocessor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALCoprocessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALCoprocessor.java
index f3d90f6..1da31da 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALCoprocessor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SampleRegionWALCoprocessor.java
@@ -123,7 +123,8 @@ public class SampleRegionWALCoprocessor implements WALCoprocessor, RegionCoproce
if (Arrays.equals(family, changedFamily) &&
Arrays.equals(qulifier, changedQualifier)) {
LOG.debug("Found the KeyValue from WALEdit which should be changed.");
- cell.getValueArray()[cell.getValueOffset()] += 1;
+ cell.getValueArray()[cell.getValueOffset()] =
+ (byte) (cell.getValueArray()[cell.getValueOffset()] + 1);
}
}
if (null != row) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
index 1394dbd..0d864b6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
@@ -558,7 +558,7 @@ public class SimpleRegionObserver implements RegionCoprocessor, RegionObserver {
TestRegionObserverInterface.TEST_TABLE)) {
assertNotNull(familyPaths);
assertEquals(1,familyPaths.size());
- assertArrayEquals(familyPaths.get(0).getFirst(), TestRegionObserverInterface.A);
+ assertArrayEquals(TestRegionObserverInterface.A, familyPaths.get(0).getFirst());
String familyPath = familyPaths.get(0).getSecond();
String familyName = Bytes.toString(TestRegionObserverInterface.A);
assertEquals(familyPath.substring(familyPath.length()-familyName.length()-1),"/"+familyName);
@@ -577,7 +577,7 @@ public class SimpleRegionObserver implements RegionCoprocessor, RegionObserver {
TestRegionObserverInterface.TEST_TABLE)) {
assertNotNull(familyPaths);
assertEquals(1,familyPaths.size());
- assertArrayEquals(familyPaths.get(0).getFirst(), TestRegionObserverInterface.A);
+ assertArrayEquals(TestRegionObserverInterface.A, familyPaths.get(0).getFirst());
String familyPath = familyPaths.get(0).getSecond();
String familyName = Bytes.toString(TestRegionObserverInterface.A);
assertEquals(familyPath.substring(familyPath.length()-familyName.length()-1),"/"+familyName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java
index 27865f7..6e6c34b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java
@@ -108,12 +108,10 @@ public class TestCoprocessorConfiguration {
tableCoprocessorLoaded.set(false);
new RegionCoprocessorHost(region, rsServices, conf);
assertEquals("System coprocessors loading default was not honored",
- systemCoprocessorLoaded.get(),
- CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED);
+ CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED, systemCoprocessorLoaded.get());
assertEquals("Table coprocessors loading default was not honored",
- tableCoprocessorLoaded.get(),
- CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED &&
- CoprocessorHost.DEFAULT_USER_COPROCESSORS_ENABLED);
+ CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED &&
+ CoprocessorHost.DEFAULT_USER_COPROCESSORS_ENABLED, tableCoprocessorLoaded.get());
}
@Test
@@ -123,8 +121,7 @@ public class TestCoprocessorConfiguration {
systemCoprocessorLoaded.set(false);
new RegionServerCoprocessorHost(rsServices, conf);
assertEquals("System coprocessors loading default was not honored",
- systemCoprocessorLoaded.get(),
- CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED);
+ CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED, systemCoprocessorLoaded.get());
}
@Test
@@ -134,8 +131,7 @@ public class TestCoprocessorConfiguration {
systemCoprocessorLoaded.set(false);
new MasterCoprocessorHost(masterServices, conf);
assertEquals("System coprocessors loading default was not honored",
- systemCoprocessorLoaded.get(),
- CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED);
+ CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED, systemCoprocessorLoaded.get());
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java
index 33a488f..de697f1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorHost.java
@@ -34,7 +34,7 @@ public class TestCoprocessorHost {
/**
* An {@link Abortable} implementation for tests.
*/
- private class TestAbortable implements Abortable {
+ private static class TestAbortable implements Abortable {
private volatile boolean aborted = false;
@Override
[02/50] [abbrv] hbase git commit: HBASE-15321 - Ability to open a
HRegion from hdfs snapshot.
Posted by zh...@apache.org.
HBASE-15321 - Ability to open a HRegion from hdfs snapshot.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0fa24ddd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0fa24ddd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0fa24ddd
Branch: refs/heads/HBASE-19397-branch-2
Commit: 0fa24dddb994ff1538691436a16ceb95383327cd
Parents: 6f8c312
Author: Rahul Gidwani <ch...@apache.org>
Authored: Mon Jan 22 12:13:13 2018 -0800
Committer: Rahul Gidwani <ch...@apache.org>
Committed: Mon Jan 22 12:13:13 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/HRegion.java | 27 ++++-
.../hbase/regionserver/HRegionFileSystem.java | 4 +-
.../regionserver/TestHdfsSnapshotHRegion.java | 117 +++++++++++++++++++
3 files changed, 145 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0fa24ddd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index aa9fa03..e18c80e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.regionserver;
import static org.apache.hadoop.hbase.HConstants.REPLICATION_SCOPE_LOCAL;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY;
import static org.apache.hadoop.hbase.util.CollectionUtils.computeIfAbsent;
-
import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -89,6 +88,7 @@ import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.NamespaceDescriptor;
@@ -7001,6 +7001,31 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
return this;
}
+ /**
+ * Open a Region on a read-only file-system (like hdfs snapshots)
+ * @param conf The Configuration object to use.
+ * @param fs Filesystem to use
+ * @param info Info for region to be opened.
+ * @param htd the table descriptor
+ * @return new HRegion
+ * @throws IOException e
+ */
+ public static HRegion openReadOnlyFileSystemHRegion(final Configuration conf, final FileSystem fs,
+ final Path tableDir, RegionInfo info, final TableDescriptor htd) throws IOException {
+ if (info == null) {
+ throw new NullPointerException("Passed region info is null");
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Opening region (readOnly filesystem): " + info);
+ }
+ if (info.getReplicaId() <= 0) {
+ info = new HRegionInfo((HRegionInfo) info, 1);
+ }
+ HRegion r = HRegion.newHRegion(tableDir, null, fs, conf, info, htd, null);
+ r.writestate.setReadOnly(true);
+ return r.openHRegion(null);
+ }
+
public static void warmupHRegion(final RegionInfo info,
final TableDescriptor htd, final WAL wal, final Configuration conf,
final RegionServerServices rsServices,
http://git-wip-us.apache.org/repos/asf/hbase/blob/0fa24ddd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
index 11833a5..00dc0d0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -27,7 +27,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -51,6 +50,7 @@ import org.apache.hadoop.hbase.util.FSHDFSUtils;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,7 +76,7 @@ public class HRegionFileSystem {
public static final String REGION_SPLITS_DIR = ".splits";
/** Temporary subdirectory of the region directory used for compaction output. */
- private static final String REGION_TEMP_DIR = ".tmp";
+ @VisibleForTesting static final String REGION_TEMP_DIR = ".tmp";
private final RegionInfo regionInfo;
//regionInfo for interacting with FS (getting encodedName, etc)
http://git-wip-us.apache.org/repos/asf/hbase/blob/0fa24ddd/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHdfsSnapshotHRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHdfsSnapshotHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHdfsSnapshotHRegion.java
new file mode 100644
index 0000000..17e698f
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHdfsSnapshotHRegion.java
@@ -0,0 +1,117 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({RegionServerTests.class, MediumTests.class})
+public class TestHdfsSnapshotHRegion {
+
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final String SNAPSHOT_NAME = "foo_snapshot";
+ private Table table;
+ public static final TableName TABLE_NAME = TableName.valueOf("foo");
+ public static final byte[] FAMILY = Bytes.toBytes("f1");
+ private DFSClient client;
+ private String baseDir;
+
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration c = TEST_UTIL.getConfiguration();
+ c.setBoolean("dfs.support.append", true);
+ TEST_UTIL.startMiniCluster(1);
+ table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAMILY);
+ TEST_UTIL.loadTable(table, FAMILY);
+
+ // setup the hdfssnapshots
+ client = new DFSClient(TEST_UTIL.getDFSCluster().getURI(), TEST_UTIL.getConfiguration());
+ String fullUrIPath = TEST_UTIL.getDefaultRootDirPath().toString();
+ String uriString = TEST_UTIL.getTestFileSystem().getUri().toString();
+ baseDir = StringUtils.removeStart(fullUrIPath, uriString);
+ client.allowSnapshot(baseDir);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ client.deleteSnapshot(baseDir, SNAPSHOT_NAME);
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testOpeningReadOnlyRegionBasic() throws Exception {
+ String snapshotDir = client.createSnapshot(baseDir, SNAPSHOT_NAME);
+ RegionInfo firstRegion = TEST_UTIL.getConnection().getRegionLocator(
+ table.getName()).getAllRegionLocations().stream().findFirst().get().getRegion();
+ Path tableDir = FSUtils.getTableDir(new Path(snapshotDir), TABLE_NAME);
+ HRegion snapshottedRegion = openSnapshotRegion(firstRegion, tableDir);
+ Assert.assertNotNull(snapshottedRegion);
+ snapshottedRegion.close();
+ }
+
+ @Test
+ public void testSnapshottingWithTmpSplitsAndMergeDirectoriesPresent() throws Exception {
+ // lets get a region and create those directories and make sure we ignore them
+ RegionInfo firstRegion = TEST_UTIL.getConnection().getRegionLocator(
+ table.getName()).getAllRegionLocations().stream().findFirst().get().getRegion();
+ String encodedName = firstRegion.getEncodedName();
+ Path tableDir = FSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), TABLE_NAME);
+ Path regionDirectoryPath = new Path(tableDir, encodedName);
+ TEST_UTIL.getTestFileSystem().create(
+ new Path(regionDirectoryPath, HRegionFileSystem.REGION_TEMP_DIR));
+ TEST_UTIL.getTestFileSystem().create(
+ new Path(regionDirectoryPath, HRegionFileSystem.REGION_SPLITS_DIR));
+ TEST_UTIL.getTestFileSystem().create(
+ new Path(regionDirectoryPath, HRegionFileSystem.REGION_MERGES_DIR));
+ // now snapshot
+ String snapshotDir = client.createSnapshot(baseDir, "foo_snapshot");
+ // everything should still open just fine
+ HRegion snapshottedRegion = openSnapshotRegion(firstRegion,
+ FSUtils.getTableDir(new Path(snapshotDir), TABLE_NAME));
+ Assert.assertNotNull(snapshottedRegion); // no errors and the region should open
+ snapshottedRegion.close();
+ }
+
+ private HRegion openSnapshotRegion(RegionInfo firstRegion, Path tableDir) throws IOException {
+ return HRegion.openReadOnlyFileSystemHRegion(
+ TEST_UTIL.getConfiguration(),
+ TEST_UTIL.getTestFileSystem(),
+ tableDir,
+ firstRegion,
+ table.getDescriptor()
+ );
+ }
+}
[31/50] [abbrv] hbase git commit: HBASE-19617 Remove
ReplicationQueues, use ReplicationQueueStorage directly
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
index b6cf54d..4b9ed74 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.List;
import java.util.UUID;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -32,9 +31,10 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WAL.Entry;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* Interface that defines a replication source
@@ -47,15 +47,10 @@ public interface ReplicationSourceInterface {
* @param conf the configuration to use
* @param fs the file system to use
* @param manager the manager to use
- * @param replicationQueues
- * @param replicationPeers
* @param server the server for this region server
- * @param peerClusterZnode
- * @param clusterId
- * @throws IOException
*/
void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueues replicationQueues, ReplicationPeers replicationPeers, Server server,
+ ReplicationQueueStorage queueStorage, ReplicationPeers replicationPeers, Server server,
String peerClusterZnode, UUID clusterId, ReplicationEndpoint replicationEndpoint,
WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 2f202fb..bf9cd30 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -34,19 +34,21 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
-
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
@@ -60,7 +62,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationTracker;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -68,6 +70,7 @@ import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -95,7 +98,7 @@ public class ReplicationSourceManager implements ReplicationListener {
private final List<ReplicationSourceInterface> sources;
// List of all the sources we got from died RSs
private final List<ReplicationSourceInterface> oldsources;
- private final ReplicationQueues replicationQueues;
+ private final ReplicationQueueStorage queueStorage;
private final ReplicationTracker replicationTracker;
private final ReplicationPeers replicationPeers;
// UUID for this cluster
@@ -130,7 +133,7 @@ public class ReplicationSourceManager implements ReplicationListener {
/**
* Creates a replication manager and sets the watch on all the other registered region servers
- * @param replicationQueues the interface for manipulating replication queues
+ * @param queueStorage the interface for manipulating replication queues
* @param replicationPeers
* @param replicationTracker
* @param conf the configuration to use
@@ -140,14 +143,14 @@ public class ReplicationSourceManager implements ReplicationListener {
* @param oldLogDir the directory where old logs are archived
* @param clusterId
*/
- public ReplicationSourceManager(ReplicationQueues replicationQueues,
+ public ReplicationSourceManager(ReplicationQueueStorage queueStorage,
ReplicationPeers replicationPeers, ReplicationTracker replicationTracker, Configuration conf,
Server server, FileSystem fs, Path logDir, Path oldLogDir, UUID clusterId,
WALFileLengthProvider walFileLengthProvider) throws IOException {
//CopyOnWriteArrayList is thread-safe.
//Generally, reading is more than modifying.
this.sources = new CopyOnWriteArrayList<>();
- this.replicationQueues = replicationQueues;
+ this.queueStorage = queueStorage;
this.replicationPeers = replicationPeers;
this.replicationTracker = replicationTracker;
this.server = server;
@@ -184,6 +187,19 @@ public class ReplicationSourceManager implements ReplicationListener {
connection = ConnectionFactory.createConnection(conf);
}
+ @FunctionalInterface
+ private interface ReplicationQueueOperation {
+ void exec() throws ReplicationException;
+ }
+
+ private void abortWhenFail(ReplicationQueueOperation op) {
+ try {
+ op.exec();
+ } catch (ReplicationException e) {
+ server.abort("Failed to operate on replication queue", e);
+ }
+ }
+
/**
* Provide the id of the peer and a log key and this method will figure which
* wal it belongs to and will log, for this region server, the current
@@ -195,12 +211,13 @@ public class ReplicationSourceManager implements ReplicationListener {
* @param queueRecovered indicates if this queue comes from another region server
* @param holdLogInZK if true then the log is retained in ZK
*/
- public void logPositionAndCleanOldLogs(Path log, String id, long position,
- boolean queueRecovered, boolean holdLogInZK) {
+ public void logPositionAndCleanOldLogs(Path log, String id, long position, boolean queueRecovered,
+ boolean holdLogInZK) {
String fileName = log.getName();
- this.replicationQueues.setLogPosition(id, fileName, position);
+ abortWhenFail(
+ () -> this.queueStorage.setWALPosition(server.getServerName(), id, fileName, position));
if (holdLogInZK) {
- return;
+ return;
}
cleanOldLogs(fileName, id, queueRecovered);
}
@@ -227,36 +244,59 @@ public class ReplicationSourceManager implements ReplicationListener {
}
}
}
- }
+ }
private void cleanOldLogs(SortedSet<String> wals, String key, String id) {
SortedSet<String> walSet = wals.headSet(key);
- LOG.debug("Removing " + walSet.size() + " logs in the list: " + walSet);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing " + walSet.size() + " logs in the list: " + walSet);
+ }
for (String wal : walSet) {
- this.replicationQueues.removeLog(id, wal);
+ abortWhenFail(() -> this.queueStorage.removeWAL(server.getServerName(), id, wal));
}
walSet.clear();
}
+ private void adoptAbandonedQueues() {
+ List<ServerName> currentReplicators = null;
+ try {
+ currentReplicators = queueStorage.getListOfReplicators();
+ } catch (ReplicationException e) {
+ server.abort("Failed to get all replicators", e);
+ return;
+ }
+ if (currentReplicators == null || currentReplicators.isEmpty()) {
+ return;
+ }
+ List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers().stream()
+ .map(ServerName::valueOf).collect(Collectors.toList());
+ LOG.info(
+ "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);
+
+ // Look if there's anything to process after a restart
+ for (ServerName rs : currentReplicators) {
+ if (!otherRegionServers.contains(rs)) {
+ transferQueues(rs);
+ }
+ }
+ }
+
/**
- * Adds a normal source per registered peer cluster and tries to process all
- * old region server wal queues
+ * Adds a normal source per registered peer cluster and tries to process all old region server wal
+ * queues
+ * <p>
+ * The returned future is for adoptAbandonedQueues task.
*/
- void init() throws IOException, ReplicationException {
+ Future<?> init() throws IOException, ReplicationException {
for (String id : this.replicationPeers.getConnectedPeerIds()) {
addSource(id);
if (replicationForBulkLoadDataEnabled) {
// Check if peer exists in hfile-refs queue, if not add it. This can happen in the case
// when a peer was added before replication for bulk loaded data was enabled.
- this.replicationQueues.addPeerToHFileRefs(id);
+ this.queueStorage.addPeerToHFileRefs(id);
}
}
- AdoptAbandonedQueuesWorker adoptionWorker = new AdoptAbandonedQueuesWorker();
- try {
- this.executor.execute(adoptionWorker);
- } catch (RejectedExecutionException ex) {
- LOG.info("Cancelling the adoption of abandoned queues because of " + ex.getMessage());
- }
+ return this.executor.submit(this::adoptAbandonedQueues);
}
/**
@@ -264,15 +304,12 @@ public class ReplicationSourceManager implements ReplicationListener {
* need to enqueue the latest log of each wal group and do replication
* @param id the id of the peer cluster
* @return the source that was created
- * @throws IOException
*/
@VisibleForTesting
ReplicationSourceInterface addSource(String id) throws IOException, ReplicationException {
ReplicationPeerConfig peerConfig = replicationPeers.getReplicationPeerConfig(id);
ReplicationPeer peer = replicationPeers.getConnectedPeer(id);
- ReplicationSourceInterface src = getReplicationSource(this.conf, this.fs, this,
- this.replicationQueues, this.replicationPeers, server, id, this.clusterId, peerConfig, peer,
- walFileLengthProvider);
+ ReplicationSourceInterface src = getReplicationSource(id, peerConfig, peer);
synchronized (this.walsById) {
this.sources.add(src);
Map<String, SortedSet<String>> walsByGroup = new HashMap<>();
@@ -287,11 +324,10 @@ public class ReplicationSourceManager implements ReplicationListener {
logs.add(name);
walsByGroup.put(walPrefix, logs);
try {
- this.replicationQueues.addLog(id, name);
+ this.queueStorage.addWAL(server.getServerName(), id, name);
} catch (ReplicationException e) {
- String message =
- "Cannot add log to queue when creating a new source, queueId=" + id
- + ", filename=" + name;
+ String message = "Cannot add log to queue when creating a new source, queueId=" + id +
+ ", filename=" + name;
server.stop(message);
throw e;
}
@@ -316,7 +352,7 @@ public class ReplicationSourceManager implements ReplicationListener {
* @param peerId Id of the peer cluster queue of wals to delete
*/
public void deleteSource(String peerId, boolean closeConnection) {
- this.replicationQueues.removeQueue(peerId);
+ abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), peerId));
if (closeConnection) {
this.replicationPeers.peerDisconnected(peerId);
}
@@ -376,8 +412,8 @@ public class ReplicationSourceManager implements ReplicationListener {
}
@VisibleForTesting
- List<String> getAllQueues() {
- return replicationQueues.getAllQueues();
+ List<String> getAllQueues() throws ReplicationException {
+ return queueStorage.getAllQueues(server.getServerName());
}
void preLogRoll(Path newLog) throws IOException {
@@ -411,10 +447,10 @@ public class ReplicationSourceManager implements ReplicationListener {
synchronized (replicationPeers) {
for (String id : replicationPeers.getConnectedPeerIds()) {
try {
- this.replicationQueues.addLog(id, logName);
+ this.queueStorage.addWAL(server.getServerName(), id, logName);
} catch (ReplicationException e) {
- throw new IOException("Cannot add log to replication queue"
- + " when creating a new source, queueId=" + id + ", filename=" + logName, e);
+ throw new IOException("Cannot add log to replication queue" +
+ " when creating a new source, queueId=" + id + ", filename=" + logName, e);
}
}
}
@@ -461,19 +497,11 @@ public class ReplicationSourceManager implements ReplicationListener {
/**
* Factory method to create a replication source
- * @param conf the configuration to use
- * @param fs the file system to use
- * @param manager the manager to use
- * @param server the server object for this region server
* @param peerId the id of the peer cluster
* @return the created source
- * @throws IOException
*/
- private ReplicationSourceInterface getReplicationSource(Configuration conf, FileSystem fs,
- ReplicationSourceManager manager, ReplicationQueues replicationQueues,
- ReplicationPeers replicationPeers, Server server, String peerId, UUID clusterId,
- ReplicationPeerConfig peerConfig, ReplicationPeer replicationPeer,
- WALFileLengthProvider walFileLengthProvider) throws IOException {
+ private ReplicationSourceInterface getReplicationSource(String peerId,
+ ReplicationPeerConfig peerConfig, ReplicationPeer replicationPeer) throws IOException {
RegionServerCoprocessorHost rsServerHost = null;
TableDescriptors tableDescriptors = null;
if (server instanceof HRegionServer) {
@@ -490,9 +518,8 @@ public class ReplicationSourceManager implements ReplicationListener {
// Default to HBase inter-cluster replication endpoint
replicationEndpointImpl = HBaseInterClusterReplicationEndpoint.class.getName();
}
- @SuppressWarnings("rawtypes")
- Class c = Class.forName(replicationEndpointImpl);
- replicationEndpoint = (ReplicationEndpoint) c.newInstance();
+ replicationEndpoint = Class.forName(replicationEndpointImpl)
+ .asSubclass(ReplicationEndpoint.class).newInstance();
if(rsServerHost != null) {
ReplicationEndpoint newReplicationEndPoint = rsServerHost
.postCreateReplicationEndPoint(replicationEndpoint);
@@ -509,7 +536,7 @@ public class ReplicationSourceManager implements ReplicationListener {
MetricsSource metrics = new MetricsSource(peerId);
// init replication source
- src.init(conf, fs, manager, replicationQueues, replicationPeers, server, peerId, clusterId,
+ src.init(conf, fs, this, queueStorage, replicationPeers, server, peerId, clusterId,
replicationEndpoint, walFileLengthProvider, metrics);
// init replication endpoint
@@ -520,21 +547,21 @@ public class ReplicationSourceManager implements ReplicationListener {
}
/**
- * Transfer all the queues of the specified to this region server.
- * First it tries to grab a lock and if it works it will move the
- * znodes and finally will delete the old znodes.
- *
+ * Transfer all the queues of the specified to this region server. First it tries to grab a lock
+ * and if it works it will move the znodes and finally will delete the old znodes.
+ * <p>
* It creates one old source for any type of source of the old rs.
- * @param rsZnode
*/
- private void transferQueues(String rsZnode) {
- NodeFailoverWorker transfer =
- new NodeFailoverWorker(rsZnode, this.replicationQueues, this.replicationPeers,
- this.clusterId);
+ private void transferQueues(ServerName deadRS) {
+ if (server.getServerName().equals(deadRS)) {
+ // it's just us, give up
+ return;
+ }
+ NodeFailoverWorker transfer = new NodeFailoverWorker(deadRS);
try {
this.executor.execute(transfer);
} catch (RejectedExecutionException ex) {
- LOG.info("Cancelling the transfer of " + rsZnode + " because of " + ex.getMessage());
+ LOG.info("Cancelling the transfer of " + deadRS + " because of " + ex.getMessage());
}
}
@@ -571,7 +598,7 @@ public class ReplicationSourceManager implements ReplicationListener {
LOG.info("Peer " + id + " connected success, trying to start the replication source thread.");
addSource(id);
if (replicationForBulkLoadDataEnabled) {
- this.replicationQueues.addPeerToHFileRefs(id);
+ this.queueStorage.addPeerToHFileRefs(id);
}
}
}
@@ -624,12 +651,12 @@ public class ReplicationSourceManager implements ReplicationListener {
deleteSource(id, true);
}
// Remove HFile Refs znode from zookeeper
- this.replicationQueues.removePeerFromHFileRefs(id);
+ abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(id));
}
@Override
public void regionServerRemoved(String regionserver) {
- transferQueues(regionserver);
+ transferQueues(ServerName.valueOf(regionserver));
}
/**
@@ -638,37 +665,21 @@ public class ReplicationSourceManager implements ReplicationListener {
*/
class NodeFailoverWorker extends Thread {
- private String rsZnode;
- private final ReplicationQueues rq;
- private final ReplicationPeers rp;
- private final UUID clusterId;
+ private final ServerName deadRS;
- /**
- * @param rsZnode
- */
- public NodeFailoverWorker(String rsZnode) {
- this(rsZnode, replicationQueues, replicationPeers, ReplicationSourceManager.this.clusterId);
- }
-
- public NodeFailoverWorker(String rsZnode, final ReplicationQueues replicationQueues,
- final ReplicationPeers replicationPeers, final UUID clusterId) {
- super("Failover-for-"+rsZnode);
- this.rsZnode = rsZnode;
- this.rq = replicationQueues;
- this.rp = replicationPeers;
- this.clusterId = clusterId;
+ @VisibleForTesting
+ public NodeFailoverWorker(ServerName deadRS) {
+ super("Failover-for-" + deadRS);
+ this.deadRS = deadRS;
}
@Override
public void run() {
- if (this.rq.isThisOurRegionServer(rsZnode)) {
- return;
- }
// Wait a bit before transferring the queues, we may be shutting down.
// This sleep may not be enough in some cases.
try {
Thread.sleep(sleepBeforeFailover +
- (long) (ThreadLocalRandom.current().nextFloat() * sleepBeforeFailover));
+ (long) (ThreadLocalRandom.current().nextFloat() * sleepBeforeFailover));
} catch (InterruptedException e) {
LOG.warn("Interrupted while waiting before transferring a queue.");
Thread.currentThread().interrupt();
@@ -679,25 +690,30 @@ public class ReplicationSourceManager implements ReplicationListener {
return;
}
Map<String, Set<String>> newQueues = new HashMap<>();
- List<String> peers = rq.getUnClaimedQueueIds(rsZnode);
- while (peers != null && !peers.isEmpty()) {
- Pair<String, SortedSet<String>> peer = this.rq.claimQueue(rsZnode,
- peers.get(ThreadLocalRandom.current().nextInt(peers.size())));
- long sleep = sleepBeforeFailover/2;
- if (peer != null) {
- newQueues.put(peer.getFirst(), peer.getSecond());
- sleep = sleepBeforeFailover;
+ try {
+ List<String> peers = queueStorage.getAllQueues(deadRS);
+ while (!peers.isEmpty()) {
+ Pair<String, SortedSet<String>> peer = queueStorage.claimQueue(deadRS,
+ peers.get(ThreadLocalRandom.current().nextInt(peers.size())), server.getServerName());
+ long sleep = sleepBeforeFailover / 2;
+ if (!peer.getSecond().isEmpty()) {
+ newQueues.put(peer.getFirst(), peer.getSecond());
+ sleep = sleepBeforeFailover;
+ }
+ try {
+ Thread.sleep(sleep);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting before transferring a queue.");
+ Thread.currentThread().interrupt();
+ }
+ peers = queueStorage.getAllQueues(deadRS);
}
- try {
- Thread.sleep(sleep);
- } catch (InterruptedException e) {
- LOG.warn("Interrupted while waiting before transferring a queue.");
- Thread.currentThread().interrupt();
+ if (!peers.isEmpty()) {
+ queueStorage.removeReplicatorIfQueueIsEmpty(deadRS);
}
- peers = rq.getUnClaimedQueueIds(rsZnode);
- }
- if (peers != null) {
- rq.removeReplicatorIfQueueIsEmpty(rsZnode);
+ } catch (ReplicationException e) {
+ server.abort("Failed to claim queue from dead regionserver", e);
+ return;
}
// Copying over the failed queue is completed.
if (newQueues.isEmpty()) {
@@ -722,8 +738,8 @@ public class ReplicationSourceManager implements ReplicationListener {
+ ex);
}
if (peer == null || peerConfig == null) {
- LOG.warn("Skipping failover for peer:" + actualPeerId + " of node" + rsZnode);
- replicationQueues.removeQueue(peerId);
+ LOG.warn("Skipping failover for peer:" + actualPeerId + " of node " + deadRS);
+ abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), peerId));
continue;
}
// track sources in walsByIdRecoveredQueues
@@ -740,13 +756,11 @@ public class ReplicationSourceManager implements ReplicationListener {
}
// enqueue sources
- ReplicationSourceInterface src =
- getReplicationSource(conf, fs, ReplicationSourceManager.this, this.rq, this.rp,
- server, peerId, this.clusterId, peerConfig, peer, walFileLengthProvider);
+ ReplicationSourceInterface src = getReplicationSource(peerId, peerConfig, peer);
// synchronized on oldsources to avoid adding recovered source for the to-be-removed peer
// see removePeer
synchronized (oldsources) {
- if (!this.rp.getConnectedPeerIds().contains(src.getPeerId())) {
+ if (!replicationPeers.getConnectedPeerIds().contains(src.getPeerId())) {
src.terminate("Recovered queue doesn't belong to any current peer");
closeRecoveredQueue(src);
continue;
@@ -765,29 +779,6 @@ public class ReplicationSourceManager implements ReplicationListener {
}
}
- class AdoptAbandonedQueuesWorker extends Thread{
-
- public AdoptAbandonedQueuesWorker() {}
-
- @Override
- public void run() {
- List<String> currentReplicators = replicationQueues.getListOfReplicators();
- if (currentReplicators == null || currentReplicators.isEmpty()) {
- return;
- }
- List<String> otherRegionServers = replicationTracker.getListOfRegionServers();
- LOG.info("Current list of replicators: " + currentReplicators + " other RSs: "
- + otherRegionServers);
-
- // Look if there's anything to process after a restart
- for (String rs : currentReplicators) {
- if (!otherRegionServers.contains(rs)) {
- transferQueues(rs);
- }
- }
- }
- }
-
/**
* Get the directory where wals are archived
* @return the directory where wals are archived
@@ -846,7 +837,11 @@ public class ReplicationSourceManager implements ReplicationListener {
}
public void cleanUpHFileRefs(String peerId, List<String> files) {
- this.replicationQueues.removeHFileRefs(peerId, files);
+ abortWhenFail(() -> this.queueStorage.removeHFileRefs(peerId, files));
+ }
+
+ int activeFailoverTaskCount() {
+ return executor.getActiveCount();
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index 21b8ac5..9ec244a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -37,22 +36,19 @@ import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
- * In a scenario of Replication based Disaster/Recovery, when hbase
- * Master-Cluster crashes, this tool is used to sync-up the delta from Master to
- * Slave using the info from ZooKeeper. The tool will run on Master-Cluser, and
- * assume ZK, Filesystem and NetWork still available after hbase crashes
+ * In a scenario of Replication based Disaster/Recovery, when hbase Master-Cluster crashes, this
+ * tool is used to sync-up the delta from Master to Slave using the info from ZooKeeper. The tool
+ * will run on Master-Cluser, and assume ZK, Filesystem and NetWork still available after hbase
+ * crashes
*
+ * <pre>
* hbase org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp
+ * </pre>
*/
-
public class ReplicationSyncUp extends Configured implements Tool {
- private static final Logger LOG = LoggerFactory.getLogger(ReplicationSyncUp.class.getName());
-
private static Configuration conf;
private static final long SLEEP_TIME = 10000;
@@ -105,13 +101,14 @@ public class ReplicationSyncUp extends Configured implements Tool {
System.out.println("Start Replication Server start");
replication = new Replication(new DummyServer(zkw), fs, logDir, oldLogDir);
manager = replication.getReplicationManager();
- manager.init();
+ manager.init().get();
try {
- int numberOfOldSource = 1; // default wait once
- while (numberOfOldSource > 0) {
+ while (manager.activeFailoverTaskCount() > 0) {
+ Thread.sleep(SLEEP_TIME);
+ }
+ while (manager.getOldSources().size() > 0) {
Thread.sleep(SLEEP_TIME);
- numberOfOldSource = manager.getOldSources().size();
}
} catch (InterruptedException e) {
System.err.println("didn't wait long enough:" + e);
@@ -121,7 +118,7 @@ public class ReplicationSyncUp extends Configured implements Tool {
manager.join();
zkw.close();
- return (0);
+ return 0;
}
static class DummyServer implements Server {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index 7e6de5f..4894b31 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -44,9 +44,8 @@ import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
import org.apache.hadoop.hbase.replication.regionserver.Replication;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -111,9 +110,8 @@ public class TestLogsCleaner {
Replication.decorateMasterConfiguration(conf);
Server server = new DummyServer();
- ReplicationQueues repQueues = ReplicationFactory.getReplicationQueues(
- new ReplicationQueuesArguments(conf, server, server.getZooKeeper()));
- repQueues.init(server.getServerName().toString());
+ ReplicationQueueStorage queueStorage =
+ ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
final Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HREGION_OLDLOGDIR_NAME);
final Path oldProcedureWALDir = new Path(oldLogDir, "masterProcedureWALs");
String fakeMachineName = URLEncoder.encode(server.getServerName().toString(), "UTF8");
@@ -144,7 +142,7 @@ public class TestLogsCleaner {
// Case 4: put 3 WALs in ZK indicating that they are scheduled for replication so these
// files would pass TimeToLiveLogCleaner but would be rejected by ReplicationLogCleaner
if (i % (30 / 3) == 1) {
- repQueues.addLog(fakeMachineName, fileName.getName());
+ queueStorage.addWAL(server.getServerName(), fakeMachineName, fileName.getName());
LOG.info("Replication log file: " + fileName);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
index ca59094..201055e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
@@ -46,9 +46,8 @@ import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
import org.apache.hadoop.hbase.replication.regionserver.Replication;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -72,19 +71,16 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
@Category({ MasterTests.class, SmallTests.class })
public class TestReplicationHFileCleaner {
- private static final Logger LOG = LoggerFactory.getLogger(ReplicationQueuesZKImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestReplicationHFileCleaner.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static Server server;
- private static ReplicationQueues rq;
+ private static ReplicationQueueStorage rq;
private static ReplicationPeers rp;
private static final String peerId = "TestReplicationHFileCleaner";
private static Configuration conf = TEST_UTIL.getConfiguration();
static FileSystem fs = null;
Path root;
- /**
- * @throws java.lang.Exception
- */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniZKCluster();
@@ -93,20 +89,10 @@ public class TestReplicationHFileCleaner {
Replication.decorateMasterConfiguration(conf);
rp = ReplicationFactory.getReplicationPeers(server.getZooKeeper(), conf, server);
rp.init();
- rq = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, server, server.getZooKeeper()));
- rq.init(server.getServerName().toString());
- try {
- fs = FileSystem.get(conf);
- } finally {
- if (fs != null) {
- fs.close();
- }
- }
+ rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
+ fs = FileSystem.get(conf);
}
- /**
- * @throws java.lang.Exception
- */
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniZKCluster();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
index 8178266..2ad8bd7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationZKNodeCleaner.java
@@ -26,10 +26,8 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -49,14 +47,12 @@ public class TestReplicationZKNodeCleaner {
private final Configuration conf;
private final ZKWatcher zkw;
- private final ReplicationQueues repQueues;
+ private final ReplicationQueueStorage repQueues;
public TestReplicationZKNodeCleaner() throws Exception {
conf = TEST_UTIL.getConfiguration();
zkw = new ZKWatcher(conf, "TestReplicationZKNodeCleaner", null);
- repQueues = ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, null,
- zkw));
- assertTrue(repQueues instanceof ReplicationQueuesZKImpl);
+ repQueues = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
}
@BeforeClass
@@ -72,9 +68,8 @@ public class TestReplicationZKNodeCleaner {
@Test
public void testReplicationZKNodeCleaner() throws Exception {
- repQueues.init(SERVER_ONE.getServerName());
// add queue for ID_ONE which isn't exist
- repQueues.addLog(ID_ONE, "file1");
+ repQueues.addWAL(SERVER_ONE, ID_ONE, "file1");
ReplicationZKNodeCleaner cleaner = new ReplicationZKNodeCleaner(conf, zkw, null);
Map<ServerName, List<String>> undeletedQueues = cleaner.getUnDeletedQueues();
@@ -84,7 +79,7 @@ public class TestReplicationZKNodeCleaner {
assertTrue(undeletedQueues.get(SERVER_ONE).contains(ID_ONE));
// add a recovery queue for ID_TWO which isn't exist
- repQueues.addLog(ID_TWO + "-" + SERVER_TWO, "file2");
+ repQueues.addWAL(SERVER_ONE, ID_TWO + "-" + SERVER_TWO, "file2");
undeletedQueues = cleaner.getUnDeletedQueues();
assertEquals(1, undeletedQueues.size());
@@ -100,11 +95,10 @@ public class TestReplicationZKNodeCleaner {
@Test
public void testReplicationZKNodeCleanerChore() throws Exception {
- repQueues.init(SERVER_ONE.getServerName());
// add queue for ID_ONE which isn't exist
- repQueues.addLog(ID_ONE, "file1");
+ repQueues.addWAL(SERVER_ONE, ID_ONE, "file1");
// add a recovery queue for ID_TWO which isn't exist
- repQueues.addLog(ID_TWO + "-" + SERVER_TWO, "file2");
+ repQueues.addWAL(SERVER_ONE, ID_TWO + "-" + SERVER_TWO, "file2");
// Wait the cleaner chore to run
Thread.sleep(20000);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
index 7ea79f9..14c5e56 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -46,9 +45,10 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
MetricsSource metrics;
WALFileLengthProvider walFileLengthProvider;
AtomicBoolean startup = new AtomicBoolean(false);
+
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueues rq, ReplicationPeers rp, Server server, String peerClusterId,
+ ReplicationQueueStorage rq, ReplicationPeers rp, Server server, String peerClusterId,
UUID clusterId, ReplicationEndpoint replicationEndpoint,
WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException {
this.manager = manager;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
index b473277..8661dd8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -64,7 +64,6 @@ public class TestReplicationSyncUpTool extends TestReplicationBase {
@Before
public void setUp() throws Exception {
-
HColumnDescriptor fam;
t1_syncupSource = new HTableDescriptor(t1_su);
@@ -100,7 +99,7 @@ public class TestReplicationSyncUpTool extends TestReplicationBase {
* check's gone Also check the puts and deletes are not replicated back to
* the originating cluster.
*/
- @Test(timeout = 300000)
+ @Test
public void testSyncUpTool() throws Exception {
/**
@@ -176,7 +175,6 @@ public class TestReplicationSyncUpTool extends TestReplicationBase {
* verify correctly replicated to Slave
*/
mimicSyncUpAfterPut();
-
}
protected void setupReplication() throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index b5a7461..325012d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -40,7 +40,6 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -67,10 +66,10 @@ import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.NodeFailoverWorker;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
@@ -95,11 +94,13 @@ import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
-import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
/**
* An abstract class that tests ReplicationSourceManager. Classes that extend this class should
* set up the proper config for this class and initialize the proper cluster using
@@ -328,18 +329,14 @@ public abstract class TestReplicationSourceManager {
@Test
public void testClaimQueues() throws Exception {
- final Server server = new DummyServer("hostname0.example.org");
-
-
- ReplicationQueues rq =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(server.getConfiguration(), server,
- server.getZooKeeper()));
- rq.init(server.getServerName().toString());
+ Server server = new DummyServer("hostname0.example.org");
+ ReplicationQueueStorage rq = ReplicationStorageFactory
+ .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
// populate some znodes in the peer znode
files.add("log1");
files.add("log2");
for (String file : files) {
- rq.addLog("1", file);
+ rq.addWAL(server.getServerName(), "1", file);
}
// create 3 DummyServers
Server s1 = new DummyServer("dummyserver1.example.org");
@@ -347,12 +344,9 @@ public abstract class TestReplicationSourceManager {
Server s3 = new DummyServer("dummyserver3.example.org");
// create 3 DummyNodeFailoverWorkers
- DummyNodeFailoverWorker w1 = new DummyNodeFailoverWorker(
- server.getServerName().getServerName(), s1);
- DummyNodeFailoverWorker w2 = new DummyNodeFailoverWorker(
- server.getServerName().getServerName(), s2);
- DummyNodeFailoverWorker w3 = new DummyNodeFailoverWorker(
- server.getServerName().getServerName(), s3);
+ DummyNodeFailoverWorker w1 = new DummyNodeFailoverWorker(server.getServerName(), s1);
+ DummyNodeFailoverWorker w2 = new DummyNodeFailoverWorker(server.getServerName(), s2);
+ DummyNodeFailoverWorker w3 = new DummyNodeFailoverWorker(server.getServerName(), s3);
latch = new CountDownLatch(3);
// start the threads
@@ -371,11 +365,9 @@ public abstract class TestReplicationSourceManager {
@Test
public void testCleanupFailoverQueues() throws Exception {
- final Server server = new DummyServer("hostname1.example.org");
- ReplicationQueues rq =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(server.getConfiguration(), server,
- server.getZooKeeper()));
- rq.init(server.getServerName().toString());
+ Server server = new DummyServer("hostname1.example.org");
+ ReplicationQueueStorage rq = ReplicationStorageFactory
+ .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
// populate some znodes in the peer znode
SortedSet<String> files = new TreeSet<>();
String group = "testgroup";
@@ -384,19 +376,14 @@ public abstract class TestReplicationSourceManager {
files.add(file1);
files.add(file2);
for (String file : files) {
- rq.addLog("1", file);
+ rq.addWAL(server.getServerName(), "1", file);
}
Server s1 = new DummyServer("dummyserver1.example.org");
- ReplicationQueues rq1 =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s1.getConfiguration(), s1,
- s1.getZooKeeper()));
- rq1.init(s1.getServerName().toString());
ReplicationPeers rp1 =
ReplicationFactory.getReplicationPeers(s1.getZooKeeper(), s1.getConfiguration(), s1);
rp1.init();
NodeFailoverWorker w1 =
- manager.new NodeFailoverWorker(server.getServerName().getServerName(), rq1, rp1, new UUID(
- new Long(1), new Long(2)));
+ manager.new NodeFailoverWorker(server.getServerName());
w1.run();
assertEquals(1, manager.getWalsByIdRecoveredQueues().size());
String id = "1-" + server.getServerName().getServerName();
@@ -408,17 +395,16 @@ public abstract class TestReplicationSourceManager {
@Test
public void testCleanupUnknownPeerZNode() throws Exception {
- final Server server = new DummyServer("hostname2.example.org");
- ReplicationQueues rq = ReplicationFactory.getReplicationQueues(
- new ReplicationQueuesArguments(server.getConfiguration(), server, server.getZooKeeper()));
- rq.init(server.getServerName().toString());
+ Server server = new DummyServer("hostname2.example.org");
+ ReplicationQueueStorage rq = ReplicationStorageFactory
+ .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
// populate some znodes in the peer znode
// add log to an unknown peer
String group = "testgroup";
- rq.addLog("2", group + ".log1");
- rq.addLog("2", group + ".log2");
+ rq.addWAL(server.getServerName(), "2", group + ".log1");
+ rq.addWAL(server.getServerName(), "2", group + ".log2");
- NodeFailoverWorker w1 = manager.new NodeFailoverWorker(server.getServerName().getServerName());
+ NodeFailoverWorker w1 = manager.new NodeFailoverWorker(server.getServerName());
w1.run();
// The log of the unknown peer should be removed from zk
@@ -481,10 +467,8 @@ public abstract class TestReplicationSourceManager {
.setClusterKey("localhost:" + utility.getZkCluster().getClientPort() + ":/hbase");
try {
DummyServer server = new DummyServer();
- final ReplicationQueues rq =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(
- server.getConfiguration(), server, server.getZooKeeper()));
- rq.init(server.getServerName().toString());
+ ReplicationQueueStorage rq = ReplicationStorageFactory
+ .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
// Purposely fail ReplicationSourceManager.addSource() by causing ReplicationSourceInterface
// initialization to throw an exception.
conf.set("replication.replicationsource.implementation",
@@ -498,11 +482,11 @@ public abstract class TestReplicationSourceManager {
assertNull(manager.getSource(peerId));
// Create a replication queue for the fake peer
- rq.addLog(peerId, "FakeFile");
+ rq.addWAL(server.getServerName(), peerId, "FakeFile");
// Unregister peer, this should remove the peer and clear all queues associated with it
// Need to wait for the ReplicationTracker to pick up the changes and notify listeners.
removePeerAndWait(peerId);
- assertFalse(rq.getAllQueues().contains(peerId));
+ assertFalse(rq.getAllQueues(server.getServerName()).contains(peerId));
} finally {
conf.set("replication.replicationsource.implementation", replicationSourceImplName);
removePeerAndWait(peerId);
@@ -625,11 +609,12 @@ public abstract class TestReplicationSourceManager {
}
}
Waiter.waitFor(conf, 20000, new Waiter.Predicate<Exception>() {
- @Override public boolean evaluate() throws Exception {
+ @Override
+ public boolean evaluate() throws Exception {
List<String> peers = rp.getAllPeerIds();
- return (!manager.getAllQueues().contains(peerId)) && (rp.getConnectedPeer(peerId) == null)
- && (!peers.contains(peerId))
- && manager.getSource(peerId) == null;
+ return (!manager.getAllQueues().contains(peerId)) &&
+ (rp.getConnectedPeer(peerId) == null) && (!peers.contains(peerId)) &&
+ manager.getSource(peerId) == null;
}
});
}
@@ -672,25 +657,24 @@ public abstract class TestReplicationSourceManager {
static class DummyNodeFailoverWorker extends Thread {
private Map<String, Set<String>> logZnodesMap;
Server server;
- private String deadRsZnode;
- ReplicationQueues rq;
+ private ServerName deadRS;
+ ReplicationQueueStorage rq;
- public DummyNodeFailoverWorker(String znode, Server s) throws Exception {
- this.deadRsZnode = znode;
+ public DummyNodeFailoverWorker(ServerName deadRS, Server s) throws Exception {
+ this.deadRS = deadRS;
this.server = s;
- this.rq =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(server.getConfiguration(), server,
- server.getZooKeeper()));
- this.rq.init(this.server.getServerName().toString());
+ this.rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(),
+ server.getConfiguration());
}
@Override
public void run() {
try {
logZnodesMap = new HashMap<>();
- List<String> queues = rq.getUnClaimedQueueIds(deadRsZnode);
- for(String queue:queues){
- Pair<String, SortedSet<String>> pair = rq.claimQueue(deadRsZnode, queue);
+ List<String> queues = rq.getAllQueues(deadRS);
+ for (String queue : queues) {
+ Pair<String, SortedSet<String>> pair =
+ rq.claimQueue(deadRS, queue, server.getServerName());
if (pair != null) {
logZnodesMap.put(pair.getFirst(), pair.getSecond());
}
@@ -729,7 +713,7 @@ public abstract class TestReplicationSourceManager {
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueues rq, ReplicationPeers rp, Server server, String peerClusterId,
+ ReplicationQueueStorage rq, ReplicationPeers rp, Server server, String peerClusterId,
UUID clusterId, ReplicationEndpoint replicationEndpoint,
WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException {
throw new IOException("Failing deliberately");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b3b92efa/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
index aeab8b0..c6d9eef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
@@ -25,11 +25,10 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueues;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.junit.BeforeClass;
@@ -41,8 +40,9 @@ import org.junit.experimental.categories.Category;
* ReplicationQueuesClientZkImpl. Also includes extra tests outside of those in
* TestReplicationSourceManager that test ReplicationQueueZkImpl-specific behaviors.
*/
-@Category({ReplicationTests.class, MediumTests.class})
+@Category({ ReplicationTests.class, MediumTests.class })
public class TestReplicationSourceManagerZkImpl extends TestReplicationSourceManager {
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf = HBaseConfiguration.create();
@@ -58,16 +58,14 @@ public class TestReplicationSourceManagerZkImpl extends TestReplicationSourceMan
// Tests the naming convention of adopted queues for ReplicationQueuesZkImpl
@Test
public void testNodeFailoverDeadServerParsing() throws Exception {
- final Server server = new DummyServer("ec2-54-234-230-108.compute-1.amazonaws.com");
- ReplicationQueues repQueues =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(conf, server,
- server.getZooKeeper()));
- repQueues.init(server.getServerName().toString());
+ Server server = new DummyServer("ec2-54-234-230-108.compute-1.amazonaws.com");
+ ReplicationQueueStorage queueStorage =
+ ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
// populate some znodes in the peer znode
files.add("log1");
files.add("log2");
for (String file : files) {
- repQueues.addLog("1", file);
+ queueStorage.addWAL(server.getServerName(), "1", file);
}
// create 3 DummyServers
@@ -76,30 +74,22 @@ public class TestReplicationSourceManagerZkImpl extends TestReplicationSourceMan
Server s3 = new DummyServer("ec2-23-20-187-167.compute-1.amazonaws.com");
// simulate three servers fail sequentially
- ReplicationQueues rq1 =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s1.getConfiguration(), s1,
- s1.getZooKeeper()));
- rq1.init(s1.getServerName().toString());
- String serverName = server.getServerName().getServerName();
- List<String> unclaimed = rq1.getUnClaimedQueueIds(serverName);
- rq1.claimQueue(serverName, unclaimed.get(0)).getSecond();
- rq1.removeReplicatorIfQueueIsEmpty(unclaimed.get(0));
- ReplicationQueues rq2 =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s2.getConfiguration(), s2,
- s2.getZooKeeper()));
- rq2.init(s2.getServerName().toString());
- serverName = s1.getServerName().getServerName();
- unclaimed = rq2.getUnClaimedQueueIds(serverName);
- rq2.claimQueue(serverName, unclaimed.get(0)).getSecond();
- rq2.removeReplicatorIfQueueIsEmpty(unclaimed.get(0));
- ReplicationQueues rq3 =
- ReplicationFactory.getReplicationQueues(new ReplicationQueuesArguments(s3.getConfiguration(), s3,
- s3.getZooKeeper()));
- rq3.init(s3.getServerName().toString());
- serverName = s2.getServerName().getServerName();
- unclaimed = rq3.getUnClaimedQueueIds(serverName);
- String queue3 = rq3.claimQueue(serverName, unclaimed.get(0)).getFirst();
- rq3.removeReplicatorIfQueueIsEmpty(unclaimed.get(0));
+ ServerName serverName = server.getServerName();
+ List<String> unclaimed = queueStorage.getAllQueues(serverName);
+ queueStorage.claimQueue(serverName, unclaimed.get(0), s1.getServerName());
+ queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
+
+ serverName = s1.getServerName();
+ unclaimed = queueStorage.getAllQueues(serverName);
+ queueStorage.claimQueue(serverName, unclaimed.get(0), s2.getServerName());
+ queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
+
+ serverName = s2.getServerName();
+ unclaimed = queueStorage.getAllQueues(serverName);
+ String queue3 =
+ queueStorage.claimQueue(serverName, unclaimed.get(0), s3.getServerName()).getFirst();
+ queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
+
ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queue3);
List<ServerName> result = replicationQueueInfo.getDeadRegionServers();
// verify
[11/50] [abbrv] hbase git commit: HBASE-19811 Fix findbugs and
error-prone warnings in hbase-server (branch-2)
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 65c4d08..ab282d5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -117,6 +117,7 @@ public class RSProcedureDispatcher
}
}
+ @Override
protected void abortPendingOperations(final ServerName serverName,
final Set<RemoteProcedure> operations) {
// TODO: Replace with a ServerNotOnlineException()
@@ -126,10 +127,12 @@ public class RSProcedureDispatcher
}
}
+ @Override
public void serverAdded(final ServerName serverName) {
addNode(serverName);
}
+ @Override
public void serverRemoved(final ServerName serverName) {
removeNode(serverName);
}
@@ -138,6 +141,7 @@ public class RSProcedureDispatcher
* Base remote call
*/
protected abstract class AbstractRSRemoteCall implements Callable<Void> {
+ @Override
public abstract Void call();
private final ServerName serverName;
@@ -269,6 +273,7 @@ public class RSProcedureDispatcher
this.remoteProcedures = remoteProcedures;
}
+ @Override
public Void call() {
request = ExecuteProceduresRequest.newBuilder();
if (LOG.isTraceEnabled()) {
@@ -290,11 +295,13 @@ public class RSProcedureDispatcher
return null;
}
+ @Override
public void dispatchOpenRequests(final MasterProcedureEnv env,
final List<RegionOpenOperation> operations) {
request.addOpenRegion(buildOpenRegionRequest(env, getServerName(), operations));
}
+ @Override
public void dispatchCloseRequests(final MasterProcedureEnv env,
final List<RegionCloseOperation> operations) {
for (RegionCloseOperation op: operations) {
@@ -471,11 +478,13 @@ public class RSProcedureDispatcher
return null;
}
+ @Override
public void dispatchOpenRequests(final MasterProcedureEnv env,
final List<RegionOpenOperation> operations) {
submitTask(new OpenRegionRemoteCall(serverName, operations));
}
+ @Override
public void dispatchCloseRequests(final MasterProcedureEnv env,
final List<RegionCloseOperation> operations) {
for (RegionCloseOperation op: operations) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
index a8475f0..559863e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
@@ -86,6 +86,7 @@ public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
return false;
}
+ @Override
public void setConf(final Configuration conf) {
super.setConf(conf);
try {
@@ -95,6 +96,7 @@ public class SnapshotHFileCleaner extends BaseHFileCleanerDelegate {
Path rootDir = FSUtils.getRootDir(conf);
cache = new SnapshotFileCache(fs, rootDir, cacheRefreshPeriod, cacheRefreshPeriod,
"snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
+ @Override
public Collection<String> filesUnderSnapshot(final Path snapshotDir)
throws IOException {
return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java
index 397570c..7436d9c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java
@@ -54,6 +54,7 @@ public class CachedMobFile extends MobFile implements Comparable<CachedMobFile>
this.accessCount = accessCount;
}
+ @Override
public int compareTo(CachedMobFile that) {
if (this.accessCount == that.accessCount) return 0;
return this.accessCount < that.accessCount ? 1 : -1;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java
index 053cba6..120f11e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java
@@ -86,6 +86,7 @@ public class ExpiredMobFileCleaner extends Configured implements Tool {
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="REC_CATCH_EXCEPTION",
justification="Intentional")
+ @Override
public int run(String[] args) throws Exception {
if (args.length != 2) {
printUsage();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
index aaf545b..ab917a2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
@@ -294,6 +294,7 @@ public class PartitionedMobCompactionRequest extends MobCompactionRequest {
this.endKey = endKey;
}
+ @Override
public int compareTo(CompactionDelPartitionId o) {
/*
* 1). Compare the start key, if the k1 < k2, then k1 is less
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java
index 6b90e6b..1b6ad91 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredRPCHandlerImpl.java
@@ -74,6 +74,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* monitored Handler.
* @return the queue timestamp or -1 if there is no RPC currently running.
*/
+ @Override
public long getRPCQueueTime() {
if (getState() != State.RUNNING) {
return -1;
@@ -86,6 +87,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* monitored Handler.
* @return the start timestamp or -1 if there is no RPC currently running.
*/
+ @Override
public long getRPCStartTime() {
if (getState() != State.RUNNING) {
return -1;
@@ -98,6 +100,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* by this Handler.
* @return a string representing the method call without parameters
*/
+ @Override
public synchronized String getRPC() {
return getRPC(false);
}
@@ -108,6 +111,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* @param withParams toggle inclusion of parameters in the RPC String
* @return A human-readable string representation of the method call.
*/
+ @Override
public synchronized String getRPC(boolean withParams) {
if (getState() != State.RUNNING) {
// no RPC is currently running
@@ -132,6 +136,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* by this Handler.
* @return A human-readable string representation of the method call.
*/
+ @Override
public long getRPCPacketLength() {
if (getState() != State.RUNNING || packet == null) {
// no RPC is currently running, or we don't have an RPC's packet info
@@ -146,6 +151,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* @return A human-readable string representation of the address and port
* of the client.
*/
+ @Override
public String getClient() {
return clientAddress + ":" + remotePort;
}
@@ -155,6 +161,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* RPC call.
* @return true if the monitored handler is currently servicing an RPC call.
*/
+ @Override
public boolean isRPCRunning() {
return getState() == State.RUNNING;
}
@@ -166,6 +173,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* @return true if the monitored handler is currently servicing an RPC call
* to a database command.
*/
+ @Override
public synchronized boolean isOperationRunning() {
if(!isRPCRunning()) {
return false;
@@ -183,6 +191,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* @param methodName The name of the method that will be called by the RPC.
* @param params The parameters that will be passed to the indicated method.
*/
+ @Override
public synchronized void setRPC(String methodName, Object [] params,
long queueTime) {
this.methodName = methodName;
@@ -197,6 +206,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* that it can later compute its size if asked for it.
* @param param The protobuf received by the RPC for this call
*/
+ @Override
public void setRPCPacket(Message param) {
this.packet = param;
}
@@ -206,6 +216,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
* @param clientAddress the address of the current client
* @param remotePort the port from which the client connected
*/
+ @Override
public void setConnection(String clientAddress, int remotePort) {
this.clientAddress = clientAddress;
this.remotePort = remotePort;
@@ -218,6 +229,7 @@ public class MonitoredRPCHandlerImpl extends MonitoredTaskImpl
this.packet = null;
}
+ @Override
public synchronized Map<String, Object> toMap() {
// only include RPC info if the Handler is actively servicing an RPC call
Map<String, Object> map = super.toMap();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
index b3869f4..bedb5e2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/MonitoredTaskImpl.java
@@ -154,6 +154,7 @@ class MonitoredTaskImpl implements MonitoredTask {
* Force the completion timestamp backwards so that
* it expires now.
*/
+ @Override
public void expireNow() {
stateTime -= 180 * 1000;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
index 6749d2f..4aff779 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java
@@ -319,7 +319,7 @@ public class TaskMonitor {
OPERATION("operation"),
ALL("all");
- private String type;
+ private final String type;
private TaskType(String type) {
this.type = type.toLowerCase();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java
index 49b344f..fe3edfa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Procedure.java
@@ -103,7 +103,7 @@ public class Procedure implements Callable<Void>, ForeignExceptionListener {
//
/** lock to prevent nodes from acquiring and then releasing before we can track them */
- private Object joinBarrierLock = new Object();
+ private final Object joinBarrierLock = new Object();
private final List<String> acquiringMembers;
private final List<String> inBarrierMembers;
private final HashMap<String, byte[]> dataFromFinishedMembers;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManagerHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManagerHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManagerHost.java
index af4d2d7..9ebb1d7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManagerHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManagerHost.java
@@ -88,11 +88,9 @@ public abstract class ProcedureManagerHost<E extends ProcedureManager> {
E impl;
Object o = null;
try {
- o = implClass.newInstance();
+ o = implClass.getDeclaredConstructor().newInstance();
impl = (E)o;
- } catch (InstantiationException e) {
- throw new IOException(e);
- } catch (IllegalAccessException e) {
+ } catch (Exception e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
index 6416e6a..d15f5ac 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/Subprocedure.java
@@ -153,6 +153,7 @@ abstract public class Subprocedure implements Callable<Void> {
* Subprocedure, ForeignException)}.
*/
@SuppressWarnings("finally")
+ @Override
final public Void call() {
LOG.debug("Starting subprocedure '" + barrierName + "' with timeout " +
executionTimeoutTimer.getMaxTime() + "ms");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinator.java
index c1fb8f5..71ba28e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinator.java
@@ -19,19 +19,21 @@ package org.apache.hadoop.hbase.procedure;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+
/**
* ZooKeeper based {@link ProcedureCoordinatorRpcs} for a {@link ProcedureCoordinator}
*/
@@ -218,8 +220,8 @@ public class ZKProcedureCoordinator implements ProcedureCoordinatorRpcs {
} else {
dataFromMember = Arrays.copyOfRange(dataFromMember, ProtobufUtil.lengthOfPBMagic(),
dataFromMember.length);
- LOG.debug("Finished data from procedure '" + procName
- + "' member '" + member + "': " + new String(dataFromMember));
+ LOG.debug("Finished data from procedure '{}' member '{}': {}", procName, member,
+ new String(dataFromMember, StandardCharsets.UTF_8));
coordinator.memberFinishedBarrier(procName, member, dataFromMember);
}
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
index ea41ae8..f29d133 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java
@@ -348,6 +348,7 @@ public class ZKProcedureMemberRpcs implements ProcedureMemberRpcs {
}
}
+ @Override
public void start(final String memberName, final ProcedureMember listener) {
LOG.debug("Starting procedure member '" + memberName + "'");
this.member = listener;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
index 976e36b..9eb3fb3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureUtil.java
@@ -157,6 +157,7 @@ public abstract class ZKProcedureUtil
return ZNodePaths.joinZNode(controller.abortZnode, opInstanceName);
}
+ @Override
public ZKWatcher getWatcher() {
return watcher;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index e68a1ce..6783e7d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -455,7 +455,7 @@ public class MasterQuotaManager implements RegionStateListener {
}
private static class NamedLock<T> {
- private HashSet<T> locks = new HashSet<>();
+ private final HashSet<T> locks = new HashSet<>();
public void lock(final T name) throws InterruptedException {
synchronized (locks) {
@@ -501,6 +501,7 @@ public class MasterQuotaManager implements RegionStateListener {
return time;
}
+ @Override
public boolean equals(Object o) {
if (o instanceof SizeSnapshotWithTimestamp) {
SizeSnapshotWithTimestamp other = (SizeSnapshotWithTimestamp) o;
@@ -509,6 +510,7 @@ public class MasterQuotaManager implements RegionStateListener {
return false;
}
+ @Override
public int hashCode() {
HashCodeBuilder hcb = new HashCodeBuilder();
return hcb.append(size).append(time).toHashCode();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 5e20ce9..869ead3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -386,7 +386,8 @@ public class QuotaObserverChore extends ScheduledChore {
for (TableName tableInNS : tablesByNamespace.get(namespace)) {
final SpaceQuotaSnapshot tableQuotaSnapshot =
tableSnapshotStore.getCurrentState(tableInNS);
- final boolean hasTableQuota = QuotaSnapshotStore.NO_QUOTA != tableQuotaSnapshot;
+ final boolean hasTableQuota =
+ !Objects.equals(QuotaSnapshotStore.NO_QUOTA, tableQuotaSnapshot);
if (hasTableQuota && tableQuotaSnapshot.getQuotaStatus().isInViolation()) {
// Table-level quota violation policy is being applied here.
if (LOG.isTraceEnabled()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RateLimiter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RateLimiter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RateLimiter.java
index 0c856b1..852d8a6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RateLimiter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RateLimiter.java
@@ -100,6 +100,7 @@ public abstract class RateLimiter {
this.avail = limit;
}
+ @Override
public String toString() {
String rateLimiter = this.getClass().getSimpleName();
if (getLimit() == Long.MAX_VALUE) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerSpaceQuotaManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerSpaceQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerSpaceQuotaManager.java
index 2d4414c..b0bdede 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerSpaceQuotaManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerSpaceQuotaManager.java
@@ -53,7 +53,7 @@ public class RegionServerSpaceQuotaManager {
private SpaceQuotaRefresherChore spaceQuotaRefresher;
private AtomicReference<Map<TableName, SpaceQuotaSnapshot>> currentQuotaSnapshots;
private boolean started = false;
- private ConcurrentHashMap<TableName,SpaceViolationPolicyEnforcement> enforcedPolicies;
+ private final ConcurrentHashMap<TableName,SpaceViolationPolicyEnforcement> enforcedPolicies;
private SpaceViolationPolicyEnforcementFactory factory;
public RegionServerSpaceQuotaManager(RegionServerServices rsServices) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaSnapshotNotifierFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaSnapshotNotifierFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaSnapshotNotifierFactory.java
index 3fb7ad3..f19595f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaSnapshotNotifierFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaSnapshotNotifierFactory.java
@@ -54,8 +54,8 @@ public class SpaceQuotaSnapshotNotifierFactory {
.getClass(SNAPSHOT_NOTIFIER_KEY, SNAPSHOT_NOTIFIER_DEFAULT,
SpaceQuotaSnapshotNotifier.class);
try {
- return clz.newInstance();
- } catch (InstantiationException | IllegalAccessException e) {
+ return clz.getDeclaredConstructor().newInstance();
+ } catch (Exception e) {
throw new IllegalArgumentException("Failed to instantiate the implementation", e);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index 8a19908..6dbe0a8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -54,7 +54,7 @@ public abstract class AbstractMemStore implements MemStore {
// Used to track when to flush
private volatile long timeOfOldestEdit;
- public final static long FIXED_OVERHEAD = ClassSize.OBJECT
+ public final static long FIXED_OVERHEAD = (long) ClassSize.OBJECT
+ (4 * ClassSize.REFERENCE)
+ (2 * Bytes.SIZEOF_LONG); // snapshotId, timeOfOldestEdit
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AdaptiveMemStoreCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AdaptiveMemStoreCompactionStrategy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AdaptiveMemStoreCompactionStrategy.java
index 232ffe3..9a866a1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AdaptiveMemStoreCompactionStrategy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AdaptiveMemStoreCompactionStrategy.java
@@ -101,10 +101,13 @@ public class AdaptiveMemStoreCompactionStrategy extends MemStoreCompactionStrate
public void resetStats() {
compactionProbability = initialCompactionProbability;
}
+
+ @Override
protected Action getMergingAction() {
return Action.MERGE_COUNT_UNIQUE_KEYS;
}
+ @Override
protected Action getFlattenAction() {
return Action.FLATTEN;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
index 523ccf2..bf9b191 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
@@ -82,7 +82,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
@Override
protected long indexEntrySize() {
- return (ClassSize.CELL_CHUNK_MAP_ENTRY - KeyValue.FIXED_OVERHEAD);
+ return ((long) ClassSize.CELL_CHUNK_MAP_ENTRY - KeyValue.FIXED_OVERHEAD);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java
index 6159385..a4fe883 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java
@@ -73,113 +73,140 @@ public class CellSet implements NavigableSet<Cell> {
return delegatee;
}
+ @Override
public Cell ceiling(Cell e) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public Iterator<Cell> descendingIterator() {
return this.delegatee.descendingMap().values().iterator();
}
+ @Override
public NavigableSet<Cell> descendingSet() {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public Cell floor(Cell e) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public SortedSet<Cell> headSet(final Cell toElement) {
return headSet(toElement, false);
}
+ @Override
public NavigableSet<Cell> headSet(final Cell toElement,
boolean inclusive) {
return new CellSet(this.delegatee.headMap(toElement, inclusive), UNKNOWN_NUM_UNIQUES);
}
+ @Override
public Cell higher(Cell e) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public Iterator<Cell> iterator() {
return this.delegatee.values().iterator();
}
+ @Override
public Cell lower(Cell e) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public Cell pollFirst() {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public Cell pollLast() {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public SortedSet<Cell> subSet(Cell fromElement, Cell toElement) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public NavigableSet<Cell> subSet(Cell fromElement,
boolean fromInclusive, Cell toElement, boolean toInclusive) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public SortedSet<Cell> tailSet(Cell fromElement) {
return tailSet(fromElement, true);
}
+ @Override
public NavigableSet<Cell> tailSet(Cell fromElement, boolean inclusive) {
return new CellSet(this.delegatee.tailMap(fromElement, inclusive), UNKNOWN_NUM_UNIQUES);
}
+ @Override
public Comparator<? super Cell> comparator() {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public Cell first() {
return this.delegatee.firstEntry().getValue();
}
+ @Override
public Cell last() {
return this.delegatee.lastEntry().getValue();
}
+ @Override
public boolean add(Cell e) {
return this.delegatee.put(e, e) == null;
}
+ @Override
public boolean addAll(Collection<? extends Cell> c) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public void clear() {
this.delegatee.clear();
}
+ @Override
public boolean contains(Object o) {
//noinspection SuspiciousMethodCalls
return this.delegatee.containsKey(o);
}
+ @Override
public boolean containsAll(Collection<?> c) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public boolean isEmpty() {
return this.delegatee.isEmpty();
}
+ @Override
public boolean remove(Object o) {
return this.delegatee.remove(o) != null;
}
+ @Override
public boolean removeAll(Collection<?> c) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public boolean retainAll(Collection<?> c) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
@@ -188,14 +215,17 @@ public class CellSet implements NavigableSet<Cell> {
return this.delegatee.get(kv);
}
+ @Override
public int size() {
return this.delegatee.size();
}
+ @Override
public Object[] toArray() {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
+ @Override
public <T> T[] toArray(T[] a) {
throw new UnsupportedOperationException(HConstants.NOT_IMPLEMENTED);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index d874b2e..3cb4103 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -143,7 +143,7 @@ public class CompactingMemStore extends AbstractMemStore {
factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
}
- inmemoryFlushSize *= factor;
+ inmemoryFlushSize = (long) (inmemoryFlushSize * factor);
LOG.info("Setting in-memory flush size threshold to " + inmemoryFlushSize
+ " and immutable segments index to be of type " + indexType);
}
@@ -365,7 +365,7 @@ public class CompactingMemStore extends AbstractMemStore {
MutableSegment activeTmp = active;
List<? extends Segment> pipelineList = pipeline.getSegments();
List<? extends Segment> snapshotList = snapshot.getAllSegments();
- long order = 1 + pipelineList.size() + snapshotList.size();
+ long order = 1L + pipelineList.size() + snapshotList.size();
// The list of elements in pipeline + the active element + the snapshot segment
// The order is the Segment ordinal
List<KeyValueScanner> list = createList((int) order);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
index 1aae068..8bd990a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
@@ -279,6 +279,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
/**
* Dumps all cells of the segment into the given log
*/
+ @Override
void dump(Logger log) {
for (ImmutableSegment s : segments) {
s.dump(log);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
index 4539ed6..daae083 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
@@ -89,6 +89,7 @@ public class DateTieredStoreEngine extends StoreEngine<DefaultStoreFlusher,
super.forceSelect(request);
}
+ @Override
public List<Path> compact(ThroughputController throughputController, User user)
throws IOException {
if (request instanceof DateTieredCompactionRequest) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java
index b3f0a44..26bf640 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java
@@ -61,11 +61,9 @@ public class DefaultStoreFlusher extends StoreFlusher {
synchronized (flushLock) {
status.setStatus("Flushing " + store + ": creating writer");
// Write the map out to the disk
- writer = store.createWriterInTmp(cellsCount, store.getColumnFamilyDescriptor().getCompressionType(),
- /* isCompaction = */ false,
- /* includeMVCCReadpoint = */ true,
- /* includesTags = */ snapshot.isTagsPresent(),
- /* shouldDropBehind = */ false);
+ writer = store.createWriterInTmp(cellsCount,
+ store.getColumnFamilyDescriptor().getCompressionType(), false, true,
+ snapshot.isTagsPresent(), false);
IOException e = null;
try {
performFlush(scanner, writer, smallestReadPoint, throughputController);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
index d56a1c2..740eb08 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.NavigableSet;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -82,15 +83,15 @@ public class HMobStore extends HStore {
private MobCacheConfig mobCacheConfig;
private Path homePath;
private Path mobFamilyPath;
- private volatile long cellsCountCompactedToMob = 0;
- private volatile long cellsCountCompactedFromMob = 0;
- private volatile long cellsSizeCompactedToMob = 0;
- private volatile long cellsSizeCompactedFromMob = 0;
- private volatile long mobFlushCount = 0;
- private volatile long mobFlushedCellsCount = 0;
- private volatile long mobFlushedCellsSize = 0;
- private volatile long mobScanCellsCount = 0;
- private volatile long mobScanCellsSize = 0;
+ private AtomicLong cellsCountCompactedToMob = new AtomicLong();
+ private AtomicLong cellsCountCompactedFromMob = new AtomicLong();
+ private AtomicLong cellsSizeCompactedToMob = new AtomicLong();
+ private AtomicLong cellsSizeCompactedFromMob = new AtomicLong();
+ private AtomicLong mobFlushCount = new AtomicLong();
+ private AtomicLong mobFlushedCellsCount = new AtomicLong();
+ private AtomicLong mobFlushedCellsSize = new AtomicLong();
+ private AtomicLong mobScanCellsCount = new AtomicLong();
+ private AtomicLong mobScanCellsSize = new AtomicLong();
private ColumnFamilyDescriptor family;
private Map<String, List<Path>> map = new ConcurrentHashMap<>();
private final IdLock keyLock = new IdLock();
@@ -453,76 +454,75 @@ public class HMobStore extends HStore {
}
public void updateCellsCountCompactedToMob(long count) {
- cellsCountCompactedToMob += count;
+ cellsCountCompactedToMob.addAndGet(count);
}
public long getCellsCountCompactedToMob() {
- return cellsCountCompactedToMob;
+ return cellsCountCompactedToMob.get();
}
public void updateCellsCountCompactedFromMob(long count) {
- cellsCountCompactedFromMob += count;
+ cellsCountCompactedFromMob.addAndGet(count);
}
public long getCellsCountCompactedFromMob() {
- return cellsCountCompactedFromMob;
+ return cellsCountCompactedFromMob.get();
}
public void updateCellsSizeCompactedToMob(long size) {
- cellsSizeCompactedToMob += size;
+ cellsSizeCompactedToMob.addAndGet(size);
}
public long getCellsSizeCompactedToMob() {
- return cellsSizeCompactedToMob;
+ return cellsSizeCompactedToMob.get();
}
public void updateCellsSizeCompactedFromMob(long size) {
- cellsSizeCompactedFromMob += size;
+ cellsSizeCompactedFromMob.addAndGet(size);
}
public long getCellsSizeCompactedFromMob() {
- return cellsSizeCompactedFromMob;
+ return cellsSizeCompactedFromMob.get();
}
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "VO_VOLATILE_INCREMENT")
public void updateMobFlushCount() {
- mobFlushCount++;
+ mobFlushCount.incrementAndGet();
}
public long getMobFlushCount() {
- return mobFlushCount;
+ return mobFlushCount.get();
}
public void updateMobFlushedCellsCount(long count) {
- mobFlushedCellsCount += count;
+ mobFlushedCellsCount.addAndGet(count);
}
public long getMobFlushedCellsCount() {
- return mobFlushedCellsCount;
+ return mobFlushedCellsCount.get();
}
public void updateMobFlushedCellsSize(long size) {
- mobFlushedCellsSize += size;
+ mobFlushedCellsSize.addAndGet(size);
}
public long getMobFlushedCellsSize() {
- return mobFlushedCellsSize;
+ return mobFlushedCellsSize.get();
}
public void updateMobScanCellsCount(long count) {
- mobScanCellsCount += count;
+ mobScanCellsCount.addAndGet(count);
}
public long getMobScanCellsCount() {
- return mobScanCellsCount;
+ return mobScanCellsCount.get();
}
public void updateMobScanCellsSize(long size) {
- mobScanCellsSize += size;
+ mobScanCellsSize.addAndGet(size);
}
public long getMobScanCellsSize() {
- return mobScanCellsSize;
+ return mobScanCellsSize.get();
}
public byte[] getRefCellTags() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index e18c80e..c0ccc1d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.InterruptedIOException;
import java.lang.reflect.Constructor;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.AbstractList;
import java.util.ArrayList;
@@ -1015,7 +1016,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
long storeMaxSequenceId = store.getMaxSequenceId().orElse(0L);
- maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(),
+ maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()),
storeMaxSequenceId);
if (maxSeqId == -1 || storeMaxSequenceId > maxSeqId) {
maxSeqId = storeMaxSequenceId;
@@ -5524,7 +5525,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
HStore store = this.stores.get(column);
if (store == null) {
throw new IllegalArgumentException(
- "No column family : " + new String(column) + " available");
+ "No column family : " + new String(column, StandardCharsets.UTF_8) + " available");
}
Collection<HStoreFile> storeFiles = store.getStorefiles();
if (storeFiles == null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 37ec595..bd7b4a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -904,7 +904,7 @@ public class HRegionServer extends HasThread implements
*/
private boolean isClusterUp() {
return this.masterless ||
- this.clusterStatusTracker != null && this.clusterStatusTracker.isClusterUp();
+ (this.clusterStatusTracker != null && this.clusterStatusTracker.isClusterUp());
}
/**
@@ -1745,7 +1745,7 @@ public class HRegionServer extends HasThread implements
if (r.shouldFlush(whyFlush)) {
FlushRequester requester = server.getFlushRequester();
if (requester != null) {
- long randomDelay = RandomUtils.nextInt(0, RANGE_OF_DELAY) + MIN_DELAY_TIME;
+ long randomDelay = (long) RandomUtils.nextInt(0, RANGE_OF_DELAY) + MIN_DELAY_TIME;
LOG.info(getName() + " requesting flush of " +
r.getRegionInfo().getRegionNameAsString() + " because " +
whyFlush.toString() +
@@ -3111,13 +3111,13 @@ public class HRegionServer extends HasThread implements
}
}
- final Boolean previous = this.regionsInTransitionInRS.putIfAbsent(encodedName.getBytes(),
+ final Boolean previous = this.regionsInTransitionInRS.putIfAbsent(Bytes.toBytes(encodedName),
Boolean.FALSE);
if (Boolean.TRUE.equals(previous)) {
LOG.info("Received CLOSE for the region:" + encodedName + " , which we are already " +
"trying to OPEN. Cancelling OPENING.");
- if (!regionsInTransitionInRS.replace(encodedName.getBytes(), previous, Boolean.FALSE)){
+ if (!regionsInTransitionInRS.replace(Bytes.toBytes(encodedName), previous, Boolean.FALSE)) {
// The replace failed. That should be an exceptional case, but theoretically it can happen.
// We're going to try to do a standard close then.
LOG.warn("The opening for region " + encodedName + " was done before we could cancel it." +
@@ -3140,7 +3140,7 @@ public class HRegionServer extends HasThread implements
if (actualRegion == null) {
LOG.debug("Received CLOSE for a region which is not online, and we're not opening.");
- this.regionsInTransitionInRS.remove(encodedName.getBytes());
+ this.regionsInTransitionInRS.remove(Bytes.toBytes(encodedName));
// The master deletes the znode when it receives this exception.
throw new NotServingRegionException("The region " + encodedName +
" is not online, and is not opening.");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
index d3509c2..afd85f8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerCommandLine.java
@@ -44,6 +44,7 @@ public class HRegionServerCommandLine extends ServerCommandLine {
this.regionServerClass = clazz;
}
+ @Override
protected String getUsage() {
return USAGE;
}
@@ -73,6 +74,7 @@ public class HRegionServerCommandLine extends ServerCommandLine {
return 0;
}
+ @Override
public int run(String args[]) throws Exception {
if (args.length != 1) {
usage(null);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 01121dd..f228d44 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -42,6 +42,7 @@ import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Predicate;
@@ -149,8 +150,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
volatile boolean forceMajor = false;
/* how many bytes to write between status checks */
static int closeCheckInterval = 0;
- private volatile long storeSize = 0L;
- private volatile long totalUncompressedBytes = 0L;
+ private AtomicLong storeSize = new AtomicLong();
+ private AtomicLong totalUncompressedBytes = new AtomicLong();
/**
* RWLock for store operations.
@@ -209,13 +210,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
private int compactionCheckMultiplier;
protected Encryption.Context cryptoContext = Encryption.Context.NONE;
- private volatile long flushedCellsCount = 0;
- private volatile long compactedCellsCount = 0;
- private volatile long majorCompactedCellsCount = 0;
- private volatile long flushedCellsSize = 0;
- private volatile long flushedOutputFileSize = 0;
- private volatile long compactedCellsSize = 0;
- private volatile long majorCompactedCellsSize = 0;
+ private AtomicLong flushedCellsCount = new AtomicLong();
+ private AtomicLong compactedCellsCount = new AtomicLong();
+ private AtomicLong majorCompactedCellsCount = new AtomicLong();
+ private AtomicLong flushedCellsSize = new AtomicLong();
+ private AtomicLong flushedOutputFileSize = new AtomicLong();
+ private AtomicLong compactedCellsSize = new AtomicLong();
+ private AtomicLong majorCompactedCellsSize = new AtomicLong();
/**
* Constructor
@@ -544,8 +545,9 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
HStoreFile storeFile = completionService.take().get();
if (storeFile != null) {
long length = storeFile.getReader().length();
- this.storeSize += length;
- this.totalUncompressedBytes += storeFile.getReader().getTotalUncompressedBytes();
+ this.storeSize.addAndGet(length);
+ this.totalUncompressedBytes
+ .addAndGet(storeFile.getReader().getTotalUncompressedBytes());
LOG.debug("loaded {}", storeFile);
results.add(storeFile);
}
@@ -844,8 +846,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
private void bulkLoadHFile(HStoreFile sf) throws IOException {
StoreFileReader r = sf.getReader();
- this.storeSize += r.length();
- this.totalUncompressedBytes += r.getTotalUncompressedBytes();
+ this.storeSize.addAndGet(r.length());
+ this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes());
// Append the new storefile into the list
this.lock.writeLock().lock();
@@ -1021,8 +1023,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
HStoreFile sf = createStoreFileAndReader(dstPath);
StoreFileReader r = sf.getReader();
- this.storeSize += r.length();
- this.totalUncompressedBytes += r.getTotalUncompressedBytes();
+ this.storeSize.addAndGet(r.length());
+ this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes());
if (LOG.isInfoEnabled()) {
LOG.info("Added " + sf + ", entries=" + r.getEntries() +
@@ -1373,11 +1375,11 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
writeCompactionWalRecord(filesToCompact, sfs);
replaceStoreFiles(filesToCompact, sfs);
if (cr.isMajor()) {
- majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;
- majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;
+ majorCompactedCellsCount.addAndGet(getCompactionProgress().totalCompactingKVs);
+ majorCompactedCellsSize.addAndGet(getCompactionProgress().totalCompactedSize);
} else {
- compactedCellsCount += getCompactionProgress().totalCompactingKVs;
- compactedCellsSize += getCompactionProgress().totalCompactedSize;
+ compactedCellsCount.addAndGet(getCompactionProgress().totalCompactingKVs);
+ compactedCellsSize.addAndGet(getCompactionProgress().totalCompactedSize);
}
long outputBytes = getTotalSize(sfs);
@@ -1478,7 +1480,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
}
}
message.append("total size for store is ")
- .append(StringUtils.TraditionalBinaryPrefix.long2String(storeSize, "", 1))
+ .append(StringUtils.TraditionalBinaryPrefix.long2String(storeSize.get(), "", 1))
.append(". This selection was in queue for ")
.append(StringUtils.formatTimeDiff(compactionStartTime, cr.getSelectionTime()))
.append(", and took ").append(StringUtils.formatTimeDiff(now, compactionStartTime))
@@ -1772,7 +1774,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
completeCompaction(delSfs);
LOG.info("Completed removal of " + delSfs.size() + " unnecessary (expired) file(s) in "
+ this + " of " + this.getRegionInfo().getRegionNameAsString()
- + "; total size for store is " + TraditionalBinaryPrefix.long2String(storeSize, "", 1));
+ + "; total size for store is "
+ + TraditionalBinaryPrefix.long2String(storeSize.get(), "", 1));
}
public void cancelRequestedCompaction(CompactionContext compaction) {
@@ -1826,16 +1829,16 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
@VisibleForTesting
protected void completeCompaction(Collection<HStoreFile> compactedFiles)
throws IOException {
- this.storeSize = 0L;
- this.totalUncompressedBytes = 0L;
+ this.storeSize.set(0L);
+ this.totalUncompressedBytes.set(0L);
for (HStoreFile hsf : this.storeEngine.getStoreFileManager().getStorefiles()) {
StoreFileReader r = hsf.getReader();
if (r == null) {
LOG.warn("StoreFile {} has a null Reader", hsf);
continue;
}
- this.storeSize += r.length();
- this.totalUncompressedBytes += r.getTotalUncompressedBytes();
+ this.storeSize.addAndGet(r.length());
+ this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes());
}
}
@@ -1896,7 +1899,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
@Override
public long getSize() {
- return storeSize;
+ return storeSize.get();
}
public void triggerMajorCompaction() {
@@ -2043,7 +2046,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
@Override
public long getStoreSizeUncompressed() {
- return this.totalUncompressedBytes;
+ return this.totalUncompressedBytes.get();
}
@Override
@@ -2235,9 +2238,9 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
committedFiles.add(sf.getPath());
}
- HStore.this.flushedCellsCount += cacheFlushCount;
- HStore.this.flushedCellsSize += cacheFlushSize;
- HStore.this.flushedOutputFileSize += outputFileSize;
+ HStore.this.flushedCellsCount.addAndGet(cacheFlushCount);
+ HStore.this.flushedCellsSize.addAndGet(cacheFlushSize);
+ HStore.this.flushedOutputFileSize.addAndGet(outputFileSize);
// Add new file to store files. Clear snapshot too while we have the Store write lock.
return HStore.this.updateStorefiles(storeFiles, snapshot.getId());
@@ -2270,8 +2273,9 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), file);
HStoreFile storeFile = createStoreFileAndReader(storeFileInfo);
storeFiles.add(storeFile);
- HStore.this.storeSize += storeFile.getReader().length();
- HStore.this.totalUncompressedBytes += storeFile.getReader().getTotalUncompressedBytes();
+ HStore.this.storeSize.addAndGet(storeFile.getReader().length());
+ HStore.this.totalUncompressedBytes
+ .addAndGet(storeFile.getReader().getTotalUncompressedBytes());
if (LOG.isInfoEnabled()) {
LOG.info("Region: " + HStore.this.getRegionInfo().getEncodedName() +
" added " + storeFile + ", entries=" + storeFile.getReader().getEntries() +
@@ -2315,7 +2319,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
}
public static final long FIXED_OVERHEAD =
- ClassSize.align(ClassSize.OBJECT + (17 * ClassSize.REFERENCE) + (11 * Bytes.SIZEOF_LONG)
+ ClassSize.align(ClassSize.OBJECT + (26 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG)
+ (5 * Bytes.SIZEOF_INT) + (2 * Bytes.SIZEOF_BOOLEAN));
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
@@ -2354,37 +2358,37 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
@Override
public long getFlushedCellsCount() {
- return flushedCellsCount;
+ return flushedCellsCount.get();
}
@Override
public long getFlushedCellsSize() {
- return flushedCellsSize;
+ return flushedCellsSize.get();
}
@Override
public long getFlushedOutputFileSize() {
- return flushedOutputFileSize;
+ return flushedOutputFileSize.get();
}
@Override
public long getCompactedCellsCount() {
- return compactedCellsCount;
+ return compactedCellsCount.get();
}
@Override
public long getCompactedCellsSize() {
- return compactedCellsSize;
+ return compactedCellsSize.get();
}
@Override
public long getMajorCompactedCellsCount() {
- return majorCompactedCellsCount;
+ return majorCompactedCellsCount.get();
}
@Override
public long getMajorCompactedCellsSize() {
- return majorCompactedCellsSize;
+ return majorCompactedCellsSize.get();
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
index 19a63b4..21446d2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/IncreasingToUpperBoundRegionSplitPolicy.java
@@ -95,7 +95,7 @@ public class IncreasingToUpperBoundRegionSplitPolicy extends ConstantSizeRegionS
}
}
- return foundABigStore | force;
+ return foundABigStore || force;
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
index 1e71bc8..fe52758 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/InternalScanner.java
@@ -67,5 +67,6 @@ public interface InternalScanner extends Closeable {
* Closes the scanner and releases any resources it has allocated
* @throws IOException
*/
+ @Override
void close() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index 779ed49..053ae99 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -104,6 +104,7 @@ public class KeyValueHeap extends NonReversedNonLazyKeyValueScanner
}
}
+ @Override
public Cell peek() {
if (this.current == null) {
return null;
@@ -111,6 +112,7 @@ public class KeyValueHeap extends NonReversedNonLazyKeyValueScanner
return this.current.peek();
}
+ @Override
public Cell next() throws IOException {
if(this.current == null) {
return null;
@@ -182,6 +184,8 @@ public class KeyValueHeap extends NonReversedNonLazyKeyValueScanner
public KVScannerComparator(CellComparator kvComparator) {
this.kvComparator = kvComparator;
}
+
+ @Override
public int compare(KeyValueScanner left, KeyValueScanner right) {
int comparison = compare(left.peek(), right.peek());
if (comparison != 0) {
@@ -210,6 +214,7 @@ public class KeyValueHeap extends NonReversedNonLazyKeyValueScanner
}
}
+ @Override
public void close() {
for (KeyValueScanner scanner : this.scannersForDelayedClose) {
scanner.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
index 796f7c9..864cc06 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
@@ -84,6 +84,7 @@ public interface KeyValueScanner extends Shipper, Closeable {
/**
* Close the KeyValue scanner.
*/
+ @Override
void close();
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
index d564e40..a8c3362 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
@@ -167,8 +167,8 @@ public class MemStoreCompactor {
// Substitute the pipeline with one segment
if (!isInterrupted.get()) {
- if (resultSwapped = compactingMemStore.swapCompactedSegments(
- versionedList, result, merge)) {
+ resultSwapped = compactingMemStore.swapCompactedSegments(versionedList, result, merge);
+ if (resultSwapped) {
// update compaction strategy
strategy.updateStats(result);
// update the wal so it can be truncated and not get too long
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index fdee404..f7493b0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.SortedMap;
import java.util.concurrent.BlockingQueue;
@@ -714,8 +715,13 @@ class MemStoreFlusher implements FlushRequester {
}
@Override
+ public int hashCode() {
+ return System.identityHashCode(this);
+ }
+
+ @Override
public boolean equals(Object obj) {
- return (this == obj);
+ return Objects.equals(this, obj);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreMergerSegmentsIterator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreMergerSegmentsIterator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreMergerSegmentsIterator.java
index f43573e..02824ba 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreMergerSegmentsIterator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreMergerSegmentsIterator.java
@@ -76,6 +76,7 @@ public class MemStoreMergerSegmentsIterator extends MemStoreSegmentsIterator {
return null;
}
+ @Override
public void close() {
if (closed) {
return;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index eaaa4ae..09929e1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -788,7 +788,8 @@ class MetricsRegionServerWrapperImpl
OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge();
if (storeAvgStoreFileAge.isPresent()) {
- avgAgeNumerator += storeAvgStoreFileAge.getAsDouble() * storeHFiles;
+ avgAgeNumerator =
+ (long) (avgAgeNumerator + storeAvgStoreFileAge.getAsDouble() * storeHFiles);
}
tempStorefileIndexSize += store.getStorefilesRootLevelIndexSize();
@@ -931,6 +932,7 @@ class MetricsRegionServerWrapperImpl
return averageRegionSize;
}
+ @Override
public long getDataMissCount() {
if (this.cacheStats == null) {
return 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
index 2aa1a82..533a05d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
@@ -249,7 +249,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge();
if (storeAvgStoreFileAge.isPresent()) {
- avgAgeNumerator += storeAvgStoreFileAge.getAsDouble() * storeHFiles;
+ avgAgeNumerator += (long) storeAvgStoreFileAge.getAsDouble() * storeHFiles;
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConcurrencyControl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConcurrencyControl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConcurrencyControl.java
index 54095e0..0c3551b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConcurrencyControl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConcurrencyControl.java
@@ -74,8 +74,12 @@ public class MultiVersionConcurrencyControl {
public void advanceTo(long newStartPoint) {
while (true) {
long seqId = this.getWritePoint();
- if (seqId >= newStartPoint) break;
- if (this.tryAdvanceTo(/* newSeqId = */ newStartPoint, /* expected = */ seqId)) break;
+ if (seqId >= newStartPoint) {
+ break;
+ }
+ if (this.tryAdvanceTo(newStartPoint, seqId)) {
+ break;
+ }
}
}
@@ -239,6 +243,7 @@ public class MultiVersionConcurrencyControl {
}
@VisibleForTesting
+ @Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("readPoint", readPoint)
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index b6c0ebe..5a01581 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -2807,7 +2807,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
private static final long serialVersionUID = -4305297078988180130L;
@Override
- public Throwable fillInStackTrace() {
+ public synchronized Throwable fillInStackTrace() {
return this;
}
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
index dc1708c..1986668 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java
@@ -279,6 +279,7 @@ public class RegionServerCoprocessorHost extends
* @return An instance of RegionServerServices, an object NOT for general user-space Coprocessor
* consumption.
*/
+ @Override
public RegionServerServices getRegionServerServices() {
return this.regionServerServices;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
index 017e0fb..6b2267f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
@@ -435,8 +435,8 @@ public class ScannerContext {
TIME_LIMIT_REACHED_MID_ROW(true, true),
BATCH_LIMIT_REACHED(true, true);
- private boolean moreValues;
- private boolean limitReached;
+ private final boolean moreValues;
+ private final boolean limitReached;
private NextState(boolean moreValues, boolean limitReached) {
this.moreValues = moreValues;
@@ -492,13 +492,13 @@ public class ScannerContext {
* limits, the checker must know their own scope (i.e. are they checking the limits between
* rows, between cells, etc...)
*/
- int depth;
+ final int depth;
LimitScope(int depth) {
this.depth = depth;
}
- int depth() {
+ final int depth() {
return depth;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java
index b67b54e..0b1d251 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ServerNonceManager.java
@@ -97,7 +97,7 @@ public class ServerNonceManager {
}
public boolean isExpired(long minRelevantTime) {
- return getActivityTime() < (minRelevantTime & (~0l >>> 3));
+ return getActivityTime() < (minRelevantTime & (~0L >>> 3));
}
public void setMvcc(long mvcc) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingSplitPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingSplitPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingSplitPolicy.java
index 4f3e0f2..9753080 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingSplitPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SteppingSplitPolicy.java
@@ -24,6 +24,7 @@ public class SteppingSplitPolicy extends IncreasingToUpperBoundRegionSplitPolicy
* This allows a table to spread quickly across servers, while avoiding creating
* too many regions.
*/
+ @Override
protected long getSizeToCheck(final int tableRegionsCount) {
return tableRegionsCount == 1 ? this.initialSize : getDesiredMaxFileSize();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index 89b2acd..80d0ad7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -187,14 +187,17 @@ public class StoreFileScanner implements KeyValueScanner {
return scanners;
}
+ @Override
public String toString() {
return "StoreFileScanner[" + hfs.toString() + ", cur=" + cur + "]";
}
+ @Override
public Cell peek() {
return cur;
}
+ @Override
public Cell next() throws IOException {
Cell retKey = cur;
@@ -215,6 +218,7 @@ public class StoreFileScanner implements KeyValueScanner {
return retKey;
}
+ @Override
public boolean seek(Cell key) throws IOException {
if (seekCount != null) seekCount.increment();
@@ -242,6 +246,7 @@ public class StoreFileScanner implements KeyValueScanner {
}
}
+ @Override
public boolean reseek(Cell key) throws IOException {
if (seekCount != null) seekCount.increment();
@@ -298,6 +303,7 @@ public class StoreFileScanner implements KeyValueScanner {
return true;
}
+ @Override
public void close() {
if (closed) return;
cur = null;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java
index 595231f..59b91d5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java
@@ -104,12 +104,8 @@ public class StripeStoreFlusher extends StoreFlusher {
return new StripeMultiFileWriter.WriterFactory() {
@Override
public StoreFileWriter createWriter() throws IOException {
- StoreFileWriter writer = store.createWriterInTmp(
- kvCount, store.getColumnFamilyDescriptor().getCompressionType(),
- /* isCompaction = */ false,
- /* includeMVCCReadpoint = */ true,
- /* includesTags = */ true,
- /* shouldDropBehind = */ false);
+ StoreFileWriter writer = store.createWriterInTmp(kvCount,
+ store.getColumnFamilyDescriptor().getCompressionType(), false, true, true, false);
return writer;
}
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index ed4a025..056f076 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
import org.apache.hadoop.hbase.regionserver.CellSink;
-import org.apache.hadoop.hbase.regionserver.CustomizedScanInfoBuilder;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -261,10 +260,8 @@ public abstract class Compactor<T extends CellSink> {
throws IOException {
// When all MVCC readpoints are 0, don't write them.
// See HBASE-8166, HBASE-12600, and HBASE-13389.
- return store.createWriterInTmp(fd.maxKeyCount, this.compactionCompression,
- /* isCompaction = */true,
- /* includeMVCCReadpoint = */fd.maxMVCCReadpoint > 0,
- /* includesTags = */fd.maxTagsLength > 0, shouldDropBehind);
+ return store.createWriterInTmp(fd.maxKeyCount, this.compactionCompression, true,
+ fd.maxMVCCReadpoint > 0, fd.maxTagsLength > 0, shouldDropBehind);
}
private ScanInfo preCompactScannerOpen(CompactionRequestImpl request, ScanType scanType,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
index 905562c..cf04d00 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
@@ -108,6 +108,7 @@ public class DateTieredCompactionPolicy extends SortedCompactionPolicy {
}
}
+ @Override
public boolean shouldPerformMajorCompaction(Collection<HStoreFile> filesToCompact)
throws IOException {
long mcTime = getNextMajorCompactTime(filesToCompact);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
index a6ea9b2..e0be6cf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
@@ -209,6 +209,7 @@ public class RatioBasedCompactionPolicy extends SortedCompactionPolicy {
* @param filesCompacting files being scheduled to compact.
* @return true to schedule a request.
*/
+ @Override
public boolean needsCompaction(Collection<HStoreFile> storeFiles,
List<HStoreFile> filesCompacting) {
int numCandidates = storeFiles.size() - filesCompacting.size();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/SortedCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/SortedCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/SortedCompactionPolicy.java
index 4f6aba9..3eb830a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/SortedCompactionPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/SortedCompactionPolicy.java
@@ -106,6 +106,7 @@ public abstract class SortedCompactionPolicy extends CompactionPolicy {
* @param filesToCompact Files to compact. Can be null.
* @return True if we should run a major compaction.
*/
+ @Override
public abstract boolean shouldPerformMajorCompaction(Collection<HStoreFile> filesToCompact)
throws IOException;
@@ -154,6 +155,7 @@ public abstract class SortedCompactionPolicy extends CompactionPolicy {
* @param compactionSize Total size of some compaction
* @return whether this should be a large or small compaction
*/
+ @Override
public boolean throttleCompaction(long compactionSize) {
return compactionSize > comConf.getThrottlePoint();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java
index 85394fd..c0f13c0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java
@@ -89,10 +89,12 @@ public class ExplicitColumnTracker implements ColumnTracker {
/**
* Done when there are no more columns to match against.
*/
+ @Override
public boolean done() {
return this.index >= columns.length;
}
+ @Override
public ColumnCount getColumnHint() {
return this.column;
}
@@ -182,6 +184,7 @@ public class ExplicitColumnTracker implements ColumnTracker {
}
// Called between every row.
+ @Override
public void reset() {
this.index = 0;
this.column = this.columns[this.index];
@@ -240,6 +243,7 @@ public class ExplicitColumnTracker implements ColumnTracker {
}
}
+ @Override
public boolean isDone(long timestamp) {
return minVersions <= 0 && isExpired(timestamp);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanWildcardColumnTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanWildcardColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanWildcardColumnTracker.java
index 419e93b..f2ad1e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanWildcardColumnTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanWildcardColumnTracker.java
@@ -180,6 +180,7 @@ public class ScanWildcardColumnTracker implements ColumnTracker {
* scanner).
* @return The column count.
*/
+ @Override
public ColumnCount getColumnHint() {
return null;
}
@@ -205,6 +206,7 @@ public class ScanWildcardColumnTracker implements ColumnTracker {
}
}
+ @Override
public boolean isDone(long timestamp) {
return minVersions <= 0 && isExpired(timestamp);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
index 0ace782..faf3b77 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java
@@ -346,7 +346,7 @@ public class AsyncFSWAL extends AbstractFSWAL<AsyncWriter> {
long currentHighestProcessedAppendTxid = highestProcessedAppendTxid;
highestProcessedAppendTxidAtLastSync = currentHighestProcessedAppendTxid;
final long startTimeNs = System.nanoTime();
- final long epoch = epochAndState >>> 2;
+ final long epoch = (long) epochAndState >>> 2L;
writer.sync().whenCompleteAsync((result, error) -> {
if (error != null) {
syncFailed(epoch, error);
[06/50] [abbrv] hbase git commit: HBASE-19811 Fix findbugs and
error-prone warnings in hbase-server (branch-2)
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
index a497bf4..59a0c31 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
@@ -71,7 +71,7 @@ public class TestRegionReplicas {
private static final int NB_SERVERS = 1;
private static Table table;
- private static final byte[] row = "TestRegionReplicas".getBytes();
+ private static final byte[] row = Bytes.toBytes("TestRegionReplicas");
private static HRegionInfo hriPrimary;
private static HRegionInfo hriSecondary;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicasWithModifyTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicasWithModifyTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicasWithModifyTable.java
index d1bf773..9a02a9d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicasWithModifyTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicasWithModifyTable.java
@@ -111,7 +111,7 @@ public class TestRegionReplicasWithModifyTable {
List<HRegion> onlineRegions2 = getSecondaryRS().getRegions(tableName);
List<HRegion> onlineRegions3 = getTertiaryRS().getRegions(tableName);
int totalRegions = onlineRegions.size() + onlineRegions2.size() + onlineRegions3.size();
- assertEquals("the number of regions should be more than 1", totalRegions, 3);
+ assertEquals("the number of regions should be more than 1", 3, totalRegions);
} finally {
disableAndDeleteTable(tableName);
}
@@ -132,7 +132,7 @@ public class TestRegionReplicasWithModifyTable {
List<HRegion> onlineRegions2 = getSecondaryRS().getRegions(tableName);
List<HRegion> onlineRegions3 = getTertiaryRS().getRegions(tableName);
int totalRegions = onlineRegions.size() + onlineRegions2.size() + onlineRegions3.size();
- assertEquals("the number of regions should be equal to 30", totalRegions, 30);
+ assertEquals("the number of regions should be equal to 30", 30, totalRegions);
} finally {
disableAndDeleteTable(tableName);
}
@@ -148,7 +148,7 @@ public class TestRegionReplicasWithModifyTable {
List<HRegion> onlineRegions2 = getSecondaryRS().getRegions(tableName);
List<HRegion> onlineRegions3 = getTertiaryRS().getRegions(tableName);
int totalRegions = onlineRegions.size() + onlineRegions2.size() + onlineRegions3.size();
- assertEquals("the number of regions should be 3", totalRegions, 3);
+ assertEquals("the number of regions should be 3", 3, totalRegions);
} finally {
disableAndDeleteTable(tableName);
}
@@ -164,7 +164,7 @@ public class TestRegionReplicasWithModifyTable {
List<HRegion> onlineRegions2 = getSecondaryRS().getRegions(tableName);
List<HRegion> onlineRegions3 = getTertiaryRS().getRegions(tableName);
int totalRegions = onlineRegions.size() + onlineRegions2.size() + onlineRegions3.size();
- assertEquals("the number of regions should be reduced to 2", totalRegions, 2);
+ assertEquals("the number of regions should be reduced to 2", 2, totalRegions);
} finally {
disableAndDeleteTable(tableName);
}
@@ -181,7 +181,7 @@ public class TestRegionReplicasWithModifyTable {
List<HRegion> onlineRegions2 = getSecondaryRS().getRegions(tableName);
List<HRegion> onlineRegions3 = getTertiaryRS().getRegions(tableName);
int totalRegions = onlineRegions.size() + onlineRegions2.size() + onlineRegions3.size();
- assertEquals("the number of regions should be reduced to 40", totalRegions, 40);
+ assertEquals("the number of regions should be reduced to 40", 40, totalRegions);
} finally {
disableAndDeleteTable(tableName);
}
@@ -198,7 +198,7 @@ public class TestRegionReplicasWithModifyTable {
List<HRegion> onlineRegions2 = getSecondaryRS().getRegions(tableName);
List<HRegion> onlineRegions3 = getTertiaryRS().getRegions(tableName);
int totalRegions = onlineRegions.size() + onlineRegions2.size() + onlineRegions3.size();
- assertEquals("the number of regions should be equal to 45", totalRegions, 3 * 15);
+ assertEquals("the number of regions should be equal to 45", 3 * 15, totalRegions);
} finally {
disableAndDeleteTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java
index 0122674..6172d61 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java
@@ -37,7 +37,7 @@ public class TestRegionServerAccounting {
// try for default cases
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
MemStoreSize memstoreSize =
- new MemStoreSize((long) (3l * 1024l * 1024l * 1024l), (long) (1l * 1024l * 1024l * 1024l));
+ new MemStoreSize(3L * 1024 * 1024 * 1024, 1L * 1024 * 1024 * 1024);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK,
regionServerAccounting.isAboveHighWaterMark());
@@ -50,7 +50,7 @@ public class TestRegionServerAccounting {
// try for default cases
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
MemStoreSize memstoreSize =
- new MemStoreSize((long) (3l * 1024l * 1024l * 1024l), (long) (1l * 1024l * 1024l * 1024l));
+ new MemStoreSize(3L * 1024 * 1024 * 1024, 1L * 1024 * 1024 * 1024);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK,
regionServerAccounting.isAboveLowWaterMark());
@@ -60,12 +60,12 @@ public class TestRegionServerAccounting {
public void testOffheapMemstoreHigherWaterMarkLimitsDueToDataSize() {
Configuration conf = HBaseConfiguration.create();
// setting 1G as offheap data size
- conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1l * 1024l));
+ conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1L * 1024));
// try for default cases
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
// this will breach offheap limit as data size is higher and not due to heap size
MemStoreSize memstoreSize =
- new MemStoreSize((long) (3l * 1024l * 1024l * 1024l), (long) (1l * 1024l * 1024l * 1024l));
+ new MemStoreSize(3L * 1024 * 1024 * 1024, 1L * 1024 * 1024 * 1024);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
assertEquals(FlushType.ABOVE_OFFHEAP_HIGHER_MARK,
regionServerAccounting.isAboveHighWaterMark());
@@ -76,12 +76,12 @@ public class TestRegionServerAccounting {
Configuration conf = HBaseConfiguration.create();
conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.2f);
// setting 1G as offheap data size
- conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1l * 1024l));
+ conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1L * 1024));
// try for default cases
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
// this will breach higher limit as heap size is higher and not due to offheap size
MemStoreSize memstoreSize =
- new MemStoreSize((long) (3l * 1024l * 1024l), (long) (2l * 1024l * 1024l * 1024l));
+ new MemStoreSize(3L * 1024 * 1024, 2L * 1024 * 1024 * 1024);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK,
regionServerAccounting.isAboveHighWaterMark());
@@ -91,12 +91,12 @@ public class TestRegionServerAccounting {
public void testOffheapMemstoreLowerWaterMarkLimitsDueToDataSize() {
Configuration conf = HBaseConfiguration.create();
// setting 1G as offheap data size
- conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1l * 1024l));
+ conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1L * 1024));
// try for default cases
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
// this will breach offheap limit as data size is higher and not due to heap size
MemStoreSize memstoreSize =
- new MemStoreSize((long) (3l * 1024l * 1024l * 1024l), (long) (1l * 1024l * 1024l * 1024l));
+ new MemStoreSize(3L * 1024 * 1024 * 1024, 1L * 1024 * 1024 * 1024);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
assertEquals(FlushType.ABOVE_OFFHEAP_LOWER_MARK,
regionServerAccounting.isAboveLowWaterMark());
@@ -107,12 +107,12 @@ public class TestRegionServerAccounting {
Configuration conf = HBaseConfiguration.create();
conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.2f);
// setting 1G as offheap data size
- conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1l * 1024l));
+ conf.setLong(MemorySizeUtil.OFFHEAP_MEMSTORE_SIZE_KEY, (1L * 1024));
// try for default cases
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
// this will breach higher limit as heap size is higher and not due to offheap size
MemStoreSize memstoreSize =
- new MemStoreSize((long) (3l * 1024l * 1024l), (long) (2l * 1024l * 1024l * 1024l));
+ new MemStoreSize(3L * 1024 * 1024, 2L * 1024 * 1024 * 1024);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK,
regionServerAccounting.isAboveLowWaterMark());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
index ea27ee5..b63b844 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
@@ -32,13 +32,10 @@ import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
@@ -51,6 +48,11 @@ import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
+
/**
* Tests on the region server, without the master.
@@ -61,7 +63,7 @@ public class TestRegionServerNoMaster {
private static final Logger LOG = LoggerFactory.getLogger(TestRegionServerNoMaster.class);
private static final int NB_SERVERS = 1;
private static Table table;
- private static final byte[] row = "ee".getBytes();
+ private static final byte[] row = Bytes.toBytes("ee");
private static HRegionInfo hri;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
index c6dce67..afbb48d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
@@ -117,7 +117,8 @@ public class TestScanner {
// Increment the least significant character so we get to next row.
secondRowBytes[START_KEY_BYTES.length - 1]++;
thirdRowBytes = START_KEY_BYTES.clone();
- thirdRowBytes[START_KEY_BYTES.length - 1] += 2;
+ thirdRowBytes[START_KEY_BYTES.length - 1] =
+ (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
col1 = Bytes.toBytes("column1");
}
@@ -589,6 +590,7 @@ public class TestScanner {
if (flushIndex == count) {
LOG.info("Starting flush at flush index " + flushIndex);
Thread t = new Thread() {
+ @Override
public void run() {
try {
region.flush(true);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java
index 0c014fd..63ea993 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java
@@ -233,6 +233,7 @@ public class TestScannerWithBulkload {
// Create a scanner and then do bulk load
final CountDownLatch latch = new CountDownLatch(1);
new Thread() {
+ @Override
public void run() {
try {
Put put1 = new Put(Bytes.toBytes("row5"));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
index 613282f..8519c3c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
@@ -76,7 +76,7 @@ public class TestSplitLogWorker {
private SplitLogWorker slw;
private ExecutorService executorService;
- class DummyServer implements Server {
+ static class DummyServer implements Server {
private ZKWatcher zkw;
private Configuration conf;
private CoordinatedStateManager cm;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index ab2d4b4..c80fc2e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -673,7 +673,7 @@ public class TestSplitTransactionOnCluster {
FileSystem fs = TESTING_UTIL.getDFSCluster().getFileSystem();
Map<String, Path> storefiles =
FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
- assertEquals("Expected nothing but found " + storefiles.toString(), storefiles.size(), 0);
+ assertEquals("Expected nothing but found " + storefiles.toString(), 0, storefiles.size());
// find a splittable region. Refresh the regions list
regions = cluster.getRegions(tableName);
@@ -696,8 +696,8 @@ public class TestSplitTransactionOnCluster {
HBaseFsck.debugLsr(conf, new Path("/"));
Map<String, Path> storefilesAfter =
FSUtils.getTableStoreFilePathMap(null, fs, rootDir, tableName);
- assertEquals("Expected nothing but found " + storefilesAfter.toString(),
- storefilesAfter.size(), 0);
+ assertEquals("Expected nothing but found " + storefilesAfter.toString(), 0,
+ storefilesAfter.size());
hri = region.getRegionInfo(); // split parent
AssignmentManager am = cluster.getMaster().getAssignmentManager();
@@ -755,7 +755,7 @@ public class TestSplitTransactionOnCluster {
region.flush(true);
HStore store = region.getStore(Bytes.toBytes("f"));
Collection<HStoreFile> storefiles = store.getStorefiles();
- assertEquals(storefiles.size(), 1);
+ assertEquals(1, storefiles.size());
assertFalse(region.hasReferences());
Path referencePath =
region.getRegionFileSystem().splitStoreFile(region.getRegionInfo(), "f",
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
index 1f5db50..2c679d8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
@@ -164,6 +164,7 @@ public class TestStoreScanner {
new KeyValueScanner[] { new KeyValueScanFixture(CellComparator.getInstance(), CELL_GRID) }));
}
+ @Override
protected void resetKVHeap(List<? extends KeyValueScanner> scanners,
CellComparator comparator) throws IOException {
if (count == null) {
@@ -172,6 +173,7 @@ public class TestStoreScanner {
heap = new KeyValueHeapWithCount(scanners, comparator, count);
}
+ @Override
protected boolean trySkipToNextRow(Cell cell) throws IOException {
boolean optimized = super.trySkipToNextRow(cell);
LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
@@ -182,6 +184,7 @@ public class TestStoreScanner {
return optimized;
}
+ @Override
protected boolean trySkipToNextColumn(Cell cell) throws IOException {
boolean optimized = super.trySkipToNextColumn(cell);
LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
@@ -227,6 +230,7 @@ public class TestStoreScanner {
new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
}
+ @Override
protected boolean trySkipToNextColumn(Cell cell) throws IOException {
boolean optimized = super.trySkipToNextColumn(cell);
LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
@@ -255,6 +259,7 @@ public class TestStoreScanner {
new KeyValueScanFixture(CellComparator.getInstance(), CELL_WITH_VERSIONS) }));
}
+ @Override
protected boolean trySkipToNextColumn(Cell cell) throws IOException {
boolean optimized = super.trySkipToNextColumn(cell);
LOG.info("Cell=" + cell + ", nextIndex=" + CellUtil.toString(getNextIndexedKey(), false)
@@ -884,6 +889,7 @@ public class TestStoreScanner {
try {
final long now = System.currentTimeMillis();
EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() {
+ @Override
public long currentTime() {
return now;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSyncTimeRangeTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSyncTimeRangeTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSyncTimeRangeTracker.java
index 52c31d9..273535b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSyncTimeRangeTracker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSyncTimeRangeTracker.java
@@ -32,10 +32,12 @@ public class TestSyncTimeRangeTracker extends TestSimpleTimeRangeTracker {
private static final int NUM_KEYS = 10000000;
private static final int NUM_OF_THREADS = 20;
+ @Override
protected TimeRangeTracker getTimeRangeTracker() {
return TimeRangeTracker.create(TimeRangeTracker.Type.SYNC);
}
+ @Override
protected TimeRangeTracker getTimeRangeTracker(long min, long max) {
return TimeRangeTracker.create(TimeRangeTracker.Type.SYNC, min, max);
}
@@ -77,7 +79,7 @@ public class TestSyncTimeRangeTracker extends TestSimpleTimeRangeTracker {
assertTrue(trr.getMin() == 0);
}
- class RandomTestData {
+ static class RandomTestData {
private long[] keys = new long[NUM_KEYS];
private long min = Long.MAX_VALUE;
private long max = 0;
@@ -107,7 +109,7 @@ public class TestSyncTimeRangeTracker extends TestSimpleTimeRangeTracker {
}
}
- class TrtUpdateRunnable implements Runnable {
+ static class TrtUpdateRunnable implements Runnable {
private TimeRangeTracker trt;
private RandomTestData data;
@@ -116,6 +118,7 @@ public class TestSyncTimeRangeTracker extends TestSimpleTimeRangeTracker {
this.data = data;
}
+ @Override
public void run() {
for (long key : data.keys) {
trt.includeTimestamp(key);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
index 62d22d2..70bdc49 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
@@ -258,6 +258,7 @@ public class TestWALLockup {
// in HBASE-14317. Flush hangs trying to get sequenceid because the ringbuffer is held up
// by the zigzaglatch waiting on syncs to come home.
Thread t = new Thread ("Flusher") {
+ @Override
public void run() {
try {
if (region.getMemStoreSize() <= 0) {
@@ -444,6 +445,7 @@ public class TestWALLockup {
dodgyWAL2.append(region.getRegionInfo(), key, edit, true);
Thread t = new Thread("Sync") {
+ @Override
public void run() {
try {
dodgyWAL2.sync();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
index f3bd7ee..290f71a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
@@ -581,7 +581,7 @@ public class TestWalAndCompactingMemStoreFlush {
// The total memstores size should be empty
assertEquals(0, totalMemstoreSizePhaseV);
// Because there is nothing in any memstore the WAL's LSN should be -1
- assertEquals(smallestSeqInRegionCurrentMemstorePhaseV, HConstants.NO_SEQNUM);
+ assertEquals(HConstants.NO_SEQNUM, smallestSeqInRegionCurrentMemstorePhaseV);
// What happens when we hit the memstore limit, but we are not able to find
// any Column Family above the threshold?
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java
index 7cdd24d..13c7a6b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java
@@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -110,7 +111,8 @@ public class PerfTestCompactionPolicies extends MockStoreFileGenerator {
final Class<? extends StoreFileListGenerator> fileGenClass,
final int inMmax,
final int inMin,
- final float inRatio) throws IllegalAccessException, InstantiationException {
+ final float inRatio) throws IllegalAccessException, InstantiationException,
+ NoSuchMethodException, InvocationTargetException {
super(PerfTestCompactionPolicies.class);
this.fileGenClass = fileGenClass;
this.max = inMmax;
@@ -138,7 +140,7 @@ public class PerfTestCompactionPolicies extends MockStoreFileGenerator {
new Class[] {Configuration.class, StoreConfigInformation.class },
new Object[] {configuration, store });
- this.generator = fileGenClass.newInstance();
+ this.generator = fileGenClass.getDeclaredConstructor().newInstance();
// Used for making paths
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java
index 932664b..f8df870 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java
@@ -92,11 +92,13 @@ public class TestCompactor {
writers.add(realWriter);
StoreFileWriter writer = mock(StoreFileWriter.class);
doAnswer(new Answer<Object>() {
+ @Override
public Object answer(InvocationOnMock invocation) {
return realWriter.kvs.add((KeyValue) invocation.getArgument(0));
}
}).when(writer).append(any());
doAnswer(new Answer<Object>() {
+ @Override
public Object answer(InvocationOnMock invocation) {
Object[] args = invocation.getArguments();
return realWriter.data.put((byte[]) args[0], (byte[]) args[1]);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/querymatcher/TestUserScanQueryMatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/querymatcher/TestUserScanQueryMatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/querymatcher/TestUserScanQueryMatcher.java
index 3ae49c0..b43a89d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/querymatcher/TestUserScanQueryMatcher.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/querymatcher/TestUserScanQueryMatcher.java
@@ -242,7 +242,7 @@ public class TestUserScanQueryMatcher extends AbstractTestScanQueryMatcher {
}
}
- private class AlwaysIncludeAndSeekNextRowFilter extends FilterBase {
+ private static class AlwaysIncludeAndSeekNextRowFilter extends FilterBase {
@Override
public ReturnCode filterKeyValue(final Cell c) throws IOException {
@@ -287,7 +287,7 @@ public class TestUserScanQueryMatcher extends AbstractTestScanQueryMatcher {
}
}
- private class AlwaysIncludeFilter extends FilterBase {
+ private static class AlwaysIncludeFilter extends FilterBase {
@Override
public ReturnCode filterKeyValue(final Cell c) throws IOException {
return ReturnCode.INCLUDE;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
index 093a512..009cca0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
@@ -289,7 +289,7 @@ public abstract class AbstractTestFSWAL {
addEdits(wal, hri2, t2, 2, mvcc, scopes2);
// get the regions to flush, it should still read region1.
regionsToFlush = wal.findRegionsToForceFlush();
- assertEquals(regionsToFlush.length, 1);
+ assertEquals(1, regionsToFlush.length);
assertEquals(hri1.getEncodedNameAsBytes(), regionsToFlush[0]);
// flush region 1, and roll the wal file. Only last wal which has entries for region1 should
// remain.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
index ededcf3..6b55adc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java
@@ -705,9 +705,8 @@ public abstract class AbstractTestWALReplay {
try {
region.flush(true);
fail("Injected exception hasn't been thrown");
- } catch (Throwable t) {
- LOG.info("Expected simulated exception when flushing region,"
- + t.getMessage());
+ } catch (IOException e) {
+ LOG.info("Expected simulated exception when flushing region, {}", e.getMessage());
// simulated to abort server
Mockito.doReturn(true).when(rsServices).isAborted();
region.setClosing(false); // region normally does not accept writes after
@@ -928,8 +927,7 @@ public abstract class AbstractTestWALReplay {
* testcase for https://issues.apache.org/jira/browse/HBASE-15252
*/
@Test
- public void testDatalossWhenInputError() throws IOException, InstantiationException,
- IllegalAccessException {
+ public void testDatalossWhenInputError() throws Exception {
final TableName tableName = TableName.valueOf("testDatalossWhenInputError");
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
@@ -964,7 +962,7 @@ public abstract class AbstractTestWALReplay {
Class<? extends AbstractFSWALProvider.Reader> logReaderClass =
conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
AbstractFSWALProvider.Reader.class);
- AbstractFSWALProvider.Reader reader = logReaderClass.newInstance();
+ AbstractFSWALProvider.Reader reader = logReaderClass.getDeclaredConstructor().newInstance();
reader.init(this.fs, editFile, conf, stream);
final long headerLength = stream.getPos();
reader.close();
@@ -1108,7 +1106,7 @@ public abstract class AbstractTestWALReplay {
// Flusher used in this test. Keep count of how often we are called and
// actually run the flush inside here.
- class TestFlusher implements FlushRequester {
+ static class TestFlusher implements FlushRequester {
private HRegion r;
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedLogWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedLogWriter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedLogWriter.java
index 2aebf2b..5a61370 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedLogWriter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/InstrumentedLogWriter.java
@@ -39,7 +39,7 @@ public class InstrumentedLogWriter extends ProtobufLogWriter {
public void append(Entry entry) throws IOException {
super.append(entry);
if (activateFailure &&
- Bytes.equals(entry.getKey().getEncodedRegionName(), "break".getBytes())) {
+ Bytes.equals(entry.getKey().getEncodedRegionName(), Bytes.toBytes("break"))) {
System.out.println(getClass().getName() + ": I will throw an exception now...");
throw(new IOException("This exception is instrumented and should only be thrown for testing"
));
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
index ed71123..0f5fda0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
@@ -220,8 +220,8 @@ public class TestNamespaceReplication extends TestReplicationBase {
if (res.isEmpty()) {
LOG.info("Row not available");
} else {
- assertEquals(res.size(), 1);
- assertArrayEquals(res.value(), val);
+ assertEquals(1, res.size());
+ assertArrayEquals(val, res.value());
break;
}
Thread.sleep(SLEEP_TIME);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestPerTableCFReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestPerTableCFReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestPerTableCFReplication.java
index bacda63..98b3fda 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestPerTableCFReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestPerTableCFReplication.java
@@ -582,8 +582,8 @@ public class TestPerTableCFReplication {
replicatedToAll = false;
break;
} else {
- assertEquals(res.size(), 1);
- assertArrayEquals(res.value(), val);
+ assertEquals(1, res.size());
+ assertArrayEquals(val, res.value());
}
}
if (replicatedToAll) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
index afb975d..3a7a575 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -270,7 +270,7 @@ public class TestReplicationBase {
LOG.info("Row not available");
Thread.sleep(SLEEP_TIME);
} else {
- assertArrayEquals(res.value(), row);
+ assertArrayEquals(row, res.value());
break;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
index 1675496..7b9dea4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDisableInactivePeer.java
@@ -84,7 +84,7 @@ public class TestReplicationDisableInactivePeer extends TestReplicationBase {
LOG.info("Row not available");
Thread.sleep(SLEEP_TIME * NB_RETRIES);
} else {
- assertArrayEquals(res.value(), row);
+ assertArrayEquals(row, res.value());
return;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java
index 30cd860..2837045 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java
@@ -111,6 +111,7 @@ public class TestReplicationKillRS extends TestReplicationBase {
private static Thread killARegionServer(final HBaseTestingUtility utility,
final long timeout, final int rs) {
Thread killer = new Thread() {
+ @Override
public void run() {
try {
Thread.sleep(timeout);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index cb47827..f46a7b1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -209,7 +209,7 @@ public class TestReplicationSmallTests extends TestReplicationBase {
LOG.info("Row not available");
Thread.sleep(SLEEP_TIME);
} else {
- assertArrayEquals(res.value(), row);
+ assertArrayEquals(row, res.value());
return;
}
}
@@ -262,7 +262,7 @@ public class TestReplicationSmallTests extends TestReplicationBase {
LOG.info("Row not available");
Thread.sleep(SLEEP_TIME * i);
} else {
- assertArrayEquals(res.value(), row);
+ assertArrayEquals(row, res.value());
break;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
index 0a602ad..b473277 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
@@ -277,14 +277,14 @@ public class TestReplicationSyncUpTool extends TestReplicationBase {
// delete half of the rows
for (int i = 0; i < NB_ROWS_IN_BATCH / 2; i++) {
String rowKey = "row" + i;
- Delete del = new Delete(rowKey.getBytes());
+ Delete del = new Delete(Bytes.toBytes(rowKey));
list.add(del);
}
ht1Source.delete(list);
for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
String rowKey = "row" + i;
- Delete del = new Delete(rowKey.getBytes());
+ Delete del = new Delete(Bytes.toBytes(rowKey));
list.add(del);
}
ht2Source.delete(list);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithTags.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithTags.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithTags.java
index 98f11f7..b2ecb67 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithTags.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationWithTags.java
@@ -187,7 +187,7 @@ public class TestReplicationWithTags {
LOG.info("Row not available");
Thread.sleep(SLEEP_TIME);
} else {
- assertArrayEquals(res.value(), ROW);
+ assertArrayEquals(ROW, res.value());
assertEquals(1, TestCoprocessorForTagsAtSink.tags.size());
Tag tag = TestCoprocessorForTagsAtSink.tags.get(0);
assertEquals(TAG_TYPE, tag.getType());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
index df84515..e69d84c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpoint.java
@@ -143,8 +143,8 @@ public class TestRegionReplicaReplicationEndpoint {
assertNotNull(peerConfig);
assertEquals(peerConfig.getClusterKey(), ZKConfig.getZooKeeperClusterKey(
HTU.getConfiguration()));
- assertEquals(peerConfig.getReplicationEndpointImpl(),
- RegionReplicaReplicationEndpoint.class.getName());
+ assertEquals(RegionReplicaReplicationEndpoint.class.getName(),
+ peerConfig.getReplicationEndpointImpl());
admin.close();
}
@@ -190,8 +190,8 @@ public class TestRegionReplicaReplicationEndpoint {
assertNotNull(peerConfig);
assertEquals(peerConfig.getClusterKey(), ZKConfig.getZooKeeperClusterKey(
HTU.getConfiguration()));
- assertEquals(peerConfig.getReplicationEndpointImpl(),
- RegionReplicaReplicationEndpoint.class.getName());
+ assertEquals(RegionReplicaReplicationEndpoint.class.getName(),
+ peerConfig.getReplicationEndpointImpl());
admin.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java
index d7044dc..375e64e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicator.java
@@ -31,27 +31,61 @@ import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.ipc.RpcServer;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.*;
-import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
-import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.TestReplicationBase;
-import org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint;
import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WAL.Entry;
-
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Test;
import org.junit.Ignore;
+import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.RollWALWriterRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.RollWALWriterResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.StopServerRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.StopServerResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateConfigurationRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateConfigurationResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WarmupRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WarmupRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse;
+
@Category(MediumTests.class)
@Ignore("Flaky, needs to be rewritten, see HBASE-19125")
@@ -88,7 +122,7 @@ public class TestReplicator extends TestReplicationBase {
// have to be replicated separately.
final byte[] valueBytes = new byte[8 *1024];
for (int i = 0; i < NUM_ROWS; i++) {
- htable1.put(new Put(("row"+Integer.toString(i)).getBytes())
+ htable1.put(new Put(Bytes.toBytes("row"+Integer.toString(i)))
.addColumn(famName, null, valueBytes)
);
}
@@ -140,7 +174,7 @@ public class TestReplicator extends TestReplicationBase {
// have to be replicated separately.
final byte[] valueBytes = new byte[8 *1024];
for (int i = 0; i < NUM_ROWS; i++) {
- htable1.put(new Put(("row"+Integer.toString(i)).getBytes())
+ htable1.put(new Put(Bytes.toBytes("row"+Integer.toString(i)))
.addColumn(famName, null, valueBytes)
);
}
@@ -409,7 +443,7 @@ public class TestReplicator extends TestReplicationBase {
@Override
public ClearRegionBlockCacheResponse clearRegionBlockCache(RpcController controller,
- ClearRegionBlockCacheRequest request) throws ServiceException {
+ ClearRegionBlockCacheRequest request) throws ServiceException {
return delegate.clearRegionBlockCache(controller, request);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java
index 43140bc..44c76f8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java
@@ -363,7 +363,7 @@ public class TestWALEntryStream {
appendToLog("foo");
entryBatch = batcher.take();
assertEquals(1, entryBatch.getNbEntries());
- assertEquals(getRow(entryBatch.getWalEntries().get(0)), "foo");
+ assertEquals("foo", getRow(entryBatch.getWalEntries().get(0)));
}
private String getRow(WAL.Entry entry) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecureIPC.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecureIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecureIPC.java
index 2414e5a..1399b21 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecureIPC.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestSecureIPC.java
@@ -153,7 +153,7 @@ public class TestSecureIPC {
UserGroupInformation ugi2 = UserGroupInformation.getCurrentUser();
// check that the login user is okay:
- assertSame(ugi, ugi2);
+ assertSame(ugi2, ugi);
assertEquals(AuthenticationMethod.KERBEROS, ugi.getAuthenticationMethod());
assertEquals(krbPrincipal, ugi.getUserName());
@@ -280,6 +280,7 @@ public class TestSecureIPC {
final Throwable exception[] = new Throwable[1];
Collections.synchronizedList(new ArrayList<Throwable>());
Thread.UncaughtExceptionHandler exceptionHandler = new Thread.UncaughtExceptionHandler() {
+ @Override
public void uncaughtException(Thread th, Throwable ex) {
exception[0] = ex;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestUser.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
index bfc82db..d5eed3c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/TestUser.java
@@ -120,6 +120,7 @@ public class TestUser {
Configuration conf = HBaseConfiguration.create();
final User user = User.createUserForTesting(conf, "testuser", new String[]{"foo"});
final PrivilegedExceptionAction<String> action = new PrivilegedExceptionAction<String>(){
+ @Override
public String run() throws IOException {
User u = User.getCurrent();
return u.getName();
@@ -138,6 +139,7 @@ public class TestUser {
// check the exception version
username = user.runAs(new PrivilegedExceptionAction<String>(){
+ @Override
public String run() throws Exception {
return User.getCurrent().getName();
}
@@ -146,6 +148,7 @@ public class TestUser {
// verify that nested contexts work
user2.runAs(new PrivilegedExceptionAction<Object>(){
+ @Override
public Object run() throws IOException, InterruptedException{
String nestedName = user.runAs(action);
assertEquals("Nest name should match nested user", "testuser", nestedName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
index 06389ab..8a1af20 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessControlFilter.java
@@ -123,6 +123,7 @@ public class TestAccessControlFilter extends SecureTestUtil {
// test read
READER.runAs(new PrivilegedExceptionAction<Object>() {
+ @Override
public Object run() throws Exception {
Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
// force a new RS connection
@@ -151,6 +152,7 @@ public class TestAccessControlFilter extends SecureTestUtil {
// test read with qualifier filter
LIMITED.runAs(new PrivilegedExceptionAction<Object>() {
+ @Override
public Object run() throws Exception {
Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
// force a new RS connection
@@ -178,6 +180,7 @@ public class TestAccessControlFilter extends SecureTestUtil {
// test as user with no permission
DENIED.runAs(new PrivilegedExceptionAction<Object>(){
+ @Override
public Object run() throws Exception {
Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
// force a new RS connection
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index be1b0e4..14e94be 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -306,12 +306,15 @@ public class TestAccessController extends SecureTestUtil {
grantGlobal(TEST_UTIL, toGroupEntry(GROUP_WRITE), Permission.Action.WRITE);
assertEquals(5, AccessControlLists.getTablePermissions(conf, TEST_TABLE).size());
+ int size = 0;
try {
- assertEquals(5, AccessControlClient.getUserPermissions(systemUserConnection,
- TEST_TABLE.toString()).size());
+ size = AccessControlClient.getUserPermissions(systemUserConnection, TEST_TABLE.toString())
+ .size();
} catch (Throwable e) {
LOG.error("error during call of AccessControlClient.getUserPermissions. ", e);
+ fail("error during call of AccessControlClient.getUserPermissions.");
}
+ assertEquals(5, size);
}
private static void cleanUp() throws Exception {
@@ -992,7 +995,7 @@ public class TestAccessController extends SecureTestUtil {
}
}
- public class BulkLoadHelper {
+ public static class BulkLoadHelper {
private final FileSystem fs;
private final Path loadPath;
private final Configuration conf;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcher.java
index 1e5ea53..252de3a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcher.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcher.java
@@ -56,11 +56,13 @@ public class TestZKSecretWatcher {
private static class MockAbortable implements Abortable {
private boolean abort;
+ @Override
public void abort(String reason, Throwable e) {
LOG.info("Aborting: "+reason, e);
abort = true;
}
+ @Override
public boolean isAborted() {
return abort;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcherRefreshKeys.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcherRefreshKeys.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcherRefreshKeys.java
index 391a844..9a249c3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcherRefreshKeys.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestZKSecretWatcherRefreshKeys.java
@@ -46,11 +46,13 @@ public class TestZKSecretWatcherRefreshKeys {
private static class MockAbortable implements Abortable {
private boolean abort;
+ @Override
public void abort(String reason, Throwable e) {
LOG.info("Aborting: "+reason, e);
abort = true;
}
+ @Override
public boolean isAborted() {
return abort;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestDefaultScanLabelGeneratorStack.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestDefaultScanLabelGeneratorStack.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestDefaultScanLabelGeneratorStack.java
index 9da2531..6dacf02 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestDefaultScanLabelGeneratorStack.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestDefaultScanLabelGeneratorStack.java
@@ -85,6 +85,7 @@ public class TestDefaultScanLabelGeneratorStack {
// Set up for the test
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
VisibilityClient.addLabels(conn, new String[] { SECRET, CONFIDENTIAL });
@@ -102,6 +103,7 @@ public class TestDefaultScanLabelGeneratorStack {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = TEST_UTIL.createTable(tableName, CF)) {
@@ -123,15 +125,13 @@ public class TestDefaultScanLabelGeneratorStack {
// Test that super user can see all the cells.
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
- Scan s = new Scan();
- ResultScanner scanner = table.getScanner(s);
- Result[] next = scanner.next(1);
+ Result[] next = getResult(table, new Scan());
// Test that super user can see all the cells.
- assertTrue(next.length == 1);
CellScanner cellScanner = next[0].cellScanner();
cellScanner.advance();
Cell current = cellScanner.current();
@@ -164,15 +164,12 @@ public class TestDefaultScanLabelGeneratorStack {
});
TESTUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
// Test scan with no auth attribute
- Scan s = new Scan();
- ResultScanner scanner = table.getScanner(s);
- Result[] next = scanner.next(1);
-
- assertTrue(next.length == 1);
+ Result[] next = getResult(table, new Scan());
CellScanner cellScanner = next[0].cellScanner();
cellScanner.advance();
Cell current = cellScanner.current();
@@ -248,6 +245,13 @@ public class TestDefaultScanLabelGeneratorStack {
});
}
+
+ private static Result [] getResult(Table table, Scan scan) throws IOException {
+ ResultScanner scanner = table.getScanner(scan);
+ Result[] next = scanner.next(1);
+ assertTrue(next.length == 1);
+ return next;
+ }
@AfterClass
public static void tearDownAfterClass() throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestEnforcingScanLabelGenerator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestEnforcingScanLabelGenerator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestEnforcingScanLabelGenerator.java
index a0703fc..c7075e3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestEnforcingScanLabelGenerator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestEnforcingScanLabelGenerator.java
@@ -82,6 +82,7 @@ public class TestEnforcingScanLabelGenerator {
// Set up for the test
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
VisibilityClient.addLabels(conn, new String[] { SECRET, CONFIDENTIAL });
@@ -99,6 +100,7 @@ public class TestEnforcingScanLabelGenerator {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = TEST_UTIL.createTable(tableName, CF)) {
@@ -120,6 +122,7 @@ public class TestEnforcingScanLabelGenerator {
// Test that super user can see all the cells.
SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
@@ -135,6 +138,7 @@ public class TestEnforcingScanLabelGenerator {
});
TESTUSER.runAs(new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelReplicationWithExpAsString.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelReplicationWithExpAsString.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelReplicationWithExpAsString.java
index 521cafe..76bba48 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelReplicationWithExpAsString.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelReplicationWithExpAsString.java
@@ -160,6 +160,7 @@ public class TestVisibilityLabelReplicationWithExpAsString extends TestVisibilit
InterruptedException {
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf1);
Table table2 = connection.getTable(TABLE_NAME)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java
index ba93d19..932f63e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java
@@ -302,6 +302,7 @@ public abstract class TestVisibilityLabels {
TEST_UTIL.getHBaseCluster().startRegionServer();
}
Thread t1 = new Thread() {
+ @Override
public void run() {
List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
.getRegionServerThreads();
@@ -320,6 +321,7 @@ public abstract class TestVisibilityLabels {
t1.start();
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
Thread t = new Thread() {
+ @Override
public void run() {
try {
while (!killedRS) {
@@ -415,6 +417,7 @@ public abstract class TestVisibilityLabels {
public void testSetAndGetUserAuths() throws Throwable {
final String user = "user1";
PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
String[] auths = { SECRET, CONFIDENTIAL };
try (Connection conn = ConnectionFactory.createConnection(conf)) {
@@ -441,6 +444,7 @@ public abstract class TestVisibilityLabels {
}
action = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
GetAuthsResponse authsResponse = null;
try (Connection conn = ConnectionFactory.createConnection(conf)) {
@@ -462,6 +466,7 @@ public abstract class TestVisibilityLabels {
// Try doing setAuths once again and there should not be any duplicates
action = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
String[] auths1 = { SECRET, CONFIDENTIAL };
GetAuthsResponse authsResponse = null;
@@ -491,7 +496,7 @@ public abstract class TestVisibilityLabels {
List<String> auths = new ArrayList<>();
for (Result result : results) {
Cell labelCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
- Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, user.getBytes());
+ Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, Bytes.toBytes(user));
if (userAuthCell != null) {
auths.add(Bytes.toString(labelCell.getValueArray(), labelCell.getValueOffset(),
labelCell.getValueLength()));
@@ -503,6 +508,7 @@ public abstract class TestVisibilityLabels {
@Test
public void testClearUserAuths() throws Throwable {
PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
String[] auths = { SECRET, CONFIDENTIAL, PRIVATE };
String user = "testUser";
@@ -700,32 +706,32 @@ public abstract class TestVisibilityLabels {
TEST_UTIL.getAdmin().createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(r1);
- put.addColumn(fam, qual, 3l, v1);
- put.addColumn(fam, qual2, 3l, v1);
- put.addColumn(fam2, qual, 3l, v1);
- put.addColumn(fam2, qual2, 3l, v1);
+ put.addColumn(fam, qual, 3L, v1);
+ put.addColumn(fam, qual2, 3L, v1);
+ put.addColumn(fam2, qual, 3L, v1);
+ put.addColumn(fam2, qual2, 3L, v1);
put.setCellVisibility(new CellVisibility(SECRET));
table.put(put);
put = new Put(r1);
- put.addColumn(fam, qual, 4l, v2);
- put.addColumn(fam, qual2, 4l, v2);
- put.addColumn(fam2, qual, 4l, v2);
- put.addColumn(fam2, qual2, 4l, v2);
+ put.addColumn(fam, qual, 4L, v2);
+ put.addColumn(fam, qual2, 4L, v2);
+ put.addColumn(fam2, qual, 4L, v2);
+ put.addColumn(fam2, qual2, 4L, v2);
put.setCellVisibility(new CellVisibility(PRIVATE));
table.put(put);
put = new Put(r2);
- put.addColumn(fam, qual, 3l, v1);
- put.addColumn(fam, qual2, 3l, v1);
- put.addColumn(fam2, qual, 3l, v1);
- put.addColumn(fam2, qual2, 3l, v1);
+ put.addColumn(fam, qual, 3L, v1);
+ put.addColumn(fam, qual2, 3L, v1);
+ put.addColumn(fam2, qual, 3L, v1);
+ put.addColumn(fam2, qual2, 3L, v1);
put.setCellVisibility(new CellVisibility(SECRET));
table.put(put);
put = new Put(r2);
- put.addColumn(fam, qual, 4l, v2);
- put.addColumn(fam, qual2, 4l, v2);
- put.addColumn(fam2, qual, 4l, v2);
- put.addColumn(fam2, qual2, 4l, v2);
+ put.addColumn(fam, qual, 4L, v2);
+ put.addColumn(fam, qual2, 4L, v2);
+ put.addColumn(fam2, qual, 4L, v2);
+ put.addColumn(fam2, qual2, 4L, v2);
put.setCellVisibility(new CellVisibility(SECRET));
table.put(put);
@@ -860,6 +866,7 @@ public abstract class TestVisibilityLabels {
public static void addLabels() throws Exception {
PrivilegedExceptionAction<VisibilityLabelsResponse> action =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
UNICODE_VIS_TAG, UC1, UC2 };
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOnNewVersionBehaviorTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOnNewVersionBehaviorTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOnNewVersionBehaviorTable.java
index 843ca99..c14438e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOnNewVersionBehaviorTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOnNewVersionBehaviorTable.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.client.Table;
public class TestVisibilityLabelsOnNewVersionBehaviorTable extends TestVisibilityLabelsWithDeletes {
+ @Override
protected Table createTable(HColumnDescriptor fam) throws IOException {
fam.setNewVersionBehavior(true);
TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOpWithDifferentUsersNoACL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOpWithDifferentUsersNoACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOpWithDifferentUsersNoACL.java
index a3c926e..8d75156 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOpWithDifferentUsersNoACL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsOpWithDifferentUsersNoACL.java
@@ -85,6 +85,7 @@ public class TestVisibilityLabelsOpWithDifferentUsersNoACL {
public void testLabelsTableOpsWithDifferentUsers() throws Throwable {
PrivilegedExceptionAction<VisibilityLabelsResponse> action =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.setAuths(conn, new String[] { CONFIDENTIAL, PRIVATE }, "user1");
@@ -99,6 +100,7 @@ public class TestVisibilityLabelsOpWithDifferentUsersNoACL {
// Ideally this should not be allowed. this operation should fail or do nothing.
action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.setAuths(conn, new String[] { CONFIDENTIAL, PRIVATE }, "user3");
@@ -115,6 +117,7 @@ public class TestVisibilityLabelsOpWithDifferentUsersNoACL {
PrivilegedExceptionAction<GetAuthsResponse> action1 =
new PrivilegedExceptionAction<GetAuthsResponse>() {
+ @Override
public GetAuthsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.getAuths(conn, "user1");
@@ -138,6 +141,7 @@ public class TestVisibilityLabelsOpWithDifferentUsersNoACL {
PrivilegedExceptionAction<VisibilityLabelsResponse> action2 =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.clearAuths(conn, new String[] {
@@ -162,6 +166,7 @@ public class TestVisibilityLabelsOpWithDifferentUsersNoACL {
private static void addLabels() throws Exception {
PrivilegedExceptionAction<VisibilityLabelsResponse> action =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
String[] labels = { SECRET, CONFIDENTIAL, PRIVATE };
try (Connection conn = ConnectionFactory.createConnection(conf)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsReplication.java
index 072a385..dce8591 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsReplication.java
@@ -301,6 +301,7 @@ public class TestVisibilityLabelsReplication {
final boolean nullExpected, final String... auths) throws IOException,
InterruptedException {
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf1);
Table table2 = connection.getTable(TABLE_NAME)) {
@@ -346,6 +347,7 @@ public class TestVisibilityLabelsReplication {
public static void addLabels() throws Exception {
PrivilegedExceptionAction<VisibilityLabelsResponse> action =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, UNICODE_VIS_TAG };
try (Connection conn = ConnectionFactory.createConnection(conf)) {
@@ -362,6 +364,7 @@ public class TestVisibilityLabelsReplication {
public static void setAuths(final Configuration conf) throws Exception {
PrivilegedExceptionAction<VisibilityLabelsResponse> action =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.setAuths(conn, new String[] { SECRET,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithACL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithACL.java
index f6ff640..ef1ae98 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithACL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithACL.java
@@ -122,6 +122,7 @@ public class TestVisibilityLabelsWithACL {
SecureTestUtil.grantOnTable(TEST_UTIL, NORMAL_USER2.getShortName(), tableName,
null, null, Permission.Action.READ);
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
Scan s = new Scan();
s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
@@ -151,6 +152,7 @@ public class TestVisibilityLabelsWithACL {
final Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL
+ "&!" + PRIVATE, SECRET + "&!" + PRIVATE);
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
Scan s = new Scan();
s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
@@ -175,6 +177,7 @@ public class TestVisibilityLabelsWithACL {
final Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL
+ "&!" + PRIVATE, SECRET + "&!" + PRIVATE);
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
Get g = new Get(row1);
g.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
@@ -204,6 +207,7 @@ public class TestVisibilityLabelsWithACL {
SecureTestUtil.grantOnTable(TEST_UTIL, NORMAL_USER2.getShortName(), tableName,
null, null, Permission.Action.READ);
PrivilegedExceptionAction<Void> getAction = new PrivilegedExceptionAction<Void>() {
+ @Override
public Void run() throws Exception {
Get g = new Get(row1);
g.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
@@ -222,6 +226,7 @@ public class TestVisibilityLabelsWithACL {
public void testLabelsTableOpsWithDifferentUsers() throws Throwable {
PrivilegedExceptionAction<VisibilityLabelsResponse> action =
new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.addLabels(conn, new String[] { "l1", "l2" });
@@ -237,6 +242,7 @@ public class TestVisibilityLabelsWithACL {
.getResult(1).getException().getName());
action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.setAuths(conn, new String[] { CONFIDENTIAL, PRIVATE }, "user1");
@@ -252,6 +258,7 @@ public class TestVisibilityLabelsWithACL {
.getResult(1).getException().getName());
action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.setAuths(conn, new String[] { CONFIDENTIAL, PRIVATE }, "user1");
@@ -265,6 +272,7 @@ public class TestVisibilityLabelsWithACL {
assertTrue(response.getResult(1).getException().getValue().isEmpty());
action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
+ @Override
public VisibilityLabelsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.clearAuths(conn, new String[] {
@@ -289,6 +297,7 @@ public class TestVisibilityLabelsWithACL {
"user3");
PrivilegedExceptionAction<GetAuthsResponse> action1 =
new PrivilegedExceptionAction<GetAuthsResponse>() {
+ @Override
public GetAuthsResponse run() throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf)) {
return VisibilityClient.getAuths(conn, "user3");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithCustomVisLabService.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithCustomVisLabService.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithCustomVisLabService.java
index 2d3f607..ea1ed10 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithCustomVisLabService.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithCustomVisLabService.java
@@ -56,6 +56,7 @@ public class TestVisibilityLabelsWithCustomVisLabService extends TestVisibilityL
}
// Extending this test from super as we don't verify predefined labels in ExpAsStringVisibilityLabelServiceImpl
+ @Override
@Test
public void testVisibilityLabelsInPutsThatDoesNotMatchAnyDefinedLabels() throws Exception {
TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
@@ -63,6 +64,7 @@ public class TestVisibilityLabelsWithCustomVisLabService extends TestVisibilityL
createTableAndWriteDataWithLabels(tableName, "SAMPLE_LABEL", "TEST");
}
+ @Override
protected List<String> extractAuths(String user, List<Result> results) {
List<String> auths = new ArrayList<>();
for (Result result : results) {
[43/50] [abbrv] hbase git commit: HBASE-19636 All rs should already
start work with the new peer change when replication peer procedure is
finished
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 0dbcc3a..ab86d7c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
@@ -33,7 +31,7 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
@@ -70,27 +68,53 @@ import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesti
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
- * This class is responsible to manage all the replication
- * sources. There are two classes of sources:
+ * This class is responsible to manage all the replication sources. There are two classes of
+ * sources:
* <ul>
- * <li> Normal sources are persistent and one per peer cluster</li>
- * <li> Old sources are recovered from a failed region server and our
- * only goal is to finish replicating the WAL queue it had up in ZK</li>
+ * <li>Normal sources are persistent and one per peer cluster</li>
+ * <li>Old sources are recovered from a failed region server and our only goal is to finish
+ * replicating the WAL queue it had</li>
+ * </ul>
+ * <p>
+ * When a region server dies, this class uses a watcher to get notified and it tries to grab a lock
+ * in order to transfer all the queues in a local old source.
+ * <p>
+ * Synchronization specification:
+ * <ul>
+ * <li>No need synchronized on {@link #sources}. {@link #sources} is a ConcurrentHashMap and there
+ * is a Lock for peer id in {@link PeerProcedureHandlerImpl}. So there is no race for peer
+ * operations.</li>
+ * <li>Need synchronized on {@link #walsById}. There are four methods which modify it,
+ * {@link #addPeer(String)}, {@link #removePeer(String)},
+ * {@link #cleanOldLogs(SortedSet, String, String)} and {@link #preLogRoll(Path)}. {@link #walsById}
+ * is a ConcurrentHashMap and there is a Lock for peer id in {@link PeerProcedureHandlerImpl}. So
+ * there is no race between {@link #addPeer(String)} and {@link #removePeer(String)}.
+ * {@link #cleanOldLogs(SortedSet, String, String)} is called by {@link ReplicationSourceInterface}.
+ * So no race with {@link #addPeer(String)}. {@link #removePeer(String)} will terminate the
+ * {@link ReplicationSourceInterface} firstly, then remove the wals from {@link #walsById}. So no
+ * race with {@link #removePeer(String)}. The only case need synchronized is
+ * {@link #cleanOldLogs(SortedSet, String, String)} and {@link #preLogRoll(Path)}.</li>
+ * <li>No need synchronized on {@link #walsByIdRecoveredQueues}. There are three methods which
+ * modify it, {@link #removePeer(String)} , {@link #cleanOldLogs(SortedSet, String, String)} and
+ * {@link ReplicationSourceManager.NodeFailoverWorker#run()}.
+ * {@link #cleanOldLogs(SortedSet, String, String)} is called by {@link ReplicationSourceInterface}.
+ * {@link #removePeer(String)} will terminate the {@link ReplicationSourceInterface} firstly, then
+ * remove the wals from {@link #walsByIdRecoveredQueues}. And
+ * {@link ReplicationSourceManager.NodeFailoverWorker#run()} will add the wals to
+ * {@link #walsByIdRecoveredQueues} firstly, then start up a {@link ReplicationSourceInterface}. So
+ * there is no race here. For {@link ReplicationSourceManager.NodeFailoverWorker#run()} and
+ * {@link #removePeer(String)}, there is already synchronized on {@link #oldsources}. So no need
+ * synchronized on {@link #walsByIdRecoveredQueues}.</li>
+ * <li>Need synchronized on {@link #latestPaths} to avoid the new open source miss new log.</li>
+ * <li>Need synchronized on {@link #oldsources} to avoid adding recovered source for the
+ * to-be-removed peer.</li>
* </ul>
- *
- * When a region server dies, this class uses a watcher to get notified and it
- * tries to grab a lock in order to transfer all the queues in a local
- * old source.
- *
- * This class implements the ReplicationListener interface so that it can track changes in
- * replication state.
*/
@InterfaceAudience.Private
public class ReplicationSourceManager implements ReplicationListener {
- private static final Logger LOG =
- LoggerFactory.getLogger(ReplicationSourceManager.class);
- // List of all the sources that read this RS's logs
- private final List<ReplicationSourceInterface> sources;
+ private static final Logger LOG = LoggerFactory.getLogger(ReplicationSourceManager.class);
+ // all the sources that read this RS's logs and every peer only has one replication source
+ private final ConcurrentMap<String, ReplicationSourceInterface> sources;
// List of all the sources we got from died RSs
private final List<ReplicationSourceInterface> oldsources;
private final ReplicationQueueStorage queueStorage;
@@ -100,11 +124,16 @@ public class ReplicationSourceManager implements ReplicationListener {
private final UUID clusterId;
// All about stopping
private final Server server;
+
// All logs we are currently tracking
- // Index structure of the map is: peer_id->logPrefix/logGroup->logs
- private final Map<String, Map<String, SortedSet<String>>> walsById;
+ // Index structure of the map is: queue_id->logPrefix/logGroup->logs
+ // For normal replication source, the peer id is same with the queue id
+ private final ConcurrentMap<String, Map<String, SortedSet<String>>> walsById;
// Logs for recovered sources we are currently tracking
- private final Map<String, Map<String, SortedSet<String>>> walsByIdRecoveredQueues;
+ // the map is: queue_id->logPrefix/logGroup->logs
+ // For recovered source, the queue id's format is peer_id-servername-*
+ private final ConcurrentMap<String, Map<String, SortedSet<String>>> walsByIdRecoveredQueues;
+
private final Configuration conf;
private final FileSystem fs;
// The paths to the latest log of each wal group, for new coming peers
@@ -142,22 +171,22 @@ public class ReplicationSourceManager implements ReplicationListener {
ReplicationPeers replicationPeers, ReplicationTracker replicationTracker, Configuration conf,
Server server, FileSystem fs, Path logDir, Path oldLogDir, UUID clusterId,
WALFileLengthProvider walFileLengthProvider) throws IOException {
- //CopyOnWriteArrayList is thread-safe.
- //Generally, reading is more than modifying.
- this.sources = new CopyOnWriteArrayList<>();
+ // CopyOnWriteArrayList is thread-safe.
+ // Generally, reading is more than modifying.
+ this.sources = new ConcurrentHashMap<>();
this.queueStorage = queueStorage;
this.replicationPeers = replicationPeers;
this.replicationTracker = replicationTracker;
this.server = server;
- this.walsById = new HashMap<>();
+ this.walsById = new ConcurrentHashMap<>();
this.walsByIdRecoveredQueues = new ConcurrentHashMap<>();
- this.oldsources = new CopyOnWriteArrayList<>();
+ this.oldsources = new ArrayList<>();
this.conf = conf;
this.fs = fs;
this.logDir = logDir;
this.oldLogDir = oldLogDir;
- this.sleepBeforeFailover =
- conf.getLong("replication.sleep.before.failover", 30000); // 30 seconds
+ this.sleepBeforeFailover = conf.getLong("replication.sleep.before.failover", 30000); // 30
+ // seconds
this.clusterId = clusterId;
this.walFileLengthProvider = walFileLengthProvider;
this.replicationTracker.registerListener(this);
@@ -166,89 +195,36 @@ public class ReplicationSourceManager implements ReplicationListener {
int nbWorkers = conf.getInt("replication.executor.workers", 1);
// use a short 100ms sleep since this could be done inline with a RS startup
// even if we fail, other region servers can take care of it
- this.executor = new ThreadPoolExecutor(nbWorkers, nbWorkers,
- 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
+ this.executor = new ThreadPoolExecutor(nbWorkers, nbWorkers, 100, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<>());
ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
tfb.setNameFormat("ReplicationExecutor-%d");
tfb.setDaemon(true);
this.executor.setThreadFactory(tfb.build());
this.latestPaths = new HashSet<Path>();
- replicationForBulkLoadDataEnabled =
- conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
- HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
+ replicationForBulkLoadDataEnabled = conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
+ HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
this.replicationWaitTime = conf.getLong(HConstants.REPLICATION_SERIALLY_WAITING_KEY,
- HConstants.REPLICATION_SERIALLY_WAITING_DEFAULT);
+ HConstants.REPLICATION_SERIALLY_WAITING_DEFAULT);
connection = ConnectionFactory.createConnection(conf);
}
- @FunctionalInterface
- private interface ReplicationQueueOperation {
- void exec() throws ReplicationException;
- }
-
- private void abortWhenFail(ReplicationQueueOperation op) {
- try {
- op.exec();
- } catch (ReplicationException e) {
- server.abort("Failed to operate on replication queue", e);
- }
- }
-
/**
- * Provide the id of the peer and a log key and this method will figure which
- * wal it belongs to and will log, for this region server, the current
- * position. It will also clean old logs from the queue.
- * @param log Path to the log currently being replicated from
- * replication status in zookeeper. It will also delete older entries.
- * @param id id of the peer cluster
- * @param position current location in the log
- * @param queueRecovered indicates if this queue comes from another region server
- * @param holdLogInZK if true then the log is retained in ZK
- */
- public void logPositionAndCleanOldLogs(Path log, String id, long position, boolean queueRecovered,
- boolean holdLogInZK) {
- String fileName = log.getName();
- abortWhenFail(
- () -> this.queueStorage.setWALPosition(server.getServerName(), id, fileName, position));
- if (holdLogInZK) {
- return;
- }
- cleanOldLogs(fileName, id, queueRecovered);
- }
-
- /**
- * Cleans a log file and all older files from ZK. Called when we are sure that a
- * log file is closed and has no more entries.
- * @param key Path to the log
- * @param id id of the peer cluster
- * @param queueRecovered Whether this is a recovered queue
+ * Adds a normal source per registered peer cluster and tries to process all old region server wal
+ * queues
+ * <p>
+ * The returned future is for adoptAbandonedQueues task.
*/
- public void cleanOldLogs(String key, String id, boolean queueRecovered) {
- String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(key);
- if (queueRecovered) {
- SortedSet<String> wals = walsByIdRecoveredQueues.get(id).get(logPrefix);
- if (wals != null && !wals.first().equals(key)) {
- cleanOldLogs(wals, key, id);
- }
- } else {
- synchronized (this.walsById) {
- SortedSet<String> wals = walsById.get(id).get(logPrefix);
- if (wals != null && !wals.first().equals(key)) {
- cleanOldLogs(wals, key, id);
- }
+ Future<?> init() throws IOException {
+ for (String id : this.replicationPeers.getAllPeerIds()) {
+ addSource(id);
+ if (replicationForBulkLoadDataEnabled) {
+ // Check if peer exists in hfile-refs queue, if not add it. This can happen in the case
+ // when a peer was added before replication for bulk loaded data was enabled.
+ throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(id));
}
}
- }
-
- private void cleanOldLogs(SortedSet<String> wals, String key, String id) {
- SortedSet<String> walSet = wals.headSet(key);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Removing " + walSet.size() + " logs in the list: " + walSet);
- }
- for (String wal : walSet) {
- abortWhenFail(() -> this.queueStorage.removeWAL(server.getServerName(), id, wal));
- }
- walSet.clear();
+ return this.executor.submit(this::adoptAbandonedQueues);
}
private void adoptAbandonedQueues() {
@@ -264,8 +240,8 @@ public class ReplicationSourceManager implements ReplicationListener {
}
List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers().stream()
.map(ServerName::valueOf).collect(Collectors.toList());
- LOG.info("Current list of replicators: " + currentReplicators + " other RSs: "
- + otherRegionServers);
+ LOG.info(
+ "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);
// Look if there's anything to process after a restart
for (ServerName rs : currentReplicators) {
@@ -276,56 +252,112 @@ public class ReplicationSourceManager implements ReplicationListener {
}
/**
- * Adds a normal source per registered peer cluster and tries to process all old region server wal
- * queues
- * <p>
- * The returned future is for adoptAbandonedQueues task.
+ * 1. Add peer to replicationPeers 2. Add the normal source and related replication queue 3. Add
+ * HFile Refs
+ * @param peerId the id of replication peer
*/
- Future<?> init() throws IOException, ReplicationException {
- for (String id : this.replicationPeers.getAllPeerIds()) {
- addSource(id);
+ public void addPeer(String peerId) throws IOException {
+ boolean added = false;
+ try {
+ added = this.replicationPeers.addPeer(peerId);
+ } catch (ReplicationException e) {
+ throw new IOException(e);
+ }
+ if (added) {
+ addSource(peerId);
if (replicationForBulkLoadDataEnabled) {
- // Check if peer exists in hfile-refs queue, if not add it. This can happen in the case
- // when a peer was added before replication for bulk loaded data was enabled.
- this.queueStorage.addPeerToHFileRefs(id);
+ throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));
}
}
- return this.executor.submit(this::adoptAbandonedQueues);
}
/**
- * Add sources for the given peer cluster on this region server. For the newly added peer, we only
- * need to enqueue the latest log of each wal group and do replication
- * @param id the id of the peer cluster
+ * 1. Remove peer for replicationPeers 2. Remove all the recovered sources for the specified id
+ * and related replication queues 3. Remove the normal source and related replication queue 4.
+ * Remove HFile Refs
+ * @param peerId the id of the replication peer
+ */
+ public void removePeer(String peerId) {
+ replicationPeers.removePeer(peerId);
+ String terminateMessage = "Replication stream was removed by a user";
+ List<ReplicationSourceInterface> oldSourcesToDelete = new ArrayList<>();
+ // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer
+ // see NodeFailoverWorker.run
+ synchronized (this.oldsources) {
+ // First close all the recovered sources for this peer
+ for (ReplicationSourceInterface src : oldsources) {
+ if (peerId.equals(src.getPeerId())) {
+ oldSourcesToDelete.add(src);
+ }
+ }
+ for (ReplicationSourceInterface src : oldSourcesToDelete) {
+ src.terminate(terminateMessage);
+ removeRecoveredSource(src);
+ }
+ }
+ LOG.info(
+ "Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());
+ // Now close the normal source for this peer
+ ReplicationSourceInterface srcToRemove = this.sources.get(peerId);
+ if (srcToRemove != null) {
+ srcToRemove.terminate(terminateMessage);
+ removeSource(srcToRemove);
+ } else {
+ // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup
+ // Delete queue from storage and memory and queue id is same with peer id for normal
+ // source
+ deleteQueue(peerId);
+ this.walsById.remove(peerId);
+ }
+
+ // Remove HFile Refs
+ abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));
+ }
+
+ /**
+ * Factory method to create a replication source
+ * @param queueId the id of the replication queue
+ * @return the created source
+ */
+ private ReplicationSourceInterface createSource(String queueId, ReplicationPeer replicationPeer)
+ throws IOException {
+ ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueId);
+
+ MetricsSource metrics = new MetricsSource(queueId);
+ // init replication source
+ src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId,
+ walFileLengthProvider, metrics);
+ return src;
+ }
+
+ /**
+ * Add a normal source for the given peer on this region server. Meanwhile, add new replication
+ * queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal
+ * group and do replication
+ * @param peerId the id of the replication peer
* @return the source that was created
*/
@VisibleForTesting
- ReplicationSourceInterface addSource(String id) throws IOException, ReplicationException {
- ReplicationPeer peer = replicationPeers.getPeer(id);
- ReplicationSourceInterface src = getReplicationSource(id, peer);
- synchronized (this.walsById) {
- this.sources.add(src);
+ ReplicationSourceInterface addSource(String peerId) throws IOException {
+ ReplicationPeer peer = replicationPeers.getPeer(peerId);
+ ReplicationSourceInterface src = createSource(peerId, peer);
+ // synchronized on latestPaths to avoid missing the new log
+ synchronized (this.latestPaths) {
+ this.sources.put(peerId, src);
Map<String, SortedSet<String>> walsByGroup = new HashMap<>();
- this.walsById.put(id, walsByGroup);
+ this.walsById.put(peerId, walsByGroup);
// Add the latest wal to that source's queue
- synchronized (latestPaths) {
- if (this.latestPaths.size() > 0) {
- for (Path logPath : latestPaths) {
- String name = logPath.getName();
- String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(name);
- SortedSet<String> logs = new TreeSet<>();
- logs.add(name);
- walsByGroup.put(walPrefix, logs);
- try {
- this.queueStorage.addWAL(server.getServerName(), id, name);
- } catch (ReplicationException e) {
- String message = "Cannot add log to queue when creating a new source, queueId=" + id +
- ", filename=" + name;
- server.stop(message);
- throw e;
- }
- src.enqueueLog(logPath);
- }
+ if (this.latestPaths.size() > 0) {
+ for (Path logPath : latestPaths) {
+ String name = logPath.getName();
+ String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(name);
+ SortedSet<String> logs = new TreeSet<>();
+ logs.add(name);
+ walsByGroup.put(walPrefix, logs);
+ // Abort RS and throw exception to make add peer failed
+ abortAndThrowIOExceptionWhenFail(
+ () -> this.queueStorage.addWAL(server.getServerName(), peerId, name));
+ src.enqueueLog(logPath);
}
}
}
@@ -333,87 +365,217 @@ public class ReplicationSourceManager implements ReplicationListener {
return src;
}
- @VisibleForTesting
- int getSizeOfLatestPath() {
- synchronized (latestPaths) {
- return latestPaths.size();
- }
- }
-
/**
- * Delete a complete queue of wals associated with a peer cluster
- * @param peerId Id of the peer cluster queue of wals to delete
+ * Close the previous replication sources of this peer id and open new sources to trigger the new
+ * replication state changes or new replication config changes. Here we don't need to change
+ * replication queue storage and only to enqueue all logs to the new replication source
+ * @param peerId the id of the replication peer
+ * @throws IOException
*/
- public void deleteSource(String peerId, boolean closeConnection) {
- abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), peerId));
- if (closeConnection) {
- this.replicationPeers.removePeer(peerId);
+ public void refreshSources(String peerId) throws IOException {
+ String terminateMessage = "Peer " + peerId +
+ " state or config changed. Will close the previous replication source and open a new one";
+ ReplicationPeer peer = replicationPeers.getPeer(peerId);
+ ReplicationSourceInterface src = createSource(peerId, peer);
+ // synchronized on latestPaths to avoid missing the new log
+ synchronized (this.latestPaths) {
+ ReplicationSourceInterface toRemove = this.sources.put(peerId, src);
+ if (toRemove != null) {
+ LOG.info("Terminate replication source for " + toRemove.getPeerId());
+ toRemove.terminate(terminateMessage);
+ }
+ for (SortedSet<String> walsByGroup : walsById.get(peerId).values()) {
+ walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));
+ }
}
- }
+ LOG.info("Startup replication source for " + src.getPeerId());
+ src.startup();
- /**
- * Terminate the replication on this region server
- */
- public void join() {
- this.executor.shutdown();
- for (ReplicationSourceInterface source : this.sources) {
- source.terminate("Region server is closing");
+ List<ReplicationSourceInterface> toStartup = new ArrayList<>();
+ // synchronized on oldsources to avoid race with NodeFailoverWorker
+ synchronized (this.oldsources) {
+ List<String> previousQueueIds = new ArrayList<>();
+ for (ReplicationSourceInterface oldSource : this.oldsources) {
+ if (oldSource.getPeerId().equals(peerId)) {
+ previousQueueIds.add(oldSource.getQueueId());
+ oldSource.terminate(terminateMessage);
+ this.oldsources.remove(oldSource);
+ }
+ }
+ for (String queueId : previousQueueIds) {
+ ReplicationSourceInterface replicationSource = createSource(queueId, peer);
+ this.oldsources.add(replicationSource);
+ for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {
+ walsByGroup.forEach(wal -> src.enqueueLog(new Path(wal)));
+ }
+ toStartup.add(replicationSource);
+ }
+ }
+ for (ReplicationSourceInterface replicationSource : oldsources) {
+ replicationSource.startup();
}
}
/**
- * Get a copy of the wals of the first source on this rs
- * @return a sorted set of wal names
+ * Clear the metrics and related replication queue of the specified old source
+ * @param src source to clear
*/
- @VisibleForTesting
- Map<String, Map<String, SortedSet<String>>> getWALs() {
- return Collections.unmodifiableMap(walsById);
+ void removeRecoveredSource(ReplicationSourceInterface src) {
+ LOG.info("Done with the recovered queue " + src.getQueueId());
+ src.getSourceMetrics().clear();
+ this.oldsources.remove(src);
+ // Delete queue from storage and memory
+ deleteQueue(src.getQueueId());
+ this.walsByIdRecoveredQueues.remove(src.getQueueId());
}
/**
- * Get a copy of the wals of the recovered sources on this rs
- * @return a sorted set of wal names
+ * Clear the metrics and related replication queue of the specified old source
+ * @param src source to clear
*/
- @VisibleForTesting
- Map<String, Map<String, SortedSet<String>>> getWalsByIdRecoveredQueues() {
- return Collections.unmodifiableMap(walsByIdRecoveredQueues);
+ void removeSource(ReplicationSourceInterface src) {
+ LOG.info("Done with the queue " + src.getQueueId());
+ src.getSourceMetrics().clear();
+ this.sources.remove(src.getPeerId());
+ // Delete queue from storage and memory
+ deleteQueue(src.getQueueId());
+ this.walsById.remove(src.getQueueId());
}
/**
- * Get a list of all the normal sources of this rs
- * @return lis of all sources
+ * Delete a complete queue of wals associated with a replication source
+ * @param queueId the id of replication queue to delete
*/
- public List<ReplicationSourceInterface> getSources() {
- return this.sources;
+ private void deleteQueue(String queueId) {
+ abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId));
+ }
+
+ @FunctionalInterface
+ private interface ReplicationQueueOperation {
+ void exec() throws ReplicationException;
+ }
+
+ private void abortWhenFail(ReplicationQueueOperation op) {
+ try {
+ op.exec();
+ } catch (ReplicationException e) {
+ server.abort("Failed to operate on replication queue", e);
+ }
+ }
+
+ private void throwIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {
+ try {
+ op.exec();
+ } catch (ReplicationException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private void abortAndThrowIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {
+ try {
+ op.exec();
+ } catch (ReplicationException e) {
+ server.abort("Failed to operate on replication queue", e);
+ throw new IOException(e);
+ }
}
/**
- * Get a list of all the old sources of this rs
- * @return list of all old sources
+ * This method will log the current position to storage. And also clean old logs from the
+ * replication queue.
+ * @param log Path to the log currently being replicated
+ * @param queueId id of the replication queue
+ * @param position current location in the log
+ * @param queueRecovered indicates if this queue comes from another region server
*/
- public List<ReplicationSourceInterface> getOldSources() {
- return this.oldsources;
+ public void logPositionAndCleanOldLogs(Path log, String queueId, long position,
+ boolean queueRecovered) {
+ String fileName = log.getName();
+ abortWhenFail(
+ () -> this.queueStorage.setWALPosition(server.getServerName(), queueId, fileName, position));
+ cleanOldLogs(fileName, queueId, queueRecovered);
}
/**
- * Get the normal source for a given peer
- * @param peerId
- * @return the normal source for the give peer if it exists, otherwise null.
+ * Cleans a log file and all older logs from replication queue. Called when we are sure that a log
+ * file is closed and has no more entries.
+ * @param log Path to the log
+ * @param queueId id of the replication queue
+ * @param queueRecovered Whether this is a recovered queue
*/
- public ReplicationSourceInterface getSource(String peerId) {
- return getSources().stream().filter(s -> s.getPeerId().equals(peerId)).findFirst().orElse(null);
+ @VisibleForTesting
+ void cleanOldLogs(String log, String queueId, boolean queueRecovered) {
+ String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(log);
+ if (queueRecovered) {
+ SortedSet<String> wals = walsByIdRecoveredQueues.get(queueId).get(logPrefix);
+ if (wals != null && !wals.first().equals(log)) {
+ cleanOldLogs(wals, log, queueId);
+ }
+ } else {
+ // synchronized on walsById to avoid race with preLogRoll
+ synchronized (this.walsById) {
+ SortedSet<String> wals = walsById.get(queueId).get(logPrefix);
+ if (wals != null && !wals.first().equals(log)) {
+ cleanOldLogs(wals, log, queueId);
+ }
+ }
+ }
}
- @VisibleForTesting
- List<String> getAllQueues() throws ReplicationException {
- return queueStorage.getAllQueues(server.getServerName());
+ private void cleanOldLogs(SortedSet<String> wals, String key, String id) {
+ SortedSet<String> walSet = wals.headSet(key);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing " + walSet.size() + " logs in the list: " + walSet);
+ }
+ for (String wal : walSet) {
+ abortWhenFail(() -> this.queueStorage.removeWAL(server.getServerName(), id, wal));
+ }
+ walSet.clear();
}
void preLogRoll(Path newLog) throws IOException {
- recordLog(newLog);
String logName = newLog.getName();
String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);
- synchronized (latestPaths) {
+ // synchronized on latestPaths to avoid the new open source miss the new log
+ synchronized (this.latestPaths) {
+ // Add log to queue storage
+ for (ReplicationSourceInterface source : this.sources.values()) {
+ // If record log to queue storage failed, abort RS and throw exception to make log roll
+ // failed
+ abortAndThrowIOExceptionWhenFail(
+ () -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));
+ }
+
+ // synchronized on walsById to avoid race with cleanOldLogs
+ synchronized (this.walsById) {
+ // Update walsById map
+ for (Map.Entry<String, Map<String, SortedSet<String>>> entry : this.walsById.entrySet()) {
+ String peerId = entry.getKey();
+ Map<String, SortedSet<String>> walsByPrefix = entry.getValue();
+ boolean existingPrefix = false;
+ for (Map.Entry<String, SortedSet<String>> walsEntry : walsByPrefix.entrySet()) {
+ SortedSet<String> wals = walsEntry.getValue();
+ if (this.sources.isEmpty()) {
+ // If there's no slaves, don't need to keep the old wals since
+ // we only consider the last one when a new slave comes in
+ wals.clear();
+ }
+ if (logPrefix.equals(walsEntry.getKey())) {
+ wals.add(logName);
+ existingPrefix = true;
+ }
+ }
+ if (!existingPrefix) {
+ // The new log belongs to a new group, add it into this peer
+ LOG.debug("Start tracking logs for wal group " + logPrefix + " for peer " + peerId);
+ SortedSet<String> wals = new TreeSet<>();
+ wals.add(logName);
+ walsByPrefix.put(logPrefix, wals);
+ }
+ }
+ }
+
+ // Add to latestPaths
Iterator<Path> iterator = latestPaths.iterator();
while (iterator.hasNext()) {
Path path = iterator.next();
@@ -426,87 +588,21 @@ public class ReplicationSourceManager implements ReplicationListener {
}
}
- /**
- * Check and enqueue the given log to the correct source. If there's still no source for the
- * group to which the given log belongs, create one
- * @param logPath the log path to check and enqueue
- * @throws IOException
- */
- private void recordLog(Path logPath) throws IOException {
- String logName = logPath.getName();
- String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);
- // update replication queues on ZK
- // synchronize on replicationPeers to avoid adding source for the to-be-removed peer
- synchronized (replicationPeers) {
- for (String id : replicationPeers.getAllPeerIds()) {
- try {
- this.queueStorage.addWAL(server.getServerName(), id, logName);
- } catch (ReplicationException e) {
- throw new IOException("Cannot add log to replication queue"
- + " when creating a new source, queueId=" + id + ", filename=" + logName, e);
- }
- }
- }
- // update walsById map
- synchronized (walsById) {
- for (Map.Entry<String, Map<String, SortedSet<String>>> entry : this.walsById.entrySet()) {
- String peerId = entry.getKey();
- Map<String, SortedSet<String>> walsByPrefix = entry.getValue();
- boolean existingPrefix = false;
- for (Map.Entry<String, SortedSet<String>> walsEntry : walsByPrefix.entrySet()) {
- SortedSet<String> wals = walsEntry.getValue();
- if (this.sources.isEmpty()) {
- // If there's no slaves, don't need to keep the old wals since
- // we only consider the last one when a new slave comes in
- wals.clear();
- }
- if (logPrefix.equals(walsEntry.getKey())) {
- wals.add(logName);
- existingPrefix = true;
- }
- }
- if (!existingPrefix) {
- // The new log belongs to a new group, add it into this peer
- LOG.debug("Start tracking logs for wal group " + logPrefix + " for peer " + peerId);
- SortedSet<String> wals = new TreeSet<>();
- wals.add(logName);
- walsByPrefix.put(logPrefix, wals);
- }
- }
- }
- }
-
void postLogRoll(Path newLog) throws IOException {
// This only updates the sources we own, not the recovered ones
- for (ReplicationSourceInterface source : this.sources) {
+ for (ReplicationSourceInterface source : this.sources.values()) {
source.enqueueLog(newLog);
}
}
- @VisibleForTesting
- public AtomicLong getTotalBufferUsed() {
- return totalBufferUsed;
- }
-
- /**
- * Factory method to create a replication source
- * @param peerId the id of the peer cluster
- * @return the created source
- */
- private ReplicationSourceInterface getReplicationSource(String peerId,
- ReplicationPeer replicationPeer) throws IOException {
- ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, peerId);
-
- MetricsSource metrics = new MetricsSource(peerId);
- // init replication source
- src.init(conf, fs, this, queueStorage, replicationPeer, server, peerId, clusterId,
- walFileLengthProvider, metrics);
- return src;
+ @Override
+ public void regionServerRemoved(String regionserver) {
+ transferQueues(ServerName.valueOf(regionserver));
}
/**
* Transfer all the queues of the specified to this region server. First it tries to grab a lock
- * and if it works it will move the znodes and finally will delete the old znodes.
+ * and if it works it will move the old queues and finally will delete the old queues.
* <p>
* It creates one old source for any type of source of the old rs.
*/
@@ -524,102 +620,8 @@ public class ReplicationSourceManager implements ReplicationListener {
}
/**
- * Clear the references to the specified old source
- * @param src source to clear
- */
- public void closeRecoveredQueue(ReplicationSourceInterface src) {
- LOG.info("Done with the recovered queue " + src.getPeerClusterZnode());
- if (src instanceof ReplicationSource) {
- ((ReplicationSource) src).getSourceMetrics().clear();
- }
- this.oldsources.remove(src);
- deleteSource(src.getPeerClusterZnode(), false);
- this.walsByIdRecoveredQueues.remove(src.getPeerClusterZnode());
- }
-
- /**
- * Clear the references to the specified old source
- * @param src source to clear
- */
- public void closeQueue(ReplicationSourceInterface src) {
- LOG.info("Done with the queue " + src.getPeerClusterZnode());
- src.getSourceMetrics().clear();
- this.sources.remove(src);
- deleteSource(src.getPeerClusterZnode(), true);
- this.walsById.remove(src.getPeerClusterZnode());
- }
-
- public void addPeer(String id) throws ReplicationException, IOException {
- LOG.info("Trying to add peer, peerId: " + id);
- boolean added = this.replicationPeers.addPeer(id);
- if (added) {
- LOG.info("Peer " + id + " connected success, trying to start the replication source thread.");
- addSource(id);
- if (replicationForBulkLoadDataEnabled) {
- this.queueStorage.addPeerToHFileRefs(id);
- }
- }
- }
-
- /**
- * Thie method first deletes all the recovered sources for the specified
- * id, then deletes the normal source (deleting all related data in ZK).
- * @param id The id of the peer cluster
- */
- public void removePeer(String id) {
- LOG.info("Closing the following queue " + id + ", currently have "
- + sources.size() + " and another "
- + oldsources.size() + " that were recovered");
- String terminateMessage = "Replication stream was removed by a user";
- List<ReplicationSourceInterface> oldSourcesToDelete = new ArrayList<>();
- // synchronized on oldsources to avoid adding recovered source for the to-be-removed peer
- // see NodeFailoverWorker.run
- synchronized (oldsources) {
- // First close all the recovered sources for this peer
- for (ReplicationSourceInterface src : oldsources) {
- if (id.equals(src.getPeerId())) {
- oldSourcesToDelete.add(src);
- }
- }
- for (ReplicationSourceInterface src : oldSourcesToDelete) {
- src.terminate(terminateMessage);
- closeRecoveredQueue(src);
- }
- }
- LOG.info("Number of deleted recovered sources for " + id + ": "
- + oldSourcesToDelete.size());
- // Now look for the one on this cluster
- List<ReplicationSourceInterface> srcToRemove = new ArrayList<>();
- // synchronize on replicationPeers to avoid adding source for the to-be-removed peer
- synchronized (this.replicationPeers) {
- for (ReplicationSourceInterface src : this.sources) {
- if (id.equals(src.getPeerId())) {
- srcToRemove.add(src);
- }
- }
- if (srcToRemove.isEmpty()) {
- LOG.error("The peer we wanted to remove is missing a ReplicationSourceInterface. " +
- "This could mean that ReplicationSourceInterface initialization failed for this peer " +
- "and that replication on this peer may not be caught up. peerId=" + id);
- }
- for (ReplicationSourceInterface toRemove : srcToRemove) {
- toRemove.terminate(terminateMessage);
- closeQueue(toRemove);
- }
- deleteSource(id, true);
- }
- // Remove HFile Refs znode from zookeeper
- abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(id));
- }
-
- @Override
- public void regionServerRemoved(String regionserver) {
- transferQueues(ServerName.valueOf(regionserver));
- }
-
- /**
- * Class responsible to setup new ReplicationSources to take care of the
- * queues from dead region servers.
+ * Class responsible to setup new ReplicationSources to take care of the queues from dead region
+ * servers.
*/
class NodeFailoverWorker extends Thread {
@@ -649,10 +651,10 @@ public class ReplicationSourceManager implements ReplicationListener {
}
Map<String, Set<String>> newQueues = new HashMap<>();
try {
- List<String> peers = queueStorage.getAllQueues(deadRS);
- while (!peers.isEmpty()) {
+ List<String> queues = queueStorage.getAllQueues(deadRS);
+ while (!queues.isEmpty()) {
Pair<String, SortedSet<String>> peer = queueStorage.claimQueue(deadRS,
- peers.get(ThreadLocalRandom.current().nextInt(peers.size())), server.getServerName());
+ queues.get(ThreadLocalRandom.current().nextInt(queues.size())), server.getServerName());
long sleep = sleepBeforeFailover / 2;
if (!peer.getSecond().isEmpty()) {
newQueues.put(peer.getFirst(), peer.getSecond());
@@ -664,9 +666,9 @@ public class ReplicationSourceManager implements ReplicationListener {
LOG.warn("Interrupted while waiting before transferring a queue.");
Thread.currentThread().interrupt();
}
- peers = queueStorage.getAllQueues(deadRS);
+ queues = queueStorage.getAllQueues(deadRS);
}
- if (!peers.isEmpty()) {
+ if (queues.isEmpty()) {
queueStorage.removeReplicatorIfQueueIsEmpty(deadRS);
}
} catch (ReplicationException e) {
@@ -681,23 +683,23 @@ public class ReplicationSourceManager implements ReplicationListener {
}
for (Map.Entry<String, Set<String>> entry : newQueues.entrySet()) {
- String peerId = entry.getKey();
+ String queueId = entry.getKey();
Set<String> walsSet = entry.getValue();
try {
// there is not an actual peer defined corresponding to peerId for the failover.
- ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(peerId);
+ ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);
String actualPeerId = replicationQueueInfo.getPeerId();
ReplicationPeer peer = replicationPeers.getPeer(actualPeerId);
if (peer == null) {
- LOG.warn("Skipping failover for peer:" + actualPeerId + " of node " + deadRS
- + ", peer is null");
- abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), peerId));
+ LOG.warn("Skipping failover for peer:" + actualPeerId + " of node " + deadRS +
+ ", peer is null");
+ abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), queueId));
continue;
}
// track sources in walsByIdRecoveredQueues
Map<String, SortedSet<String>> walsByGroup = new HashMap<>();
- walsByIdRecoveredQueues.put(peerId, walsByGroup);
+ walsByIdRecoveredQueues.put(queueId, walsByGroup);
for (String wal : walsSet) {
String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal);
SortedSet<String> wals = walsByGroup.get(walPrefix);
@@ -708,14 +710,12 @@ public class ReplicationSourceManager implements ReplicationListener {
wals.add(wal);
}
- // enqueue sources
- ReplicationSourceInterface src = getReplicationSource(peerId, peer);
+ ReplicationSourceInterface src = createSource(queueId, peer);
// synchronized on oldsources to avoid adding recovered source for the to-be-removed peer
- // see removePeer
synchronized (oldsources) {
if (!replicationPeers.getAllPeerIds().contains(src.getPeerId())) {
src.terminate("Recovered queue doesn't belong to any current peer");
- closeRecoveredQueue(src);
+ removeRecoveredSource(src);
continue;
}
oldsources.add(src);
@@ -733,6 +733,82 @@ public class ReplicationSourceManager implements ReplicationListener {
}
/**
+ * Terminate the replication on this region server
+ */
+ public void join() {
+ this.executor.shutdown();
+ for (ReplicationSourceInterface source : this.sources.values()) {
+ source.terminate("Region server is closing");
+ }
+ }
+
+ /**
+ * Get a copy of the wals of the normal sources on this rs
+ * @return a sorted set of wal names
+ */
+ @VisibleForTesting
+ Map<String, Map<String, SortedSet<String>>> getWALs() {
+ return Collections.unmodifiableMap(walsById);
+ }
+
+ /**
+ * Get a copy of the wals of the recovered sources on this rs
+ * @return a sorted set of wal names
+ */
+ @VisibleForTesting
+ Map<String, Map<String, SortedSet<String>>> getWalsByIdRecoveredQueues() {
+ return Collections.unmodifiableMap(walsByIdRecoveredQueues);
+ }
+
+ /**
+ * Get a list of all the normal sources of this rs
+ * @return list of all normal sources
+ */
+ public List<ReplicationSourceInterface> getSources() {
+ return new ArrayList<>(this.sources.values());
+ }
+
+ /**
+ * Get a list of all the recovered sources of this rs
+ * @return list of all recovered sources
+ */
+ public List<ReplicationSourceInterface> getOldSources() {
+ return this.oldsources;
+ }
+
+ /**
+ * Get the normal source for a given peer
+ * @return the normal source for the give peer if it exists, otherwise null.
+ */
+ @VisibleForTesting
+ public ReplicationSourceInterface getSource(String peerId) {
+ return this.sources.get(peerId);
+ }
+
+ @VisibleForTesting
+ List<String> getAllQueues() throws IOException {
+ List<String> allQueues = Collections.emptyList();
+ try {
+ allQueues = queueStorage.getAllQueues(server.getServerName());
+ } catch (ReplicationException e) {
+ throw new IOException(e);
+ }
+ return allQueues;
+ }
+
+ @VisibleForTesting
+ int getSizeOfLatestPath() {
+ synchronized (latestPaths) {
+ return latestPaths.size();
+ }
+ }
+
+ @VisibleForTesting
+ public AtomicLong getTotalBufferUsed() {
+ return totalBufferUsed;
+ }
+
+ /**
* Get the directory where wals are archived
* @return the directory where wals are archived
*/
@@ -764,28 +840,30 @@ public class ReplicationSourceManager implements ReplicationListener {
* Get the ReplicationPeers used by this ReplicationSourceManager
* @return the ReplicationPeers used by this ReplicationSourceManager
*/
- public ReplicationPeers getReplicationPeers() {return this.replicationPeers;}
+ public ReplicationPeers getReplicationPeers() {
+ return this.replicationPeers;
+ }
/**
* Get a string representation of all the sources' metrics
*/
public String getStats() {
StringBuilder stats = new StringBuilder();
- for (ReplicationSourceInterface source : sources) {
+ for (ReplicationSourceInterface source : this.sources.values()) {
stats.append("Normal source for cluster " + source.getPeerId() + ": ");
stats.append(source.getStats() + "\n");
}
for (ReplicationSourceInterface oldSource : oldsources) {
- stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerId()+": ");
- stats.append(oldSource.getStats()+ "\n");
+ stats.append("Recovered source for cluster/machine(s) " + oldSource.getPeerId() + ": ");
+ stats.append(oldSource.getStats() + "\n");
}
return stats.toString();
}
public void addHFileRefs(TableName tableName, byte[] family, List<Pair<Path, Path>> pairs)
- throws ReplicationException {
- for (ReplicationSourceInterface source : this.sources) {
- source.addHFileRefs(tableName, family, pairs);
+ throws IOException {
+ for (ReplicationSourceInterface source : this.sources.values()) {
+ throwIOExceptionWhenFail(() -> source.addHFileRefs(tableName, family, pairs));
}
}
@@ -798,11 +876,10 @@ public class ReplicationSourceManager implements ReplicationListener {
}
/**
- * Whether an entry can be pushed to the peer or not right now.
- * If we enable serial replication, we can not push the entry until all entries in its region
- * whose sequence numbers are smaller than this entry have been pushed.
- * For each ReplicationSource, we need only check the first entry in each region, as long as it
- * can be pushed, we can push all in this ReplicationSource.
+ * Whether an entry can be pushed to the peer or not right now. If we enable serial replication,
+ * we can not push the entry until all entries in its region whose sequence numbers are smaller
+ * than this entry have been pushed. For each ReplicationSource, we need only check the first
+ * entry in each region, as long as it can be pushed, we can push all in this ReplicationSource.
* This method will be blocked until we can push.
* @return the first barrier of entry's region, or -1 if there is no barrier. It is used to
* prevent saving positions in the region of no barrier.
@@ -813,22 +890,18 @@ public class ReplicationSourceManager implements ReplicationListener {
/**
* There are barriers for this region and position for this peer. N barriers form N intervals,
* (b1,b2) (b2,b3) ... (bn,max). Generally, there is no logs whose seq id is not greater than
- * the first barrier and the last interval is start from the last barrier.
- *
- * There are several conditions that we can push now, otherwise we should block:
- * 1) "Serial replication" is not enabled, we can push all logs just like before. This case
- * should not call this method.
- * 2) There is no barriers for this region, or the seq id is smaller than the first barrier.
- * It is mainly because we alter REPLICATION_SCOPE = 2. We can not guarantee the
- * order of logs that is written before altering.
- * 3) This entry is in the first interval of barriers. We can push them because it is the
- * start of a region. But if the region is created by region split, we should check
- * if the parent regions are fully pushed.
- * 4) If the entry's seq id and the position are in same section, or the pos is the last
- * number of previous section. Because when open a region we put a barrier the number
- * is the last log's id + 1.
- * 5) Log's seq is smaller than pos in meta, we are retrying. It may happen when a RS crashes
- * after save replication meta and before save zk offset.
+ * the first barrier and the last interval is start from the last barrier. There are several
+ * conditions that we can push now, otherwise we should block: 1) "Serial replication" is not
+ * enabled, we can push all logs just like before. This case should not call this method. 2)
+ * There is no barriers for this region, or the seq id is smaller than the first barrier. It is
+ * mainly because we alter REPLICATION_SCOPE = 2. We can not guarantee the order of logs that is
+ * written before altering. 3) This entry is in the first interval of barriers. We can push them
+ * because it is the start of a region. But if the region is created by region split, we should
+ * check if the parent regions are fully pushed. 4) If the entry's seq id and the position are
+ * in same section, or the pos is the last number of previous section. Because when open a
+ * region we put a barrier the number is the last log's id + 1. 5) Log's seq is smaller than pos
+ * in meta, we are retrying. It may happen when a RS crashes after save replication meta and
+ * before save zk offset.
*/
List<Long> barriers = MetaTableAccessor.getReplicationBarriers(connection, encodedName);
if (barriers.isEmpty() || seq <= barriers.get(0)) {
@@ -842,8 +915,8 @@ public class ReplicationSourceManager implements ReplicationListener {
if (interval == 1) {
// Case 3
// Check if there are parent regions
- String parentValue = MetaTableAccessor.getSerialReplicationParentRegion(connection,
- encodedName);
+ String parentValue =
+ MetaTableAccessor.getSerialReplicationParentRegion(connection, encodedName);
if (parentValue == null) {
// This region has no parent or the parent's log entries are fully pushed.
return;
@@ -855,16 +928,17 @@ public class ReplicationSourceManager implements ReplicationListener {
byte[] region = Bytes.toBytes(parent);
long pos = MetaTableAccessor.getReplicationPositionForOnePeer(connection, region, peerId);
List<Long> parentBarriers = MetaTableAccessor.getReplicationBarriers(connection, region);
- if (parentBarriers.size() > 0
- && parentBarriers.get(parentBarriers.size() - 1) - 1 > pos) {
+ if (parentBarriers.size() > 0 &&
+ parentBarriers.get(parentBarriers.size() - 1) - 1 > pos) {
allParentDone = false;
// For a closed region, we will write a close event marker to WAL whose sequence id is
// larger than final barrier but still smaller than next region's openSeqNum.
// So if the pos is larger than last barrier, we can say we have read the event marker
// which means the parent region has been fully pushed.
- LOG.info(Bytes.toString(encodedName) + " can not start pushing because parent region's"
- + " log has not been fully pushed: parent=" + Bytes.toString(region) + " pos=" + pos
- + " barriers=" + Arrays.toString(barriers.toArray()));
+ LOG.info(
+ Bytes.toString(encodedName) + " can not start pushing because parent region's" +
+ " log has not been fully pushed: parent=" + Bytes.toString(region) + " pos=" + pos +
+ " barriers=" + Arrays.toString(barriers.toArray()));
break;
}
}
@@ -878,7 +952,8 @@ public class ReplicationSourceManager implements ReplicationListener {
}
while (true) {
- long pos = MetaTableAccessor.getReplicationPositionForOnePeer(connection, encodedName, peerId);
+ long pos =
+ MetaTableAccessor.getReplicationPositionForOnePeer(connection, encodedName, peerId);
if (seq <= pos) {
// Case 5
}
@@ -893,9 +968,9 @@ public class ReplicationSourceManager implements ReplicationListener {
}
}
- LOG.info(Bytes.toString(encodedName) + " can not start pushing to peer " + peerId
- + " because previous log has not been pushed: sequence=" + seq + " pos=" + pos
- + " barriers=" + Arrays.toString(barriers.toArray()));
+ LOG.info(Bytes.toString(encodedName) + " can not start pushing to peer " + peerId +
+ " because previous log has not been pushed: sequence=" + seq + " pos=" + pos +
+ " barriers=" + Arrays.toString(barriers.toArray()));
Thread.sleep(replicationWaitTime);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
index ea98cda..808f738 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
@@ -277,8 +277,8 @@ public class ReplicationSourceShipper extends Thread {
}
protected void updateLogPosition(long lastReadPosition) {
- source.getSourceManager().logPositionAndCleanOldLogs(currentPath, source.getPeerClusterZnode(),
- lastReadPosition, false, false);
+ source.getSourceManager().logPositionAndCleanOldLogs(currentPath, source.getQueueId(),
+ lastReadPosition, false);
lastLoggedPosition = lastReadPosition;
}
@@ -295,7 +295,7 @@ public class ReplicationSourceShipper extends Thread {
public void startup(UncaughtExceptionHandler handler) {
String name = Thread.currentThread().getName();
Threads.setDaemonThreadRunning(this, name + ".replicationSource." + walGroupId + ","
- + source.getPeerClusterZnode(), handler);
+ + source.getQueueId(), handler);
}
public PriorityBlockingQueue<Path> getLogQueue() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
index e56fab2..c12dcb6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java
@@ -115,7 +115,7 @@ public class ReplicationSourceWALReader extends Thread {
this.conf.getInt("replication.source.maxretriesmultiplier", 300); // 5 minutes @ 1 sec per
this.eofAutoRecovery = conf.getBoolean("replication.source.eof.autorecovery", false);
this.entryBatchQueue = new LinkedBlockingQueue<>(batchCount);
- LOG.info("peerClusterZnode=" + source.getPeerClusterZnode()
+ LOG.info("peerClusterZnode=" + source.getQueueId()
+ ", ReplicationSourceWALReaderThread : " + source.getPeerId()
+ " inited, replicationBatchSizeCapacity=" + replicationBatchSizeCapacity
+ ", replicationBatchCountCapacity=" + replicationBatchCountCapacity
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
index 38ec598..ff20ddc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
@@ -89,7 +89,7 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
}
@Override
- public String getPeerClusterZnode() {
+ public String getQueueId() {
return peerClusterId;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
index 0f5fda0..b63a555 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplication.java
@@ -30,12 +30,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
@@ -43,6 +42,8 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
@@ -68,9 +69,6 @@ public class TestNamespaceReplication extends TestReplicationBase {
private static final byte[] val = Bytes.toBytes("myval");
- private static HTableDescriptor tabA;
- private static HTableDescriptor tabB;
-
private static Connection connection1;
private static Connection connection2;
private static Admin admin1;
@@ -90,23 +88,21 @@ public class TestNamespaceReplication extends TestReplicationBase {
admin2.createNamespace(NamespaceDescriptor.create(ns1).build());
admin2.createNamespace(NamespaceDescriptor.create(ns2).build());
- tabA = new HTableDescriptor(tabAName);
- HColumnDescriptor fam = new HColumnDescriptor(f1Name);
- fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
- tabA.addFamily(fam);
- fam = new HColumnDescriptor(f2Name);
- fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
- tabA.addFamily(fam);
+ TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tabAName);
+ builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+ .newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+ builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+ .newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+ TableDescriptor tabA = builder.build();
admin1.createTable(tabA);
admin2.createTable(tabA);
- tabB = new HTableDescriptor(tabBName);
- fam = new HColumnDescriptor(f1Name);
- fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
- tabB.addFamily(fam);
- fam = new HColumnDescriptor(f2Name);
- fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
- tabB.addFamily(fam);
+ builder = TableDescriptorBuilder.newBuilder(tabBName);
+ builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+ .newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+ builder.addColumnFamily(ColumnFamilyDescriptorBuilder
+ .newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
+ TableDescriptor tabB = builder.build();
admin1.createTable(tabB);
admin2.createTable(tabB);
}
@@ -134,22 +130,24 @@ public class TestNamespaceReplication extends TestReplicationBase {
@Test
public void testNamespaceReplication() throws Exception {
+ String peerId = "2";
+
Table htab1A = connection1.getTable(tabAName);
Table htab2A = connection2.getTable(tabAName);
Table htab1B = connection1.getTable(tabBName);
Table htab2B = connection2.getTable(tabBName);
- ReplicationPeerConfig rpc = admin.getPeerConfig("2");
- rpc.setReplicateAllUserTables(false);
- admin.updatePeerConfig("2", rpc);
+ ReplicationPeerConfig rpc = admin1.getReplicationPeerConfig(peerId);
+ admin1.updateReplicationPeerConfig(peerId,
+ ReplicationPeerConfig.newBuilder(rpc).setReplicateAllUserTables(false).build());
// add ns1 to peer config which replicate to cluster2
- rpc = admin.getPeerConfig("2");
+ rpc = admin1.getReplicationPeerConfig(peerId);
Set<String> namespaces = new HashSet<>();
namespaces.add(ns1);
- rpc.setNamespaces(namespaces);
- admin.updatePeerConfig("2", rpc);
+ admin1.updateReplicationPeerConfig(peerId,
+ ReplicationPeerConfig.newBuilder(rpc).setNamespaces(namespaces).build());
LOG.info("update peer config");
// Table A can be replicated to cluster2
@@ -163,15 +161,14 @@ public class TestNamespaceReplication extends TestReplicationBase {
ensureRowNotExisted(htab2B, row, f1Name, f2Name);
// add ns1:TA => 'f1' and ns2 to peer config which replicate to cluster2
- rpc = admin.getPeerConfig("2");
+ rpc = admin1.getReplicationPeerConfig(peerId);
namespaces = new HashSet<>();
namespaces.add(ns2);
- rpc.setNamespaces(namespaces);
Map<TableName, List<String>> tableCfs = new HashMap<>();
tableCfs.put(tabAName, new ArrayList<>());
tableCfs.get(tabAName).add("f1");
- rpc.setTableCFsMap(tableCfs);
- admin.updatePeerConfig("2", rpc);
+ admin1.updateReplicationPeerConfig(peerId, ReplicationPeerConfig.newBuilder(rpc)
+ .setNamespaces(namespaces).setTableCFsMap(tableCfs).build());
LOG.info("update peer config");
// Only family f1 of Table A can replicated to cluster2
@@ -186,7 +183,7 @@ public class TestNamespaceReplication extends TestReplicationBase {
delete(htab1B, row, f1Name, f2Name);
ensureRowNotExisted(htab2B, row, f1Name, f2Name);
- admin.removePeer("2");
+ admin1.removeReplicationPeer(peerId);
}
private void put(Table source, byte[] row, byte[]... families)
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 1001aa5..33216cb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -95,12 +94,12 @@ import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
-
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
+
/**
* An abstract class that tests ReplicationSourceManager. Classes that extend this class should
* set up the proper config for this class and initialize the proper cluster using
@@ -313,7 +312,7 @@ public abstract class TestReplicationSourceManager {
wal.rollWriter();
manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
- "1", 0, false, false);
+ "1", 0, false);
wal.append(hri,
new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
http://git-wip-us.apache.org/repos/asf/hbase/blob/aab18b45/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
index c6d9eef..490c4b5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.replication.regionserver;
import static org.junit.Assert.assertTrue;
import java.util.List;
-
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.Server;
[40/50] [abbrv] hbase git commit: HBASE-19686 Use KeyLocker instead
of ReentrantLock in PeerProcedureHandlerImpl
Posted by zh...@apache.org.
HBASE-19686 Use KeyLocker instead of ReentrantLock in PeerProcedureHandlerImpl
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/54659f2c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/54659f2c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/54659f2c
Branch: refs/heads/HBASE-19397-branch-2
Commit: 54659f2cf5a69c7333ac5bc086681c2f7d4b4c76
Parents: e560998
Author: zhangduo <zh...@apache.org>
Authored: Tue Jan 2 16:13:55 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../regionserver/PeerProcedureHandlerImpl.java | 41 ++++++++++++++------
1 file changed, 29 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/54659f2c/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index 1efe180..c09c6a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -19,10 +19,10 @@
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
-import java.util.concurrent.locks.ReentrantLock;
-
+import java.util.concurrent.locks.Lock;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.util.KeyLocker;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +32,7 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
private static final Logger LOG = LoggerFactory.getLogger(PeerProcedureHandlerImpl.class);
private final ReplicationSourceManager replicationSourceManager;
- private final ReentrantLock peersLock = new ReentrantLock();
+ private final KeyLocker<String> peersLock = new KeyLocker<>();
public PeerProcedureHandlerImpl(ReplicationSourceManager replicationSourceManager) {
this.replicationSourceManager = replicationSourceManager;
@@ -40,40 +40,57 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
@Override
public void addPeer(String peerId) throws ReplicationException, IOException {
- peersLock.lock();
+ Lock peerLock = peersLock.acquireLock(peerId);
try {
replicationSourceManager.addPeer(peerId);
} finally {
- peersLock.unlock();
+ peerLock.unlock();
}
}
@Override
public void removePeer(String peerId) throws ReplicationException, IOException {
- peersLock.lock();
+ Lock peerLock = peersLock.acquireLock(peerId);
try {
if (replicationSourceManager.getReplicationPeers().getPeer(peerId) != null) {
replicationSourceManager.removePeer(peerId);
}
} finally {
- peersLock.unlock();
+ peerLock.unlock();
}
}
@Override
public void disablePeer(String peerId) throws ReplicationException, IOException {
- PeerState newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
- LOG.info("disable replication peer, id: " + peerId + ", new state: " + newState);
+ PeerState newState;
+ Lock peerLock = peersLock.acquireLock(peerId);
+ try {
+ newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+ } finally {
+ peerLock.unlock();
+ }
+ LOG.info("disable replication peer, id: {}, new state: {}", peerId, newState);
}
@Override
public void enablePeer(String peerId) throws ReplicationException, IOException {
- PeerState newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
- LOG.info("enable replication peer, id: " + peerId + ", new state: " + newState);
+ PeerState newState;
+ Lock peerLock = peersLock.acquireLock(peerId);
+ try {
+ newState = replicationSourceManager.getReplicationPeers().refreshPeerState(peerId);
+ } finally {
+ peerLock.unlock();
+ }
+ LOG.info("enable replication peer, id: {}, new state: {}", peerId, newState);
}
@Override
public void updatePeerConfig(String peerId) throws ReplicationException, IOException {
- replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+ Lock peerLock = peersLock.acquireLock(peerId);
+ try {
+ replicationSourceManager.getReplicationPeers().refreshPeerConfig(peerId);
+ } finally {
+ peerLock.unlock();
+ }
}
}
[37/50] [abbrv] hbase git commit: HBASE-19697 Remove
TestReplicationAdminUsingProcedure
Posted by zh...@apache.org.
HBASE-19697 Remove TestReplicationAdminUsingProcedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ca706683
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ca706683
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ca706683
Branch: refs/heads/HBASE-19397-branch-2
Commit: ca70668309f32262ba4c8edfce34ec10b2340a51
Parents: 424fa6c
Author: zhangduo <zh...@apache.org>
Authored: Wed Jan 3 21:13:57 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../TestReplicationAdminUsingProcedure.java | 225 -------------------
1 file changed, 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ca706683/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
deleted file mode 100644
index 1300376..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdminUsingProcedure.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client.replication;
-
-import java.io.IOException;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.TestReplicationBase;
-import org.apache.hadoop.hbase.testclassification.ClientTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.log4j.Logger;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-
-@Category({ MediumTests.class, ClientTests.class })
-public class TestReplicationAdminUsingProcedure extends TestReplicationBase {
-
- private static final String PEER_ID = "2";
- private static final Logger LOG = Logger.getLogger(TestReplicationAdminUsingProcedure.class);
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- conf1.setInt("hbase.multihconnection.threads.max", 10);
-
- // Start the master & slave mini cluster.
- TestReplicationBase.setUpBeforeClass();
-
- // Remove the replication peer
- hbaseAdmin.removeReplicationPeer(PEER_ID);
- }
-
- private void loadData(int startRowKey, int endRowKey) throws IOException {
- for (int i = startRowKey; i < endRowKey; i++) {
- byte[] rowKey = Bytes.add(row, Bytes.toBytes(i));
- Put put = new Put(rowKey);
- put.addColumn(famName, null, Bytes.toBytes(i));
- htable1.put(put);
- }
- }
-
- private void waitForReplication(int expectedRows, int retries)
- throws IOException, InterruptedException {
- Scan scan;
- for (int i = 0; i < retries; i++) {
- scan = new Scan();
- if (i == retries - 1) {
- throw new IOException("Waited too much time for normal batch replication");
- }
- try (ResultScanner scanner = htable2.getScanner(scan)) {
- int count = 0;
- for (Result res : scanner) {
- count++;
- }
- if (count != expectedRows) {
- LOG.info("Only got " + count + " rows, expected rows: " + expectedRows);
- Thread.sleep(SLEEP_TIME);
- } else {
- return;
- }
- }
- }
- }
-
- @Before
- public void setUp() throws IOException {
- ReplicationPeerConfig rpc = new ReplicationPeerConfig();
- rpc.setClusterKey(utility2.getClusterKey());
- hbaseAdmin.addReplicationPeer(PEER_ID, rpc);
-
- utility1.waitUntilAllRegionsAssigned(tableName);
- utility2.waitUntilAllRegionsAssigned(tableName);
- }
-
- @After
- public void tearDown() throws IOException {
- hbaseAdmin.removeReplicationPeer(PEER_ID);
- truncateBoth();
- }
-
- private void truncateBoth() throws IOException {
- utility1.deleteTableData(tableName);
- utility2.deleteTableData(tableName);
- }
-
- @Test
- public void testAddPeer() throws Exception {
- // Load data
- loadData(0, NB_ROWS_IN_BATCH);
-
- // Wait the replication finished
- waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
- }
-
- @Test
- public void testRemovePeer() throws Exception {
- // prev-check
- waitForReplication(0, NB_RETRIES);
-
- // Load data
- loadData(0, NB_ROWS_IN_BATCH);
-
- // Wait the replication finished
- waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
-
- // Remove the peer id
- hbaseAdmin.removeReplicationPeer(PEER_ID);
-
- // Load data again
- loadData(NB_ROWS_IN_BATCH, 2 * NB_ROWS_IN_BATCH);
-
- // Wait the replication again
- boolean foundException = false;
- try {
- waitForReplication(NB_ROWS_IN_BATCH * 2, NB_RETRIES);
- } catch (IOException e) {
- foundException = true;
- }
- Assert.assertTrue(foundException);
-
- // Truncate the table in source cluster
- truncateBoth();
-
- // Add peer again
- ReplicationPeerConfig rpc = new ReplicationPeerConfig();
- rpc.setClusterKey(utility2.getClusterKey());
- hbaseAdmin.addReplicationPeer(PEER_ID, rpc);
-
- // Load data again
- loadData(0, NB_ROWS_IN_BATCH);
-
- // Wait the replication finished
- waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
- }
-
- @Test
- public void testDisableAndEnablePeer() throws Exception {
- // disable peer
- hbaseAdmin.disableReplicationPeer(PEER_ID);
-
- // Load data
- loadData(0, NB_ROWS_IN_BATCH);
-
- // Will failed to wait the replication.
- boolean foundException = false;
- try {
- waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
- } catch (IOException e) {
- foundException = true;
- }
- Assert.assertTrue(foundException);
-
- // Enable the peer
- hbaseAdmin.enableReplicationPeer(PEER_ID);
- waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
-
- // Load more data
- loadData(NB_ROWS_IN_BATCH, NB_ROWS_IN_BATCH * 2);
-
- // Wait replication again.
- waitForReplication(NB_ROWS_IN_BATCH * 2, NB_RETRIES);
- }
-
- @Test
- public void testUpdatePeerConfig() throws Exception {
- ReplicationPeerConfig rpc = new ReplicationPeerConfig();
- rpc.setClusterKey(utility2.getClusterKey());
- rpc.setExcludeTableCFsMap(
- ImmutableMap.of(tableName, ImmutableList.of(Bytes.toString(famName))));
-
- // Update the peer config to exclude the test table name.
- hbaseAdmin.updateReplicationPeerConfig(PEER_ID, rpc);
-
- // Load data
- loadData(0, NB_ROWS_IN_BATCH);
-
- // Will failed to wait the replication
- boolean foundException = false;
- try {
- waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
- } catch (IOException e) {
- foundException = true;
- }
- Assert.assertTrue(foundException);
-
- // Truncate the table in source cluster
- truncateBoth();
-
- // Update the peer config to include the test table name.
- ReplicationPeerConfig rpc2 = new ReplicationPeerConfig();
- rpc2.setClusterKey(utility2.getClusterKey());
- hbaseAdmin.updateReplicationPeerConfig(PEER_ID, rpc2);
-
- // Load data again
- loadData(0, NB_ROWS_IN_BATCH);
-
- // Wait the replication finished
- waitForReplication(NB_ROWS_IN_BATCH, NB_RETRIES);
- }
-}
[17/50] [abbrv] hbase git commit: HBASE-19216 Implement a general
framework to execute remote procedure on RS
Posted by zh...@apache.org.
HBASE-19216 Implement a general framework to execute remote procedure on RS
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41e57c3c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41e57c3c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41e57c3c
Branch: refs/heads/HBASE-19397-branch-2
Commit: 41e57c3c8e8fe048cc2fa7b422e4a195f40662e7
Parents: 739b9b4
Author: zhangduo <zh...@apache.org>
Authored: Fri Dec 15 21:06:44 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../hbase/procedure2/LockedResourceType.java | 4 +-
.../procedure2/RemoteProcedureDispatcher.java | 23 +-
.../src/main/protobuf/Admin.proto | 9 +-
.../src/main/protobuf/MasterProcedure.proto | 30 +++
.../src/main/protobuf/RegionServerStatus.proto | 15 ++
.../apache/hadoop/hbase/executor/EventType.java | 26 ++-
.../hadoop/hbase/executor/ExecutorType.java | 3 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 33 ++-
.../hadoop/hbase/master/MasterRpcServices.java | 13 ++
.../assignment/RegionTransitionProcedure.java | 18 +-
.../procedure/MasterProcedureScheduler.java | 224 +++++++++++++------
.../procedure/PeerProcedureInterface.java | 34 +++
.../master/procedure/RSProcedureDispatcher.java | 101 +++++----
.../master/replication/ModifyPeerProcedure.java | 127 +++++++++++
.../master/replication/RefreshPeerCallable.java | 67 ++++++
.../replication/RefreshPeerProcedure.java | 197 ++++++++++++++++
.../hbase/procedure2/RSProcedureCallable.java | 43 ++++
.../hbase/regionserver/HRegionServer.java | 75 ++++++-
.../hbase/regionserver/RSRpcServices.java | 56 +++--
.../handler/RSProcedureHandler.java | 51 +++++
.../assignment/TestAssignmentManager.java | 20 +-
.../replication/DummyModifyPeerProcedure.java | 41 ++++
.../TestDummyModifyPeerProcedure.java | 80 +++++++
.../security/access/TestAccessController.java | 1 +
24 files changed, 1114 insertions(+), 177 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
index c5fe62b..dc9b5d4 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -22,5 +22,5 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public enum LockedResourceType {
- SERVER, NAMESPACE, TABLE, REGION
+ SERVER, NAMESPACE, TABLE, REGION, PEER
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 72a2ebb..735e899 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -226,13 +226,30 @@ public abstract class RemoteProcedureDispatcher<TEnv, TRemote extends Comparable
/**
* Remote procedure reference.
- * @param <TEnv>
- * @param <TRemote>
*/
public interface RemoteProcedure<TEnv, TRemote> {
+ /**
+ * For building the remote operation.
+ */
RemoteOperation remoteCallBuild(TEnv env, TRemote remote);
- void remoteCallCompleted(TEnv env, TRemote remote, RemoteOperation response);
+
+ /**
+ * Called when the executeProcedure call is failed.
+ */
void remoteCallFailed(TEnv env, TRemote remote, IOException exception);
+
+ /**
+ * Called when RS tells the remote procedure is succeeded through the
+ * {@code reportProcedureDone} method.
+ */
+ void remoteOperationCompleted(TEnv env);
+
+ /**
+ * Called when RS tells the remote procedure is failed through the {@code reportProcedureDone}
+ * method.
+ * @param error the error message
+ */
+ void remoteOperationFailed(TEnv env, String error);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-protocol-shaded/src/main/protobuf/Admin.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/Admin.proto
index 118c79b..ddcc266 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Admin.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Admin.proto
@@ -256,14 +256,19 @@ message ClearRegionBlockCacheResponse {
required CacheEvictionStats stats = 1;
}
+message RemoteProcedureRequest {
+ required uint64 proc_id = 1;
+ required string proc_class = 2;
+ optional bytes proc_data = 3;
+}
+
message ExecuteProceduresRequest {
repeated OpenRegionRequest open_region = 1;
repeated CloseRegionRequest close_region = 2;
+ repeated RemoteProcedureRequest proc = 3;
}
message ExecuteProceduresResponse {
- repeated OpenRegionResponse open_region = 1;
- repeated CloseRegionResponse close_region = 2;
}
service AdminService {
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index f9b8807..0e2bdba 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -365,3 +365,33 @@ message GCMergedRegionsStateData {
required RegionInfo parent_b = 2;
required RegionInfo merged_child = 3;
}
+
+enum PeerModificationState {
+ UPDATE_PEER_STORAGE = 1;
+ REFRESH_PEER_ON_RS = 2;
+ POST_PEER_MODIFICATION = 3;
+}
+
+message PeerModificationStateData {
+ required string peer_id = 1;
+}
+
+enum PeerModificationType {
+ ADD_PEER = 1;
+ REMOVE_PEER = 2;
+ ENABLE_PEER = 3;
+ DISABLE_PEER = 4;
+ UPDATE_PEER_CONFIG = 5;
+}
+
+message RefreshPeerStateData {
+ required string peer_id = 1;
+ required PeerModificationType type = 2;
+ required ServerName target_server = 3;
+}
+
+message RefreshPeerParameter {
+ required string peer_id = 1;
+ required PeerModificationType type = 2;
+ required ServerName target_server = 3;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index f83bb20..eb396ac 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -143,7 +143,19 @@ message RegionSpaceUseReportRequest {
}
message RegionSpaceUseReportResponse {
+}
+message ReportProcedureDoneRequest {
+ required uint64 proc_id = 1;
+ enum Status {
+ SUCCESS = 1;
+ ERROR = 2;
+ }
+ required Status status = 2;
+ optional string error = 3;
+}
+
+message ReportProcedureDoneResponse {
}
service RegionServerStatusService {
@@ -181,4 +193,7 @@ service RegionServerStatusService {
*/
rpc ReportRegionSpaceUse(RegionSpaceUseReportRequest)
returns(RegionSpaceUseReportResponse);
+
+ rpc ReportProcedureDone(ReportProcedureDoneRequest)
+ returns(ReportProcedureDoneResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
index 26fb63a..922deb8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
@@ -20,15 +20,14 @@ package org.apache.hadoop.hbase.executor;
import org.apache.yetus.audience.InterfaceAudience;
/**
- * List of all HBase event handler types. Event types are named by a
- * convention: event type names specify the component from which the event
- * originated and then where its destined -- e.g. RS2ZK_ prefix means the
- * event came from a regionserver destined for zookeeper -- and then what
- * the even is; e.g. REGION_OPENING.
- *
- * <p>We give the enums indices so we can add types later and keep them
- * grouped together rather than have to add them always to the end as we
- * would have to if we used raw enum ordinals.
+ * List of all HBase event handler types.
+ * <p>
+ * Event types are named by a convention: event type names specify the component from which the
+ * event originated and then where its destined -- e.g. RS_ZK_ prefix means the event came from a
+ * regionserver destined for zookeeper -- and then what the even is; e.g. REGION_OPENING.
+ * <p>
+ * We give the enums indices so we can add types later and keep them grouped together rather than
+ * have to add them always to the end as we would have to if we used raw enum ordinals.
*/
@InterfaceAudience.Private
public enum EventType {
@@ -275,7 +274,14 @@ public enum EventType {
*
* RS_COMPACTED_FILES_DISCHARGER
*/
- RS_COMPACTED_FILES_DISCHARGER (83, ExecutorType.RS_COMPACTED_FILES_DISCHARGER);
+ RS_COMPACTED_FILES_DISCHARGER (83, ExecutorType.RS_COMPACTED_FILES_DISCHARGER),
+
+ /**
+ * RS refresh peer.<br>
+ *
+ * RS_REFRESH_PEER
+ */
+ RS_REFRESH_PEER (84, ExecutorType.RS_REFRESH_PEER);
private final int code;
private final ExecutorType executor;
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
index c75a0a9..7f130d1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/executor/ExecutorType.java
@@ -46,7 +46,8 @@ public enum ExecutorType {
RS_LOG_REPLAY_OPS (27),
RS_REGION_REPLICA_FLUSH_OPS (28),
RS_COMPACTED_FILES_DISCHARGER (29),
- RS_OPEN_PRIORITY_REGION (30);
+ RS_OPEN_PRIORITY_REGION (30),
+ RS_REFRESH_PEER (31);
ExecutorType(int value) {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index bd52bd1..5f182ff 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -141,6 +140,7 @@ import org.apache.hadoop.hbase.procedure2.LockedResource;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import org.apache.hadoop.hbase.quotas.MasterSpaceQuotaObserver;
@@ -329,8 +329,7 @@ public class HMaster extends HRegionServer implements MasterServices {
// flag set after we become the active master (used for testing)
private volatile boolean activeMaster = false;
- // flag set after we complete initialization once active,
- // it is not private since it's used in unit tests
+ // flag set after we complete initialization once active
private final ProcedureEvent initialized = new ProcedureEvent("master initialized");
// flag set after master services are started,
@@ -3561,4 +3560,28 @@ public class HMaster extends HRegionServer implements MasterServices {
public SpaceQuotaSnapshotNotifier getSpaceQuotaSnapshotNotifier() {
return this.spaceQuotaSnapshotNotifier;
}
-}
+
+ @SuppressWarnings("unchecked")
+ private RemoteProcedure<MasterProcedureEnv, ?> getRemoteProcedure(long procId) {
+ Procedure<?> procedure = procedureExecutor.getProcedure(procId);
+ if (procedure == null) {
+ return null;
+ }
+ assert procedure instanceof RemoteProcedure;
+ return (RemoteProcedure<MasterProcedureEnv, ?>) procedure;
+ }
+
+ public void remoteProcedureCompleted(long procId) {
+ RemoteProcedure<MasterProcedureEnv, ?> procedure = getRemoteProcedure(procId);
+ if (procedure != null) {
+ procedure.remoteOperationCompleted(procedureExecutor.getEnvironment());
+ }
+ }
+
+ public void remoteProcedureFailed(long procId, String error) {
+ RemoteProcedure<MasterProcedureEnv, ?> procedure = getRemoteProcedure(procId);
+ if (procedure != null) {
+ procedure.remoteOperationFailed(procedureExecutor.getEnvironment(), error);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 907ca9b..f875e20 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -264,6 +264,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
@@ -2247,4 +2249,15 @@ public class MasterRpcServices extends RSRpcServices
}
return response.build();
}
+
+ @Override
+ public ReportProcedureDoneResponse reportProcedureDone(RpcController controller,
+ ReportProcedureDoneRequest request) throws ServiceException {
+ if (request.getStatus() == ReportProcedureDoneRequest.Status.SUCCESS) {
+ master.remoteProcedureCompleted(request.getProcId());
+ } else {
+ master.remoteProcedureFailed(request.getProcId(), request.getError());
+ }
+ return ReportProcedureDoneResponse.getDefaultInstance();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index 4a88e3b..04dccc4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -173,12 +173,6 @@ public abstract class RegionTransitionProcedure
RegionStateNode regionNode, IOException exception);
@Override
- public void remoteCallCompleted(final MasterProcedureEnv env,
- final ServerName serverName, final RemoteOperation response) {
- // Ignore the response? reportTransition() is the one that count?
- }
-
- @Override
public void remoteCallFailed(final MasterProcedureEnv env,
final ServerName serverName, final IOException exception) {
final RegionStateNode regionNode = getRegionState(env);
@@ -413,4 +407,16 @@ public abstract class RegionTransitionProcedure
* @return ServerName the Assign or Unassign is going against.
*/
public abstract ServerName getServer(final MasterProcedureEnv env);
+
+ @Override
+ public void remoteOperationCompleted(MasterProcedureEnv env) {
+ // should not be called for region operation until we modified the open/close region procedure
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void remoteOperationFailed(MasterProcedureEnv env, String error) {
+ // should not be called for region operation until we modified the open/close region procedure
+ throw new UnsupportedOperationException();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index dc9c69d..8ff2d12 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -24,7 +24,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
+import java.util.function.Function;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ServerName;
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.master.locking.LockProcedure;
+import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface.PeerOperationType;
import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface.TableOperationType;
import org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler;
import org.apache.hadoop.hbase.procedure2.LockAndQueue;
@@ -109,12 +110,17 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
new ServerQueueKeyComparator();
private final static TableQueueKeyComparator TABLE_QUEUE_KEY_COMPARATOR =
new TableQueueKeyComparator();
+ private final static PeerQueueKeyComparator PEER_QUEUE_KEY_COMPARATOR =
+ new PeerQueueKeyComparator();
private final FairQueue<ServerName> serverRunQueue = new FairQueue<>();
private final FairQueue<TableName> tableRunQueue = new FairQueue<>();
+ private final FairQueue<String> peerRunQueue = new FairQueue<>();
private final ServerQueue[] serverBuckets = new ServerQueue[128];
private TableQueue tableMap = null;
+ private PeerQueue peerMap = null;
+
private final SchemaLocking locking = new SchemaLocking();
/**
@@ -161,6 +167,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
doAdd(tableRunQueue, getTableQueue(getTableName(proc)), proc, addFront);
} else if (isServerProcedure(proc)) {
doAdd(serverRunQueue, getServerQueue(getServerName(proc)), proc, addFront);
+ } else if (isPeerProcedure(proc)) {
+ doAdd(peerRunQueue, getPeerQueue(getPeerId(proc)), proc, addFront);
} else {
// TODO: at the moment we only have Table and Server procedures
// if you are implementing a non-table/non-server procedure, you have two options: create
@@ -172,7 +180,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
}
private <T extends Comparable<T>> void doAdd(final FairQueue<T> fairq,
- final Queue<T> queue, final Procedure proc, final boolean addFront) {
+ final Queue<T> queue, final Procedure<?> proc, final boolean addFront) {
queue.add(proc, addFront);
if (!queue.getLockStatus().hasExclusiveLock() || queue.getLockStatus().isLockOwner(proc.getProcId())) {
// if the queue was not remove for an xlock execution
@@ -189,7 +197,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
@Override
protected boolean queueHasRunnables() {
- return tableRunQueue.hasRunnables() || serverRunQueue.hasRunnables();
+ return tableRunQueue.hasRunnables() || serverRunQueue.hasRunnables() ||
+ peerRunQueue.hasRunnables();
}
@Override
@@ -197,7 +206,10 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
// For now, let server handling have precedence over table handling; presumption is that it
// is more important handling crashed servers than it is running the
// enabling/disabling tables, etc.
- Procedure pollResult = doPoll(serverRunQueue);
+ Procedure<?> pollResult = doPoll(serverRunQueue);
+ if (pollResult == null) {
+ pollResult = doPoll(peerRunQueue);
+ }
if (pollResult == null) {
pollResult = doPoll(tableRunQueue);
}
@@ -267,60 +279,30 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
exclusiveLockOwnerProcedure, sharedLockCount, waitingProcedures);
}
+ private <T> void addToLockedResources(List<LockedResource> lockedResources,
+ Map<T, LockAndQueue> locks, Function<T, String> keyTransformer,
+ LockedResourceType resourcesType) {
+ locks.entrySet().stream().filter(e -> e.getValue().isLocked())
+ .map(
+ e -> createLockedResource(resourcesType, keyTransformer.apply(e.getKey()), e.getValue()))
+ .forEachOrdered(lockedResources::add);
+ }
+
@Override
public List<LockedResource> getLocks() {
schedLock();
-
try {
List<LockedResource> lockedResources = new ArrayList<>();
-
- for (Entry<ServerName, LockAndQueue> entry : locking.serverLocks
- .entrySet()) {
- String serverName = entry.getKey().getServerName();
- LockAndQueue queue = entry.getValue();
-
- if (queue.isLocked()) {
- LockedResource lockedResource =
- createLockedResource(LockedResourceType.SERVER, serverName, queue);
- lockedResources.add(lockedResource);
- }
- }
-
- for (Entry<String, LockAndQueue> entry : locking.namespaceLocks
- .entrySet()) {
- String namespaceName = entry.getKey();
- LockAndQueue queue = entry.getValue();
-
- if (queue.isLocked()) {
- LockedResource lockedResource =
- createLockedResource(LockedResourceType.NAMESPACE, namespaceName, queue);
- lockedResources.add(lockedResource);
- }
- }
-
- for (Entry<TableName, LockAndQueue> entry : locking.tableLocks
- .entrySet()) {
- String tableName = entry.getKey().getNameAsString();
- LockAndQueue queue = entry.getValue();
-
- if (queue.isLocked()) {
- LockedResource lockedResource =
- createLockedResource(LockedResourceType.TABLE, tableName, queue);
- lockedResources.add(lockedResource);
- }
- }
-
- for (Entry<String, LockAndQueue> entry : locking.regionLocks.entrySet()) {
- String regionName = entry.getKey();
- LockAndQueue queue = entry.getValue();
-
- if (queue.isLocked()) {
- LockedResource lockedResource =
- createLockedResource(LockedResourceType.REGION, regionName, queue);
- lockedResources.add(lockedResource);
- }
- }
-
+ addToLockedResources(lockedResources, locking.serverLocks, sn -> sn.getServerName(),
+ LockedResourceType.SERVER);
+ addToLockedResources(lockedResources, locking.namespaceLocks, Function.identity(),
+ LockedResourceType.NAMESPACE);
+ addToLockedResources(lockedResources, locking.tableLocks, tn -> tn.getNameAsString(),
+ LockedResourceType.TABLE);
+ addToLockedResources(lockedResources, locking.regionLocks, Function.identity(),
+ LockedResourceType.REGION);
+ addToLockedResources(lockedResources, locking.peerLocks, Function.identity(),
+ LockedResourceType.PEER);
return lockedResources;
} finally {
schedUnlock();
@@ -328,8 +310,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
}
@Override
- public LockedResource getLockResource(LockedResourceType resourceType,
- String resourceName) {
+ public LockedResource getLockResource(LockedResourceType resourceType, String resourceName) {
LockAndQueue queue = null;
schedLock();
try {
@@ -346,8 +327,10 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
case REGION:
queue = locking.regionLocks.get(resourceName);
break;
+ case PEER:
+ queue = locking.peerLocks.get(resourceName);
+ break;
}
-
return queue != null ? createLockedResource(resourceType, resourceName, queue) : null;
} finally {
schedUnlock();
@@ -431,6 +414,11 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
markTableAsDeleted(iProcTable.getTableName(), proc);
return;
}
+ } else if (proc instanceof PeerProcedureInterface) {
+ PeerProcedureInterface iProcPeer = (PeerProcedureInterface) proc;
+ if (iProcPeer.getPeerOperationType() == PeerOperationType.REMOVE) {
+ removePeerQueue(iProcPeer.getPeerId());
+ }
} else {
// No cleanup for ServerProcedureInterface types, yet.
return;
@@ -468,12 +456,11 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
locking.removeTableLock(tableName);
}
-
- private static boolean isTableProcedure(Procedure proc) {
+ private static boolean isTableProcedure(Procedure<?> proc) {
return proc instanceof TableProcedureInterface;
}
- private static TableName getTableName(Procedure proc) {
+ private static TableName getTableName(Procedure<?> proc) {
return ((TableProcedureInterface)proc).getTableName();
}
@@ -494,15 +481,42 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
return Math.abs(hashCode) % buckets.length;
}
- private static boolean isServerProcedure(Procedure proc) {
+ private static boolean isServerProcedure(Procedure<?> proc) {
return proc instanceof ServerProcedureInterface;
}
- private static ServerName getServerName(Procedure proc) {
+ private static ServerName getServerName(Procedure<?> proc) {
return ((ServerProcedureInterface)proc).getServerName();
}
// ============================================================================
+ // Peer Queue Lookup Helpers
+ // ============================================================================
+ private PeerQueue getPeerQueue(String peerId) {
+ PeerQueue node = AvlTree.get(peerMap, peerId, PEER_QUEUE_KEY_COMPARATOR);
+ if (node != null) {
+ return node;
+ }
+ node = new PeerQueue(peerId, locking.getPeerLock(peerId));
+ peerMap = AvlTree.insert(peerMap, node);
+ return node;
+ }
+
+ private void removePeerQueue(String peerId) {
+ peerMap = AvlTree.remove(peerMap, peerId, PEER_QUEUE_KEY_COMPARATOR);
+ locking.removePeerLock(peerId);
+ }
+
+
+ private static boolean isPeerProcedure(Procedure<?> proc) {
+ return proc instanceof PeerProcedureInterface;
+ }
+
+ private static String getPeerId(Procedure<?> proc) {
+ return ((PeerProcedureInterface) proc).getPeerId();
+ }
+
+ // ============================================================================
// Table and Server Queue Implementation
// ============================================================================
private static class ServerQueueKeyComparator implements AvlKeyComparator<ServerQueue> {
@@ -571,6 +585,26 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
}
}
+ private static class PeerQueueKeyComparator implements AvlKeyComparator<PeerQueue> {
+
+ @Override
+ public int compareKey(PeerQueue node, Object key) {
+ return node.compareKey((String) key);
+ }
+ }
+
+ public static class PeerQueue extends Queue<String> {
+
+ public PeerQueue(String peerId, LockStatus lockStatus) {
+ super(peerId, lockStatus);
+ }
+
+ @Override
+ public boolean requireExclusiveLock(Procedure proc) {
+ return requirePeerExclusiveLock((PeerProcedureInterface) proc);
+ }
+ }
+
// ============================================================================
// Table Locking Helpers
// ============================================================================
@@ -958,7 +992,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
* @param serverName Server to lock
* @return true if the procedure has to wait for the server to be available
*/
- public boolean waitServerExclusiveLock(final Procedure procedure, final ServerName serverName) {
+ public boolean waitServerExclusiveLock(final Procedure<?> procedure,
+ final ServerName serverName) {
schedLock();
try {
final LockAndQueue lock = locking.getServerLock(serverName);
@@ -980,7 +1015,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
* @param procedure the procedure releasing the lock
* @param serverName the server that has the exclusive lock
*/
- public void wakeServerExclusiveLock(final Procedure procedure, final ServerName serverName) {
+ public void wakeServerExclusiveLock(final Procedure<?> procedure, final ServerName serverName) {
schedLock();
try {
final LockAndQueue lock = locking.getServerLock(serverName);
@@ -994,6 +1029,56 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
}
// ============================================================================
+ // Peer Locking Helpers
+ // ============================================================================
+
+ private static boolean requirePeerExclusiveLock(PeerProcedureInterface proc) {
+ return proc.getPeerOperationType() != PeerOperationType.REFRESH;
+ }
+
+ /**
+ * Try to acquire the exclusive lock on the specified peer.
+ * @see #wakePeerExclusiveLock(Procedure, String)
+ * @param procedure the procedure trying to acquire the lock
+ * @param peerId peer to lock
+ * @return true if the procedure has to wait for the per to be available
+ */
+ public boolean waitPeerExclusiveLock(Procedure<?> procedure, String peerId) {
+ schedLock();
+ try {
+ final LockAndQueue lock = locking.getPeerLock(peerId);
+ if (lock.tryExclusiveLock(procedure)) {
+ removeFromRunQueue(peerRunQueue, getPeerQueue(peerId));
+ return false;
+ }
+ waitProcedure(lock, procedure);
+ logLockedResource(LockedResourceType.PEER, peerId);
+ return true;
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ /**
+ * Wake the procedures waiting for the specified peer
+ * @see #waitPeerExclusiveLock(Procedure, String)
+ * @param procedure the procedure releasing the lock
+ * @param peerId the peer that has the exclusive lock
+ */
+ public void wakePeerExclusiveLock(Procedure<?> procedure, String peerId) {
+ schedLock();
+ try {
+ final LockAndQueue lock = locking.getPeerLock(peerId);
+ lock.releaseExclusiveLock(procedure);
+ addToRunQueue(peerRunQueue, getPeerQueue(peerId));
+ int waitingCount = wakeWaitingProcedures(lock);
+ wakePollIfNeeded(waitingCount);
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ // ============================================================================
// Generic Helpers
// ============================================================================
private static abstract class Queue<TKey extends Comparable<TKey>>
@@ -1098,6 +1183,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
final Map<TableName, LockAndQueue> tableLocks = new HashMap<>();
// Single map for all regions irrespective of tables. Key is encoded region name.
final Map<String, LockAndQueue> regionLocks = new HashMap<>();
+ final Map<String, LockAndQueue> peerLocks = new HashMap<>();
private <T> LockAndQueue getLock(Map<T, LockAndQueue> map, T key) {
LockAndQueue lock = map.get(key);
@@ -1132,6 +1218,14 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
return getLock(serverLocks, serverName);
}
+ LockAndQueue getPeerLock(String peerId) {
+ return getLock(peerLocks, peerId);
+ }
+
+ LockAndQueue removePeerLock(String peerId) {
+ return peerLocks.remove(peerId);
+ }
+
/**
* Removes all locks by clearing the maps.
* Used when procedure executor is stopped for failure and recovery testing.
@@ -1142,6 +1236,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
namespaceLocks.clear();
tableLocks.clear();
regionLocks.clear();
+ peerLocks.clear();
}
@Override
@@ -1149,7 +1244,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
return "serverLocks=" + filterUnlocked(this.serverLocks) +
", namespaceLocks=" + filterUnlocked(this.namespaceLocks) +
", tableLocks=" + filterUnlocked(this.tableLocks) +
- ", regionLocks=" + filterUnlocked(this.regionLocks);
+ ", regionLocks=" + filterUnlocked(this.regionLocks) +
+ ", peerLocks=" + filterUnlocked(this.peerLocks);
}
private String filterUnlocked(Map<?, LockAndQueue> locks) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/PeerProcedureInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/PeerProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/PeerProcedureInterface.java
new file mode 100644
index 0000000..4abc9ad
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/PeerProcedureInterface.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface PeerProcedureInterface {
+
+ enum PeerOperationType {
+ ADD, REMOVE, ENABLE, DISABLE, UPDATE_CONFIG, REFRESH
+ }
+
+ String getPeerId();
+
+ PeerOperationType getPeerOperationType();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index ab282d5..eb4680c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -24,7 +24,6 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
-
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -36,10 +35,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.ipc.RemoteException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
-import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
-import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
-import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
@@ -49,6 +45,13 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProc
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.RemoteProcedureRequest;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
+import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
/**
* A remote procecdure dispatcher for regionservers.
@@ -226,7 +229,10 @@ public class RSProcedureDispatcher
private interface RemoteProcedureResolver {
void dispatchOpenRequests(MasterProcedureEnv env, List<RegionOpenOperation> operations);
+
void dispatchCloseRequests(MasterProcedureEnv env, List<RegionCloseOperation> operations);
+
+ void dispatchServerOperations(MasterProcedureEnv env, List<ServerOperation> operations);
}
/**
@@ -235,22 +241,28 @@ public class RSProcedureDispatcher
* Then {@code resolver} is used to dispatch {@link RegionOpenOperation}s and
* {@link RegionCloseOperation}s.
* @param serverName RegionServer to which the remote operations are sent
- * @param remoteProcedures Remote procedures which are dispatched to the given server
+ * @param operations Remote procedures which are dispatched to the given server
* @param resolver Used to dispatch remote procedures to given server.
*/
- public void splitAndResolveOperation(final ServerName serverName,
- final Set<RemoteProcedure> remoteProcedures, final RemoteProcedureResolver resolver) {
- final ArrayListMultimap<Class<?>, RemoteOperation> reqsByType =
- buildAndGroupRequestByType(procedureEnv, serverName, remoteProcedures);
+ public void splitAndResolveOperation(ServerName serverName, Set<RemoteProcedure> operations,
+ RemoteProcedureResolver resolver) {
+ MasterProcedureEnv env = master.getMasterProcedureExecutor().getEnvironment();
+ ArrayListMultimap<Class<?>, RemoteOperation> reqsByType =
+ buildAndGroupRequestByType(env, serverName, operations);
- final List<RegionOpenOperation> openOps = fetchType(reqsByType, RegionOpenOperation.class);
+ List<RegionOpenOperation> openOps = fetchType(reqsByType, RegionOpenOperation.class);
if (!openOps.isEmpty()) {
- resolver.dispatchOpenRequests(procedureEnv, openOps);
+ resolver.dispatchOpenRequests(env, openOps);
}
- final List<RegionCloseOperation> closeOps = fetchType(reqsByType, RegionCloseOperation.class);
+ List<RegionCloseOperation> closeOps = fetchType(reqsByType, RegionCloseOperation.class);
if (!closeOps.isEmpty()) {
- resolver.dispatchCloseRequests(procedureEnv, closeOps);
+ resolver.dispatchCloseRequests(env, closeOps);
+ }
+
+ List<ServerOperation> refreshOps = fetchType(reqsByType, ServerOperation.class);
+ if (!refreshOps.isEmpty()) {
+ resolver.dispatchServerOperations(env, refreshOps);
}
if (!reqsByType.isEmpty()) {
@@ -282,8 +294,7 @@ public class RSProcedureDispatcher
splitAndResolveOperation(getServerName(), remoteProcedures, this);
try {
- final ExecuteProceduresResponse response = sendRequest(getServerName(), request.build());
- remoteCallCompleted(procedureEnv, response);
+ sendRequest(getServerName(), request.build());
} catch (IOException e) {
e = unwrapException(e);
// TODO: In the future some operation may want to bail out early.
@@ -309,6 +320,11 @@ public class RSProcedureDispatcher
}
}
+ @Override
+ public void dispatchServerOperations(MasterProcedureEnv env, List<ServerOperation> operations) {
+ operations.stream().map(o -> o.buildRequest()).forEachOrdered(request::addProc);
+ }
+
protected ExecuteProceduresResponse sendRequest(final ServerName serverName,
final ExecuteProceduresRequest request) throws IOException {
try {
@@ -318,17 +334,8 @@ public class RSProcedureDispatcher
}
}
-
- private void remoteCallCompleted(final MasterProcedureEnv env,
- final ExecuteProceduresResponse response) {
- /*
- for (RemoteProcedure proc: operations) {
- proc.remoteCallCompleted(env, getServerName(), response);
- }*/
- }
-
private void remoteCallFailed(final MasterProcedureEnv env, final IOException e) {
- for (RemoteProcedure proc: remoteProcedures) {
+ for (RemoteProcedure proc : remoteProcedures) {
proc.remoteCallFailed(env, getServerName(), e);
}
}
@@ -369,8 +376,7 @@ public class RSProcedureDispatcher
buildOpenRegionRequest(procedureEnv, getServerName(), operations);
try {
- OpenRegionResponse response = sendRequest(getServerName(), request);
- remoteCallCompleted(procedureEnv, response);
+ sendRequest(getServerName(), request);
} catch (IOException e) {
e = unwrapException(e);
// TODO: In the future some operation may want to bail out early.
@@ -391,16 +397,6 @@ public class RSProcedureDispatcher
}
}
- private void remoteCallCompleted(final MasterProcedureEnv env,
- final OpenRegionResponse response) {
- int index = 0;
- for (RegionOpenOperation op: operations) {
- OpenRegionResponse.RegionOpeningState state = response.getOpeningState(index++);
- op.setFailedOpen(state == OpenRegionResponse.RegionOpeningState.FAILED_OPENING);
- op.getRemoteProcedure().remoteCallCompleted(env, getServerName(), op);
- }
- }
-
private void remoteCallFailed(final MasterProcedureEnv env, final IOException e) {
for (RegionOpenOperation op: operations) {
op.getRemoteProcedure().remoteCallFailed(env, getServerName(), e);
@@ -450,7 +446,6 @@ public class RSProcedureDispatcher
private void remoteCallCompleted(final MasterProcedureEnv env,
final CloseRegionResponse response) {
operation.setClosed(response.getClosed());
- operation.getRemoteProcedure().remoteCallCompleted(env, getServerName(), operation);
}
private void remoteCallFailed(final MasterProcedureEnv env, final IOException e) {
@@ -491,6 +486,11 @@ public class RSProcedureDispatcher
submitTask(new CloseRegionRemoteCall(serverName, op));
}
}
+
+ @Override
+ public void dispatchServerOperations(MasterProcedureEnv env, List<ServerOperation> operations) {
+ throw new UnsupportedOperationException();
+ }
}
// ==========================================================================
@@ -498,13 +498,28 @@ public class RSProcedureDispatcher
// - ServerOperation: refreshConfig, grant, revoke, ... (TODO)
// - RegionOperation: open, close, flush, snapshot, ...
// ==========================================================================
- /* Currently unused
- public static abstract class ServerOperation extends RemoteOperation {
- protected ServerOperation(final RemoteProcedure remoteProcedure) {
+
+ public static final class ServerOperation extends RemoteOperation {
+
+ private final long procId;
+
+ private final Class<?> rsProcClass;
+
+ private final byte[] rsProcData;
+
+ public ServerOperation(RemoteProcedure remoteProcedure, long procId, Class<?> rsProcClass,
+ byte[] rsProcData) {
super(remoteProcedure);
+ this.procId = procId;
+ this.rsProcClass = rsProcClass;
+ this.rsProcData = rsProcData;
+ }
+
+ public RemoteProcedureRequest buildRequest() {
+ return RemoteProcedureRequest.newBuilder().setProcId(procId)
+ .setProcClass(rsProcClass.getName()).setProcData(ByteString.copyFrom(rsProcData)).build();
}
}
- */
public static abstract class RegionOperation extends RemoteOperation {
private final RegionInfo regionInfo;
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
new file mode 100644
index 0000000..fca05a7
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationState;
+
+@InterfaceAudience.Private
+public abstract class ModifyPeerProcedure
+ extends StateMachineProcedure<MasterProcedureEnv, PeerModificationState>
+ implements PeerProcedureInterface {
+
+ private static final Log LOG = LogFactory.getLog(ModifyPeerProcedure.class);
+
+ protected String peerId;
+
+ protected ModifyPeerProcedure() {
+ }
+
+ protected ModifyPeerProcedure(String peerId) {
+ this.peerId = peerId;
+ }
+
+ @Override
+ public String getPeerId() {
+ return peerId;
+ }
+
+ /**
+ * Return {@code false} means that the operation is invalid and we should give up, otherwise
+ * {@code true}.
+ * <p>
+ * You need to call {@link #setFailure(String, Throwable)} to give the detail failure information.
+ */
+ protected abstract boolean updatePeerStorage() throws IOException;
+
+ protected void postPeerModification() {
+ }
+
+ @Override
+ protected Flow executeFromState(MasterProcedureEnv env, PeerModificationState state)
+ throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+ switch (state) {
+ case UPDATE_PEER_STORAGE:
+ try {
+ if (!updatePeerStorage()) {
+ assert isFailed() : "setFailure is not called";
+ return Flow.NO_MORE_STATE;
+ }
+ } catch (IOException e) {
+ LOG.warn("update peer storage failed, retry", e);
+ throw new ProcedureYieldException();
+ }
+ setNextState(PeerModificationState.REFRESH_PEER_ON_RS);
+ return Flow.HAS_MORE_STATE;
+ case REFRESH_PEER_ON_RS:
+ addChildProcedure(env.getMasterServices().getServerManager().getOnlineServersList().stream()
+ .map(sn -> new RefreshPeerProcedure(peerId, getPeerOperationType(), sn))
+ .toArray(RefreshPeerProcedure[]::new));
+ setNextState(PeerModificationState.POST_PEER_MODIFICATION);
+ return Flow.HAS_MORE_STATE;
+ case POST_PEER_MODIFICATION:
+ postPeerModification();
+ return Flow.NO_MORE_STATE;
+ default:
+ throw new UnsupportedOperationException("unhandled state=" + state);
+ }
+ }
+
+ @Override
+ protected LockState acquireLock(MasterProcedureEnv env) {
+ return env.getProcedureScheduler().waitPeerExclusiveLock(this, peerId)
+ ? LockState.LOCK_EVENT_WAIT
+ : LockState.LOCK_ACQUIRED;
+ }
+
+ @Override
+ protected void releaseLock(MasterProcedureEnv env) {
+ env.getProcedureScheduler().wakePeerExclusiveLock(this, peerId);
+ }
+
+ @Override
+ protected void rollbackState(MasterProcedureEnv env, PeerModificationState state)
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected PeerModificationState getState(int stateId) {
+ return PeerModificationState.forNumber(stateId);
+ }
+
+ @Override
+ protected int getStateId(PeerModificationState state) {
+ return state.getNumber();
+ }
+
+ @Override
+ protected PeerModificationState getInitialState() {
+ return PeerModificationState.UPDATE_PEER_STORAGE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerCallable.java
new file mode 100644
index 0000000..4e09107
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerCallable.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.executor.EventType;
+import org.apache.hadoop.hbase.procedure2.RSProcedureCallable;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshPeerParameter;
+
+/**
+ * The callable executed at RS side to refresh the peer config/state.
+ * <p>
+ * TODO: only a dummy implementation for verifying the framework, will add implementation later.
+ */
+@InterfaceAudience.Private
+public class RefreshPeerCallable implements RSProcedureCallable {
+
+ private HRegionServer rs;
+
+ private String peerId;
+
+ private Exception initError;
+
+ @Override
+ public Void call() throws Exception {
+ if (initError != null) {
+ throw initError;
+ }
+ rs.getFileSystem().create(new Path("/" + peerId + "/" + rs.getServerName().toString())).close();
+ return null;
+ }
+
+ @Override
+ public void init(byte[] parameter, HRegionServer rs) {
+ this.rs = rs;
+ try {
+ this.peerId = RefreshPeerParameter.parseFrom(parameter).getPeerId();
+ } catch (InvalidProtocolBufferException e) {
+ initError = e;
+ return;
+ }
+ }
+
+ @Override
+ public EventType getEventType() {
+ return EventType.RS_REFRESH_PEER;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
new file mode 100644
index 0000000..18da487
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
+import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher.ServerOperation;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperation;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationType;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshPeerParameter;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshPeerStateData;
+
+@InterfaceAudience.Private
+public class RefreshPeerProcedure extends Procedure<MasterProcedureEnv>
+ implements PeerProcedureInterface, RemoteProcedure<MasterProcedureEnv, ServerName> {
+
+ private static final Log LOG = LogFactory.getLog(RefreshPeerProcedure.class);
+
+ private String peerId;
+
+ private PeerOperationType type;
+
+ private ServerName targetServer;
+
+ private boolean dispatched;
+
+ private ProcedureEvent<?> event;
+
+ private boolean succ;
+
+ public RefreshPeerProcedure() {
+ }
+
+ public RefreshPeerProcedure(String peerId, PeerOperationType type, ServerName targetServer) {
+ this.peerId = peerId;
+ this.type = type;
+ this.targetServer = targetServer;
+ }
+
+ @Override
+ public String getPeerId() {
+ return peerId;
+ }
+
+ @Override
+ public PeerOperationType getPeerOperationType() {
+ return PeerOperationType.REFRESH;
+ }
+
+ private static PeerModificationType toPeerModificationType(PeerOperationType type) {
+ switch (type) {
+ case ADD:
+ return PeerModificationType.ADD_PEER;
+ case REMOVE:
+ return PeerModificationType.REMOVE_PEER;
+ case ENABLE:
+ return PeerModificationType.ENABLE_PEER;
+ case DISABLE:
+ return PeerModificationType.DISABLE_PEER;
+ case UPDATE_CONFIG:
+ return PeerModificationType.UPDATE_PEER_CONFIG;
+ default:
+ throw new IllegalArgumentException("Unknown type: " + type);
+ }
+ }
+
+ private static PeerOperationType toPeerOperationType(PeerModificationType type) {
+ switch (type) {
+ case ADD_PEER:
+ return PeerOperationType.ADD;
+ case REMOVE_PEER:
+ return PeerOperationType.REMOVE;
+ case ENABLE_PEER:
+ return PeerOperationType.ENABLE;
+ case DISABLE_PEER:
+ return PeerOperationType.DISABLE;
+ case UPDATE_PEER_CONFIG:
+ return PeerOperationType.UPDATE_CONFIG;
+ default:
+ throw new IllegalArgumentException("Unknown type: " + type);
+ }
+ }
+
+ @Override
+ public RemoteOperation remoteCallBuild(MasterProcedureEnv env, ServerName remote) {
+ assert targetServer.equals(remote);
+ return new ServerOperation(this, getProcId(), RefreshPeerCallable.class,
+ RefreshPeerParameter.newBuilder().setPeerId(peerId).setType(toPeerModificationType(type))
+ .setTargetServer(ProtobufUtil.toServerName(remote)).build().toByteArray());
+ }
+
+ private void complete(MasterProcedureEnv env, boolean succ) {
+ if (event == null) {
+ LOG.warn("procedure event for " + getProcId() +
+ " is null, maybe the procedure is created when recovery", new Exception());
+ return;
+ }
+ LOG.info("Refresh peer " + peerId + " for " + type + " on " + targetServer +
+ (succ ? " suceeded" : " failed"));
+ this.succ = succ;
+ event.wake(env.getProcedureScheduler());
+ event = null;
+ }
+
+ @Override
+ public synchronized void remoteCallFailed(MasterProcedureEnv env, ServerName remote,
+ IOException exception) {
+ complete(env, false);
+ }
+
+ @Override
+ public synchronized void remoteOperationCompleted(MasterProcedureEnv env) {
+ complete(env, true);
+ }
+
+ @Override
+ public synchronized void remoteOperationFailed(MasterProcedureEnv env, String error) {
+ complete(env, false);
+ }
+
+ @Override
+ protected synchronized Procedure<MasterProcedureEnv>[] execute(MasterProcedureEnv env)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ if (dispatched) {
+ if (succ) {
+ return null;
+ }
+ // retry
+ dispatched = false;
+ }
+ if (!env.getRemoteDispatcher().addOperationToNode(targetServer, this)) {
+ LOG.info("Can not add remote operation for refreshing peer " + peerId + " for " + type +
+ " to " + targetServer + ", this usually because the server is already dead," +
+ " give up and mark the procedure as complete");
+ return null;
+ }
+ dispatched = true;
+ event = new ProcedureEvent<>(this);
+ event.suspendIfNotReady(this);
+ throw new ProcedureSuspendedException();
+ }
+
+ @Override
+ protected void rollback(MasterProcedureEnv env) throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected boolean abort(MasterProcedureEnv env) {
+ // TODO: no correctness problem if we just ignore this, implement later.
+ return false;
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ serializer.serialize(
+ RefreshPeerStateData.newBuilder().setPeerId(peerId).setType(toPeerModificationType(type))
+ .setTargetServer(ProtobufUtil.toServerName(targetServer)).build());
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ RefreshPeerStateData data = serializer.deserialize(RefreshPeerStateData.class);
+ peerId = data.getPeerId();
+ type = toPeerOperationType(data.getType());
+ targetServer = ProtobufUtil.toServerName(data.getTargetServer());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/RSProcedureCallable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/RSProcedureCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/RSProcedureCallable.java
new file mode 100644
index 0000000..62c2e36
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/RSProcedureCallable.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2;
+
+import java.util.concurrent.Callable;
+
+import org.apache.hadoop.hbase.executor.EventType;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * A general interface for a sub procedure runs at RS side.
+ */
+@InterfaceAudience.Private
+public interface RSProcedureCallable extends Callable<Void> {
+
+ /**
+ * Initialize the callable
+ * @param parameter the parameter passed from master.
+ * @param rs the regionserver instance
+ */
+ void init(byte[] parameter, HRegionServer rs);
+
+ /**
+ * Event type used to select thread pool.
+ */
+ EventType getEventType();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1c1ce82..b439f38 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
-import javax.servlet.http.HttpServlet;
import java.io.IOException;
import java.lang.Thread.UncaughtExceptionHandler;
import java.lang.management.MemoryType;
@@ -49,7 +46,9 @@ import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
-
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import javax.servlet.http.HttpServlet;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
@@ -117,6 +116,7 @@ import org.apache.hadoop.hbase.master.LoadBalancer;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.mob.MobCacheConfig;
import org.apache.hadoop.hbase.procedure.RegionServerProcedureManagerHost;
+import org.apache.hadoop.hbase.procedure2.RSProcedureCallable;
import org.apache.hadoop.hbase.quotas.FileSystemUtilizationChore;
import org.apache.hadoop.hbase.quotas.QuotaUtil;
import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager;
@@ -127,6 +127,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequester;
import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler;
import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler;
+import org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler;
import org.apache.hadoop.hbase.regionserver.handler.RegionReplicaFlushHandler;
import org.apache.hadoop.hbase.regionserver.throttle.FlushThroughputControllerFactory;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
@@ -172,6 +173,9 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import sun.misc.Signal;
+import sun.misc.SignalHandler;
+
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
@@ -181,6 +185,7 @@ import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall;
@@ -206,13 +211,11 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse;
-import sun.misc.Signal;
-import sun.misc.SignalHandler;
-
/**
* HRegionServer makes a set of HRegions available to clients. It checks in with
* the HMaster. There are many HRegionServers in a single HBase deployment.
@@ -1920,6 +1923,8 @@ public class HRegionServer extends HasThread implements
conf.getInt("hbase.regionserver.region.replica.flusher.threads",
conf.getInt("hbase.regionserver.executor.openregion.threads", 3)));
}
+ this.executorService.startExecutorService(ExecutorType.RS_REFRESH_PEER,
+ conf.getInt("hbase.regionserver.executor.refresh.peer.threads", 2));
Threads.setDaemonThreadRunning(this.walRoller.getThread(), getName() + ".logRoller",
uncaughtExceptionHandler);
@@ -3701,4 +3706,60 @@ public class HRegionServer extends HasThread implements
return ConnectionUtils.createShortCircuitConnection(conf, null, user, this.serverName,
this.rpcServices, this.rpcServices);
}
+
+ public void executeProcedure(long procId, RSProcedureCallable callable) {
+ executorService.submit(new RSProcedureHandler(this, procId, callable));
+ }
+
+ public void reportProcedureDone(long procId, Throwable error) {
+ ReportProcedureDoneRequest.Builder builder =
+ ReportProcedureDoneRequest.newBuilder().setProcId(procId);
+ if (error != null) {
+ builder.setStatus(ReportProcedureDoneRequest.Status.ERROR)
+ .setError(Throwables.getStackTraceAsString(error));
+ } else {
+ builder.setStatus(ReportProcedureDoneRequest.Status.SUCCESS);
+ }
+ ReportProcedureDoneRequest request = builder.build();
+ int tries = 0;
+ long pauseTime = INIT_PAUSE_TIME_MS;
+ while (keepLooping()) {
+ RegionServerStatusService.BlockingInterface rss = rssStub;
+ try {
+ if (rss == null) {
+ createRegionServerStatusStub();
+ continue;
+ }
+ rss.reportProcedureDone(null, request);
+ // Log if we had to retry else don't log unless TRACE. We want to
+ // know if were successful after an attempt showed in logs as failed.
+ if (tries > 0 || LOG.isTraceEnabled()) {
+ LOG.info("PROCEDURE REPORTED " + request);
+ }
+ return;
+ } catch (ServiceException se) {
+ IOException ioe = ProtobufUtil.getRemoteException(se);
+ boolean pause =
+ ioe instanceof ServerNotRunningYetException || ioe instanceof PleaseHoldException;
+ if (pause) {
+ // Do backoff else we flood the Master with requests.
+ pauseTime = ConnectionUtils.getPauseTime(INIT_PAUSE_TIME_MS, tries);
+ } else {
+ pauseTime = INIT_PAUSE_TIME_MS; // Reset.
+ }
+ LOG.info(
+ "Failed to report transition " + TextFormat.shortDebugString(request) + "; retry (#" +
+ tries + ")" + (pause ? " after " + pauseTime + "ms delay (Master is coming online...)."
+ : " immediately."),
+ ioe);
+ if (pause) {
+ Threads.sleep(pauseTime);
+ }
+ tries++;
+ if (rssStub == rss) {
+ rssStub = null;
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 5a01581..5391a82 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -100,6 +99,7 @@ import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.log.HBaseMarkers;
import org.apache.hadoop.hbase.master.MasterRpcServices;
+import org.apache.hadoop.hbase.procedure2.RSProcedureCallable;
import org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement;
import org.apache.hadoop.hbase.quotas.OperationQuota;
import org.apache.hadoop.hbase.quotas.QuotaUtil;
@@ -172,6 +172,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionR
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.RemoteProcedureRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.RollWALWriterRequest;
@@ -3435,23 +3436,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
}
@Override
- public ExecuteProceduresResponse executeProcedures(RpcController controller,
- ExecuteProceduresRequest request) throws ServiceException {
- ExecuteProceduresResponse.Builder builder = ExecuteProceduresResponse.newBuilder();
- if (request.getOpenRegionCount() > 0) {
- for (OpenRegionRequest req : request.getOpenRegionList()) {
- builder.addOpenRegion(openRegion(controller, req));
- }
- }
- if (request.getCloseRegionCount() > 0) {
- for (CloseRegionRequest req : request.getCloseRegionList()) {
- builder.addCloseRegion(closeRegion(controller, req));
- }
- }
- return builder.build();
- }
-
- @Override
public ClearRegionBlockCacheResponse clearRegionBlockCache(RpcController controller,
ClearRegionBlockCacheRequest request) {
ClearRegionBlockCacheResponse.Builder builder =
@@ -3468,4 +3452,38 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
stats.withMaxCacheSize(regionServer.getCacheConfig().getBlockCache().getMaxSize());
return builder.setStats(ProtobufUtil.toCacheEvictionStats(stats.build())).build();
}
+
+ @Override
+ public ExecuteProceduresResponse executeProcedures(RpcController controller,
+ ExecuteProceduresRequest request) throws ServiceException {
+ if (request.getOpenRegionCount() > 0) {
+ for (OpenRegionRequest req : request.getOpenRegionList()) {
+ openRegion(controller, req);
+ }
+ }
+ if (request.getCloseRegionCount() > 0) {
+ for (CloseRegionRequest req : request.getCloseRegionList()) {
+ closeRegion(controller, req);
+ }
+ }
+ if (request.getProcCount() > 0) {
+ for (RemoteProcedureRequest req : request.getProcList()) {
+ RSProcedureCallable callable;
+ try {
+ callable =
+ Class.forName(req.getProcClass()).asSubclass(RSProcedureCallable.class).newInstance();
+ } catch (Exception e) {
+ // here we just ignore the error as this should not happen and we do not provide a general
+ // way to report errors for all types of remote procedure. The procedure will hang at
+ // master side but after you solve the problem and restart master it will be executed
+ // again and pass.
+ LOG.warn("create procedure of type " + req.getProcClass() + " failed, give up", e);
+ continue;
+ }
+ callable.init(req.getProcData().toByteArray(), regionServer);
+ regionServer.executeProcedure(req.getProcId(), callable);
+ }
+ }
+ return ExecuteProceduresResponse.getDefaultInstance();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
new file mode 100644
index 0000000..94bcfec
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver.handler;
+
+import org.apache.hadoop.hbase.executor.EventHandler;
+import org.apache.hadoop.hbase.procedure2.RSProcedureCallable;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * A event handler for running procedure.
+ */
+@InterfaceAudience.Private
+public class RSProcedureHandler extends EventHandler {
+
+ private final long procId;
+
+ private final RSProcedureCallable callable;
+
+ public RSProcedureHandler(HRegionServer rs, long procId, RSProcedureCallable callable) {
+ super(rs, callable.getEventType());
+ this.procId = procId;
+ this.callable = callable;
+ }
+
+ @Override
+ public void process() {
+ Exception error = null;
+ try {
+ callable.call();
+ } catch (Exception e) {
+ error = e;
+ }
+ ((HRegionServer) server).reportProcedureDone(procId, error);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
index faa2114..9b9f624 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java
@@ -538,26 +538,16 @@ public class TestAssignmentManager {
@Override
public ExecuteProceduresResponse sendRequest(ServerName server,
ExecuteProceduresRequest request) throws IOException {
- ExecuteProceduresResponse.Builder builder = ExecuteProceduresResponse.newBuilder();
if (request.getOpenRegionCount() > 0) {
- for (OpenRegionRequest req: request.getOpenRegionList()) {
- OpenRegionResponse.Builder resp = OpenRegionResponse.newBuilder();
- for (RegionOpenInfo openReq: req.getOpenInfoList()) {
- RegionOpeningState state = execOpenRegion(server, openReq);
- if (state != null) {
- resp.addOpeningState(state);
- }
+ for (OpenRegionRequest req : request.getOpenRegionList()) {
+ for (RegionOpenInfo openReq : req.getOpenInfoList()) {
+ execOpenRegion(server, openReq);
}
- builder.addOpenRegion(resp.build());
}
}
if (request.getCloseRegionCount() > 0) {
- for (CloseRegionRequest req: request.getCloseRegionList()) {
- CloseRegionResponse resp = execCloseRegion(server,
- req.getRegion().getValue().toByteArray());
- if (resp != null) {
- builder.addCloseRegion(resp);
- }
+ for (CloseRegionRequest req : request.getCloseRegionList()) {
+ execCloseRegion(server, req.getRegion().getValue().toByteArray());
}
}
return ExecuteProceduresResponse.newBuilder().build();
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
new file mode 100644
index 0000000..44343d7
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+
+public class DummyModifyPeerProcedure extends ModifyPeerProcedure {
+
+ public DummyModifyPeerProcedure() {
+ }
+
+ public DummyModifyPeerProcedure(String peerId) {
+ super(peerId);
+ }
+
+ @Override
+ public PeerOperationType getPeerOperationType() {
+ return PeerOperationType.ADD;
+ }
+
+ @Override
+ protected boolean updatePeerStorage() throws IOException {
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestDummyModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestDummyModifyPeerProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestDummyModifyPeerProcedure.java
new file mode 100644
index 0000000..ec06306
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestDummyModifyPeerProcedure.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, LargeTests.class })
+public class TestDummyModifyPeerProcedure {
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static String PEER_ID;
+
+ private static Path DIR;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.startMiniCluster(3);
+ PEER_ID = "testPeer";
+ DIR = new Path("/" + PEER_ID);
+ UTIL.getTestFileSystem().mkdirs(DIR);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void test() throws Exception {
+ ProcedureExecutor<?> executor =
+ UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
+ long procId = executor.submitProcedure(new DummyModifyPeerProcedure(PEER_ID));
+ UTIL.waitFor(30000, new Waiter.Predicate<Exception>() {
+
+ @Override
+ public boolean evaluate() throws Exception {
+ return executor.isFinished(procId);
+ }
+ });
+ Set<String> serverNames = UTIL.getHBaseCluster().getRegionServerThreads().stream()
+ .map(t -> t.getRegionServer().getServerName().toString())
+ .collect(Collectors.toCollection(HashSet::new));
+ for (FileStatus s : UTIL.getTestFileSystem().listStatus(DIR)) {
+ assertTrue(serverNames.remove(s.getPath().getName()));
+ }
+ assertTrue(serverNames.isEmpty());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41e57c3c/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 14e94be..9a1bd74 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -30,6 +30,7 @@ import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
import com.google.protobuf.ServiceException;
+
import java.io.IOException;
import java.security.PrivilegedAction;
import java.util.ArrayList;
[27/50] [abbrv] hbase git commit: HBASE-19630 Add peer cluster key
check when add new replication peer
Posted by zh...@apache.org.
HBASE-19630 Add peer cluster key check when add new replication peer
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/78317db4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/78317db4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/78317db4
Branch: refs/heads/HBASE-19397-branch-2
Commit: 78317db4ded17bc43181a6172e3aace5e24139de
Parents: dcc7793
Author: Guanghao Zhang <zg...@apache.org>
Authored: Tue Dec 26 21:10:00 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../replication/ReplicationPeerManager.java | 54 ++++++++++++--------
.../replication/TestReplicationAdmin.java | 22 ++++++++
2 files changed, 54 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/78317db4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 84abfeb..b78cbce 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master.replication;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -42,6 +43,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -216,36 +218,36 @@ public final class ReplicationPeerManager {
return desc != null ? Optional.of(desc.getPeerConfig()) : Optional.empty();
}
- /**
- * If replicate_all flag is true, it means all user tables will be replicated to peer cluster.
- * Then allow config exclude namespaces or exclude table-cfs which can't be replicated to peer
- * cluster.
- * <p>
- * If replicate_all flag is false, it means all user tables can't be replicated to peer cluster.
- * Then allow to config namespaces or table-cfs which will be replicated to peer cluster.
- */
- private static void checkPeerConfig(ReplicationPeerConfig peerConfig)
- throws DoNotRetryIOException {
+ private void checkPeerConfig(ReplicationPeerConfig peerConfig) throws DoNotRetryIOException {
+ checkClusterKey(peerConfig.getClusterKey());
+
if (peerConfig.replicateAllUserTables()) {
- if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty()) ||
- (peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) {
- throw new DoNotRetryIOException("Need clean namespaces or table-cfs config firstly " +
- "when you want replicate all cluster");
+ // If replicate_all flag is true, it means all user tables will be replicated to peer cluster.
+ // Then allow config exclude namespaces or exclude table-cfs which can't be replicated to peer
+ // cluster.
+ if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty())
+ || (peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) {
+ throw new DoNotRetryIOException("Need clean namespaces or table-cfs config firstly "
+ + "when you want replicate all cluster");
}
checkNamespacesAndTableCfsConfigConflict(peerConfig.getExcludeNamespaces(),
peerConfig.getExcludeTableCFsMap());
} else {
- if ((peerConfig.getExcludeNamespaces() != null &&
- !peerConfig.getExcludeNamespaces().isEmpty()) ||
- (peerConfig.getExcludeTableCFsMap() != null &&
- !peerConfig.getExcludeTableCFsMap().isEmpty())) {
+ // If replicate_all flag is false, it means all user tables can't be replicated to peer
+ // cluster. Then allow to config namespaces or table-cfs which will be replicated to peer
+ // cluster.
+ if ((peerConfig.getExcludeNamespaces() != null
+ && !peerConfig.getExcludeNamespaces().isEmpty())
+ || (peerConfig.getExcludeTableCFsMap() != null
+ && !peerConfig.getExcludeTableCFsMap().isEmpty())) {
throw new DoNotRetryIOException(
- "Need clean exclude-namespaces or exclude-table-cfs config firstly" +
- " when replicate_all flag is false");
+ "Need clean exclude-namespaces or exclude-table-cfs config firstly"
+ + " when replicate_all flag is false");
}
checkNamespacesAndTableCfsConfigConflict(peerConfig.getNamespaces(),
peerConfig.getTableCFsMap());
}
+
checkConfiguredWALEntryFilters(peerConfig);
}
@@ -268,7 +270,7 @@ public final class ReplicationPeerManager {
* exclude namespace.</li>
* </ol>
*/
- private static void checkNamespacesAndTableCfsConfigConflict(Set<String> namespaces,
+ private void checkNamespacesAndTableCfsConfigConflict(Set<String> namespaces,
Map<TableName, ? extends Collection<String>> tableCfs) throws DoNotRetryIOException {
if (namespaces == null || namespaces.isEmpty()) {
return;
@@ -285,7 +287,7 @@ public final class ReplicationPeerManager {
}
}
- private static void checkConfiguredWALEntryFilters(ReplicationPeerConfig peerConfig)
+ private void checkConfiguredWALEntryFilters(ReplicationPeerConfig peerConfig)
throws DoNotRetryIOException {
String filterCSV = peerConfig.getConfiguration()
.get(BaseReplicationEndpoint.REPLICATION_WALENTRYFILTER_CONFIG_KEY);
@@ -302,6 +304,14 @@ public final class ReplicationPeerManager {
}
}
+ private void checkClusterKey(String clusterKey) throws DoNotRetryIOException {
+ try {
+ ZKConfig.validateClusterKey(clusterKey);
+ } catch (IOException e) {
+ throw new DoNotRetryIOException("Invalid cluster key: " + clusterKey, e);
+ }
+ }
+
public static ReplicationPeerManager create(ZKWatcher zk, Configuration conf)
throws ReplicationException {
ReplicationPeerStorage peerStorage =
http://git-wip-us.apache.org/repos/asf/hbase/blob/78317db4/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index 56a6cfe..dbbc78f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -113,6 +113,28 @@ public class TestReplicationAdmin {
}
}
+ @Test
+ public void testAddInvalidPeer() {
+ ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder();
+ builder.setClusterKey(KEY_ONE);
+ try {
+ String invalidPeerId = "1-2";
+ hbaseAdmin.addReplicationPeer(invalidPeerId, builder.build());
+ fail("Should fail as the peer id: " + invalidPeerId + " is invalid");
+ } catch (Exception e) {
+ // OK
+ }
+
+ try {
+ String invalidClusterKey = "2181:/hbase";
+ builder.setClusterKey(invalidClusterKey);
+ hbaseAdmin.addReplicationPeer(ID_ONE, builder.build());
+ fail("Should fail as the peer cluster key: " + invalidClusterKey + " is invalid");
+ } catch (Exception e) {
+ // OK
+ }
+ }
+
/**
* Simple testing of adding and removing peers, basically shows that
* all interactions with ZK work
[12/50] [abbrv] hbase git commit: HBASE-19811 Fix findbugs and
error-prone warnings in hbase-server (branch-2)
Posted by zh...@apache.org.
HBASE-19811 Fix findbugs and error-prone warnings in hbase-server (branch-2)
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b1269ec5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b1269ec5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b1269ec5
Branch: refs/heads/HBASE-19397-branch-2
Commit: b1269ec57ff77f4077516fb4f866e2fe7ede3a3e
Parents: 1e5fc1e
Author: Peter Somogyi <ps...@cloudera.com>
Authored: Mon Jan 22 15:32:06 2018 +0100
Committer: Michael Stack <st...@apache.org>
Committed: Mon Jan 22 17:14:15 2018 -0800
----------------------------------------------------------------------
.../hbase/SslRMIServerSocketFactorySecure.java | 1 +
.../hadoop/hbase/client/locking/EntityLock.java | 1 +
.../hadoop/hbase/conf/ConfigurationManager.java | 2 +-
.../ZkSplitLogWorkerCoordination.java | 2 +-
.../coprocessor/BaseRowProcessorEndpoint.java | 8 +-
.../hbase/coprocessor/CoprocessorHost.java | 1 +
.../hbase/coprocessor/ObserverContextImpl.java | 3 +
.../coprocessor/ReadOnlyConfiguration.java | 14 +-
.../hadoop/hbase/filter/FilterWrapper.java | 1 +
.../org/apache/hadoop/hbase/fs/HFileSystem.java | 83 ++--
.../hbase/io/FSDataInputStreamWrapper.java | 11 +-
.../hadoop/hbase/io/HalfStoreFileReader.java | 8 +
.../org/apache/hadoop/hbase/io/Reference.java | 1 +
.../asyncfs/FanOutOneBlockAsyncDFSOutput.java | 1 +
.../hadoop/hbase/io/hfile/BlockCache.java | 1 +
.../hbase/io/hfile/CompoundBloomFilter.java | 1 +
.../hadoop/hbase/io/hfile/FixedFileTrailer.java | 10 +-
.../hadoop/hbase/io/hfile/HFileBlock.java | 68 +--
.../hadoop/hbase/io/hfile/HFileReaderImpl.java | 2 +-
.../hadoop/hbase/io/hfile/HFileScanner.java | 1 +
.../hadoop/hbase/io/hfile/LruBlockCache.java | 4 +
.../hadoop/hbase/io/hfile/LruCachedBlock.java | 1 +
.../hbase/io/hfile/LruCachedBlockQueue.java | 1 +
.../hbase/io/hfile/bucket/BucketAllocator.java | 1 +
.../hbase/io/hfile/bucket/BucketCache.java | 18 +-
.../hbase/io/hfile/bucket/CachedEntryQueue.java | 1 +
.../apache/hadoop/hbase/ipc/BufferChain.java | 2 +-
.../ipc/FastPathBalancedQueueRpcExecutor.java | 1 +
.../hadoop/hbase/ipc/ServerRpcConnection.java | 5 +-
.../hadoop/hbase/ipc/SimpleRpcServer.java | 1 +
.../hbase/master/ClusterStatusPublisher.java | 9 +-
.../apache/hadoop/hbase/master/DeadServer.java | 1 +
.../hadoop/hbase/master/HMasterCommandLine.java | 3 +-
.../hadoop/hbase/master/LoadBalancer.java | 1 +
...MasterAnnotationReadingPriorityFunction.java | 1 +
.../hbase/master/MasterCoprocessorHost.java | 1 +
.../hadoop/hbase/master/MobCompactionChore.java | 2 +-
.../hbase/master/RegionServerTracker.java | 2 +-
.../hadoop/hbase/master/SplitLogManager.java | 2 +-
.../assignment/MergeTableRegionsProcedure.java | 2 +-
.../hbase/master/assignment/RegionStates.java | 7 +-
.../balancer/FavoredStochasticBalancer.java | 5 +-
.../master/balancer/RegionLocationFinder.java | 27 +-
.../master/balancer/SimpleLoadBalancer.java | 1 +
.../master/balancer/StochasticLoadBalancer.java | 6 +-
.../hbase/master/cleaner/CleanerChore.java | 4 +-
.../hbase/master/cleaner/HFileCleaner.java | 2 +-
.../hadoop/hbase/master/cleaner/LogCleaner.java | 2 +-
.../hbase/master/locking/LockProcedure.java | 2 +-
.../master/procedure/ProcedurePrepareLatch.java | 4 +
.../master/procedure/RSProcedureDispatcher.java | 9 +
.../master/snapshot/SnapshotHFileCleaner.java | 2 +
.../apache/hadoop/hbase/mob/CachedMobFile.java | 1 +
.../hadoop/hbase/mob/ExpiredMobFileCleaner.java | 1 +
.../PartitionedMobCompactionRequest.java | 1 +
.../monitoring/MonitoredRPCHandlerImpl.java | 12 +
.../hbase/monitoring/MonitoredTaskImpl.java | 1 +
.../hadoop/hbase/monitoring/TaskMonitor.java | 2 +-
.../hadoop/hbase/procedure/Procedure.java | 2 +-
.../hbase/procedure/ProcedureManagerHost.java | 6 +-
.../hadoop/hbase/procedure/Subprocedure.java | 1 +
.../hbase/procedure/ZKProcedureCoordinator.java | 12 +-
.../hbase/procedure/ZKProcedureMemberRpcs.java | 1 +
.../hadoop/hbase/procedure/ZKProcedureUtil.java | 1 +
.../hadoop/hbase/quotas/MasterQuotaManager.java | 4 +-
.../hadoop/hbase/quotas/QuotaObserverChore.java | 3 +-
.../apache/hadoop/hbase/quotas/RateLimiter.java | 1 +
.../quotas/RegionServerSpaceQuotaManager.java | 2 +-
.../SpaceQuotaSnapshotNotifierFactory.java | 4 +-
.../hbase/regionserver/AbstractMemStore.java | 2 +-
.../AdaptiveMemStoreCompactionStrategy.java | 3 +
.../regionserver/CellChunkImmutableSegment.java | 2 +-
.../hadoop/hbase/regionserver/CellSet.java | 30 ++
.../hbase/regionserver/CompactingMemStore.java | 4 +-
.../regionserver/CompositeImmutableSegment.java | 1 +
.../regionserver/DateTieredStoreEngine.java | 1 +
.../hbase/regionserver/DefaultStoreFlusher.java | 8 +-
.../hadoop/hbase/regionserver/HMobStore.java | 56 +--
.../hadoop/hbase/regionserver/HRegion.java | 5 +-
.../hbase/regionserver/HRegionServer.java | 10 +-
.../regionserver/HRegionServerCommandLine.java | 2 +
.../hadoop/hbase/regionserver/HStore.java | 84 ++--
...IncreasingToUpperBoundRegionSplitPolicy.java | 2 +-
.../hbase/regionserver/InternalScanner.java | 1 +
.../hadoop/hbase/regionserver/KeyValueHeap.java | 5 +
.../hbase/regionserver/KeyValueScanner.java | 1 +
.../hbase/regionserver/MemStoreCompactor.java | 4 +-
.../hbase/regionserver/MemStoreFlusher.java | 8 +-
.../MemStoreMergerSegmentsIterator.java | 1 +
.../MetricsRegionServerWrapperImpl.java | 4 +-
.../regionserver/MetricsRegionWrapperImpl.java | 2 +-
.../MultiVersionConcurrencyControl.java | 9 +-
.../hbase/regionserver/RSRpcServices.java | 2 +-
.../RegionServerCoprocessorHost.java | 1 +
.../hbase/regionserver/ScannerContext.java | 8 +-
.../hbase/regionserver/ServerNonceManager.java | 2 +-
.../hbase/regionserver/SteppingSplitPolicy.java | 1 +
.../hbase/regionserver/StoreFileScanner.java | 6 +
.../hbase/regionserver/StripeStoreFlusher.java | 8 +-
.../regionserver/compactions/Compactor.java | 7 +-
.../compactions/DateTieredCompactionPolicy.java | 1 +
.../compactions/RatioBasedCompactionPolicy.java | 1 +
.../compactions/SortedCompactionPolicy.java | 2 +
.../querymatcher/ExplicitColumnTracker.java | 4 +
.../querymatcher/ScanWildcardColumnTracker.java | 2 +
.../hbase/regionserver/wal/AsyncFSWAL.java | 2 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 20 +-
.../hbase/regionserver/wal/FSWALEntry.java | 1 +
.../regionserver/wal/ProtobufLogReader.java | 6 +-
.../replication/BaseReplicationEndpoint.java | 2 +-
.../DefaultSourceFSConfigurationProvider.java | 2 +-
.../replication/regionserver/Replication.java | 5 +
.../regionserver/ReplicationSink.java | 4 +-
.../regionserver/ReplicationSource.java | 19 +-
.../regionserver/ReplicationSourceManager.java | 2 +-
.../ReplicationSourceWALReader.java | 3 +-
.../security/access/AccessControlFilter.java | 1 +
.../hbase/security/access/AccessController.java | 4 +-
.../hbase/security/access/AuthResult.java | 2 +
.../token/AuthenticationTokenSecretManager.java | 1 +
.../security/visibility/ExpressionParser.java | 4 +-
.../visibility/VisibilityController.java | 4 +-
.../VisibilityNewVersionBehaivorTracker.java | 2 +-
.../expression/LeafExpressionNode.java | 1 +
.../expression/NonLeafExpressionNode.java | 1 +
.../visibility/expression/Operator.java | 3 +-
.../hadoop/hbase/snapshot/SnapshotInfo.java | 6 +-
.../hbase/snapshot/SnapshotManifestV1.java | 5 +
.../hbase/snapshot/SnapshotManifestV2.java | 5 +
.../hbase/tool/LoadIncrementalHFiles.java | 2 +-
.../util/BoundedPriorityBlockingQueue.java | 14 +
.../apache/hadoop/hbase/util/FSMapRUtils.java | 1 +
.../hadoop/hbase/util/FSRegionScanner.java | 2 +-
.../hadoop/hbase/util/FSTableDescriptors.java | 2 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 6 +-
.../org/apache/hadoop/hbase/util/IdLock.java | 1 +
.../hadoop/hbase/util/RegionSplitter.java | 2 +-
.../hadoop/hbase/util/RowBloomContext.java | 1 +
.../hadoop/hbase/util/ShutdownHookManager.java | 6 +-
.../hadoop/hbase/wal/AbstractFSWALProvider.java | 3 +-
.../hadoop/hbase/wal/DisabledWALProvider.java | 3 +-
.../apache/hadoop/hbase/wal/FSHLogProvider.java | 2 +-
.../hbase/wal/RegionGroupingProvider.java | 13 +-
.../java/org/apache/hadoop/hbase/wal/WAL.java | 1 +
.../org/apache/hadoop/hbase/wal/WALFactory.java | 16 +-
.../org/apache/hadoop/hbase/wal/WALKeyImpl.java | 4 +-
.../apache/hadoop/hbase/wal/WALSplitter.java | 6 +-
.../hadoop/hbase/AcidGuaranteesTestTool.java | 4 +
.../org/apache/hadoop/hbase/HBaseTestCase.java | 3 +-
.../hadoop/hbase/HBaseTestingUtility.java | 9 +-
.../apache/hadoop/hbase/MiniHBaseCluster.java | 7 +-
.../hadoop/hbase/MultithreadedTestUtil.java | 2 +
.../hbase/TestHDFSBlocksDistribution.java | 1 +
.../org/apache/hadoop/hbase/TestIOFencing.java | 11 +-
.../hbase/TestMetaTableAccessorNoCluster.java | 1 +
.../hadoop/hbase/TestMetaTableLocator.java | 6 +-
.../hadoop/hbase/TestMovedRegionsCleaner.java | 1 +
.../apache/hadoop/hbase/TestMultiVersions.java | 2 +
.../org/apache/hadoop/hbase/TestServerName.java | 5 +-
...TestServerSideScanMetricsFromClientSide.java | 8 +-
.../apache/hadoop/hbase/client/TestAdmin1.java | 2 +-
.../apache/hadoop/hbase/client/TestAdmin2.java | 11 +-
.../hbase/client/TestAsyncClusterAdminApi2.java | 2 +
.../client/TestAsyncDecommissionAdminApi.java | 2 +-
.../client/TestAsyncProcedureAdminApi.java | 2 +-
.../hbase/client/TestAsyncRegionAdminApi.java | 25 +-
...estAsyncReplicationAdminApiWithClusters.java | 1 +
.../hbase/client/TestAsyncSnapshotAdminApi.java | 105 +++--
.../hbase/client/TestAsyncTableAdminApi.java | 2 +-
.../hbase/client/TestAsyncTableBatch.java | 6 +-
...estAvoidCellReferencesIntoShippedBlocks.java | 15 +-
.../hadoop/hbase/client/TestClientPushback.java | 8 +-
.../client/TestConnectionImplementation.java | 3 +-
.../hadoop/hbase/client/TestFastFail.java | 3 +-
.../hadoop/hbase/client/TestFromClientSide.java | 356 ++++++++--------
.../hbase/client/TestFromClientSide3.java | 58 +--
.../hadoop/hbase/client/TestMetaCache.java | 3 +
.../hbase/client/TestMultipleTimestamps.java | 4 +-
.../hadoop/hbase/client/TestReplicasClient.java | 14 +-
.../client/TestRestoreSnapshotFromClient.java | 2 +-
.../apache/hadoop/hbase/client/TestResult.java | 4 +-
.../hbase/client/TestServerBusyException.java | 4 +-
.../hadoop/hbase/client/TestSizeFailures.java | 4 +-
.../hbase/client/TestSmallReversedScanner.java | 4 +-
.../hbase/client/TestSnapshotFromClient.java | 2 +-
.../hbase/client/TestSnapshotMetadata.java | 4 +-
.../hbase/client/TestSnapshotWithAcl.java | 2 +-
.../hbase/client/TestSplitOrMergeStatus.java | 8 +-
.../hbase/client/TestTimestampsFilter.java | 20 +-
.../hbase/client/TestUpdateConfiguration.java | 17 +-
.../TestReplicationAdminWithClusters.java | 10 +-
.../hbase/conf/TestConfigurationManager.java | 1 +
.../hadoop/hbase/constraint/TestConstraint.java | 10 +-
.../coprocessor/SampleRegionWALCoprocessor.java | 3 +-
.../hbase/coprocessor/SimpleRegionObserver.java | 4 +-
.../TestCoprocessorConfiguration.java | 14 +-
.../hbase/coprocessor/TestCoprocessorHost.java | 2 +-
...TestMasterCoprocessorExceptionWithAbort.java | 2 +-
.../hbase/coprocessor/TestMasterObserver.java | 1 +
.../hbase/coprocessor/TestWALObserver.java | 1 +
.../hbase/executor/TestExecutorService.java | 2 +-
.../hadoop/hbase/filter/TestBitComparator.java | 8 +-
.../apache/hadoop/hbase/filter/TestFilter.java | 9 +-
.../hbase/filter/TestFilterFromRegionSide.java | 1 +
.../hadoop/hbase/filter/TestFilterList.java | 3 +-
.../hbase/filter/TestFilterSerialization.java | 4 +-
.../filter/TestFuzzyRowFilterEndToEnd.java | 2 +-
.../filter/TestInvocationRecordFilter.java | 4 +
.../hadoop/hbase/filter/TestParseFilter.java | 144 ++++---
.../TestSingleColumnValueExcludeFilter.java | 2 +-
.../hadoop/hbase/fs/TestBlockReorder.java | 5 +-
.../apache/hadoop/hbase/io/TestHFileLink.java | 2 +-
.../apache/hadoop/hbase/io/TestHeapSize.java | 4 +-
.../encoding/TestBufferedDataBlockEncoder.java | 12 +-
.../io/encoding/TestDataBlockEncoders.java | 18 +-
.../encoding/TestLoadAndSwitchEncodeOnDisk.java | 2 +
.../encoding/TestSeekBeforeWithReverseScan.java | 12 +-
.../hadoop/hbase/io/hfile/CacheTestUtils.java | 9 +-
.../apache/hadoop/hbase/io/hfile/NanoTimer.java | 1 +
.../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 2 +-
.../hadoop/hbase/io/hfile/TestHFileBlock.java | 2 +-
.../hbase/io/hfile/TestHFileEncryption.java | 2 +-
.../hbase/io/hfile/TestLruBlockCache.java | 4 +-
.../hbase/io/hfile/bucket/TestBucketCache.java | 20 +-
.../apache/hadoop/hbase/ipc/TestNettyIPC.java | 2 +-
.../hadoop/hbase/ipc/TestProtoBufRpc.java | 2 +-
.../hadoop/hbase/ipc/TestRpcClientLeaks.java | 7 +-
.../hbase/ipc/TestSimpleRpcScheduler.java | 8 +-
.../hbase/mapreduce/MapreduceTestingShim.java | 4 +
.../hbase/master/TestAssignmentListener.java | 11 +-
.../hadoop/hbase/master/TestMasterFailover.java | 4 +-
.../hbase/master/TestMasterNoCluster.java | 1 +
.../TestMasterOperationsForRegionReplicas.java | 5 +-
.../hadoop/hbase/master/TestMasterShutdown.java | 1 +
.../hbase/master/TestMetaShutdownHandler.java | 6 +-
.../hbase/master/TestRegionPlacement.java | 4 +-
.../hbase/master/TestSplitLogManager.java | 12 +-
.../hbase/master/TestTableStateManager.java | 5 +-
.../master/assignment/MockMasterServices.java | 2 +
.../TestSplitTableRegionProcedure.java | 4 +-
.../hbase/master/balancer/BalancerTestBase.java | 1 +
.../hbase/master/cleaner/TestLogsCleaner.java | 1 +
.../cleaner/TestReplicationHFileCleaner.java | 1 +
.../master/cleaner/TestSnapshotFromMaster.java | 2 +-
.../hbase/master/locking/TestLockProcedure.java | 13 +-
...ProcedureSchedulerPerformanceEvaluation.java | 4 +
.../procedure/TestModifyNamespaceProcedure.java | 8 +-
.../procedure/TestModifyTableProcedure.java | 25 +-
.../procedure/TestWALProcedureStoreOnHDFS.java | 2 +-
.../master/snapshot/TestSnapshotFileCache.java | 1 +
.../snapshot/TestSnapshotHFileCleaner.java | 3 +-
.../hadoop/hbase/mob/TestCachedMobFile.java | 36 +-
.../apache/hadoop/hbase/mob/TestMobFile.java | 14 +-
.../hadoop/hbase/mob/TestMobFileCache.java | 15 +-
.../hadoop/hbase/mob/TestMobFileName.java | 11 +-
.../hbase/namespace/TestNamespaceAuditor.java | 4 +-
.../hadoop/hbase/procedure/TestProcedure.java | 6 +-
.../procedure/TestProcedureCoordinator.java | 3 +-
.../hadoop/hbase/procedure/TestZKProcedure.java | 2 +-
.../hbase/quotas/TestQuotaStatusRPCs.java | 2 +-
.../quotas/TestSuperUserQuotaPermissions.java | 1 +
.../EncodedSeekPerformanceTest.java | 2 +-
.../hbase/regionserver/MockHStoreFile.java | 2 +
.../regionserver/StatefulStoreMockMaker.java | 1 +
.../hbase/regionserver/TestAtomicOperation.java | 6 +-
.../hbase/regionserver/TestBlocksScanned.java | 1 +
.../hadoop/hbase/regionserver/TestBulkLoad.java | 6 +-
.../hbase/regionserver/TestCellFlatSet.java | 25 +-
.../hbase/regionserver/TestCellSkipListSet.java | 1 +
.../regionserver/TestCompactingMemStore.java | 5 +-
.../TestCompactingToCellFlatMapMemStore.java | 4 +-
.../hbase/regionserver/TestCompaction.java | 7 +-
.../TestCompactionArchiveConcurrentClose.java | 2 +
.../TestCompactionFileNotFound.java | 2 +-
.../hbase/regionserver/TestDefaultMemStore.java | 4 +-
.../regionserver/TestGetClosestAtOrBefore.java | 3 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 79 ++--
.../regionserver/TestHRegionReplayEvents.java | 8 +-
.../regionserver/TestHRegionServerBulkLoad.java | 2 +
.../hadoop/hbase/regionserver/TestHStore.java | 9 +-
.../hbase/regionserver/TestHStoreFile.java | 33 +-
.../hbase/regionserver/TestKeyValueHeap.java | 3 +-
.../hbase/regionserver/TestMajorCompaction.java | 9 +-
.../hbase/regionserver/TestMemStoreLAB.java | 4 +-
.../hbase/regionserver/TestMinorCompaction.java | 3 +-
.../TestMultiVersionConcurrencyControl.java | 2 +
.../hbase/regionserver/TestPriorityRpc.java | 30 +-
.../hbase/regionserver/TestRegionReplicas.java | 2 +-
.../TestRegionReplicasWithModifyTable.java | 12 +-
.../TestRegionServerAccounting.java | 20 +-
.../regionserver/TestRegionServerNoMaster.java | 12 +-
.../hadoop/hbase/regionserver/TestScanner.java | 4 +-
.../regionserver/TestScannerWithBulkload.java | 1 +
.../hbase/regionserver/TestSplitLogWorker.java | 2 +-
.../TestSplitTransactionOnCluster.java | 8 +-
.../hbase/regionserver/TestStoreScanner.java | 6 +
.../regionserver/TestSyncTimeRangeTracker.java | 7 +-
.../hbase/regionserver/TestWALLockup.java | 2 +
.../TestWalAndCompactingMemStoreFlush.java | 2 +-
.../compactions/PerfTestCompactionPolicies.java | 6 +-
.../regionserver/compactions/TestCompactor.java | 2 +
.../querymatcher/TestUserScanQueryMatcher.java | 4 +-
.../regionserver/wal/AbstractTestFSWAL.java | 2 +-
.../regionserver/wal/AbstractTestWALReplay.java | 12 +-
.../regionserver/wal/InstrumentedLogWriter.java | 2 +-
.../replication/TestNamespaceReplication.java | 4 +-
.../replication/TestPerTableCFReplication.java | 4 +-
.../hbase/replication/TestReplicationBase.java | 2 +-
.../TestReplicationDisableInactivePeer.java | 2 +-
.../replication/TestReplicationKillRS.java | 1 +
.../replication/TestReplicationSmallTests.java | 4 +-
.../replication/TestReplicationSyncUpTool.java | 4 +-
.../replication/TestReplicationWithTags.java | 2 +-
.../TestRegionReplicaReplicationEndpoint.java | 8 +-
.../regionserver/TestReplicator.java | 60 ++-
.../regionserver/TestWALEntryStream.java | 2 +-
.../hadoop/hbase/security/TestSecureIPC.java | 3 +-
.../apache/hadoop/hbase/security/TestUser.java | 3 +
.../access/TestAccessControlFilter.java | 3 +
.../security/access/TestAccessController.java | 9 +-
.../security/token/TestZKSecretWatcher.java | 2 +
.../token/TestZKSecretWatcherRefreshKeys.java | 2 +
.../TestDefaultScanLabelGeneratorStack.java | 22 +-
.../TestEnforcingScanLabelGenerator.java | 4 +
...sibilityLabelReplicationWithExpAsString.java | 1 +
.../visibility/TestVisibilityLabels.java | 41 +-
...sibilityLabelsOnNewVersionBehaviorTable.java | 1 +
...sibilityLabelsOpWithDifferentUsersNoACL.java | 5 +
.../TestVisibilityLabelsReplication.java | 3 +
.../visibility/TestVisibilityLabelsWithACL.java | 9 +
...VisibilityLabelsWithCustomVisLabService.java | 2 +
...ibilityLabelsWithDefaultVisLabelService.java | 4 +
.../TestVisibilityLabelsWithDeletes.java | 426 +++++++++----------
.../TestVisibilityLabelsWithSLGStack.java | 1 +
.../TestVisibilityLablesWithGroups.java | 8 +
.../TestVisibilityWithCheckAuths.java | 2 +-
.../TestWithDisabledAuthorization.java | 8 +-
.../snapshot/TestFlushSnapshotFromClient.java | 2 +
.../hadoop/hbase/tool/MapreduceTestingShim.java | 4 +
.../hbase/tool/TestLoadIncrementalHFiles.java | 2 +-
.../TestLoadIncrementalHFilesSplitRecovery.java | 2 +-
.../hadoop/hbase/util/MultiThreadedAction.java | 4 +-
.../hadoop/hbase/util/MultiThreadedReader.java | 3 +-
.../util/TestBoundedPriorityBlockingQueue.java | 6 +-
.../hadoop/hbase/util/TestByteBuffUtils.java | 4 +-
.../hadoop/hbase/util/TestFSHDFSUtils.java | 3 +-
.../apache/hadoop/hbase/util/TestFSVisitor.java | 7 +-
.../hbase/util/TestHBaseFsckEncryption.java | 10 +-
.../hadoop/hbase/util/TestHBaseFsckMOB.java | 22 +-
.../hadoop/hbase/util/TestJSONMetricUtil.java | 2 +-
.../hbase/util/TestMiniClusterLoadParallel.java | 1 +
.../hbase/util/TestRegionSplitCalculator.java | 4 +-
.../hbase/util/test/LoadTestDataGenerator.java | 7 +-
.../apache/hadoop/hbase/wal/TestWALFactory.java | 2 +-
.../apache/hadoop/hbase/wal/TestWALSplit.java | 112 ++---
.../TestWALSplitBoundedLogWriterCreation.java | 1 +
.../hbase/zookeeper/TestZooKeeperACL.java | 52 +--
357 files changed, 1959 insertions(+), 1495 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/SslRMIServerSocketFactorySecure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/SslRMIServerSocketFactorySecure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/SslRMIServerSocketFactorySecure.java
index 8560ddc..3583afe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/SslRMIServerSocketFactorySecure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/SslRMIServerSocketFactorySecure.java
@@ -33,6 +33,7 @@ public class SslRMIServerSocketFactorySecure extends SslRMIServerSocketFactory {
@Override
public ServerSocket createServerSocket(int port) throws IOException {
return new ServerSocket(port) {
+ @Override
public Socket accept() throws IOException {
Socket socket = super.accept();
SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/client/locking/EntityLock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/locking/EntityLock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/locking/EntityLock.java
index 3fea1a2..b956e33 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/locking/EntityLock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/locking/EntityLock.java
@@ -227,6 +227,7 @@ public class EntityLock {
return this;
}
+ @Override
public void run() {
final LockHeartbeatRequest lockHeartbeatRequest =
LockHeartbeatRequest.newBuilder().setProcId(procId).build();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/conf/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/conf/ConfigurationManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/conf/ConfigurationManager.java
index 2bbb90b..555a5c0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/conf/ConfigurationManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/conf/ConfigurationManager.java
@@ -79,7 +79,7 @@ public class ConfigurationManager {
// notified when the configuration is reloaded from disk. This is a set
// constructed from a WeakHashMap, whose entries would be removed if the
// observer classes go out of scope.
- private Set<ConfigurationObserver> configurationObservers =
+ private final Set<ConfigurationObserver> configurationObservers =
Collections.newSetFromMap(new WeakHashMap<ConfigurationObserver,
Boolean>());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
index bcba101..2143f80 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
@@ -74,7 +74,7 @@ public class ZkSplitLogWorkerCoordination extends ZKListener implements
private TaskExecutor splitTaskExecutor;
- private AtomicInteger taskReadySeq = new AtomicInteger(0);
+ private final AtomicInteger taskReadySeq = new AtomicInteger(0);
private volatile String currentTask = null;
private int currentVersion;
private volatile boolean shouldStop = false;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRowProcessorEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRowProcessorEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRowProcessorEndpoint.java
index f460ac9..ef91bf2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRowProcessorEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRowProcessorEndpoint.java
@@ -118,7 +118,7 @@ extends RowProcessorService implements RegionCoprocessor {
Class<?> cls;
try {
cls = Class.forName(className);
- RowProcessor<S,T> ci = (RowProcessor<S,T>) cls.newInstance();
+ RowProcessor<S,T> ci = (RowProcessor<S,T>) cls.getDeclaredConstructor().newInstance();
if (request.hasRowProcessorInitializerMessageName()) {
Class<?> imn = Class.forName(request.getRowProcessorInitializerMessageName())
.asSubclass(Message.class);
@@ -141,11 +141,7 @@ extends RowProcessorService implements RegionCoprocessor {
ci.initialize(s);
}
return ci;
- } catch (ClassNotFoundException e) {
- throw new IOException(e);
- } catch (InstantiationException e) {
- throw new IOException(e);
- } catch (IllegalAccessException e) {
+ } catch (Exception e) {
throw new IOException(e);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 42da86a..05ac9f6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -572,6 +572,7 @@ public abstract class CoprocessorHost<C extends Coprocessor, E extends Coprocess
return this.result;
}
+ @Override
void callObserver() throws IOException {
Optional<O> observer = observerGetter.apply(getEnvironment().getInstance());
if (observer.isPresent()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContextImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContextImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContextImpl.java
index 7de6f0b..6ed1ad3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContextImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ObserverContextImpl.java
@@ -48,6 +48,7 @@ public class ObserverContextImpl<E extends CoprocessorEnvironment> implements Ob
this.bypassable = bypassable;
}
+ @Override
public E getEnvironment() {
return env;
}
@@ -60,6 +61,7 @@ public class ObserverContextImpl<E extends CoprocessorEnvironment> implements Ob
return this.bypassable;
};
+ @Override
public void bypass() {
if (!this.bypassable) {
throw new UnsupportedOperationException("This method does not support 'bypass'.");
@@ -82,6 +84,7 @@ public class ObserverContextImpl<E extends CoprocessorEnvironment> implements Ob
return false;
}
+ @Override
public Optional<User> getCaller() {
return Optional.ofNullable(caller);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ReadOnlyConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ReadOnlyConfiguration.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ReadOnlyConfiguration.java
index b073ada..b805c50 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ReadOnlyConfiguration.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/ReadOnlyConfiguration.java
@@ -87,7 +87,7 @@ class ReadOnlyConfiguration extends Configuration {
}
@Override
- public void reloadConfiguration() {
+ public synchronized void reloadConfiguration() {
// This is a write operation. We need to allow it though because if any Configuration in
// current JVM context calls addDefaultResource, this forces a reload of all Configurations
// (all Configurations are 'registered' by the default constructor. Rather than turn
@@ -100,10 +100,12 @@ class ReadOnlyConfiguration extends Configuration {
return conf.get(name);
}
+ // Do not add @Override because it is not in Hadoop 2.6.5
public void setAllowNullValueProperties(boolean val) {
throw new UnsupportedOperationException("Read-only Configuration");
}
+ @Override
public String getTrimmed(String name) {
return conf.getTrimmed(name);
}
@@ -129,12 +131,12 @@ class ReadOnlyConfiguration extends Configuration {
}
@Override
- public void unset(String name) {
+ public synchronized void unset(String name) {
throw new UnsupportedOperationException("Read-only Configuration");
}
@Override
- public void setIfUnset(String name, String value) {
+ public synchronized void setIfUnset(String name, String value) {
throw new UnsupportedOperationException("Read-only Configuration");
}
@@ -239,7 +241,7 @@ class ReadOnlyConfiguration extends Configuration {
}
@Override
- public String[] getPropertySources(String name) {
+ public synchronized String[] getPropertySources(String name) {
return conf.getPropertySources(name);
}
@@ -326,7 +328,7 @@ class ReadOnlyConfiguration extends Configuration {
}
@Override
- public Class<?>[] getClasses(String name, Class<?>[] defaultValue) {
+ public Class<?>[] getClasses(String name, Class<?>... defaultValue) {
return conf.getClasses(name, defaultValue);
}
@@ -422,7 +424,7 @@ class ReadOnlyConfiguration extends Configuration {
}
@Override
- public void setQuietMode(boolean quietmode) {
+ public synchronized void setQuietMode(boolean quietmode) {
throw new UnsupportedOperationException("Read-only Configuration");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/filter/FilterWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/filter/FilterWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/filter/FilterWrapper.java
index b7c56e0..9bc072a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/filter/FilterWrapper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/filter/FilterWrapper.java
@@ -119,6 +119,7 @@ final public class FilterWrapper extends Filter {
return filterCell(c);
}
+ @Override
public ReturnCode filterCell(final Cell c) throws IOException {
return this.filter.filterCell(c);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
index b89470f..9ea67c1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/fs/HFileSystem.java
@@ -235,16 +235,15 @@ public class HFileSystem extends FilterFileSystem {
}
}
- /**
+ /**
* Returns a brand new instance of the FileSystem. It does not use
* the FileSystem.Cache. In newer versions of HDFS, we can directly
* invoke FileSystem.newInstance(Configuration).
- *
+ *
* @param conf Configuration
* @return A new instance of the filesystem
*/
- private static FileSystem newInstanceFileSystem(Configuration conf)
- throws IOException {
+ private static FileSystem newInstanceFileSystem(Configuration conf) throws IOException {
URI uri = FileSystem.getDefaultUri(conf);
FileSystem fs = null;
Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
@@ -361,47 +360,43 @@ public class HFileSystem extends FilterFileSystem {
private static ClientProtocol createReorderingProxy(final ClientProtocol cp,
final ReorderBlocks lrb, final Configuration conf) {
- return (ClientProtocol) Proxy.newProxyInstance
- (cp.getClass().getClassLoader(),
- new Class[]{ClientProtocol.class, Closeable.class},
- new InvocationHandler() {
- public Object invoke(Object proxy, Method method,
- Object[] args) throws Throwable {
- try {
- if ((args == null || args.length == 0)
- && "close".equals(method.getName())) {
- RPC.stopProxy(cp);
- return null;
- } else {
- Object res = method.invoke(cp, args);
- if (res != null && args != null && args.length == 3
- && "getBlockLocations".equals(method.getName())
- && res instanceof LocatedBlocks
- && args[0] instanceof String
- && args[0] != null) {
- lrb.reorderBlocks(conf, (LocatedBlocks) res, (String) args[0]);
- }
- return res;
- }
- } catch (InvocationTargetException ite) {
- // We will have this for all the exception, checked on not, sent
- // by any layer, including the functional exception
- Throwable cause = ite.getCause();
- if (cause == null){
- throw new RuntimeException(
- "Proxy invocation failed and getCause is null", ite);
- }
- if (cause instanceof UndeclaredThrowableException) {
- Throwable causeCause = cause.getCause();
- if (causeCause == null) {
- throw new RuntimeException("UndeclaredThrowableException had null cause!");
- }
- cause = cause.getCause();
- }
- throw cause;
+ return (ClientProtocol) Proxy.newProxyInstance(cp.getClass().getClassLoader(),
+ new Class[]{ClientProtocol.class, Closeable.class}, new InvocationHandler() {
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ try {
+ if ((args == null || args.length == 0) && "close".equals(method.getName())) {
+ RPC.stopProxy(cp);
+ return null;
+ } else {
+ Object res = method.invoke(cp, args);
+ if (res != null && args != null && args.length == 3
+ && "getBlockLocations".equals(method.getName())
+ && res instanceof LocatedBlocks
+ && args[0] instanceof String
+ && args[0] != null) {
+ lrb.reorderBlocks(conf, (LocatedBlocks) res, (String) args[0]);
}
+ return res;
}
- });
+ } catch (InvocationTargetException ite) {
+ // We will have this for all the exception, checked on not, sent
+ // by any layer, including the functional exception
+ Throwable cause = ite.getCause();
+ if (cause == null){
+ throw new RuntimeException("Proxy invocation failed and getCause is null", ite);
+ }
+ if (cause instanceof UndeclaredThrowableException) {
+ Throwable causeCause = cause.getCause();
+ if (causeCause == null) {
+ throw new RuntimeException("UndeclaredThrowableException had null cause!");
+ }
+ cause = cause.getCause();
+ }
+ throw cause;
+ }
+ }
+ });
}
/**
@@ -424,6 +419,7 @@ public class HFileSystem extends FilterFileSystem {
* datanode is actually dead, so if we use it it will timeout.
*/
static class ReorderWALBlocks implements ReorderBlocks {
+ @Override
public void reorderBlocks(Configuration conf, LocatedBlocks lbs, String src)
throws IOException {
@@ -481,6 +477,7 @@ public class HFileSystem extends FilterFileSystem {
* createNonRecursive. This is a hadoop bug and when it is fixed in Hadoop,
* this definition will go away.
*/
+ @Override
@SuppressWarnings("deprecation")
public FSDataOutputStream createNonRecursive(Path f,
boolean overwrite,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java
index 5fd9d36..6c73405 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java
@@ -22,15 +22,16 @@ import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
@@ -73,7 +74,7 @@ public class FSDataInputStreamWrapper implements Closeable {
*/
private volatile FSDataInputStream stream = null;
private volatile FSDataInputStream streamNoFsChecksum = null;
- private Object streamNoFsChecksumFirstCreateLock = new Object();
+ private final Object streamNoFsChecksumFirstCreateLock = new Object();
// The configuration states that we should validate hbase checksums
private boolean useHBaseChecksumConfigured;
@@ -86,7 +87,7 @@ public class FSDataInputStreamWrapper implements Closeable {
// In the case of a checksum failure, do these many succeeding
// reads without hbase checksum verification.
- private volatile int hbaseChecksumOffCount = -1;
+ private AtomicInteger hbaseChecksumOffCount = new AtomicInteger(-1);
private Boolean instanceOfCanUnbuffer = null;
// Using reflection to get org.apache.hadoop.fs.CanUnbuffer#unbuffer method to avoid compilation
@@ -216,7 +217,7 @@ public class FSDataInputStreamWrapper implements Closeable {
}
if (!partOfConvoy) {
this.useHBaseChecksum = false;
- this.hbaseChecksumOffCount = offCount;
+ this.hbaseChecksumOffCount.set(offCount);
}
return this.stream;
}
@@ -224,7 +225,7 @@ public class FSDataInputStreamWrapper implements Closeable {
/** Report that checksum was ok, so we may ponder going back to HBase checksum. */
public void checksumOk() {
if (this.useHBaseChecksumConfigured && !this.useHBaseChecksum
- && (this.hbaseChecksumOffCount-- < 0)) {
+ && (this.hbaseChecksumOffCount.getAndDecrement() < 0)) {
// The stream we need is already open (because we were using HBase checksum in the past).
assert this.streamNoFsChecksum != null;
this.useHBaseChecksum = true;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index 80207eb..f30d488 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -129,35 +129,41 @@ public class HalfStoreFileReader extends StoreFileReader {
final HFileScanner delegate = s;
public boolean atEnd = false;
+ @Override
public Cell getKey() {
if (atEnd) return null;
return delegate.getKey();
}
+ @Override
public String getKeyString() {
if (atEnd) return null;
return delegate.getKeyString();
}
+ @Override
public ByteBuffer getValue() {
if (atEnd) return null;
return delegate.getValue();
}
+ @Override
public String getValueString() {
if (atEnd) return null;
return delegate.getValueString();
}
+ @Override
public Cell getCell() {
if (atEnd) return null;
return delegate.getCell();
}
+ @Override
public boolean next() throws IOException {
if (atEnd) return false;
@@ -200,10 +206,12 @@ public class HalfStoreFileReader extends StoreFileReader {
return (this.delegate.getReader().getComparator().compare(splitCell, getKey())) > 0;
}
+ @Override
public org.apache.hadoop.hbase.io.hfile.HFile.Reader getReader() {
return this.delegate.getReader();
}
+ @Override
public boolean isSeeked() {
return this.delegate.isSeeked();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
index 6dbfd2f..6dce132 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
@@ -222,6 +222,7 @@ public class Reference {
return Arrays.hashCode(splitkey) + region.hashCode();
}
+ @Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null) return false;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
index 5f4bb76..1645d68 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
@@ -507,6 +507,7 @@ public class FanOutOneBlockAsyncDFSOutput implements AsyncFSOutput {
* @param syncBlock will call hsync if true, otherwise hflush.
* @return A CompletableFuture that hold the acked length after flushing.
*/
+ @Override
public CompletableFuture<Long> flush(boolean syncBlock) {
CompletableFuture<Long> future = new CompletableFuture<>();
flush0(future, syncBlock);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
index 50d8508..dccfe39 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
@@ -126,6 +126,7 @@ public interface BlockCache extends Iterable<CachedBlock> {
/**
* @return Iterator over the blocks in the cache.
*/
+ @Override
Iterator<CachedBlock> iterator();
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java
index 768b37f..2aceed7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java
@@ -156,6 +156,7 @@ public class CompoundBloomFilter extends CompoundBloomFilterBase
return result;
}
+ @Override
public boolean supportsAutoLoading() {
return true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
index 70a3d4d..a0d3df3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
@@ -545,7 +545,7 @@ public class FixedFileTrailer {
try {
// If null, it should be the Bytes.BYTES_RAWCOMPARATOR
if (klass != null) {
- CellComparator comp = klass.newInstance();
+ CellComparator comp = klass.getDeclaredConstructor().newInstance();
// if the name wasn't one of the legacy names, maybe its a legit new
// kind of comparator.
comparatorClassName = klass.getName();
@@ -589,12 +589,8 @@ public class FixedFileTrailer {
public static CellComparator createComparator(
String comparatorClassName) throws IOException {
try {
- Class<? extends CellComparator> comparatorClass = getComparatorClass(comparatorClassName);
- return comparatorClass != null ? comparatorClass.newInstance() : null;
- } catch (InstantiationException e) {
- throw new IOException("Comparator class " + comparatorClassName +
- " is not instantiable", e);
- } catch (IllegalAccessException e) {
+ return getComparatorClass(comparatorClassName).getDeclaredConstructor().newInstance();
+ } catch (Exception e) {
throw new IOException("Comparator class " + comparatorClassName +
" is not instantiable", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index 992ebbd..5674414 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -255,42 +255,43 @@ public class HFileBlock implements Cacheable {
*/
static final CacheableDeserializer<Cacheable> BLOCK_DESERIALIZER =
new CacheableDeserializer<Cacheable>() {
- public HFileBlock deserialize(ByteBuff buf, boolean reuse, MemoryType memType)
+ @Override
+ public HFileBlock deserialize(ByteBuff buf, boolean reuse, MemoryType memType)
throws IOException {
- // The buf has the file block followed by block metadata.
- // Set limit to just before the BLOCK_METADATA_SPACE then rewind.
- buf.limit(buf.limit() - BLOCK_METADATA_SPACE).rewind();
- // Get a new buffer to pass the HFileBlock for it to 'own'.
- ByteBuff newByteBuff;
- if (reuse) {
- newByteBuff = buf.slice();
- } else {
- int len = buf.limit();
- newByteBuff = new SingleByteBuff(ByteBuffer.allocate(len));
- newByteBuff.put(0, buf, buf.position(), len);
- }
- // Read out the BLOCK_METADATA_SPACE content and shove into our HFileBlock.
- buf.position(buf.limit());
- buf.limit(buf.limit() + HFileBlock.BLOCK_METADATA_SPACE);
- boolean usesChecksum = buf.get() == (byte)1;
- long offset = buf.getLong();
- int nextBlockOnDiskSize = buf.getInt();
- HFileBlock hFileBlock =
- new HFileBlock(newByteBuff, usesChecksum, memType, offset, nextBlockOnDiskSize, null);
- return hFileBlock;
- }
+ // The buf has the file block followed by block metadata.
+ // Set limit to just before the BLOCK_METADATA_SPACE then rewind.
+ buf.limit(buf.limit() - BLOCK_METADATA_SPACE).rewind();
+ // Get a new buffer to pass the HFileBlock for it to 'own'.
+ ByteBuff newByteBuff;
+ if (reuse) {
+ newByteBuff = buf.slice();
+ } else {
+ int len = buf.limit();
+ newByteBuff = new SingleByteBuff(ByteBuffer.allocate(len));
+ newByteBuff.put(0, buf, buf.position(), len);
+ }
+ // Read out the BLOCK_METADATA_SPACE content and shove into our HFileBlock.
+ buf.position(buf.limit());
+ buf.limit(buf.limit() + HFileBlock.BLOCK_METADATA_SPACE);
+ boolean usesChecksum = buf.get() == (byte) 1;
+ long offset = buf.getLong();
+ int nextBlockOnDiskSize = buf.getInt();
+ HFileBlock hFileBlock =
+ new HFileBlock(newByteBuff, usesChecksum, memType, offset, nextBlockOnDiskSize, null);
+ return hFileBlock;
+ }
- @Override
- public int getDeserialiserIdentifier() {
- return DESERIALIZER_IDENTIFIER;
- }
+ @Override
+ public int getDeserialiserIdentifier() {
+ return DESERIALIZER_IDENTIFIER;
+ }
- @Override
- public HFileBlock deserialize(ByteBuff b) throws IOException {
- // Used only in tests
- return deserialize(b, false, MemoryType.EXCLUSIVE);
- }
- };
+ @Override
+ public HFileBlock deserialize(ByteBuff b) throws IOException {
+ // Used only in tests
+ return deserialize(b, false, MemoryType.EXCLUSIVE);
+ }
+ };
private static final int DESERIALIZER_IDENTIFIER;
static {
@@ -1480,6 +1481,7 @@ public class HFileBlock implements Cacheable {
this(new FSDataInputStreamWrapper(istream), fileSize, null, null, fileContext);
}
+ @Override
public BlockIterator blockRange(final long startOffset, final long endOffset) {
final FSReader owner = this; // handle for inner class
return new BlockIterator() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
index a16565e..1f591a0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
@@ -947,7 +947,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
Cell ret;
int cellBufSize = getKVBufSize();
- long seqId = 0l;
+ long seqId = 0L;
if (this.reader.shouldIncludeMemStoreTS()) {
seqId = currMemstoreTS;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
index 032c1ad..a2a35fe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java
@@ -166,5 +166,6 @@ public interface HFileScanner extends Shipper, Closeable {
/**
* Close this HFile scanner and do necessary cleanup.
*/
+ @Override
void close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 284447a..d26b90a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -448,6 +448,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
* @param cacheKey block's cache key
* @param buf block buffer
*/
+ @Override
public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
cacheBlock(cacheKey, buf, false);
}
@@ -794,6 +795,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
return totalSize;
}
+ @Override
public int compareTo(BlockBucket that) {
return Long.compare(this.overflow(), that.overflow());
}
@@ -970,6 +972,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
* <p>Includes: total accesses, hits, misses, evicted blocks, and runs
* of the eviction processes.
*/
+ @Override
public CacheStats getStats() {
return this.stats;
}
@@ -1096,6 +1099,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
return (long) Math.floor(this.maxSize * this.memoryFactor * this.minFactor);
}
+ @Override
public void shutdown() {
if (victimHandler != null) {
victimHandler.shutdown();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlock.java
index 21b3bfd..32a277d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlock.java
@@ -90,6 +90,7 @@ public class LruCachedBlock implements HeapSize, Comparable<LruCachedBlock> {
return this.cachedTime;
}
+ @Override
public long heapSize() {
return size;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlockQueue.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlockQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlockQueue.java
index bed08fe..4c67c9a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlockQueue.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruCachedBlockQueue.java
@@ -103,6 +103,7 @@ public class LruCachedBlockQueue implements HeapSize {
* Total size of all elements in this queue.
* @return size of all elements currently in queue, in bytes
*/
+ @Override
public long heapSize() {
return heapSize;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java
index 7bbb4ed..e31b1cb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java
@@ -414,6 +414,7 @@ public final class BucketAllocator {
}
}
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder(1024);
for (int i = 0; i < buckets.length; ++i) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index d07c30d..bd2b9c8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -36,6 +36,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
+import java.util.Objects;
import java.util.PriorityQueue;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
@@ -904,6 +905,7 @@ public class BucketCache implements BlockCache, HeapSize {
this.writerEnabled = false;
}
+ @Override
public void run() {
List<RAMQueueEntry> entries = new ArrayList<>();
try {
@@ -1395,10 +1397,22 @@ public class BucketCache implements BlockCache, HeapSize {
}
@Override
- public boolean equals(Object that) {
- return this == that;
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ BucketEntryGroup that = (BucketEntryGroup) o;
+ return totalSize == that.totalSize && bucketSize == that.bucketSize
+ && Objects.equals(queue, that.queue);
}
+ @Override
+ public int hashCode() {
+ return Objects.hash(queue, totalSize, bucketSize);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/CachedEntryQueue.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/CachedEntryQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/CachedEntryQueue.java
index fa39202..29721ab 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/CachedEntryQueue.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/CachedEntryQueue.java
@@ -59,6 +59,7 @@ public class CachedEntryQueue {
}
queue = MinMaxPriorityQueue.orderedBy(new Comparator<Map.Entry<BlockCacheKey, BucketEntry>>() {
+ @Override
public int compare(Entry<BlockCacheKey, BucketEntry> entry1,
Entry<BlockCacheKey, BucketEntry> entry2) {
return BucketEntry.COMPARATOR.compare(entry1.getValue(), entry2.getValue());
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
index c340c06..915b82d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
@@ -101,7 +101,7 @@ class BufferChain {
try {
long ret = channel.write(buffers, bufferOffset, bufCount);
if (ret > 0) {
- remaining -= ret;
+ remaining = (int) (remaining - ret);
}
return ret;
} finally {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FastPathBalancedQueueRpcExecutor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FastPathBalancedQueueRpcExecutor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FastPathBalancedQueueRpcExecutor.java
index 9a01a0a..eaea34d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FastPathBalancedQueueRpcExecutor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/FastPathBalancedQueueRpcExecutor.java
@@ -91,6 +91,7 @@ public class FastPathBalancedQueueRpcExecutor extends BalancedQueueRpcExecutor {
this.fastPathHandlerStack = fastPathHandlerStack;
}
+ @Override
protected CallRunner getCallRunner() throws InterruptedException {
// Get a callrunner if one in the Q.
CallRunner cr = this.q.poll();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcConnection.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcConnection.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcConnection.java
index 096efa3..17bb362 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcConnection.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcConnection.java
@@ -193,14 +193,15 @@ abstract class ServerRpcConnection implements Closeable {
String className = header.getCellBlockCodecClass();
if (className == null || className.length() == 0) return;
try {
- this.codec = (Codec)Class.forName(className).newInstance();
+ this.codec = (Codec)Class.forName(className).getDeclaredConstructor().newInstance();
} catch (Exception e) {
throw new UnsupportedCellCodecException(className, e);
}
if (!header.hasCellBlockCompressorClass()) return;
className = header.getCellBlockCompressorClass();
try {
- this.compressionCodec = (CompressionCodec)Class.forName(className).newInstance();
+ this.compressionCodec =
+ (CompressionCodec)Class.forName(className).getDeclaredConstructor().newInstance();
} catch (Exception e) {
throw new UnsupportedCompressionCodecException(className, e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleRpcServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleRpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleRpcServer.java
index b14c934..13a3cf7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleRpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/SimpleRpcServer.java
@@ -548,6 +548,7 @@ public class SimpleRpcServer extends RpcServer {
* The number of open RPC conections
* @return the number of open rpc connections
*/
+ @Override
public int getNumOpenConnections() {
return connectionManager.size();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
index 6b87194..5e97204 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
@@ -118,10 +118,8 @@ public class ClusterStatusPublisher extends ScheduledChore {
this.master = master;
this.messagePeriod = conf.getInt(STATUS_PUBLISH_PERIOD, DEFAULT_STATUS_PUBLISH_PERIOD);
try {
- this.publisher = publisherClass.newInstance();
- } catch (InstantiationException e) {
- throw new IOException("Can't create publisher " + publisherClass.getName(), e);
- } catch (IllegalAccessException e) {
+ this.publisher = publisherClass.getDeclaredConstructor().newInstance();
+ } catch (Exception e) {
throw new IOException("Can't create publisher " + publisherClass.getName(), e);
}
this.publisher.connect(conf);
@@ -166,7 +164,8 @@ public class ClusterStatusPublisher extends ScheduledChore {
.build());
}
- protected void cleanup() {
+ @Override
+ protected synchronized void cleanup() {
connected = false;
publisher.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
index db04c60..116d24e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java
@@ -155,6 +155,7 @@ public class DeadServer {
}
}
+ @Override
public synchronized String toString() {
StringBuilder sb = new StringBuilder();
for (ServerName sn : deadServers.keySet()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
index 3ec70d3..a591025 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -69,11 +69,12 @@ public class HMasterCommandLine extends ServerCommandLine {
this.masterClass = masterClass;
}
+ @Override
protected String getUsage() {
return USAGE;
}
-
+ @Override
public int run(String args[]) throws Exception {
Options opt = new Options();
opt.addOption("localRegionServers", true,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
index f1a0593..917da08 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
@@ -156,6 +156,7 @@ public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObse
* Notification that config has changed
* @param conf
*/
+ @Override
void onConfigurationChange(Configuration conf);
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterAnnotationReadingPriorityFunction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterAnnotationReadingPriorityFunction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterAnnotationReadingPriorityFunction.java
index 18aab6e..f25f3bf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterAnnotationReadingPriorityFunction.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterAnnotationReadingPriorityFunction.java
@@ -52,6 +52,7 @@ public class MasterAnnotationReadingPriorityFunction extends AnnotationReadingPr
super(rpcServices, clz);
}
+ @Override
public int getPriority(RPCProtos.RequestHeader header, Message param, User user) {
// Yes this is copy pasted from the base class but it keeps from having to look in the
// annotatedQos table twice something that could get costly since this is called for
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 10e1d0a..8396145 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -133,6 +133,7 @@ public class MasterCoprocessorHost
* @return An instance of MasterServices, an object NOT for general user-space Coprocessor
* consumption.
*/
+ @Override
public MasterServices getMasterServices() {
return this.masterServices;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java
index 8a7c4e1..6c5d677 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java
@@ -89,7 +89,7 @@ public class MobCompactionChore extends ScheduledChore {
}
@Override
- protected void cleanup() {
+ protected synchronized void cleanup() {
super.cleanup();
pool.shutdown();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
index 2f2d536..29218e2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionServerTracker.java
@@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class RegionServerTracker extends ZKListener {
private static final Logger LOG = LoggerFactory.getLogger(RegionServerTracker.class);
- private NavigableMap<ServerName, RegionServerInfo> regionServers = new TreeMap<>();
+ private final NavigableMap<ServerName, RegionServerInfo> regionServers = new TreeMap<>();
private ServerManager serverManager;
private MasterServices server;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index d1c1612..2b88fb1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -637,7 +637,7 @@ public class SplitLogManager {
public enum TerminationStatus {
IN_PROGRESS("in_progress"), SUCCESS("success"), FAILURE("failure"), DELETED("deleted");
- String statusMsg;
+ final String statusMsg;
TerminationStatus(String msg) {
statusMsg = msg;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
index 3810403..8c59776 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java
@@ -612,7 +612,7 @@ public class MergeTableRegionsProcedure
final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName());
for (String family: regionFs.getFamilies()) {
- final ColumnFamilyDescriptor hcd = htd.getColumnFamily(family.getBytes());
+ final ColumnFamilyDescriptor hcd = htd.getColumnFamily(Bytes.toBytes(family));
final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family);
if (storeFiles != null && storeFiles.size() > 0) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
index 1cf9a54..fa94495 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
@@ -33,6 +33,7 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
@@ -864,7 +865,7 @@ public class RegionStates {
private final RegionStateNode regionNode;
private volatile Exception exception = null;
- private volatile int retries = 0;
+ private AtomicInteger retries = new AtomicInteger();
public RegionFailedOpen(final RegionStateNode regionNode) {
this.regionNode = regionNode;
@@ -879,11 +880,11 @@ public class RegionStates {
}
public int incrementAndGetRetries() {
- return ++this.retries;
+ return this.retries.incrementAndGet();
}
public int getRetries() {
- return retries;
+ return retries.get();
}
public void setException(final Exception exception) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java
index b3fbc57..a72478c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java
@@ -86,7 +86,7 @@ public class FavoredStochasticBalancer extends StochasticLoadBalancer implements
}
@Override
- public void setMasterServices(MasterServices masterServices) {
+ public synchronized void setMasterServices(MasterServices masterServices) {
super.setMasterServices(masterServices);
fnm = masterServices.getFavoredNodesManager();
}
@@ -692,7 +692,8 @@ public class FavoredStochasticBalancer extends StochasticLoadBalancer implements
* implementation. For the misplaced regions, we assign a bogus server to it and AM takes care.
*/
@Override
- public List<RegionPlan> balanceCluster(Map<ServerName, List<RegionInfo>> clusterState) {
+ public synchronized List<RegionPlan> balanceCluster(Map<ServerName,
+ List<RegionInfo>> clusterState) {
if (this.services != null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
index a9b1bb7..07e9600 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
@@ -72,21 +72,22 @@ class RegionLocationFinder {
private CacheLoader<RegionInfo, HDFSBlocksDistribution> loader =
new CacheLoader<RegionInfo, HDFSBlocksDistribution>() {
- public ListenableFuture<HDFSBlocksDistribution> reload(final RegionInfo hri,
- HDFSBlocksDistribution oldValue) throws Exception {
- return executor.submit(new Callable<HDFSBlocksDistribution>() {
- @Override
- public HDFSBlocksDistribution call() throws Exception {
- return internalGetTopBlockLocation(hri);
- }
- });
- }
-
+ @Override
+ public ListenableFuture<HDFSBlocksDistribution> reload(final RegionInfo hri,
+ HDFSBlocksDistribution oldValue) throws Exception {
+ return executor.submit(new Callable<HDFSBlocksDistribution>() {
@Override
- public HDFSBlocksDistribution load(RegionInfo key) throws Exception {
- return internalGetTopBlockLocation(key);
+ public HDFSBlocksDistribution call() throws Exception {
+ return internalGetTopBlockLocation(hri);
}
- };
+ });
+ }
+
+ @Override
+ public HDFSBlocksDistribution load(RegionInfo key) throws Exception {
+ return internalGetTopBlockLocation(key);
+ }
+ };
// The cache for where regions are located.
private LoadingCache<RegionInfo, HDFSBlocksDistribution> cache = null;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
index f53683f..b0d3f19 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java
@@ -106,6 +106,7 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
}
+ @Override
public void setClusterLoad(Map<TableName, Map<ServerName, List<RegionInfo>>> clusterLoad){
serverLoadList = new ArrayList<>();
float sum = 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index 6b4f943..dca9cbb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -349,8 +349,8 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
// Allow turning this feature off if the locality cost is not going to
// be used in any computations.
RegionLocationFinder finder = null;
- if (this.localityCost != null && this.localityCost.getMultiplier() > 0
- || this.rackLocalityCost != null && this.rackLocalityCost.getMultiplier() > 0) {
+ if ((this.localityCost != null && this.localityCost.getMultiplier() > 0)
+ || (this.rackLocalityCost != null && this.rackLocalityCost.getMultiplier() > 0)) {
finder = this.regionFinder;
}
@@ -1401,7 +1401,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
// Now if we found a region load get the type of cost that was requested.
if (regionLoadList != null) {
- cost += getRegionLoadCost(regionLoadList);
+ cost = (long) (cost + getRegionLoadCost(regionLoadList));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index 775d8f9..21f8251 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -205,7 +205,7 @@ public abstract class CleanerChore<T extends FileCleanerDelegate> extends Schedu
Class<? extends FileCleanerDelegate> c = Class.forName(className).asSubclass(
FileCleanerDelegate.class);
@SuppressWarnings("unchecked")
- T cleaner = (T) c.newInstance();
+ T cleaner = (T) c.getDeclaredConstructor().newInstance();
cleaner.setConf(conf);
cleaner.init(this.params);
return cleaner;
@@ -360,7 +360,7 @@ public abstract class CleanerChore<T extends FileCleanerDelegate> extends Schedu
}
@Override
- public void cleanup() {
+ public synchronized void cleanup() {
for (T lc : this.cleanersChain) {
try {
lc.stop("Exiting");
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index 640c8f7..08640a7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -182,7 +182,7 @@ public class HFileCleaner extends CleanerChore<BaseHFileCleanerDelegate> {
}
@Override
- public void cleanup() {
+ public synchronized void cleanup() {
super.cleanup();
stopHFileDeleteThreads();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
index db364ee..9beed58 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java
@@ -108,7 +108,7 @@ public class LogCleaner extends CleanerChore<BaseLogCleanerDelegate> {
}
@Override
- public void cleanup() {
+ public synchronized void cleanup() {
super.cleanup();
interruptOldWALsCleaner();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/locking/LockProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/locking/LockProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/locking/LockProcedure.java
index edf7642..b4c55f4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/locking/LockProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/locking/LockProcedure.java
@@ -202,7 +202,7 @@ public final class LockProcedure extends Procedure<MasterProcedureEnv>
* @return false, so procedure framework doesn't mark this procedure as failure.
*/
@Override
- protected boolean setTimeoutFailure(final MasterProcedureEnv env) {
+ protected synchronized boolean setTimeoutFailure(final MasterProcedureEnv env) {
synchronized (event) {
if (LOG.isDebugEnabled()) LOG.debug("Timeout failure " + this.event);
if (!event.isReady()) { // Maybe unlock() awakened the event.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b1269ec5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java
index 09d05e6..535f288 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java
@@ -71,7 +71,9 @@ public abstract class ProcedurePrepareLatch {
}
private static class NoopLatch extends ProcedurePrepareLatch {
+ @Override
protected void countDown(final Procedure proc) {}
+ @Override
public void await() throws IOException {}
}
@@ -80,6 +82,7 @@ public abstract class ProcedurePrepareLatch {
private IOException exception = null;
+ @Override
protected void countDown(final Procedure proc) {
if (proc.hasException()) {
exception = proc.getException().unwrapRemoteIOException();
@@ -87,6 +90,7 @@ public abstract class ProcedurePrepareLatch {
latch.countDown();
}
+ @Override
public void await() throws IOException {
try {
latch.await();
[33/50] [abbrv] hbase git commit: HBASE-19635 Introduce a thread at
RS side to call reportProcedureDone
Posted by zh...@apache.org.
HBASE-19635 Introduce a thread at RS side to call reportProcedureDone
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/52458290
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/52458290
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/52458290
Branch: refs/heads/HBASE-19397-branch-2
Commit: 52458290ed4f483c5737d86a8dc8a3c9cfbaf59e
Parents: b3b92ef
Author: zhangduo <zh...@apache.org>
Authored: Wed Dec 27 20:13:42 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:04 2018 +0800
----------------------------------------------------------------------
.../src/main/protobuf/RegionServerStatus.proto | 5 +-
.../hadoop/hbase/master/MasterRpcServices.java | 15 ++-
.../hbase/regionserver/HRegionServer.java | 72 ++++--------
.../RemoteProcedureResultReporter.java | 111 +++++++++++++++++++
.../handler/RSProcedureHandler.java | 2 +-
5 files changed, 149 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/52458290/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 4f75941..3f836cd 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -146,7 +146,7 @@ message RegionSpaceUseReportRequest {
message RegionSpaceUseReportResponse {
}
-message ReportProcedureDoneRequest {
+message RemoteProcedureResult {
required uint64 proc_id = 1;
enum Status {
SUCCESS = 1;
@@ -155,6 +155,9 @@ message ReportProcedureDoneRequest {
required Status status = 2;
optional ForeignExceptionMessage error = 3;
}
+message ReportProcedureDoneRequest {
+ repeated RemoteProcedureResult result = 1;
+}
message ReportProcedureDoneResponse {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52458290/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 72bf2d1..377a9c6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -265,6 +265,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RemoteProcedureResult;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
@@ -2254,12 +2255,14 @@ public class MasterRpcServices extends RSRpcServices
@Override
public ReportProcedureDoneResponse reportProcedureDone(RpcController controller,
ReportProcedureDoneRequest request) throws ServiceException {
- if (request.getStatus() == ReportProcedureDoneRequest.Status.SUCCESS) {
- master.remoteProcedureCompleted(request.getProcId());
- } else {
- master.remoteProcedureFailed(request.getProcId(),
- RemoteProcedureException.fromProto(request.getError()));
- }
+ request.getResultList().forEach(result -> {
+ if (result.getStatus() == RemoteProcedureResult.Status.SUCCESS) {
+ master.remoteProcedureCompleted(result.getProcId());
+ } else {
+ master.remoteProcedureFailed(result.getProcId(),
+ RemoteProcedureException.fromProto(result.getError()));
+ }
+ });
return ReportProcedureDoneResponse.getDefaultInstance();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52458290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 3c8ec17..3844415 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -146,7 +146,6 @@ import org.apache.hadoop.hbase.util.CompressionTest;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.JvmPauseMonitor;
import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig;
@@ -382,6 +381,9 @@ public class HRegionServer extends HasThread implements
// eclipse warning when accessed by inner classes
protected LogRoller walRoller;
+ // A thread which calls reportProcedureDone
+ private RemoteProcedureResultReporter procedureResultReporter;
+
// flag set after we're done setting up server threads
final AtomicBoolean online = new AtomicBoolean(false);
@@ -1887,6 +1889,7 @@ public class HRegionServer extends HasThread implements
this.walRoller = new LogRoller(this, this);
this.flushThroughputController = FlushThroughputControllerFactory.create(this, conf);
+ this.procedureResultReporter = new RemoteProcedureResultReporter(this);
// Create the CompactedFileDischarger chore executorService. This chore helps to
// remove the compacted files
@@ -1930,6 +1933,8 @@ public class HRegionServer extends HasThread implements
Threads.setDaemonThreadRunning(this.walRoller.getThread(), getName() + ".logRoller",
uncaughtExceptionHandler);
this.cacheFlusher.start(uncaughtExceptionHandler);
+ Threads.setDaemonThreadRunning(this.procedureResultReporter,
+ getName() + ".procedureResultReporter", uncaughtExceptionHandler);
if (this.compactionChecker != null) choreService.scheduleChore(compactionChecker);
if (this.periodicFlusher != null) choreService.scheduleChore(periodicFlusher);
@@ -3712,55 +3717,26 @@ public class HRegionServer extends HasThread implements
executorService.submit(new RSProcedureHandler(this, procId, callable));
}
- public void reportProcedureDone(long procId, Throwable error) {
- ReportProcedureDoneRequest.Builder builder =
- ReportProcedureDoneRequest.newBuilder().setProcId(procId);
- if (error != null) {
- builder.setStatus(ReportProcedureDoneRequest.Status.ERROR)
- .setError(ForeignExceptionUtil.toProtoForeignException(serverName.toString(), error));
- } else {
- builder.setStatus(ReportProcedureDoneRequest.Status.SUCCESS);
+ public void remoteProcedureComplete(long procId, Throwable error) {
+ procedureResultReporter.complete(procId, error);
+ }
+
+ void reportProcedureDone(ReportProcedureDoneRequest request) throws IOException {
+ RegionServerStatusService.BlockingInterface rss = rssStub;
+ for (;;) {
+ rss = rssStub;
+ if (rss != null) {
+ break;
+ }
+ createRegionServerStatusStub();
}
- ReportProcedureDoneRequest request = builder.build();
- int tries = 0;
- long pauseTime = INIT_PAUSE_TIME_MS;
- while (keepLooping()) {
- RegionServerStatusService.BlockingInterface rss = rssStub;
- try {
- if (rss == null) {
- createRegionServerStatusStub();
- continue;
- }
- rss.reportProcedureDone(null, request);
- // Log if we had to retry else don't log unless TRACE. We want to
- // know if were successful after an attempt showed in logs as failed.
- if (tries > 0 || LOG.isTraceEnabled()) {
- LOG.info("PROCEDURE REPORTED " + request);
- }
- return;
- } catch (ServiceException se) {
- IOException ioe = ProtobufUtil.getRemoteException(se);
- boolean pause =
- ioe instanceof ServerNotRunningYetException || ioe instanceof PleaseHoldException;
- if (pause) {
- // Do backoff else we flood the Master with requests.
- pauseTime = ConnectionUtils.getPauseTime(INIT_PAUSE_TIME_MS, tries);
- } else {
- pauseTime = INIT_PAUSE_TIME_MS; // Reset.
- }
- LOG.info(
- "Failed to report transition " + TextFormat.shortDebugString(request) + "; retry (#" +
- tries + ")" + (pause ? " after " + pauseTime + "ms delay (Master is coming online...)."
- : " immediately."),
- ioe);
- if (pause) {
- Threads.sleep(pauseTime);
- }
- tries++;
- if (rssStub == rss) {
- rssStub = null;
- }
+ try {
+ rss.reportProcedureDone(null, request);
+ } catch (ServiceException se) {
+ if (rssStub == rss) {
+ rssStub = null;
}
+ throw ProtobufUtil.getRemoteException(se);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52458290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
new file mode 100644
index 0000000..e4be422
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RemoteProcedureResultReporter.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.hadoop.hbase.PleaseHoldException;
+import org.apache.hadoop.hbase.client.ConnectionUtils;
+import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
+import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.TextFormat;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RemoteProcedureResult;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportProcedureDoneRequest;
+
+/**
+ * A thread which calls {@code reportProcedureDone} to tell master the result of a remote procedure.
+ */
+@InterfaceAudience.Private
+class RemoteProcedureResultReporter extends Thread {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RemoteProcedureResultReporter.class);
+
+ // Time to pause if master says 'please hold'. Make configurable if needed.
+ private static final int INIT_PAUSE_TIME_MS = 1000;
+
+ private static final int MAX_BATCH = 100;
+
+ private final HRegionServer server;
+
+ private final LinkedBlockingQueue<RemoteProcedureResult> results = new LinkedBlockingQueue<>();
+
+ public RemoteProcedureResultReporter(HRegionServer server) {
+ this.server = server;
+ }
+
+ public void complete(long procId, Throwable error) {
+ RemoteProcedureResult.Builder builder = RemoteProcedureResult.newBuilder().setProcId(procId);
+ if (error != null) {
+ builder.setStatus(RemoteProcedureResult.Status.ERROR).setError(
+ ForeignExceptionUtil.toProtoForeignException(server.getServerName().toString(), error));
+ } else {
+ builder.setStatus(RemoteProcedureResult.Status.SUCCESS);
+ }
+ results.add(builder.build());
+ }
+
+ @Override
+ public void run() {
+ ReportProcedureDoneRequest.Builder builder = ReportProcedureDoneRequest.newBuilder();
+ int tries = 0;
+ while (!server.isStopped()) {
+ if (builder.getResultCount() == 0) {
+ try {
+ builder.addResult(results.take());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ continue;
+ }
+ }
+ while (builder.getResultCount() < MAX_BATCH) {
+ RemoteProcedureResult result = results.poll();
+ if (result == null) {
+ break;
+ }
+ builder.addResult(result);
+ }
+ ReportProcedureDoneRequest request = builder.build();
+ try {
+ server.reportProcedureDone(builder.build());
+ builder.clear();
+ tries = 0;
+ } catch (IOException e) {
+ boolean pause =
+ e instanceof ServerNotRunningYetException || e instanceof PleaseHoldException;
+ long pauseTime;
+ if (pause) {
+ // Do backoff else we flood the Master with requests.
+ pauseTime = ConnectionUtils.getPauseTime(INIT_PAUSE_TIME_MS, tries);
+ } else {
+ pauseTime = INIT_PAUSE_TIME_MS; // Reset.
+ }
+ LOG.info("Failed report procedure " + TextFormat.shortDebugString(request) + "; retry (#" +
+ tries + ")" + (pause ? " after " + pauseTime + "ms delay (Master is coming online...)."
+ : " immediately."),
+ e);
+ Threads.sleep(pauseTime);
+ tries++;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52458290/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
index 240b0a7..d2175d0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/RSProcedureHandler.java
@@ -49,6 +49,6 @@ public class RSProcedureHandler extends EventHandler {
LOG.error("Catch exception when call RSProcedureCallable: ", e);
error = e;
}
- ((HRegionServer) server).reportProcedureDone(procId, error);
+ ((HRegionServer) server).remoteProcedureComplete(procId, error);
}
}
[26/50] [abbrv] hbase git commit: HBASE-19579 Add peer lock test for
shell command list_locks
Posted by zh...@apache.org.
HBASE-19579 Add peer lock test for shell command list_locks
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0541dbe3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0541dbe3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0541dbe3
Branch: refs/heads/HBASE-19397-branch-2
Commit: 0541dbe302bdc5f0ec29a14ce8412a7889557446
Parents: 0a754e9
Author: Guanghao Zhang <zg...@apache.org>
Authored: Sat Dec 23 21:04:27 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../src/main/protobuf/LockService.proto | 1 +
.../src/test/ruby/shell/list_locks_test.rb | 19 +++++++++++++++++++
2 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0541dbe3/hbase-protocol-shaded/src/main/protobuf/LockService.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/LockService.proto b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
index b8d180c..0675070 100644
--- a/hbase-protocol-shaded/src/main/protobuf/LockService.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
@@ -77,6 +77,7 @@ enum LockedResourceType {
NAMESPACE = 2;
TABLE = 3;
REGION = 4;
+ PEER = 5;
}
message LockedResource {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0541dbe3/hbase-shell/src/test/ruby/shell/list_locks_test.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/shell/list_locks_test.rb b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
index f465a6b..ef1c0ce 100644
--- a/hbase-shell/src/test/ruby/shell/list_locks_test.rb
+++ b/hbase-shell/src/test/ruby/shell/list_locks_test.rb
@@ -67,6 +67,25 @@ module Hbase
proc_id)
end
+ define_test 'list peer locks' do
+ lock = create_exclusive_lock(0)
+ peer_id = '1'
+
+ @scheduler.waitPeerExclusiveLock(lock, peer_id)
+ output = capture_stdout { @list_locks.command }
+ @scheduler.wakePeerExclusiveLock(lock, peer_id)
+
+ assert_equal(
+ "PEER(1)\n" \
+ "Lock type: EXCLUSIVE, procedure: {" \
+ "\"className\"=>\"org.apache.hadoop.hbase.master.locking.LockProcedure\", " \
+ "\"procId\"=>\"0\", \"submittedTime\"=>\"0\", \"state\"=>\"RUNNABLE\", " \
+ "\"lastUpdate\"=>\"0\", " \
+ "\"stateMessage\"=>[{\"lockType\"=>\"EXCLUSIVE\", \"description\"=>\"description\"}]" \
+ "}\n\n",
+ output)
+ end
+
define_test 'list server locks' do
lock = create_exclusive_lock(0)
[30/50] [abbrv] hbase git commit: HBASE-19592 Add UTs to test retry
on update zk failure
Posted by zh...@apache.org.
HBASE-19592 Add UTs to test retry on update zk failure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2417c32c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2417c32c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2417c32c
Branch: refs/heads/HBASE-19397-branch-2
Commit: 2417c32cebd7633c88b44a314dc65b2221759092
Parents: 78317db
Author: zhangduo <zh...@apache.org>
Authored: Tue Dec 26 20:39:00 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../replication/ReplicationPeerManager.java | 5 +-
.../TestReplicationProcedureRetry.java | 200 +++++++++++++++++++
2 files changed, 202 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2417c32c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index b78cbce..f4ccce8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -53,7 +53,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* Used to add/remove a replication peer.
*/
@InterfaceAudience.Private
-public final class ReplicationPeerManager {
+public class ReplicationPeerManager {
private final ReplicationPeerStorage peerStorage;
@@ -61,8 +61,7 @@ public final class ReplicationPeerManager {
private final ConcurrentMap<String, ReplicationPeerDescription> peers;
- private ReplicationPeerManager(ReplicationPeerStorage peerStorage,
- ReplicationQueueStorage queueStorage,
+ ReplicationPeerManager(ReplicationPeerStorage peerStorage, ReplicationQueueStorage queueStorage,
ConcurrentMap<String, ReplicationPeerDescription> peers) {
this.peerStorage = peerStorage;
this.queueStorage = queueStorage;
http://git-wip-us.apache.org/repos/asf/hbase/blob/2417c32c/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
new file mode 100644
index 0000000..ab35b46
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationProcedureRetry.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.invocation.InvocationOnMock;
+
+/**
+ * All the modification method will fail once in the test and should finally succeed.
+ */
+@Category({ ReplicationTests.class, MediumTests.class })
+public class TestReplicationProcedureRetry {
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, MockHMaster.class, HMaster.class);
+ UTIL.startMiniCluster(3);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @After
+ public void tearDownAfterTest() throws IOException {
+ for (ReplicationPeerDescription desc : UTIL.getAdmin().listReplicationPeers()) {
+ UTIL.getAdmin().removeReplicationPeer(desc.getPeerId());
+ }
+ }
+
+ private void doTest() throws IOException {
+ Admin admin = UTIL.getAdmin();
+ String peerId = "1";
+ ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
+ .setClusterKey("localhost:" + UTIL.getZkCluster().getClientPort() + ":/hbase2").build();
+ admin.addReplicationPeer(peerId, peerConfig, true);
+
+ assertEquals(peerConfig.getClusterKey(),
+ admin.getReplicationPeerConfig(peerId).getClusterKey());
+ ReplicationPeerConfig newPeerConfig =
+ ReplicationPeerConfig.newBuilder(peerConfig).setBandwidth(123456).build();
+ admin.updateReplicationPeerConfig(peerId, newPeerConfig);
+ assertEquals(newPeerConfig.getBandwidth(),
+ admin.getReplicationPeerConfig(peerId).getBandwidth());
+
+ admin.disableReplicationPeer(peerId);
+ assertFalse(admin.listReplicationPeers().get(0).isEnabled());
+
+ admin.enableReplicationPeer(peerId);
+ assertTrue(admin.listReplicationPeers().get(0).isEnabled());
+
+ admin.removeReplicationPeer(peerId);
+ assertTrue(admin.listReplicationPeers().isEmpty());
+
+ // make sure that we have run into the mocked method
+ MockHMaster master = (MockHMaster) UTIL.getHBaseCluster().getMaster();
+ assertTrue(master.addPeerCalled);
+ assertTrue(master.removePeerCalled);
+ assertTrue(master.updatePeerConfigCalled);
+ assertTrue(master.enablePeerCalled);
+ assertTrue(master.disablePeerCalled);
+ }
+
+ @Test
+ public void testErrorBeforeUpdate() throws IOException, ReplicationException {
+ ((MockHMaster) UTIL.getHBaseCluster().getMaster()).reset(true);
+ doTest();
+ }
+
+ @Test
+ public void testErrorAfterUpdate() throws IOException, ReplicationException {
+ ((MockHMaster) UTIL.getHBaseCluster().getMaster()).reset(false);
+ doTest();
+ }
+
+ public static final class MockHMaster extends HMaster {
+
+ volatile boolean addPeerCalled;
+
+ volatile boolean removePeerCalled;
+
+ volatile boolean updatePeerConfigCalled;
+
+ volatile boolean enablePeerCalled;
+
+ volatile boolean disablePeerCalled;
+
+ private ReplicationPeerManager manager;
+
+ public MockHMaster(Configuration conf) throws IOException, KeeperException {
+ super(conf);
+ }
+
+ private Object invokeWithError(InvocationOnMock invocation, boolean errorBeforeUpdate)
+ throws Throwable {
+ if (errorBeforeUpdate) {
+ throw new ReplicationException("mock error before update");
+ }
+ invocation.callRealMethod();
+ throw new ReplicationException("mock error after update");
+ }
+
+ public void reset(boolean errorBeforeUpdate) throws ReplicationException {
+ addPeerCalled = false;
+ removePeerCalled = false;
+ updatePeerConfigCalled = false;
+ enablePeerCalled = false;
+ disablePeerCalled = false;
+ ReplicationPeerManager m = super.getReplicationPeerManager();
+ manager = spy(m);
+ doAnswer(invocation -> {
+ if (!addPeerCalled) {
+ addPeerCalled = true;
+ return invokeWithError(invocation, errorBeforeUpdate);
+ } else {
+ return invocation.callRealMethod();
+ }
+ }).when(manager).addPeer(anyString(), any(ReplicationPeerConfig.class), anyBoolean());
+ doAnswer(invocation -> {
+ if (!removePeerCalled) {
+ removePeerCalled = true;
+ return invokeWithError(invocation, errorBeforeUpdate);
+ } else {
+ return invocation.callRealMethod();
+ }
+ }).when(manager).removePeer(anyString());
+ doAnswer(invocation -> {
+ if (!updatePeerConfigCalled) {
+ updatePeerConfigCalled = true;
+ return invokeWithError(invocation, errorBeforeUpdate);
+ } else {
+ return invocation.callRealMethod();
+ }
+ }).when(manager).updatePeerConfig(anyString(), any(ReplicationPeerConfig.class));
+ doAnswer(invocation -> {
+ if (!enablePeerCalled) {
+ enablePeerCalled = true;
+ return invokeWithError(invocation, errorBeforeUpdate);
+ } else {
+ return invocation.callRealMethod();
+ }
+ }).when(manager).enablePeer(anyString());
+ doAnswer(invocation -> {
+ if (!disablePeerCalled) {
+ disablePeerCalled = true;
+ return invokeWithError(invocation, errorBeforeUpdate);
+ } else {
+ return invocation.callRealMethod();
+ }
+ }).when(manager).disablePeer(anyString());
+ }
+
+ @Override
+ public ReplicationPeerManager getReplicationPeerManager() {
+ return manager;
+ }
+ }
+}
[39/50] [abbrv] hbase git commit: HBASE-19623 Create replication
endpoint asynchronously when adding a replication source
Posted by zh...@apache.org.
HBASE-19623 Create replication endpoint asynchronously when adding a replication source
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e560998e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e560998e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e560998e
Branch: refs/heads/HBASE-19397-branch-2
Commit: e560998e8b9b3bc5bf0d2c702580583f83fac103
Parents: 16d080c
Author: zhangduo <zh...@apache.org>
Authored: Tue Jan 2 13:25:58 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:19:45 2018 +0800
----------------------------------------------------------------------
.../hbase/replication/ReplicationPeer.java | 8 ++
.../hbase/replication/ReplicationPeers.java | 18 +--
.../replication/ZKReplicationPeerStorage.java | 7 +-
.../replication/TestReplicationStateBasic.java | 20 +---
.../TestZKReplicationPeerStorage.java | 14 +--
.../HBaseInterClusterReplicationEndpoint.java | 17 ++-
.../RecoveredReplicationSource.java | 13 +--
.../regionserver/ReplicationSource.java | 110 +++++++++++--------
.../ReplicationSourceInterface.java | 8 +-
.../regionserver/ReplicationSourceManager.java | 47 +-------
.../client/TestAsyncReplicationAdminApi.java | 2 -
.../replication/TestReplicationAdmin.java | 2 -
.../replication/ReplicationSourceDummy.java | 7 +-
.../replication/TestReplicationSource.java | 27 +++--
.../TestReplicationSourceManager.java | 8 +-
15 files changed, 127 insertions(+), 181 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
index 4846018..2da3cce 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -54,6 +54,14 @@ public interface ReplicationPeer {
PeerState getPeerState();
/**
+ * Test whether the peer is enabled.
+ * @return {@code true} if enabled, otherwise {@code false}.
+ */
+ default boolean isPeerEnabled() {
+ return getPeerState() == PeerState.ENABLED;
+ }
+
+ /**
* Get the peer config object
* @return the ReplicationPeerConfig for this peer
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 422801b..45940a5 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.replication;
import java.io.IOException;
+import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -86,21 +87,6 @@ public class ReplicationPeers {
}
/**
- * Get the peer state for the specified connected remote slave cluster. The value might be read
- * from cache, so it is recommended to use {@link #peerStorage } to read storage directly if
- * reading the state after enabling or disabling it.
- * @param peerId a short that identifies the cluster
- * @return true if replication is enabled, false otherwise.
- */
- public boolean isPeerEnabled(String peerId) {
- ReplicationPeer replicationPeer = this.peerCache.get(peerId);
- if (replicationPeer == null) {
- throw new IllegalArgumentException("Peer with id= " + peerId + " is not cached");
- }
- return replicationPeer.getPeerState() == PeerState.ENABLED;
- }
-
- /**
* Returns the ReplicationPeerImpl for the specified cached peer. This ReplicationPeer will
* continue to track changes to the Peer's state and config. This method returns null if no peer
* has been cached with the given peerId.
@@ -117,7 +103,7 @@ public class ReplicationPeers {
* @return a Set of Strings for peerIds
*/
public Set<String> getAllPeerIds() {
- return peerCache.keySet();
+ return Collections.unmodifiableSet(peerCache.keySet());
}
public static Configuration getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
index bf448e8..42d4b3f 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationPeerStorage.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.replication;
import java.util.Arrays;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -30,8 +29,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
@@ -41,8 +38,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
@InterfaceAudience.Private
class ZKReplicationPeerStorage extends ZKReplicationStorageBase implements ReplicationPeerStorage {
- private static final Logger LOG = LoggerFactory.getLogger(ZKReplicationPeerStorage.class);
-
public static final byte[] ENABLED_ZNODE_BYTES =
toByteArray(ReplicationProtos.ReplicationState.State.ENABLED);
public static final byte[] DISABLED_ZNODE_BYTES =
@@ -126,7 +121,7 @@ class ZKReplicationPeerStorage extends ZKReplicationStorageBase implements Repli
@Override
public List<String> listPeerIds() throws ReplicationException {
try {
- return CollectionUtils.nullToEmpty(ZKUtil.listChildrenAndWatchThem(zookeeper, peersZNode));
+ return CollectionUtils.nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, peersZNode));
} catch (KeeperException e) {
throw new ReplicationException("Cannot get the list of peers", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
index 07c6c15..f3eeccc 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
@@ -225,11 +225,6 @@ public abstract class TestReplicationStateBasic {
fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
} catch (ReplicationException e) {
}
- try {
- rp.isPeerEnabled("bogus");
- fail("Should have thrown an IllegalArgumentException when passed a bogus peerId");
- } catch (IllegalArgumentException e) {
- }
try {
assertFalse(rp.addPeer("bogus"));
@@ -245,12 +240,6 @@ public abstract class TestReplicationStateBasic {
rp.getPeerStorage().addPeer(ID_TWO, new ReplicationPeerConfig().setClusterKey(KEY_TWO), true);
assertNumberOfPeers(2);
- // Test methods with a peer that is added but not connected
- try {
- rp.isPeerEnabled(ID_ONE);
- fail("There are no connected peers, should have thrown an IllegalArgumentException");
- } catch (IllegalArgumentException e) {
- }
assertEquals(KEY_ONE, ZKConfig.getZooKeeperClusterKey(ReplicationPeers
.getPeerClusterConfiguration(rp.getPeerStorage().getPeerConfig(ID_ONE), rp.getConf())));
rp.getPeerStorage().removePeer(ID_ONE);
@@ -261,7 +250,7 @@ public abstract class TestReplicationStateBasic {
rp.getPeerStorage().addPeer(ID_ONE, new ReplicationPeerConfig().setClusterKey(KEY_ONE), true);
rp.addPeer(ID_ONE);
assertNumberOfPeers(2);
- assertTrue(rp.isPeerEnabled(ID_ONE));
+ assertTrue(rp.getPeer(ID_ONE).isPeerEnabled());
rp.getPeerStorage().setPeerState(ID_ONE, false);
// now we do not rely on zk watcher to trigger the state change so we need to trigger it
// manually...
@@ -279,11 +268,6 @@ public abstract class TestReplicationStateBasic {
// Disconnect peer
rp.removePeer(ID_ONE);
assertNumberOfPeers(2);
- try {
- rp.isPeerEnabled(ID_ONE);
- fail("There are no connected peers, should have thrown an IllegalArgumentException");
- } catch (IllegalArgumentException e) {
- }
}
protected void assertConnectedPeerStatus(boolean status, String peerId) throws Exception {
@@ -292,7 +276,7 @@ public abstract class TestReplicationStateBasic {
fail("ConnectedPeerStatus was " + !status + " but expected " + status + " in ZK");
}
while (true) {
- if (status == rp.isPeerEnabled(peerId)) {
+ if (status == rp.getPeer(peerId).isPeerEnabled()) {
return;
}
if (zkTimeoutCount < ZK_MAX_COUNT) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
index e8098c8..3eb11da 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationPeerStorage.java
@@ -80,15 +80,11 @@ public class TestZKReplicationPeerStorage {
private ReplicationPeerConfig getConfig(int seed) {
Random rand = new Random(seed);
- ReplicationPeerConfig config = new ReplicationPeerConfig();
- config.setClusterKey(Long.toHexString(rand.nextLong()));
- config.setReplicationEndpointImpl(Long.toHexString(rand.nextLong()));
- config.setNamespaces(randNamespaces(rand));
- config.setExcludeNamespaces(randNamespaces(rand));
- config.setTableCFsMap(randTableCFs(rand));
- config.setReplicateAllUserTables(rand.nextBoolean());
- config.setBandwidth(rand.nextInt(1000));
- return config;
+ return ReplicationPeerConfig.newBuilder().setClusterKey(Long.toHexString(rand.nextLong()))
+ .setReplicationEndpointImpl(Long.toHexString(rand.nextLong()))
+ .setNamespaces(randNamespaces(rand)).setExcludeNamespaces(randNamespaces(rand))
+ .setTableCFsMap(randTableCFs(rand)).setReplicateAllUserTables(rand.nextBoolean())
+ .setBandwidth(rand.nextInt(1000)).build();
}
private void assertSetEquals(Set<String> expected, Set<String> actual) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
index 5872d08..49ea1f7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.hbase.replication.regionserver;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-
import java.io.IOException;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
@@ -39,7 +37,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -48,22 +45,24 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.protobuf.ReplicationProtbufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint;
-import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.replication.regionserver.ReplicationSinkManager.SinkPeer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.wal.WAL.Entry;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService.BlockingInterface;
/**
* A {@link org.apache.hadoop.hbase.replication.ReplicationEndpoint}
@@ -416,7 +415,7 @@ public class HBaseInterClusterReplicationEndpoint extends HBaseReplicationEndpoi
}
protected boolean isPeerEnabled() {
- return ctx.getReplicationPeer().getPeerState() == PeerState.ENABLED;
+ return ctx.getReplicationPeer().isPeerEnabled();
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index e0c45d5..7bceb78 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -28,8 +28,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
+import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Threads;
@@ -51,11 +50,11 @@ public class RecoveredReplicationSource extends ReplicationSource {
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage queueStorage, ReplicationPeers replicationPeers, Server server,
- String peerClusterZnode, UUID clusterId, ReplicationEndpoint replicationEndpoint,
- WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException {
- super.init(conf, fs, manager, queueStorage, replicationPeers, server, peerClusterZnode,
- clusterId, replicationEndpoint, walFileLengthProvider, metrics);
+ ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
+ String peerClusterZnode, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
+ MetricsSource metrics) throws IOException {
+ super.init(conf, fs, manager, queueStorage, replicationPeer, server, peerClusterZnode,
+ clusterId, walFileLengthProvider, metrics);
this.actualPeerId = this.replicationQueueInfo.getPeerId();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 8250992..ffed88d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -38,14 +38,16 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
+import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
import org.apache.hadoop.hbase.replication.ChainWALEntryFilter;
import org.apache.hadoop.hbase.replication.ClusterMarkingEntryFilter;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.SystemTableWALEntryFilter;
@@ -82,7 +84,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
// per group queue size, keep no more than this number of logs in each wal group
protected int queueSizePerGroup;
protected ReplicationQueueStorage queueStorage;
- private ReplicationPeers replicationPeers;
+ private ReplicationPeer replicationPeer;
protected Configuration conf;
protected ReplicationQueueInfo replicationQueueInfo;
@@ -110,8 +112,10 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
private volatile boolean sourceRunning = false;
// Metrics for this source
private MetricsSource metrics;
- //WARN threshold for the number of queued logs, defaults to 2
+ // WARN threshold for the number of queued logs, defaults to 2
private int logQueueWarnThreshold;
+ // whether the replication endpoint has been initialized
+ private volatile boolean endpointInitialized = false;
// ReplicationEndpoint which will handle the actual replication
private ReplicationEndpoint replicationEndpoint;
// A filter (or a chain of filters) for the WAL entries.
@@ -133,22 +137,19 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
/**
* Instantiation method used by region servers
- *
* @param conf configuration to use
* @param fs file system to use
* @param manager replication manager to ping to
* @param server the server for this region server
* @param peerClusterZnode the name of our znode
* @param clusterId unique UUID for the cluster
- * @param replicationEndpoint the replication endpoint implementation
* @param metrics metrics for replication source
- * @throws IOException
*/
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage queueStorage, ReplicationPeers replicationPeers, Server server,
- String peerClusterZnode, UUID clusterId, ReplicationEndpoint replicationEndpoint,
- WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException {
+ ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
+ String peerClusterZnode, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
+ MetricsSource metrics) throws IOException {
this.server = server;
this.conf = HBaseConfiguration.create(conf);
this.waitOnEndpointSeconds =
@@ -160,7 +161,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
this.conf.getInt("replication.source.maxretriesmultiplier", 300); // 5 minutes @ 1 sec per
this.queueSizePerGroup = this.conf.getInt("hbase.regionserver.maxlogs", 32);
this.queueStorage = queueStorage;
- this.replicationPeers = replicationPeers;
+ this.replicationPeer = replicationPeer;
this.manager = manager;
this.fs = fs;
this.metrics = metrics;
@@ -171,7 +172,6 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
// ReplicationQueueInfo parses the peerId out of the znode for us
this.peerId = this.replicationQueueInfo.getPeerId();
this.logQueueWarnThreshold = this.conf.getInt("replication.source.log.queue.warn", 2);
- this.replicationEndpoint = replicationEndpoint;
defaultBandwidth = this.conf.getLong("replication.source.per.peer.node.bandwidth", 0);
currentBandwidth = getCurrentBandwidth();
@@ -196,7 +196,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
if (queue == null) {
queue = new PriorityBlockingQueue<>(queueSizePerGroup, new LogsComparator());
queues.put(logPrefix, queue);
- if (this.sourceRunning) {
+ if (this.isSourceActive() && this.endpointInitialized) {
// new wal group observed after source startup, start a new worker thread to track it
// notice: it's possible that log enqueued when this.running is set but worker thread
// still not launched, so it's necessary to check workerThreads before start the worker
@@ -222,7 +222,7 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
// A peerId will not have "-" in its name, see HBASE-11394
peerId = peerClusterZnode.split("-")[0];
}
- Map<TableName, List<String>> tableCFMap = replicationPeers.getPeer(peerId).getTableCFs();
+ Map<TableName, List<String>> tableCFMap = replicationPeer.getTableCFs();
if (tableCFMap != null) {
List<String> tableCfs = tableCFMap.get(tableName);
if (tableCFMap.containsKey(tableName)
@@ -241,21 +241,59 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
}
}
+ private void initAndStartReplicationEndpoint() throws Exception {
+ RegionServerCoprocessorHost rsServerHost = null;
+ TableDescriptors tableDescriptors = null;
+ if (server instanceof HRegionServer) {
+ rsServerHost = ((HRegionServer) server).getRegionServerCoprocessorHost();
+ tableDescriptors = ((HRegionServer) server).getTableDescriptors();
+ }
+ String replicationEndpointImpl = replicationPeer.getPeerConfig().getReplicationEndpointImpl();
+ if (replicationEndpointImpl == null) {
+ // Default to HBase inter-cluster replication endpoint
+ replicationEndpointImpl = HBaseInterClusterReplicationEndpoint.class.getName();
+ }
+ replicationEndpoint =
+ Class.forName(replicationEndpointImpl).asSubclass(ReplicationEndpoint.class).newInstance();
+ if (rsServerHost != null) {
+ ReplicationEndpoint newReplicationEndPoint =
+ rsServerHost.postCreateReplicationEndPoint(replicationEndpoint);
+ if (newReplicationEndPoint != null) {
+ // Override the newly created endpoint from the hook with configured end point
+ replicationEndpoint = newReplicationEndPoint;
+ }
+ }
+ replicationEndpoint
+ .init(new ReplicationEndpoint.Context(conf, replicationPeer.getConfiguration(), fs, peerId,
+ clusterId, replicationPeer, metrics, tableDescriptors, server));
+ replicationEndpoint.start();
+ replicationEndpoint.awaitRunning(waitOnEndpointSeconds, TimeUnit.SECONDS);
+ }
+
@Override
public void run() {
// mark we are running now
this.sourceRunning = true;
- try {
- // start the endpoint, connect to the cluster
- this.replicationEndpoint.start();
- this.replicationEndpoint.awaitRunning(this.waitOnEndpointSeconds, TimeUnit.SECONDS);
- } catch (Exception ex) {
- LOG.warn("Error starting ReplicationEndpoint, exiting", ex);
- uninitialize();
- throw new RuntimeException(ex);
- }
int sleepMultiplier = 1;
+ while (this.isSourceActive()) {
+ try {
+ initAndStartReplicationEndpoint();
+ break;
+ } catch (Exception e) {
+ LOG.warn("Error starting ReplicationEndpoint, retrying", e);
+ if (replicationEndpoint != null) {
+ replicationEndpoint.stop();
+ replicationEndpoint = null;
+ }
+ if (sleepForRetries("Error starting ReplicationEndpoint", sleepMultiplier)) {
+ sleepMultiplier++;
+ }
+ }
+ }
+ this.endpointInitialized = true;
+
+ sleepMultiplier = 1;
// delay this until we are in an asynchronous thread
while (this.isSourceActive() && this.peerClusterId == null) {
this.peerClusterId = replicationEndpoint.getPeerUUID();
@@ -288,8 +326,8 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
private void initializeWALEntryFilter() {
// get the WALEntryFilter from ReplicationEndpoint and add it to default filters
- ArrayList<WALEntryFilter> filters = Lists.newArrayList(
- (WALEntryFilter)new SystemTableWALEntryFilter());
+ ArrayList<WALEntryFilter> filters =
+ Lists.<WALEntryFilter> newArrayList(new SystemTableWALEntryFilter());
WALEntryFilter filterFromEndpoint = this.replicationEndpoint.getWALEntryfilter();
if (filterFromEndpoint != null) {
filters.add(filterFromEndpoint);
@@ -309,7 +347,6 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
worker.startup(getUncaughtExceptionHandler());
worker.setWALReader(startNewWALReader(worker.getName(), walGroupId, queue,
worker.getStartPosition()));
- workerThreads.put(walGroupId, worker);
}
}
@@ -370,25 +407,11 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
}
private long getCurrentBandwidth() {
- ReplicationPeer replicationPeer = this.replicationPeers.getPeer(peerId);
- long peerBandwidth = replicationPeer != null ? replicationPeer.getPeerBandwidth() : 0;
+ long peerBandwidth = replicationPeer.getPeerBandwidth();
// user can set peer bandwidth to 0 to use default bandwidth
return peerBandwidth != 0 ? peerBandwidth : defaultBandwidth;
}
- private void uninitialize() {
- LOG.debug("Source exiting " + this.peerId);
- metrics.clear();
- if (this.replicationEndpoint.isRunning() || this.replicationEndpoint.isStarting()) {
- this.replicationEndpoint.stop();
- try {
- this.replicationEndpoint.awaitTerminated(this.waitOnEndpointSeconds, TimeUnit.SECONDS);
- } catch (TimeoutException e) {
- LOG.warn("Failed termination after " + this.waitOnEndpointSeconds + " seconds.");
- }
- }
- }
-
/**
* Do the sleeping logic
* @param msg Why we sleep
@@ -410,12 +433,11 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
/**
* check whether the peer is enabled or not
- *
* @return true if the peer is enabled, otherwise false
*/
@Override
public boolean isPeerEnabled() {
- return this.replicationPeers.isPeerEnabled(this.peerId);
+ return replicationPeer.isPeerEnabled();
}
@Override
@@ -427,8 +449,8 @@ public class ReplicationSource extends Thread implements ReplicationSourceInterf
LOG.error("Unexpected exception in ReplicationSource", e);
}
};
- Threads
- .setDaemonThreadRunning(this, n + ".replicationSource," + this.peerClusterZnode, handler);
+ Threads.setDaemonThreadRunning(this, n + ".replicationSource," + this.peerClusterZnode,
+ handler);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
index 4b9ed74..4f10c73 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
+import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WAL.Entry;
@@ -50,9 +50,9 @@ public interface ReplicationSourceInterface {
* @param server the server for this region server
*/
void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage queueStorage, ReplicationPeers replicationPeers, Server server,
- String peerClusterZnode, UUID clusterId, ReplicationEndpoint replicationEndpoint,
- WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException;
+ ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
+ String peerClusterZnode, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
+ MetricsSource metrics) throws IOException;
/**
* Add a log to the list of logs to replicate
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index c289c74..0dbcc3a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -49,13 +49,9 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
-import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationListener;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
@@ -497,49 +493,14 @@ public class ReplicationSourceManager implements ReplicationListener {
* @param peerId the id of the peer cluster
* @return the created source
*/
- private ReplicationSourceInterface getReplicationSource(String peerId, ReplicationPeer peer)
- throws IOException {
- RegionServerCoprocessorHost rsServerHost = null;
- TableDescriptors tableDescriptors = null;
- if (server instanceof HRegionServer) {
- rsServerHost = ((HRegionServer) server).getRegionServerCoprocessorHost();
- tableDescriptors = ((HRegionServer) server).getTableDescriptors();
- }
-
+ private ReplicationSourceInterface getReplicationSource(String peerId,
+ ReplicationPeer replicationPeer) throws IOException {
ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, peerId);
- ReplicationEndpoint replicationEndpoint = null;
- try {
- String replicationEndpointImpl = peer.getPeerConfig().getReplicationEndpointImpl();
- if (replicationEndpointImpl == null) {
- // Default to HBase inter-cluster replication endpoint
- replicationEndpointImpl = HBaseInterClusterReplicationEndpoint.class.getName();
- }
- replicationEndpoint = Class.forName(replicationEndpointImpl)
- .asSubclass(ReplicationEndpoint.class).newInstance();
- if (rsServerHost != null) {
- ReplicationEndpoint newReplicationEndPoint =
- rsServerHost.postCreateReplicationEndPoint(replicationEndpoint);
- if (newReplicationEndPoint != null) {
- // Override the newly created endpoint from the hook with configured end point
- replicationEndpoint = newReplicationEndPoint;
- }
- }
- } catch (Exception e) {
- LOG.warn("Passed replication endpoint implementation throws errors" +
- " while initializing ReplicationSource for peer: " + peerId, e);
- throw new IOException(e);
- }
-
MetricsSource metrics = new MetricsSource(peerId);
// init replication source
- src.init(conf, fs, this, queueStorage, replicationPeers, server, peerId, clusterId,
- replicationEndpoint, walFileLengthProvider, metrics);
-
- // init replication endpoint
- replicationEndpoint.init(new ReplicationEndpoint.Context(conf, peer.getConfiguration(), fs,
- peerId, clusterId, peer, metrics, tableDescriptors, server));
-
+ src.init(conf, fs, this, queueStorage, replicationPeer, server, peerId, clusterId,
+ walFileLengthProvider, metrics);
return src;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
index b28eaaf..5225add 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -70,7 +69,6 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 120000);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
TEST_UTIL.getConfiguration().setInt(START_LOG_ERRORS_AFTER_COUNT_KEY, 0);
- TEST_UTIL.getConfiguration().setInt(ReadOnlyZKClient.RECOVERY_RETRY, 1);
TEST_UTIL.startMiniCluster();
ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index dbbc78f..f9629bd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.hbase.replication.TestReplicationEndpoint.InterClusterR
import org.apache.hadoop.hbase.replication.TestReplicationEndpoint.ReplicationEndpointForTest;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -84,7 +83,6 @@ public class TestReplicationAdmin {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
- TEST_UTIL.getConfiguration().setInt(ReadOnlyZKClient.RECOVERY_RETRY, 1);
TEST_UTIL.startMiniCluster();
admin = new ReplicationAdmin(TEST_UTIL.getConfiguration());
hbaseAdmin = TEST_UTIL.getAdmin();
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
index 14c5e56..38ec598 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -48,9 +49,9 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage rq, ReplicationPeers rp, Server server, String peerClusterId,
- UUID clusterId, ReplicationEndpoint replicationEndpoint,
- WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException {
+ ReplicationQueueStorage rq, ReplicationPeer rp, Server server, String peerClusterId,
+ UUID clusterId, WALFileLengthProvider walFileLengthProvider, MetricsSource metrics)
+ throws IOException {
this.manager = manager;
this.peerClusterId = peerClusterId;
this.metrics = metrics;
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
index 48d8924..2194000 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSource.java
@@ -32,28 +32,28 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.replication.regionserver.Replication;
-import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager;
-import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.wal.WALKeyImpl;
-import org.apache.hadoop.hbase.wal.WALProvider;
-import org.apache.hadoop.hbase.wal.WALFactory;
-import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint;
+import org.apache.hadoop.hbase.replication.regionserver.Replication;
import org.apache.hadoop.hbase.replication.regionserver.ReplicationSource;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.hadoop.hbase.wal.WALEdit;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hbase.wal.WALKeyImpl;
+import org.apache.hadoop.hbase.wal.WALProvider;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -159,15 +159,14 @@ public class TestReplicationSource {
}
};
replicationEndpoint.start();
- ReplicationPeers mockPeers = Mockito.mock(ReplicationPeers.class);
ReplicationPeer mockPeer = Mockito.mock(ReplicationPeer.class);
Mockito.when(mockPeer.getPeerBandwidth()).thenReturn(0L);
Configuration testConf = HBaseConfiguration.create();
testConf.setInt("replication.source.maxretriesmultiplier", 1);
ReplicationSourceManager manager = Mockito.mock(ReplicationSourceManager.class);
Mockito.when(manager.getTotalBufferUsed()).thenReturn(new AtomicLong());
- source.init(testConf, null, manager, null, mockPeers, null, "testPeer", null,
- replicationEndpoint, p -> OptionalLong.empty(), null);
+ source.init(testConf, null, manager, null, mockPeer, null, "testPeer", null,
+ p -> OptionalLong.empty(), null);
ExecutorService executor = Executors.newSingleThreadExecutor();
Future<?> future = executor.submit(new Runnable() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e560998e/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index dd56a76..f4d3901 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -62,8 +62,8 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
-import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
+import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
@@ -712,9 +712,9 @@ public abstract class TestReplicationSourceManager {
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage rq, ReplicationPeers rp, Server server, String peerClusterId,
- UUID clusterId, ReplicationEndpoint replicationEndpoint,
- WALFileLengthProvider walFileLengthProvider, MetricsSource metrics) throws IOException {
+ ReplicationQueueStorage rq, ReplicationPeer rp, Server server, String peerClusterId,
+ UUID clusterId, WALFileLengthProvider walFileLengthProvider, MetricsSource metrics)
+ throws IOException {
throw new IOException("Failing deliberately");
}
}
[47/50] [abbrv] hbase git commit: HBASE-19711
TestReplicationAdmin.testConcurrentPeerOperations hangs
Posted by zh...@apache.org.
HBASE-19711 TestReplicationAdmin.testConcurrentPeerOperations hangs
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e4687c90
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e4687c90
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e4687c90
Branch: refs/heads/HBASE-19397-branch-2
Commit: e4687c9055b93e4e5001c54276f39c0f72cced00
Parents: b0e88ed
Author: Guanghao Zhang <zg...@apache.org>
Authored: Fri Jan 5 15:39:06 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:20:38 2018 +0800
----------------------------------------------------------------------
.../procedure/MasterProcedureScheduler.java | 23 ++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e4687c90/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index 4ecb3b1..0400de4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -402,7 +402,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
@Override
public void completionCleanup(final Procedure proc) {
if (proc instanceof TableProcedureInterface) {
- TableProcedureInterface iProcTable = (TableProcedureInterface)proc;
+ TableProcedureInterface iProcTable = (TableProcedureInterface) proc;
boolean tableDeleted;
if (proc.hasException()) {
Exception procEx = proc.getException().unwrapRemoteException();
@@ -423,9 +423,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
}
} else if (proc instanceof PeerProcedureInterface) {
PeerProcedureInterface iProcPeer = (PeerProcedureInterface) proc;
- if (iProcPeer.getPeerOperationType() == PeerOperationType.REMOVE) {
- removePeerQueue(iProcPeer.getPeerId());
- }
+ tryCleanupPeerQueue(iProcPeer.getPeerId(), proc);
} else {
// No cleanup for ServerProcedureInterface types, yet.
return;
@@ -514,6 +512,23 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
locking.removePeerLock(peerId);
}
+ private void tryCleanupPeerQueue(String peerId, Procedure procedure) {
+ schedLock();
+ try {
+ PeerQueue queue = AvlTree.get(peerMap, peerId, PEER_QUEUE_KEY_COMPARATOR);
+ if (queue == null) {
+ return;
+ }
+
+ final LockAndQueue lock = locking.getPeerLock(peerId);
+ if (queue.isEmpty() && lock.tryExclusiveLock(procedure)) {
+ removeFromRunQueue(peerRunQueue, queue);
+ removePeerQueue(peerId);
+ }
+ } finally {
+ schedUnlock();
+ }
+ }
private static boolean isPeerProcedure(Procedure<?> proc) {
return proc instanceof PeerProcedureInterface;
[16/50] [abbrv] hbase git commit: HBASE-19524 Master side changes for
moving peer modification from zk watcher to procedure
Posted by zh...@apache.org.
HBASE-19524 Master side changes for moving peer modification from zk watcher to procedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e8dbb5dd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e8dbb5dd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e8dbb5dd
Branch: refs/heads/HBASE-19397-branch-2
Commit: e8dbb5ddba8fbaae6065e5bde88b742d22579d9a
Parents: 41e57c3
Author: zhangduo <zh...@apache.org>
Authored: Mon Dec 18 15:22:36 2017 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Jan 23 18:18:00 2018 +0800
----------------------------------------------------------------------
.../procedure2/RemoteProcedureDispatcher.java | 3 +-
.../src/main/protobuf/MasterProcedure.proto | 21 ++++-
.../src/main/protobuf/RegionServerStatus.proto | 3 +-
.../src/main/protobuf/Replication.proto | 5 +
.../replication/ReplicationPeersZKImpl.java | 4 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 98 ++++++++------------
.../hadoop/hbase/master/MasterRpcServices.java | 4 +-
.../hadoop/hbase/master/MasterServices.java | 26 +++---
.../assignment/RegionTransitionProcedure.java | 13 +--
.../master/procedure/MasterProcedureEnv.java | 5 +
.../master/procedure/ProcedurePrepareLatch.java | 2 +-
.../master/replication/AddPeerProcedure.java | 97 +++++++++++++++++++
.../replication/DisablePeerProcedure.java | 70 ++++++++++++++
.../master/replication/EnablePeerProcedure.java | 69 ++++++++++++++
.../master/replication/ModifyPeerProcedure.java | 97 ++++++++++++++++---
.../master/replication/RefreshPeerCallable.java | 67 -------------
.../replication/RefreshPeerProcedure.java | 28 ++++--
.../master/replication/RemovePeerProcedure.java | 69 ++++++++++++++
.../master/replication/ReplicationManager.java | 76 +++++++--------
.../replication/UpdatePeerConfigProcedure.java | 92 ++++++++++++++++++
.../hbase/regionserver/HRegionServer.java | 5 +-
.../regionserver/RefreshPeerCallable.java | 70 ++++++++++++++
.../hbase/master/MockNoopMasterServices.java | 23 +++--
.../replication/DummyModifyPeerProcedure.java | 13 ++-
24 files changed, 735 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
index 735e899..e4c57a5 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.java
@@ -247,9 +247,8 @@ public abstract class RemoteProcedureDispatcher<TEnv, TRemote extends Comparable
/**
* Called when RS tells the remote procedure is failed through the {@code reportProcedureDone}
* method.
- * @param error the error message
*/
- void remoteOperationFailed(TEnv env, String error);
+ void remoteOperationFailed(TEnv env, RemoteProcedureException error);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 0e2bdba..ae676ea 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -27,6 +27,7 @@ option optimize_for = SPEED;
import "HBase.proto";
import "RPC.proto";
import "Snapshot.proto";
+import "Replication.proto";
// ============================================================================
// WARNING - Compatibility rules
@@ -367,9 +368,10 @@ message GCMergedRegionsStateData {
}
enum PeerModificationState {
- UPDATE_PEER_STORAGE = 1;
- REFRESH_PEER_ON_RS = 2;
- POST_PEER_MODIFICATION = 3;
+ PRE_PEER_MODIFICATION = 1;
+ UPDATE_PEER_STORAGE = 2;
+ REFRESH_PEER_ON_RS = 3;
+ POST_PEER_MODIFICATION = 4;
}
message PeerModificationStateData {
@@ -394,4 +396,17 @@ message RefreshPeerParameter {
required string peer_id = 1;
required PeerModificationType type = 2;
required ServerName target_server = 3;
+}
+
+message ModifyPeerStateData {
+ required string peer_id = 1;
+}
+
+message AddPeerStateData {
+ required ReplicationPeer peer_config = 1;
+ required bool enabled = 2;
+}
+
+message UpdatePeerConfigStateData {
+ required ReplicationPeer peer_config = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index eb396ac..4f75941 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -28,6 +28,7 @@ option optimize_for = SPEED;
import "HBase.proto";
import "ClusterStatus.proto";
+import "ErrorHandling.proto";
message RegionServerStartupRequest {
/** Port number this regionserver is up on */
@@ -152,7 +153,7 @@ message ReportProcedureDoneRequest {
ERROR = 2;
}
required Status status = 2;
- optional string error = 3;
+ optional ForeignExceptionMessage error = 3;
}
message ReportProcedureDoneResponse {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-protocol-shaded/src/main/protobuf/Replication.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Replication.proto b/hbase-protocol-shaded/src/main/protobuf/Replication.proto
index 8657c25..9f7b4c2 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Replication.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Replication.proto
@@ -84,6 +84,7 @@ message AddReplicationPeerRequest {
}
message AddReplicationPeerResponse {
+ optional uint64 proc_id = 1;
}
message RemoveReplicationPeerRequest {
@@ -91,6 +92,7 @@ message RemoveReplicationPeerRequest {
}
message RemoveReplicationPeerResponse {
+ optional uint64 proc_id = 1;
}
message EnableReplicationPeerRequest {
@@ -98,6 +100,7 @@ message EnableReplicationPeerRequest {
}
message EnableReplicationPeerResponse {
+ optional uint64 proc_id = 1;
}
message DisableReplicationPeerRequest {
@@ -105,6 +108,7 @@ message DisableReplicationPeerRequest {
}
message DisableReplicationPeerResponse {
+ optional uint64 proc_id = 1;
}
message GetReplicationPeerConfigRequest {
@@ -122,6 +126,7 @@ message UpdateReplicationPeerConfigRequest {
}
message UpdateReplicationPeerConfigResponse {
+ optional uint64 proc_id = 1;
}
message ListReplicationPeersRequest {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 4e469ed..e4b1875 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -530,7 +530,7 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
for (String queueId : queueIds) {
ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
if (queueInfo.getPeerId().equals(peerId)) {
- throw new ReplicationException("undeleted queue for peerId: " + peerId
+ throw new IllegalArgumentException("undeleted queue for peerId: " + peerId
+ ", replicator: " + replicator + ", queueId: " + queueId);
}
}
@@ -538,7 +538,7 @@ public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements Re
// Check for hfile-refs queue
if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
&& queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
- throw new ReplicationException("Undeleted queue for peerId: " + peerId
+ throw new IllegalArgumentException("Undeleted queue for peerId: " + peerId
+ ", found in hfile-refs node path " + hfileRefsZNode);
}
} catch (KeeperException e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 5f182ff..da3464a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -128,7 +128,13 @@ import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.master.procedure.RecoverMetaProcedure;
import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
+import org.apache.hadoop.hbase.master.replication.AddPeerProcedure;
+import org.apache.hadoop.hbase.master.replication.DisablePeerProcedure;
+import org.apache.hadoop.hbase.master.replication.EnablePeerProcedure;
+import org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure;
+import org.apache.hadoop.hbase.master.replication.RemovePeerProcedure;
import org.apache.hadoop.hbase.master.replication.ReplicationManager;
+import org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
@@ -141,6 +147,7 @@ import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureException;
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import org.apache.hadoop.hbase.quotas.MasterSpaceQuotaObserver;
@@ -169,7 +176,6 @@ import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CompressionTest;
import org.apache.hadoop.hbase.util.EncryptionTest;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.IdLock;
@@ -198,6 +204,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionServerInfo;
@@ -330,15 +337,15 @@ public class HMaster extends HRegionServer implements MasterServices {
private volatile boolean activeMaster = false;
// flag set after we complete initialization once active
- private final ProcedureEvent initialized = new ProcedureEvent("master initialized");
+ private final ProcedureEvent<?> initialized = new ProcedureEvent<>("master initialized");
// flag set after master services are started,
// initialization may have not completed yet.
volatile boolean serviceStarted = false;
// flag set after we complete assignMeta.
- private final ProcedureEvent serverCrashProcessingEnabled =
- new ProcedureEvent("server crash processing");
+ private final ProcedureEvent<?> serverCrashProcessingEnabled =
+ new ProcedureEvent<>("server crash processing");
// Maximum time we should run balancer for
private final int maxBlancingTime;
@@ -1219,7 +1226,6 @@ public class HMaster extends HRegionServer implements MasterServices {
private void startProcedureExecutor() throws IOException {
final MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
- final Path rootDir = FSUtils.getRootDir(conf);
procedureStore = new WALProcedureStore(conf,
new MasterProcedureEnv.WALStoreLeaseRecovery(this));
@@ -2326,11 +2332,8 @@ public class HMaster extends HRegionServer implements MasterServices {
return true;
}
Pair<RegionInfo, ServerName> pair =
- new Pair(MetaTableAccessor.getRegionInfo(data),
+ new Pair<>(MetaTableAccessor.getRegionInfo(data),
MetaTableAccessor.getServerName(data,0));
- if (pair == null) {
- return false;
- }
if (!pair.getFirst().getTable().equals(tableName)) {
return false;
}
@@ -2779,7 +2782,7 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
- public ProcedureEvent getInitializedEvent() {
+ public ProcedureEvent<?> getInitializedEvent() {
return initialized;
}
@@ -2798,7 +2801,7 @@ public class HMaster extends HRegionServer implements MasterServices {
procedureExecutor.getEnvironment().setEventReady(serverCrashProcessingEnabled, b);
}
- public ProcedureEvent getServerCrashProcessingEnabledEvent() {
+ public ProcedureEvent<?> getServerCrashProcessingEnabledEvent() {
return serverCrashProcessingEnabled;
}
@@ -3349,54 +3352,36 @@ public class HMaster extends HRegionServer implements MasterServices {
return favoredNodesManager;
}
+ private long executePeerProcedure(ModifyPeerProcedure procedure) throws IOException {
+ long procId = procedureExecutor.submitProcedure(procedure);
+ procedure.getLatch().await();
+ return procId;
+ }
+
@Override
- public void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
+ public long addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
throws ReplicationException, IOException {
- if (cpHost != null) {
- cpHost.preAddReplicationPeer(peerId, peerConfig);
- }
- LOG.info(getClientIdAuditPrefix() + " creating replication peer, id=" + peerId + ", config="
- + peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"));
- this.replicationManager.addReplicationPeer(peerId, peerConfig, enabled);
- if (cpHost != null) {
- cpHost.postAddReplicationPeer(peerId, peerConfig);
- }
+ LOG.info(getClientIdAuditPrefix() + " creating replication peer, id=" + peerId + ", config=" +
+ peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"));
+ return executePeerProcedure(new AddPeerProcedure(peerId, peerConfig, enabled));
}
@Override
- public void removeReplicationPeer(String peerId) throws ReplicationException, IOException {
- if (cpHost != null) {
- cpHost.preRemoveReplicationPeer(peerId);
- }
+ public long removeReplicationPeer(String peerId) throws ReplicationException, IOException {
LOG.info(getClientIdAuditPrefix() + " removing replication peer, id=" + peerId);
- this.replicationManager.removeReplicationPeer(peerId);
- if (cpHost != null) {
- cpHost.postRemoveReplicationPeer(peerId);
- }
+ return executePeerProcedure(new RemovePeerProcedure(peerId));
}
@Override
- public void enableReplicationPeer(String peerId) throws ReplicationException, IOException {
- if (cpHost != null) {
- cpHost.preEnableReplicationPeer(peerId);
- }
+ public long enableReplicationPeer(String peerId) throws ReplicationException, IOException {
LOG.info(getClientIdAuditPrefix() + " enable replication peer, id=" + peerId);
- this.replicationManager.enableReplicationPeer(peerId);
- if (cpHost != null) {
- cpHost.postEnableReplicationPeer(peerId);
- }
+ return executePeerProcedure(new EnablePeerProcedure(peerId));
}
@Override
- public void disableReplicationPeer(String peerId) throws ReplicationException, IOException {
- if (cpHost != null) {
- cpHost.preDisableReplicationPeer(peerId);
- }
+ public long disableReplicationPeer(String peerId) throws ReplicationException, IOException {
LOG.info(getClientIdAuditPrefix() + " disable replication peer, id=" + peerId);
- this.replicationManager.disableReplicationPeer(peerId);
- if (cpHost != null) {
- cpHost.postDisableReplicationPeer(peerId);
- }
+ return executePeerProcedure(new DisablePeerProcedure(peerId));
}
@Override
@@ -3415,17 +3400,11 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
- public void updateReplicationPeerConfig(String peerId, ReplicationPeerConfig peerConfig)
+ public long updateReplicationPeerConfig(String peerId, ReplicationPeerConfig peerConfig)
throws ReplicationException, IOException {
- if (cpHost != null) {
- cpHost.preUpdateReplicationPeerConfig(peerId, peerConfig);
- }
- LOG.info(getClientIdAuditPrefix() + " update replication peer config, id=" + peerId
- + ", config=" + peerConfig);
- this.replicationManager.updatePeerConfig(peerId, peerConfig);
- if (cpHost != null) {
- cpHost.postUpdateReplicationPeerConfig(peerId, peerConfig);
- }
+ LOG.info(getClientIdAuditPrefix() + " update replication peer config, id=" + peerId +
+ ", config=" + peerConfig);
+ return executePeerProcedure(new UpdatePeerConfigProcedure(peerId, peerConfig));
}
@Override
@@ -3578,10 +3557,15 @@ public class HMaster extends HRegionServer implements MasterServices {
}
}
- public void remoteProcedureFailed(long procId, String error) {
+ public void remoteProcedureFailed(long procId, RemoteProcedureException error) {
RemoteProcedure<MasterProcedureEnv, ?> procedure = getRemoteProcedure(procId);
if (procedure != null) {
procedure.remoteOperationFailed(procedureExecutor.getEnvironment(), error);
}
}
-}
\ No newline at end of file
+
+ @Override
+ public ReplicationManager getReplicationManager() {
+ return replicationManager;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index f875e20..8025a51 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hbase.procedure2.LockedResource;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureException;
import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
@@ -2256,7 +2257,8 @@ public class MasterRpcServices extends RSRpcServices
if (request.getStatus() == ReportProcedureDoneRequest.Status.SUCCESS) {
master.remoteProcedureCompleted(request.getProcId());
} else {
- master.remoteProcedureFailed(request.getProcId(), request.getError());
+ master.remoteProcedureFailed(request.getProcId(),
+ RemoteProcedureException.fromProto(request.getError()));
}
return ReportProcedureDoneResponse.getDefaultInstance();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index 9786fde..e798455 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,10 +17,11 @@
*/
package org.apache.hadoop.hbase.master;
+import com.google.protobuf.Service;
+
import java.io.IOException;
import java.util.List;
-import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.locking.LockManager;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.replication.ReplicationManager;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
import org.apache.hadoop.hbase.procedure2.LockedResource;
@@ -52,8 +53,6 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.Service;
-
/**
* A curated subset of services provided by {@link HMaster}.
* For use internally only. Passed to Managers, Services and Chores so can pass less-than-a
@@ -136,7 +135,7 @@ public interface MasterServices extends Server {
* @return Tripped when Master has finished initialization.
*/
@VisibleForTesting
- public ProcedureEvent getInitializedEvent();
+ public ProcedureEvent<?> getInitializedEvent();
/**
* @return Master's instance of {@link MetricsMaster}
@@ -430,26 +429,26 @@ public interface MasterServices extends Server {
* @param peerConfig configuration for the replication slave cluster
* @param enabled peer state, true if ENABLED and false if DISABLED
*/
- void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
+ long addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
throws ReplicationException, IOException;
/**
* Removes a peer and stops the replication
* @param peerId a short name that identifies the peer
*/
- void removeReplicationPeer(String peerId) throws ReplicationException, IOException;
+ long removeReplicationPeer(String peerId) throws ReplicationException, IOException;
/**
* Restart the replication stream to the specified peer
* @param peerId a short name that identifies the peer
*/
- void enableReplicationPeer(String peerId) throws ReplicationException, IOException;
+ long enableReplicationPeer(String peerId) throws ReplicationException, IOException;
/**
* Stop the replication stream to the specified peer
* @param peerId a short name that identifies the peer
*/
- void disableReplicationPeer(String peerId) throws ReplicationException, IOException;
+ long disableReplicationPeer(String peerId) throws ReplicationException, IOException;
/**
* Returns the configured ReplicationPeerConfig for the specified peer
@@ -460,11 +459,16 @@ public interface MasterServices extends Server {
IOException;
/**
+ * Returns the {@link ReplicationManager}.
+ */
+ ReplicationManager getReplicationManager();
+
+ /**
* Update the peerConfig for the specified peer
* @param peerId a short name that identifies the peer
* @param peerConfig new config for the peer
*/
- void updateReplicationPeerConfig(String peerId, ReplicationPeerConfig peerConfig)
+ long updateReplicationPeerConfig(String peerId, ReplicationPeerConfig peerConfig)
throws ReplicationException, IOException;
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index 04dccc4..1724a38 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -16,12 +16,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.master.assignment;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
-
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -33,13 +31,16 @@ import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperation;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionTransitionState;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureException;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionTransitionState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
+
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
/**
* Base class for the Assign and Unassign Procedure.
*
@@ -415,7 +416,7 @@ public abstract class RegionTransitionProcedure
}
@Override
- public void remoteOperationFailed(MasterProcedureEnv env, String error) {
+ public void remoteOperationFailed(MasterProcedureEnv env, RemoteProcedureException error) {
// should not be called for region operation until we modified the open/close region procedure
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
index 0a4c97d..fa4d371 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
+import org.apache.hadoop.hbase.master.replication.ReplicationManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
@@ -137,6 +138,10 @@ public class MasterProcedureEnv implements ConfigurationObserver {
return remoteDispatcher;
}
+ public ReplicationManager getReplicationManager() {
+ return master.getReplicationManager();
+ }
+
public boolean isRunning() {
if (this.master == null || this.master.getMasterProcedureExecutor() == null) return false;
return master.getMasterProcedureExecutor().isRunning();
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java
index 535f288..f572cef 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedurePrepareLatch.java
@@ -64,7 +64,7 @@ public abstract class ProcedurePrepareLatch {
protected abstract void countDown(final Procedure proc);
public abstract void await() throws IOException;
- protected static void releaseLatch(final ProcedurePrepareLatch latch, final Procedure proc) {
+ public static void releaseLatch(final ProcedurePrepareLatch latch, final Procedure proc) {
if (latch != null) {
latch.countDown(proc);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
new file mode 100644
index 0000000..c3862d8
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AddPeerStateData;
+
+/**
+ * The procedure for adding a new replication peer.
+ */
+@InterfaceAudience.Private
+public class AddPeerProcedure extends ModifyPeerProcedure {
+
+ private static final Log LOG = LogFactory.getLog(AddPeerProcedure.class);
+
+ private ReplicationPeerConfig peerConfig;
+
+ private boolean enabled;
+
+ public AddPeerProcedure() {
+ }
+
+ public AddPeerProcedure(String peerId, ReplicationPeerConfig peerConfig, boolean enabled) {
+ super(peerId);
+ this.peerConfig = peerConfig;
+ this.enabled = enabled;
+ }
+
+ @Override
+ public PeerOperationType getPeerOperationType() {
+ return PeerOperationType.ADD;
+ }
+
+ @Override
+ protected void prePeerModification(MasterProcedureEnv env) throws IOException {
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.preAddReplicationPeer(peerId, peerConfig);
+ }
+ }
+
+ @Override
+ protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
+ env.getReplicationManager().addReplicationPeer(peerId, peerConfig, enabled);
+ }
+
+ @Override
+ protected void postPeerModification(MasterProcedureEnv env) throws IOException {
+ LOG.info("Successfully added " + (enabled ? "ENABLED" : "DISABLED") + " peer " + peerId +
+ ", config " + peerConfig);
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ env.getMasterCoprocessorHost().postAddReplicationPeer(peerId, peerConfig);
+ }
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.serializeStateData(serializer);
+ serializer.serialize(AddPeerStateData.newBuilder()
+ .setPeerConfig(ReplicationPeerConfigUtil.convert(peerConfig)).setEnabled(enabled).build());
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.deserializeStateData(serializer);
+ AddPeerStateData data = serializer.deserialize(AddPeerStateData.class);
+ peerConfig = ReplicationPeerConfigUtil.convert(data.getPeerConfig());
+ enabled = data.getEnabled();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
new file mode 100644
index 0000000..0b32db9
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/DisablePeerProcedure.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * The procedure for disabling a replication peer.
+ */
+@InterfaceAudience.Private
+public class DisablePeerProcedure extends ModifyPeerProcedure {
+
+ private static final Log LOG = LogFactory.getLog(DisablePeerProcedure.class);
+
+ public DisablePeerProcedure() {
+ }
+
+ public DisablePeerProcedure(String peerId) {
+ super(peerId);
+ }
+
+ @Override
+ public PeerOperationType getPeerOperationType() {
+ return PeerOperationType.DISABLE;
+ }
+
+ @Override
+ protected void prePeerModification(MasterProcedureEnv env) throws IOException {
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.preDisableReplicationPeer(peerId);
+ }
+ }
+
+ @Override
+ protected void updatePeerStorage(MasterProcedureEnv env)
+ throws IllegalArgumentException, Exception {
+ env.getReplicationManager().disableReplicationPeer(peerId);
+ }
+
+ @Override
+ protected void postPeerModification(MasterProcedureEnv env) throws IOException {
+ LOG.info("Successfully disabled peer " + peerId);
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.postDisableReplicationPeer(peerId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
new file mode 100644
index 0000000..92ba000
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/EnablePeerProcedure.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * The procedure for enabling a replication peer.
+ */
+@InterfaceAudience.Private
+public class EnablePeerProcedure extends ModifyPeerProcedure {
+
+ private static final Log LOG = LogFactory.getLog(EnablePeerProcedure.class);
+
+ public EnablePeerProcedure() {
+ }
+
+ public EnablePeerProcedure(String peerId) {
+ super(peerId);
+ }
+
+ @Override
+ public PeerOperationType getPeerOperationType() {
+ return PeerOperationType.ENABLE;
+ }
+
+ @Override
+ protected void prePeerModification(MasterProcedureEnv env) throws IOException {
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.preEnableReplicationPeer(peerId);
+ }
+ }
+
+ @Override
+ protected void updatePeerStorage(MasterProcedureEnv env) throws Exception {
+ env.getReplicationManager().enableReplicationPeer(peerId);
+ }
+
+ @Override
+ protected void postPeerModification(MasterProcedureEnv env) throws IOException {
+ LOG.info("Successfully enabled peer " + peerId);
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.postEnableReplicationPeer(peerId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index fca05a7..7076bab 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -21,15 +21,22 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
+import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.ModifyPeerStateData;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationState;
+/**
+ * The base class for all replication peer related procedure.
+ */
@InterfaceAudience.Private
public abstract class ModifyPeerProcedure
extends StateMachineProcedure<MasterProcedureEnv, PeerModificationState>
@@ -39,11 +46,21 @@ public abstract class ModifyPeerProcedure
protected String peerId;
+ // used to keep compatible with old client where we can only returns after updateStorage.
+ protected ProcedurePrepareLatch latch;
+
protected ModifyPeerProcedure() {
}
protected ModifyPeerProcedure(String peerId) {
this.peerId = peerId;
+ // TODO: temporarily set a 4.0 here to always wait for the procedure exection completed. Change
+ // to 3.0 or 2.0 after the client modification is done.
+ this.latch = ProcedurePrepareLatch.createLatch(4, 0);
+ }
+
+ public ProcedurePrepareLatch getLatch() {
+ return latch;
}
@Override
@@ -52,28 +69,58 @@ public abstract class ModifyPeerProcedure
}
/**
- * Return {@code false} means that the operation is invalid and we should give up, otherwise
- * {@code true}.
+ * Called before we start the actual processing. If an exception is thrown then we will give up
+ * and mark the procedure as failed directly.
+ */
+ protected abstract void prePeerModification(MasterProcedureEnv env) throws IOException;
+
+ /**
+ * We will give up and mark the procedure as failure if {@link IllegalArgumentException} is
+ * thrown, for other type of Exception we will retry.
+ */
+ protected abstract void updatePeerStorage(MasterProcedureEnv env)
+ throws IllegalArgumentException, Exception;
+
+ /**
+ * Called before we finish the procedure. The implementation can do some logging work, and also
+ * call the coprocessor hook if any.
* <p>
- * You need to call {@link #setFailure(String, Throwable)} to give the detail failure information.
+ * Notice that, since we have already done the actual work, throwing exception here will not fail
+ * this procedure, we will just ignore it and finish the procedure as suceeded.
*/
- protected abstract boolean updatePeerStorage() throws IOException;
+ protected abstract void postPeerModification(MasterProcedureEnv env) throws IOException;
- protected void postPeerModification() {
+ private void releaseLatch() {
+ ProcedurePrepareLatch.releaseLatch(latch, this);
}
@Override
protected Flow executeFromState(MasterProcedureEnv env, PeerModificationState state)
throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
switch (state) {
- case UPDATE_PEER_STORAGE:
+ case PRE_PEER_MODIFICATION:
try {
- if (!updatePeerStorage()) {
- assert isFailed() : "setFailure is not called";
- return Flow.NO_MORE_STATE;
- }
+ prePeerModification(env);
} catch (IOException e) {
- LOG.warn("update peer storage failed, retry", e);
+ LOG.warn(getClass().getName() + " failed to call prePeerModification for peer " + peerId +
+ ", mark the procedure as failure and give up", e);
+ setFailure("prePeerModification", e);
+ releaseLatch();
+ return Flow.NO_MORE_STATE;
+ }
+ setNextState(PeerModificationState.UPDATE_PEER_STORAGE);
+ return Flow.HAS_MORE_STATE;
+ case UPDATE_PEER_STORAGE:
+ try {
+ updatePeerStorage(env);
+ } catch (IllegalArgumentException e) {
+ setFailure("master-" + getPeerOperationType().name().toLowerCase() + "-peer",
+ new DoNotRetryIOException(e));
+ releaseLatch();
+ return Flow.NO_MORE_STATE;
+ } catch (Exception e) {
+ LOG.warn(
+ getClass().getName() + " update peer storage for peer " + peerId + " failed, retry", e);
throw new ProcedureYieldException();
}
setNextState(PeerModificationState.REFRESH_PEER_ON_RS);
@@ -85,7 +132,13 @@ public abstract class ModifyPeerProcedure
setNextState(PeerModificationState.POST_PEER_MODIFICATION);
return Flow.HAS_MORE_STATE;
case POST_PEER_MODIFICATION:
- postPeerModification();
+ try {
+ postPeerModification(env);
+ } catch (IOException e) {
+ LOG.warn(getClass().getName() + " failed to call prePeerModification for peer " + peerId +
+ ", ignore since the procedure has already done", e);
+ }
+ releaseLatch();
return Flow.NO_MORE_STATE;
default:
throw new UnsupportedOperationException("unhandled state=" + state);
@@ -107,6 +160,12 @@ public abstract class ModifyPeerProcedure
@Override
protected void rollbackState(MasterProcedureEnv env, PeerModificationState state)
throws IOException, InterruptedException {
+ if (state == PeerModificationState.PRE_PEER_MODIFICATION ||
+ state == PeerModificationState.UPDATE_PEER_STORAGE) {
+ // actually the peer related operations has no rollback, but if we haven't done any
+ // modifications on the peer storage, we can just return.
+ return;
+ }
throw new UnsupportedOperationException();
}
@@ -122,6 +181,18 @@ public abstract class ModifyPeerProcedure
@Override
protected PeerModificationState getInitialState() {
- return PeerModificationState.UPDATE_PEER_STORAGE;
+ return PeerModificationState.PRE_PEER_MODIFICATION;
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.serializeStateData(serializer);
+ serializer.serialize(ModifyPeerStateData.newBuilder().setPeerId(peerId).build());
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.deserializeStateData(serializer);
+ peerId = serializer.deserialize(ModifyPeerStateData.class).getPeerId();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerCallable.java
deleted file mode 100644
index 4e09107..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerCallable.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.replication;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.executor.EventType;
-import org.apache.hadoop.hbase.procedure2.RSProcedureCallable;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.yetus.audience.InterfaceAudience;
-
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshPeerParameter;
-
-/**
- * The callable executed at RS side to refresh the peer config/state.
- * <p>
- * TODO: only a dummy implementation for verifying the framework, will add implementation later.
- */
-@InterfaceAudience.Private
-public class RefreshPeerCallable implements RSProcedureCallable {
-
- private HRegionServer rs;
-
- private String peerId;
-
- private Exception initError;
-
- @Override
- public Void call() throws Exception {
- if (initError != null) {
- throw initError;
- }
- rs.getFileSystem().create(new Path("/" + peerId + "/" + rs.getServerName().toString())).close();
- return null;
- }
-
- @Override
- public void init(byte[] parameter, HRegionServer rs) {
- this.rs = rs;
- try {
- this.peerId = RefreshPeerParameter.parseFrom(parameter).getPeerId();
- } catch (InvalidProtocolBufferException e) {
- initError = e;
- return;
- }
- }
-
- @Override
- public EventType getEventType() {
- return EventType.RS_REFRESH_PEER;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
index 18da487..ddc2401 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RefreshPeerProcedure.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperation;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
+import org.apache.hadoop.hbase.procedure2.RemoteProcedureException;
+import org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
@@ -118,15 +120,22 @@ public class RefreshPeerProcedure extends Procedure<MasterProcedureEnv>
.setTargetServer(ProtobufUtil.toServerName(remote)).build().toByteArray());
}
- private void complete(MasterProcedureEnv env, boolean succ) {
+ private void complete(MasterProcedureEnv env, Throwable error) {
if (event == null) {
LOG.warn("procedure event for " + getProcId() +
- " is null, maybe the procedure is created when recovery", new Exception());
+ " is null, maybe the procedure is created when recovery",
+ new Exception());
return;
}
- LOG.info("Refresh peer " + peerId + " for " + type + " on " + targetServer +
- (succ ? " suceeded" : " failed"));
- this.succ = succ;
+ if (error != null) {
+ LOG.warn("Refresh peer " + peerId + " for " + type + " on " + targetServer + " failed",
+ error);
+ this.succ = false;
+ } else {
+ LOG.info("Refresh peer " + peerId + " for " + type + " on " + targetServer + " suceeded");
+ this.succ = true;
+ }
+
event.wake(env.getProcedureScheduler());
event = null;
}
@@ -134,17 +143,18 @@ public class RefreshPeerProcedure extends Procedure<MasterProcedureEnv>
@Override
public synchronized void remoteCallFailed(MasterProcedureEnv env, ServerName remote,
IOException exception) {
- complete(env, false);
+ complete(env, exception);
}
@Override
public synchronized void remoteOperationCompleted(MasterProcedureEnv env) {
- complete(env, true);
+ complete(env, null);
}
@Override
- public synchronized void remoteOperationFailed(MasterProcedureEnv env, String error) {
- complete(env, false);
+ public synchronized void remoteOperationFailed(MasterProcedureEnv env,
+ RemoteProcedureException error) {
+ complete(env, error);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
new file mode 100644
index 0000000..3daad6d
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * The procedure for removing a replication peer.
+ */
+@InterfaceAudience.Private
+public class RemovePeerProcedure extends ModifyPeerProcedure {
+
+ private static final Log LOG = LogFactory.getLog(RemovePeerProcedure.class);
+
+ public RemovePeerProcedure() {
+ }
+
+ public RemovePeerProcedure(String peerId) {
+ super(peerId);
+ }
+
+ @Override
+ public PeerOperationType getPeerOperationType() {
+ return PeerOperationType.REMOVE;
+ }
+
+ @Override
+ protected void prePeerModification(MasterProcedureEnv env) throws IOException {
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.preRemoveReplicationPeer(peerId);
+ }
+ }
+
+ @Override
+ protected void updatePeerStorage(MasterProcedureEnv env) throws Exception {
+ env.getReplicationManager().removeReplicationPeer(peerId);
+ }
+
+ @Override
+ protected void postPeerModification(MasterProcedureEnv env) throws IOException {
+ LOG.info("Successfully removed peer " + peerId);
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.postRemoveReplicationPeer(peerId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
index f36b2e2..b6f8784 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationManager.java
@@ -27,10 +27,8 @@ import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
@@ -39,24 +37,21 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
/**
* Manages and performs all replication admin operations.
+ * <p>
* Used to add/remove a replication peer.
*/
@InterfaceAudience.Private
public class ReplicationManager {
-
- private final Configuration conf;
- private final ZKWatcher zkw;
private final ReplicationQueuesClient replicationQueuesClient;
private final ReplicationPeers replicationPeers;
public ReplicationManager(Configuration conf, ZKWatcher zkw, Abortable abortable)
throws IOException {
- this.conf = conf;
- this.zkw = zkw;
try {
this.replicationQueuesClient = ReplicationFactory
.getReplicationQueuesClient(new ReplicationQueuesClientArguments(conf, abortable, zkw));
@@ -70,7 +65,7 @@ public class ReplicationManager {
}
public void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
- throws ReplicationException, IOException {
+ throws ReplicationException {
checkPeerConfig(peerConfig);
replicationPeers.registerPeer(peerId, peerConfig, enabled);
replicationPeers.peerConnected(peerId);
@@ -89,8 +84,8 @@ public class ReplicationManager {
this.replicationPeers.disablePeer(peerId);
}
- public ReplicationPeerConfig getPeerConfig(String peerId) throws ReplicationException,
- ReplicationPeerNotFoundException {
+ public ReplicationPeerConfig getPeerConfig(String peerId)
+ throws ReplicationException, ReplicationPeerNotFoundException {
ReplicationPeerConfig peerConfig = replicationPeers.getReplicationPeerConfig(peerId);
if (peerConfig == null) {
throw new ReplicationPeerNotFoundException(peerId);
@@ -110,9 +105,9 @@ public class ReplicationManager {
List<String> peerIds = replicationPeers.getAllPeerIds();
for (String peerId : peerIds) {
if (pattern == null || (pattern != null && pattern.matcher(peerId).matches())) {
- peers.add(new ReplicationPeerDescription(peerId, replicationPeers
- .getStatusOfPeerFromBackingStore(peerId), replicationPeers
- .getReplicationPeerConfig(peerId)));
+ peers.add(new ReplicationPeerDescription(peerId,
+ replicationPeers.getStatusOfPeerFromBackingStore(peerId),
+ replicationPeers.getReplicationPeerConfig(peerId)));
}
}
return peers;
@@ -126,13 +121,12 @@ public class ReplicationManager {
* If replicate_all flag is false, it means all user tables can't be replicated to peer cluster.
* Then allow to config namespaces or table-cfs which will be replicated to peer cluster.
*/
- private void checkPeerConfig(ReplicationPeerConfig peerConfig)
- throws ReplicationException, IOException {
+ private void checkPeerConfig(ReplicationPeerConfig peerConfig) {
if (peerConfig.replicateAllUserTables()) {
- if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty())
- || (peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) {
- throw new ReplicationException("Need clean namespaces or table-cfs config firstly"
- + " when replicate_all flag is true");
+ if ((peerConfig.getNamespaces() != null && !peerConfig.getNamespaces().isEmpty()) ||
+ (peerConfig.getTableCFsMap() != null && !peerConfig.getTableCFsMap().isEmpty())) {
+ throw new IllegalArgumentException("Need clean namespaces or table-cfs config firstly " +
+ "when you want replicate all cluster");
}
checkNamespacesAndTableCfsConfigConflict(peerConfig.getExcludeNamespaces(),
peerConfig.getExcludeTableCFsMap());
@@ -141,7 +135,7 @@ public class ReplicationManager {
&& !peerConfig.getExcludeNamespaces().isEmpty())
|| (peerConfig.getExcludeTableCFsMap() != null
&& !peerConfig.getExcludeTableCFsMap().isEmpty())) {
- throw new ReplicationException(
+ throw new IllegalArgumentException(
"Need clean exclude-namespaces or exclude-table-cfs config firstly"
+ " when replicate_all flag is false");
}
@@ -154,20 +148,24 @@ public class ReplicationManager {
/**
* Set a namespace in the peer config means that all tables in this namespace will be replicated
* to the peer cluster.
- * 1. If peer config already has a namespace, then not allow set any table of this namespace
- * to the peer config.
- * 2. If peer config already has a table, then not allow set this table's namespace to the peer
- * config.
- *
+ * <ol>
+ * <li>If peer config already has a namespace, then not allow set any table of this namespace to
+ * the peer config.</li>
+ * <li>If peer config already has a table, then not allow set this table's namespace to the peer
+ * config.</li>
+ * </ol>
+ * <p>
* Set a exclude namespace in the peer config means that all tables in this namespace can't be
* replicated to the peer cluster.
- * 1. If peer config already has a exclude namespace, then not allow set any exclude table of
- * this namespace to the peer config.
- * 2. If peer config already has a exclude table, then not allow set this table's namespace
- * as a exclude namespace.
+ * <ol>
+ * <li>If peer config already has a exclude namespace, then not allow set any exclude table of
+ * this namespace to the peer config.</li>
+ * <li>If peer config already has a exclude table, then not allow set this table's namespace as a
+ * exclude namespace.</li>
+ * </ol>
*/
private void checkNamespacesAndTableCfsConfigConflict(Set<String> namespaces,
- Map<TableName, ? extends Collection<String>> tableCfs) throws ReplicationException {
+ Map<TableName, ? extends Collection<String>> tableCfs) {
if (namespaces == null || namespaces.isEmpty()) {
return;
}
@@ -177,24 +175,22 @@ public class ReplicationManager {
for (Map.Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
TableName table = entry.getKey();
if (namespaces.contains(table.getNamespaceAsString())) {
- throw new ReplicationException("Table-cfs " + table + " is conflict with namespaces "
+ throw new IllegalArgumentException("Table-cfs " + table + " is conflict with namespaces "
+ table.getNamespaceAsString() + " in peer config");
}
}
}
- private void checkConfiguredWALEntryFilters(ReplicationPeerConfig peerConfig)
- throws IOException {
- String filterCSV = peerConfig.getConfiguration().
- get(BaseReplicationEndpoint.REPLICATION_WALENTRYFILTER_CONFIG_KEY);
- if (filterCSV != null && !filterCSV.isEmpty()){
- String [] filters = filterCSV.split(",");
+ private void checkConfiguredWALEntryFilters(ReplicationPeerConfig peerConfig) {
+ String filterCSV = peerConfig.getConfiguration()
+ .get(BaseReplicationEndpoint.REPLICATION_WALENTRYFILTER_CONFIG_KEY);
+ if (filterCSV != null && !filterCSV.isEmpty()) {
+ String[] filters = filterCSV.split(",");
for (String filter : filters) {
try {
- Class clazz = Class.forName(filter);
- Object o = clazz.newInstance();
+ Class.forName(filter).newInstance();
} catch (Exception e) {
- throw new DoNotRetryIOException("Configured WALEntryFilter " + filter +
+ throw new IllegalArgumentException("Configured WALEntryFilter " + filter +
" could not be created. Failing add/update " + "peer operation.", e);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
new file mode 100644
index 0000000..435eefc
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/UpdatePeerConfigProcedure.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.UpdatePeerConfigStateData;
+
+/**
+ * The procedure for updating the config for a replication peer.
+ */
+@InterfaceAudience.Private
+public class UpdatePeerConfigProcedure extends ModifyPeerProcedure {
+
+ private static final Log LOG = LogFactory.getLog(UpdatePeerConfigProcedure.class);
+
+ private ReplicationPeerConfig peerConfig;
+
+ public UpdatePeerConfigProcedure() {
+ }
+
+ public UpdatePeerConfigProcedure(String peerId, ReplicationPeerConfig peerConfig) {
+ super(peerId);
+ this.peerConfig = peerConfig;
+ }
+
+ @Override
+ public PeerOperationType getPeerOperationType() {
+ return PeerOperationType.UPDATE_CONFIG;
+ }
+
+ @Override
+ protected void prePeerModification(MasterProcedureEnv env) throws IOException {
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.preUpdateReplicationPeerConfig(peerId, peerConfig);
+ }
+ }
+
+ @Override
+ protected void updatePeerStorage(MasterProcedureEnv env)
+ throws IllegalArgumentException, Exception {
+ env.getReplicationManager().updatePeerConfig(peerId, peerConfig);
+ }
+
+ @Override
+ protected void postPeerModification(MasterProcedureEnv env) throws IOException {
+ LOG.info("Successfully updated peer config of " + peerId + " to " + peerConfig);
+ MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.postUpdateReplicationPeerConfig(peerId, peerConfig);
+ }
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.serializeStateData(serializer);
+ serializer.serialize(UpdatePeerConfigStateData.newBuilder()
+ .setPeerConfig(ReplicationPeerConfigUtil.convert(peerConfig)).build());
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.deserializeStateData(serializer);
+ peerConfig = ReplicationPeerConfigUtil
+ .convert(serializer.deserialize(UpdatePeerConfigStateData.class).getPeerConfig());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b439f38..3c8ec17 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -146,6 +146,7 @@ import org.apache.hadoop.hbase.util.CompressionTest;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.hadoop.hbase.util.HasThread;
import org.apache.hadoop.hbase.util.JvmPauseMonitor;
import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig;
@@ -3716,7 +3717,7 @@ public class HRegionServer extends HasThread implements
ReportProcedureDoneRequest.newBuilder().setProcId(procId);
if (error != null) {
builder.setStatus(ReportProcedureDoneRequest.Status.ERROR)
- .setError(Throwables.getStackTraceAsString(error));
+ .setError(ForeignExceptionUtil.toProtoForeignException(serverName.toString(), error));
} else {
builder.setStatus(ReportProcedureDoneRequest.Status.SUCCESS);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
new file mode 100644
index 0000000..a47a483
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RefreshPeerCallable.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.executor.EventType;
+import org.apache.hadoop.hbase.procedure2.RSProcedureCallable;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RefreshPeerParameter;
+
+/**
+ * The callable executed at RS side to refresh the peer config/state.
+ * <p>
+ * TODO: only a dummy implementation for verifying the framework, will add implementation later.
+ */
+@InterfaceAudience.Private
+public class RefreshPeerCallable implements RSProcedureCallable {
+
+ private HRegionServer rs;
+
+ private String peerId;
+
+ private Exception initError;
+
+ @Override
+ public Void call() throws Exception {
+ if (initError != null) {
+ throw initError;
+ }
+ Path dir = new Path("/" + peerId);
+ if (rs.getFileSystem().exists(dir)) {
+ rs.getFileSystem().create(new Path(dir, rs.getServerName().toString())).close();
+ }
+ return null;
+ }
+
+ @Override
+ public void init(byte[] parameter, HRegionServer rs) {
+ this.rs = rs;
+ try {
+ this.peerId = RefreshPeerParameter.parseFrom(parameter).getPeerId();
+ } catch (InvalidProtocolBufferException e) {
+ initError = e;
+ return;
+ }
+ }
+
+ @Override
+ public EventType getEventType() {
+ return EventType.RS_REFRESH_PEER;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index be91aa0..e88710e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.locking.LockManager;
import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.replication.ReplicationManager;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
import org.apache.hadoop.hbase.procedure2.LockedResource;
@@ -368,7 +369,6 @@ public class MockNoopMasterServices implements MasterServices {
@Override
public ClusterConnection getClusterConnection() {
- // TODO Auto-generated method stub
return null;
}
@@ -398,20 +398,24 @@ public class MockNoopMasterServices implements MasterServices {
}
@Override
- public void addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
+ public long addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
throws ReplicationException {
+ return 0;
}
@Override
- public void removeReplicationPeer(String peerId) throws ReplicationException {
+ public long removeReplicationPeer(String peerId) throws ReplicationException {
+ return 0;
}
@Override
- public void enableReplicationPeer(String peerId) throws ReplicationException, IOException {
+ public long enableReplicationPeer(String peerId) throws ReplicationException, IOException {
+ return 0;
}
@Override
- public void disableReplicationPeer(String peerId) throws ReplicationException, IOException {
+ public long disableReplicationPeer(String peerId) throws ReplicationException, IOException {
+ return 0;
}
@Override
@@ -421,8 +425,9 @@ public class MockNoopMasterServices implements MasterServices {
}
@Override
- public void updateReplicationPeerConfig(String peerId, ReplicationPeerConfig peerConfig)
+ public long updateReplicationPeerConfig(String peerId, ReplicationPeerConfig peerConfig)
throws ReplicationException, IOException {
+ return 0;
}
@Override
@@ -457,7 +462,6 @@ public class MockNoopMasterServices implements MasterServices {
@Override
public ProcedureEvent getInitializedEvent() {
- // TODO Auto-generated method stub
return null;
}
@@ -470,4 +474,9 @@ public class MockNoopMasterServices implements MasterServices {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public ReplicationManager getReplicationManager() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8dbb5dd/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
index 44343d7..ed7c6fa 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/DummyModifyPeerProcedure.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hbase.master.replication;
-import java.io.IOException;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
public class DummyModifyPeerProcedure extends ModifyPeerProcedure {
@@ -34,8 +34,15 @@ public class DummyModifyPeerProcedure extends ModifyPeerProcedure {
}
@Override
- protected boolean updatePeerStorage() throws IOException {
- return true;
+ protected void prePeerModification(MasterProcedureEnv env) {
+ }
+
+ @Override
+ protected void updatePeerStorage(MasterProcedureEnv env) {
+ }
+
+ @Override
+ protected void postPeerModification(MasterProcedureEnv env) {
}
}